starred/shelfmark
1
0
Fork 0
mirror of https://github.com/calibrain/shelfmark.git synced 2026-09-24 11:10:19 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Add new python tooling + apply ruff linter cleanup (#845)

Browse Source 
- Adds `uv`, `ruff`, `pyright`, `vulture` and `pytest-xdist`
- Move project, lockfile, docker build etc to uv
- Align python tooling on 3.14
- Huge bulk of ruff linter fixes applied. Still in progress but all the
core types are now enforced
- Update CI and test helpers
This commit is contained in:
Alex
2026-04-10 13:03:25 +01:00
committed by GitHub
parent ff094bed56
commit 3a3a3ce449
144 changed files with 10639 additions and 6424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/dependabot.yml
+1 -1
View File
@@ -1,7 +1,7 @@
version: 2
updates:
# Python dependencies
- package-ecosystem: "pip"
- package-ecosystem: "uv"
directory: "/"
schedule:
interval: "weekly"
.github/workflows/ci.yml
+14 -10
View File
@@ -14,20 +14,24 @@ jobs:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- name: Install uv and Python
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
with:
python-version: "3.10"
cache: "pip"
version: "0.11.3"
python-version: "3.14"
enable-cache: true
- name: Install dependencies
run: |
pip install -r requirements-base.txt
pip install -r requirements-shelfmark.txt
pip install pytest
- name: Sync dependencies
run: uv sync --locked --extra browser
- name: Lint backend
run: uv run ruff check shelfmark
- name: Check backend formatting
run: uv run ruff format --check shelfmark
- name: Run tests
run: pytest tests/ -x --tb=short
run: uv run pytest tests/ -x --tb=short
frontend-checks:
runs-on: ubuntu-latest
Dockerfile
+11 -12
View File
@@ -27,6 +27,8 @@ RUN npm run build
# Use python-slim as the base image
FROM python:3.14-slim AS base
COPY --from=ghcr.io/astral-sh/uv:0.11.3 /uv /uvx /bin/
# Add build argument for version
ARG BUILD_VERSION
ENV BUILD_VERSION=${BUILD_VERSION}
@@ -39,13 +41,12 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
# Consistent environment variables grouped together
ENV DEBIAN_FRONTEND=noninteractive \
DOCKERMODE=true \
UV_LINK_MODE=copy \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONIOENCODING=UTF-8 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_DEFAULT_TIMEOUT=100 \
NAME=Shelfmark \
PATH=/app/.venv/bin:$PATH \
PYTHONPATH=/app \
# PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed
LANG=en_US.UTF-8 \
@@ -91,12 +92,10 @@ RUN apt-get update && \
# Set working directory
WORKDIR /app
# Install Python dependencies using pip
# Copying requirements files separately leverages build cache
# Cache mount persists pip cache between builds for faster installs
COPY requirements-base.txt requirements-shelfmark.txt ./
RUN --mount=type=cache,target=/root/.cache/pip \
pip install -r requirements-base.txt
# Install core Python dependencies first for better layer caching
COPY pyproject.toml uv.lock ./
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --locked --no-default-groups
# Copy application code *after* dependencies are installed
COPY . .
@@ -146,9 +145,9 @@ RUN apt-get update && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Install additional dependencies (requirements file already copied in base stage)
RUN --mount=type=cache,target=/root/.cache/pip \
pip install -r requirements-shelfmark.txt
# Install the browser automation stack used by the full image
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --locked --no-default-groups --extra browser
# Grant read/execute permissions to others
RUN chmod -R o+rx /usr/bin/chromium
Makefile
+28 -1
View File
@@ -1,4 +1,4 @@
.PHONY: help install dev build preview typecheck frontend-test clean up down docker-build refresh restart build-serve
.PHONY: help install install-python-dev dev build preview typecheck frontend-test clean up up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-check
# Frontend directory
FRONTEND_DIR := src/frontend
@@ -18,6 +18,11 @@ help:
@echo " preview - Preview production build"
@echo " typecheck - Run TypeScript type checking"
@echo " frontend-test - Run frontend unit tests"
@echo " install-python-dev - Sync Python runtime + dev tooling with uv"
@echo " python-lint - Run Ruff against Python backend code"
@echo " python-lint-fix - Run Ruff with safe auto-fixes"
@echo " python-format - Format Python backend code with Ruff"
@echo " python-format-check - Check Python backend formatting with Ruff"
@echo " clean - Remove node_modules and build artifacts"
@echo ""
@echo "Backend (Docker):"
@@ -32,6 +37,11 @@ install:
@echo "Installing frontend dependencies..."
cd $(FRONTEND_DIR) && npm install
# Install Python development dependencies
install-python-dev:
@echo "Syncing Python runtime and dev tooling with uv..."
uv sync --locked --extra browser
# Start development server
dev:
@echo "Starting development server..."
@@ -59,6 +69,23 @@ typecheck:
@echo "Running TypeScript type checking..."
cd $(FRONTEND_DIR) && npm run typecheck
# Python linting
python-lint:
@echo "Running Ruff..."
uv run ruff check shelfmark
python-lint-fix:
@echo "Running Ruff with safe auto-fixes..."
uv run ruff check shelfmark --fix
python-format:
@echo "Formatting Python backend code with Ruff..."
uv run ruff format shelfmark
python-format-check:
@echo "Checking Python backend formatting with Ruff..."
uv run ruff format --check shelfmark
# Run frontend unit tests
frontend-test:
@echo "Running frontend unit tests..."
docker-compose.test-clients.yml
+1
View File
@@ -54,6 +54,7 @@ services:
# Mount tests for running pytest in container
- ./tests:/app/tests:ro
- ./pyproject.toml:/app/pyproject.toml:ro
- ./uv.lock:/app/uv.lock:ro
# Mount client configs for integration tests to read credentials
- ./.local/test-clients/qbittorrent/config:/qbittorrent-config:ro
- ./.local/test-clients/sabnzbd/config:/sabnzbd-config:ro
entrypoint.sh
+8 -3
View File
@@ -68,6 +68,11 @@ else
fi
set -e
PYTHON_BIN="/app/.venv/bin/python"
if [ ! -x "$PYTHON_BIN" ]; then
PYTHON_BIN="python3"
fi
# Print build version
echo "Build version: $BUILD_VERSION"
echo "Release version: $RELEASE_VERSION"
@@ -294,7 +299,7 @@ if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
# Keep SeleniumBase's bundled drivers directory writable as well for
# compatibility with legacy UC code paths that still probe bundled assets.
set +e
SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
SELENIUMBASE_DRIVERS_DIR=$("$PYTHON_BIN" -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
set -e
if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
@@ -317,7 +322,7 @@ make_writable "${INGEST_DIR:-/books}" root
# Check any additional configured destination roots from saved settings
echo "Checking for additional configured destination roots..."
if [ -f /app/scripts/fix_permissions.py ]; then
configured_dirs=$(python3 /app/scripts/fix_permissions.py 2>/dev/null || echo "")
configured_dirs=$("$PYTHON_BIN" /app/scripts/fix_permissions.py 2>/dev/null || echo "")
if [ -n "$configured_dirs" ]; then
echo "$configured_dirs" | while read -r dir; do
if [ -n "$dir" ] && [ -d "$dir" ]; then
@@ -372,7 +377,7 @@ if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then
set -x
echo "vvvvvvvvvvvv DEBUG MODE vvvvvvvvvvvv"
echo "Starting Xvfb for debugging"
python3 -c "from pyvirtualdisplay import Display; Display(visible=False, size=(1440,1880)).start()"
"$PYTHON_BIN" -c "from pyvirtualdisplay import Display; Display(visible=False, size=(1440,1880)).start()"
id
free -h
uname -a
pyproject.toml
+87 -6
View File
@@ -2,7 +2,44 @@
name = "shelfmark"
version = "0.1.0"
description = "Shelfmark - Book Downloader"
requires-python = ">=3.10"
requires-python = ">=3.14"
dependencies = [
"flask",
"flask-cors",
"flask-socketio",
"python-socketio",
"requests[socks]",
"defusedxml",
"beautifulsoup4",
"tqdm",
"dnspython",
"gunicorn",
"gevent",
"gevent-websocket",
"psutil",
"emoji",
"rarfile",
"qbittorrent-api",
"transmission-rpc",
"authlib>=1.6.6,<1.7",
"apprise>=1.9.0",
]
[project.optional-dependencies]
browser = [
"pyvirtualdisplay",
"pyautogui",
"seleniumbase==4.47.9",
"python-xlib",
]
[dependency-groups]
dev = [
"pyright>=1.1.408",
"pytest",
"pytest-xdist>=3.8.0",
"ruff==0.15.9",
]
[tool.pytest.ini_options]
testpaths = ["tests"]
@@ -12,6 +49,8 @@ python_functions = ["test_*"]
addopts = [
"-v",
"--tb=short",
"-n",
"auto",
]
markers = [
"integration: marks tests that require running services (deselect with '-m \"not integration\"')",
@@ -19,8 +58,50 @@ markers = [
"e2e: marks end-to-end tests that require the full application stack",
]
[tool.mypy]
python_version = "3.10"
warn_return_any = true
warn_unused_ignores = true
ignore_missing_imports = true
[tool.ruff]
line-length = 100
extend-exclude = [".local"]
[tool.ruff.lint]
select = [
"F", "I", "UP", "B", "C4", "SIM", "PTH", "RET", "PIE", "FURB", "PERF", "TRY",
"ANN001", "ANN201", "ANN202", "ANN204",
"E731",
"FBT002", "FBT003",
"G003", "G004",
"PLR1714",
"PLW0108",
"Q000",
"RUF005", "RUF012", "RUF013", "RUF059", "RUF100",
"TC001", "TC003",
]
ignore = ["UP035", "TRY003", "E501"]
[tool.pyright]
include = ["shelfmark"]
exclude = [".local", "tests", "**/__pycache__", "**/node_modules"]
pythonVersion = "3.14"
typeCheckingMode = "off"
[tool.vulture]
paths = ["shelfmark"]
exclude = [".local", "tests"]
ignore_decorators = [
"@app.route",
"@app.before_request",
"@app.after_request",
"@app.errorhandler",
"@socketio.on",
"@register_provider",
"@register_provider_kwargs",
"@register_settings",
"@register_source",
"@register_handler",
"@register_client",
"@register_output",
]
min_confidence = 90
sort_by_size = true
[tool.uv]
package = false
requirements-base.txt
-19
View File
@@ -1,19 +0,0 @@
flask
flask-cors
flask-socketio
python-socketio
requests[socks]
defusedxml
beautifulsoup4
tqdm
dnspython
gunicorn
gevent
gevent-websocket
psutil
emoji
rarfile
qbittorrent-api
transmission-rpc
authlib>=1.6.6,<1.7
apprise>=1.9.0
requirements-shelfmark.txt
-4
View File
@@ -1,4 +0,0 @@
pyvirtualdisplay
pyautogui
seleniumbase==4.47.9
python-xlib
scripts/test_clients.py
+4 -4
View File
@@ -9,7 +9,7 @@ Usage:
2. Wait for containers to initialize (first run takes ~30s)
3. Run this script to verify clients are accessible:
python scripts/test_clients.py
uv run python scripts/test_clients.py
4. Access cwabd at http://localhost:8084
- Go to Settings > Prowlarr > Download Clients
@@ -26,7 +26,7 @@ Web UIs:
- rTorrent: http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
Prerequisites (for running this script locally):
pip install requests transmission-rpc qbittorrent-api
uv sync --locked
First-Time Setup:
qBittorrent:
@@ -260,7 +260,7 @@ def test_qbittorrent():
except ImportError:
print(" ERROR: qbittorrent-api not installed")
print(" Run: pip install qbittorrent-api")
print(" Run: uv sync --locked")
return False
except Exception as e:
print(f" ERROR: {e}")
@@ -317,7 +317,7 @@ def test_transmission():
except ImportError:
print(" ERROR: transmission-rpc not installed")
print(" Run: pip install transmission-rpc")
print(" Run: uv sync --locked")
return False
except Exception as e:
print(f" ERROR: {e}")
shelfmark/__main__.py
+1 -1
View File
@@ -1,8 +1,8 @@
"""Package entry point for `python -m shelfmark`."""
from shelfmark.main import app, socketio
from shelfmark.config.env import FLASK_HOST, FLASK_PORT
from shelfmark.core.config import config
from shelfmark.main import app, socketio
if __name__ == "__main__":
socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=config.get("DEBUG", False))
shelfmark/api/websocket.py
+75 -124
View File
@@ -2,123 +2,71 @@
import logging
import threading
from typing import Optional, Dict, Any, Callable, List
from typing import TYPE_CHECKING, Any
from flask import Flask
from flask_socketio import SocketIO, join_room, leave_room
if TYPE_CHECKING:
from collections.abc import Callable
logger = logging.getLogger(__name__)
class WebSocketManager:
"""Manages WebSocket connections and broadcasts."""
def __init__(self):
self.socketio: Optional[SocketIO] = None
def __init__(self) -> None:
self.socketio: SocketIO | None = None
self._enabled = False
self._connection_count = 0
self._connection_lock = threading.Lock()
self._on_first_connect_callbacks: List[Callable[[], None]] = []
self._on_all_disconnect_callbacks: List[Callable[[], None]] = []
self._needs_rewarm = False # Flag to trigger warmup callbacks on next connect
self._user_rooms: Dict[str, int] = {} # room_name -> ref count
self._sid_rooms: Dict[str, str] = {} # sid -> room_name
self._user_rooms: dict[str, int] = {} # room_name -> ref count
self._sid_rooms: dict[str, str] = {} # sid -> room_name
self._rooms_lock = threading.Lock()
self._queue_status_fn: Optional[Callable] = None # Reference to queue_status()
self._queue_status_fn: Callable | None = None # Reference to queue_status()
def init_app(self, app, socketio: SocketIO):
def init_app(self, app: Flask, socketio: SocketIO) -> None:
"""Initialize the WebSocket manager with Flask-SocketIO instance."""
self.socketio = socketio
self._enabled = True
logger.info("WebSocket manager initialized")
def register_on_first_connect(self, callback: Callable[[], None]):
"""Register a callback for when the first client connects."""
self._on_first_connect_callbacks.append(callback)
logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
def register_on_all_disconnect(self, callback: Callable[[], None]):
"""Register a callback for when all clients disconnect."""
self._on_all_disconnect_callbacks.append(callback)
logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
def request_warmup_on_next_connect(self):
"""Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
with self._connection_lock:
self._needs_rewarm = True
logger.debug("Warmup requested for next client connect")
def client_connected(self):
def client_connected(self) -> None:
"""Track a new client connection. Call this from the connect event handler."""
with self._connection_lock:
was_zero = self._connection_count == 0
needs_rewarm = self._needs_rewarm
self._connection_count += 1
current_count = self._connection_count
# Clear rewarm flag if we're going to trigger warmup
if was_zero or needs_rewarm:
self._needs_rewarm = False
logger.debug(f"Client connected. Active connections: {current_count}")
logger.debug("Client connected. Active connections: %s", current_count)
# Trigger warmup callbacks if this is the first connection OR if rewarm was requested
# (rewarm is requested when bypasser shuts down due to idle while clients are connected)
if was_zero or needs_rewarm:
reason = "First client connected" if was_zero else "Rewarm requested after idle shutdown"
logger.info(f"{reason}, triggering warmup callbacks...")
for callback in self._on_first_connect_callbacks:
try:
# Run callbacks in a separate thread to not block the connection
thread = threading.Thread(target=callback, daemon=True)
thread.start()
except Exception as e:
logger.error(f"Error in on_first_connect callback {callback.__name__}: {e}")
def client_disconnected(self):
def client_disconnected(self) -> None:
"""Track a client disconnection. Call this from the disconnect event handler."""
with self._connection_lock:
self._connection_count = max(0, self._connection_count - 1)
current_count = self._connection_count
is_now_zero = current_count == 0
logger.debug(f"Client disconnected. Active connections: {current_count}")
# If all clients have disconnected, trigger cleanup callbacks
if is_now_zero:
logger.info("All clients disconnected, triggering disconnect callbacks...")
for callback in self._on_all_disconnect_callbacks:
try:
callback()
except Exception as e:
logger.error(f"Error in on_all_disconnect callback {callback.__name__}: {e}")
def get_connection_count(self) -> int:
"""Get the current number of active WebSocket connections."""
with self._connection_lock:
return self._connection_count
def has_active_connections(self) -> bool:
"""Check if there are any active WebSocket connections."""
return self.get_connection_count() > 0
logger.debug("Client disconnected. Active connections: %s", current_count)
def is_enabled(self) -> bool:
"""Check if WebSocket is enabled and ready."""
return self._enabled and self.socketio is not None
def set_queue_status_fn(self, fn: Callable):
def set_queue_status_fn(self, fn: Callable) -> None:
"""Set the queue_status function reference for per-room filtering."""
self._queue_status_fn = fn
def _increment_user_room_locked(self, room: str):
def _increment_user_room_locked(self, room: str) -> None:
self._user_rooms[room] = self._user_rooms.get(room, 0) + 1
def _decrement_user_room_locked(self, room: str):
def _decrement_user_room_locked(self, room: str) -> None:
count = self._user_rooms.get(room, 1) - 1
if count <= 0:
self._user_rooms.pop(room, None)
else:
self._user_rooms[room] = count
def _set_sid_room_locked(self, sid: str, room: Optional[str]):
def _set_sid_room_locked(self, sid: str, room: str | None) -> None:
current_room = self._sid_rooms.get(sid)
if current_room == room:
return
@@ -135,9 +83,14 @@ class WebSocketManager:
if room.startswith("user_"):
self._increment_user_room_locked(room)
def sync_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
def sync_user_room(
self,
sid: str,
is_admin: bool,
db_user_id: int | None = None,
) -> None:
"""Ensure a SID is in exactly one room matching the current session scope."""
room: Optional[str] = None
room: str | None = None
if is_admin:
room = "admins"
elif db_user_id is not None:
@@ -146,24 +99,35 @@ class WebSocketManager:
with self._rooms_lock:
self._set_sid_room_locked(sid, room)
def join_user_room(self, sid: str, is_admin: bool, db_user_id: Optional[int] = None):
def join_user_room(
self,
sid: str,
is_admin: bool,
db_user_id: int | None = None,
) -> None:
"""Join the appropriate room based on user role."""
self.sync_user_room(sid, is_admin, db_user_id)
self.sync_user_room(sid, is_admin=is_admin, db_user_id=db_user_id)
def leave_user_room(self, sid: str, is_admin: bool = False, db_user_id: Optional[int] = None):
def leave_user_room(
self,
sid: str,
*,
is_admin: bool = False,
db_user_id: int | None = None,
) -> None:
"""Leave whichever room the SID currently belongs to."""
del is_admin, db_user_id # Backward-compatible signature; routing is SID-based.
with self._rooms_lock:
self._set_sid_room_locked(sid, None)
def broadcast_status_update(self, status_data: Dict[str, Any]):
def broadcast_status_update(self, status_data: dict[str, Any]) -> None:
"""Broadcast status update to all connected clients, filtered by user room."""
if not self.is_enabled():
return
try:
# Admins (and no-auth users) get full status
self.socketio.emit('status_update', status_data, to="admins")
self.socketio.emit("status_update", status_data, to="admins")
# Each user room gets filtered status
with self._rooms_lock:
@@ -171,56 +135,43 @@ class WebSocketManager:
if active_rooms and self._queue_status_fn:
for room in active_rooms:
try:
# Extract user_id from room name "user_123"
uid = int(room.split("_", 1)[1])
filtered = self._queue_status_fn(user_id=uid)
self.socketio.emit('status_update', filtered, to=room)
except Exception as e:
logger.error(f"Failed to send status update for room {room}: {e}")
self._broadcast_status_update_to_room(room)
logger.debug("Broadcasted status update to all rooms")
except Exception as e:
logger.error(f"Error broadcasting status update: {e}")
except Exception:
logger.exception("Error broadcasting status update")
def broadcast_download_progress(self, book_id: str, progress: float, status: str, user_id: Optional[int] = None):
def _broadcast_status_update_to_room(self, room: str) -> None:
"""Broadcast status update to one user room."""
try:
# Extract user_id from room name "user_123"
uid = int(room.split("_", 1)[1])
filtered = self._queue_status_fn(user_id=uid) if self._queue_status_fn else None
if filtered is not None:
self.socketio.emit("status_update", filtered, to=room)
except Exception:
logger.exception("Failed to send status update for room %s", room)
def broadcast_download_progress(
self, book_id: str, progress: float, status: str, user_id: int | None = None
) -> None:
"""Broadcast download progress update for a specific book."""
if not self.is_enabled():
return
try:
data = {
'book_id': book_id,
'progress': progress,
'status': status
}
data = {"book_id": book_id, "progress": progress, "status": status}
# Admins always see all progress
self.socketio.emit('download_progress', data, to="admins")
self.socketio.emit("download_progress", data, to="admins")
# If task belongs to a specific user, send to their room too
if user_id is not None:
room = f"user_{user_id}"
with self._rooms_lock:
if room in self._user_rooms:
self.socketio.emit('download_progress', data, to=room)
logger.debug(f"Broadcasted progress for book {book_id}: {progress}%")
except Exception as e:
logger.error(f"Error broadcasting download progress: {e}")
def broadcast_notification(self, message: str, notification_type: str = 'info'):
"""Broadcast a notification message to all clients."""
if not self.is_enabled():
return
try:
data = {
'message': message,
'type': notification_type
}
# When calling socketio.emit() outside event handlers, it broadcasts by default
self.socketio.emit('notification', data)
logger.debug(f"Broadcasted notification: {message}")
except Exception as e:
logger.error(f"Error broadcasting notification: {e}")
self.socketio.emit("download_progress", data, to=room)
logger.debug("Broadcasted progress for book %s: %s%%", book_id, progress)
except Exception:
logger.exception("Error broadcasting download progress")
def broadcast_search_status(
self,
@@ -228,23 +179,23 @@ class WebSocketManager:
provider: str,
book_id: str,
message: str,
phase: str = 'searching'
):
phase: str = "searching",
) -> None:
"""Broadcast search status update for a release source search."""
if not self.is_enabled():
return
try:
data = {
'source': source,
'provider': provider,
'book_id': book_id,
'message': message,
'phase': phase,
"source": source,
"provider": provider,
"book_id": book_id,
"message": message,
"phase": phase,
}
self.socketio.emit('search_status', data)
except Exception as e:
logger.error(f"Error broadcasting search status: {e}")
self.socketio.emit("search_status", data)
except Exception:
logger.exception("Error broadcasting search status")
# Global WebSocket manager instance
shelfmark/bypass/external_bypasser.py
+50 -27
View File
@@ -2,8 +2,7 @@
import random
import time
from threading import Event
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING
import requests
@@ -14,6 +13,8 @@ from shelfmark.core.utils import normalize_http_url
from shelfmark.download.network import get_ssl_verify
if TYPE_CHECKING:
from threading import Event
from shelfmark.download import network
logger = setup_logger(__name__)
@@ -29,7 +30,7 @@ BACKOFF_BASE = 1.0
BACKOFF_CAP = 10.0
def _fetch_via_bypasser(target_url: str) -> Optional[str]:
def _fetch_via_bypasser(target_url: str) -> str | None:
"""Make a single request to the external bypasser service. Returns HTML or None."""
raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
@@ -37,7 +38,9 @@ def _fetch_via_bypasser(target_url: str) -> Optional[str]:
bypasser_url = normalize_http_url(raw_bypasser_url)
if not bypasser_url or not bypasser_path:
logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
logger.error(
"External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH."
)
return None
read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
@@ -46,48 +49,63 @@ def _fetch_via_bypasser(target_url: str) -> Optional[str]:
response = requests.post(
f"{bypasser_url}{bypasser_path}",
headers={"Content-Type": "application/json"},
json={"cmd": "request.get", "url": target_url, "maxTimeout": bypasser_timeout},
json={
"cmd": "request.get",
"url": target_url,
"maxTimeout": bypasser_timeout,
},
timeout=(CONNECT_TIMEOUT, read_timeout),
verify=get_ssl_verify(bypasser_url),
)
response.raise_for_status()
result = response.json()
status = result.get('status', 'unknown')
message = result.get('message', '')
logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
status = result.get("status", "unknown")
message = result.get("message", "")
logger.debug("External bypasser response for '%s': %s - %s", target_url, status, message)
if status != 'ok':
logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
if status != "ok":
logger.warning(
"External bypasser failed for '%s': %s - %s",
target_url,
status,
message,
)
return None
solution = result.get('solution')
html = solution.get('response', '') if solution else ''
solution = result.get("solution")
html = solution.get("response", "") if solution else ""
if not html:
logger.warning(f"External bypasser returned empty response for '{target_url}'")
logger.warning("External bypasser returned empty response for '%s'", target_url)
return None
return html
except requests.exceptions.Timeout:
logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
logger.warning(
"External bypasser timed out for '%s' (connect: %ss, read: %.0fs)",
target_url,
CONNECT_TIMEOUT,
read_timeout,
)
except requests.exceptions.RequestException as e:
logger.warning(f"External bypasser request failed for '{target_url}': {e}")
logger.warning("External bypasser request failed for '%s': %s", target_url, e)
except (KeyError, TypeError, ValueError) as e:
logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
logger.warning("External bypasser returned malformed response for '%s': %s", target_url, e)
else:
return html
return None
def _check_cancelled(cancel_flag: Optional[Event], context: str) -> None:
def _check_cancelled(cancel_flag: Event | None, context: str) -> None:
"""Check if operation was cancelled and raise exception if so."""
if cancel_flag and cancel_flag.is_set():
logger.info(f"External bypasser cancelled {context}")
raise BypassCancelledException("Bypass cancelled")
logger.info("External bypasser cancelled %s", context)
msg = "Bypass cancelled"
raise BypassCancelledException(msg)
def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> None:
def _sleep_with_cancellation(seconds: float, cancel_flag: Event | None) -> None:
"""Sleep for the specified duration, checking for cancellation each second."""
for _ in range(int(seconds)):
_check_cancelled(cancel_flag, "during backoff")
@@ -99,9 +117,9 @@ def _sleep_with_cancellation(seconds: float, cancel_flag: Optional[Event]) -> No
def get_bypassed_page(
url: str,
selector: Optional["network.AAMirrorSelector"] = None,
cancel_flag: Optional[Event] = None
) -> Optional[str]:
selector: network.AAMirrorSelector | None = None,
cancel_flag: Event | None = None,
) -> str | None:
"""Fetch HTML via external bypasser with retries and mirror rotation."""
from shelfmark.download import network as network_module
@@ -119,12 +137,17 @@ def get_bypassed_page(
break
delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
logger.info(
"External bypasser attempt %s/%s failed, retrying in %.1fs",
attempt,
MAX_RETRY,
delay,
)
_sleep_with_cancellation(delay, cancel_flag)
new_base, action = sel.next_mirror_or_rotate_dns()
if action in ("mirror", "dns") and new_base:
logger.info(f"Rotated {action} for retry")
logger.info("Rotated %s for retry", action)
return None
shelfmark/bypass/fingerprint.py
+24 -15
View File
@@ -1,34 +1,37 @@
"""Browser fingerprint profile management for bypass stealth."""
import random
from typing import Optional
from shelfmark.core.logger import setup_logger
logger = setup_logger(__name__)
COMMON_RESOLUTIONS = [
(1920, 1080, 0.35),
(1366, 768, 0.18),
(1536, 864, 0.10),
(1440, 900, 0.08),
(1280, 720, 0.07),
(1600, 900, 0.06),
(1280, 800, 0.05),
(2560, 1440, 0.04),
(1680, 1050, 0.04),
(1920, 1200, 0.03),
(1920, 1080, 0.35),
(1366, 768, 0.18),
(1536, 864, 0.10),
(1440, 900, 0.08),
(1280, 720, 0.07),
(1600, 900, 0.06),
(1280, 800, 0.05),
(2560, 1440, 0.04),
(1680, 1050, 0.04),
(1920, 1200, 0.03),
]
# Current screen size (module-level singleton)
_current_screen_size: Optional[tuple[int, int]] = None
_current_screen_size: tuple[int, int] | None = None
def get_screen_size() -> tuple[int, int]:
global _current_screen_size
if _current_screen_size is None:
_current_screen_size = _generate_screen_size()
logger.debug(f"Generated initial screen size: {_current_screen_size[0]}x{_current_screen_size[1]}")
logger.debug(
"Generated initial screen size: %sx%s",
_current_screen_size[0],
_current_screen_size[1],
)
return _current_screen_size
@@ -39,9 +42,15 @@ def rotate_screen_size() -> tuple[int, int]:
width, height = _current_screen_size
if old_size:
logger.info(f"Rotated screen size: {old_size[0]}x{old_size[1]} -> {width}x{height}")
logger.info(
"Rotated screen size: %sx%s -> %sx%s",
old_size[0],
old_size[1],
width,
height,
)
else:
logger.info(f"Generated screen size: {width}x{height}")
logger.info("Generated screen size: %sx%s", width, height)
return _current_screen_size
shelfmark/bypass/internal_bypasser.py
+210 -162
View File
@@ -8,9 +8,11 @@ import subprocess
import threading
import time
import traceback
from contextlib import suppress
from datetime import datetime
from pathlib import Path
from threading import Event
from typing import Any, Optional
from typing import Any
from urllib.parse import urlparse
import requests
@@ -28,8 +30,8 @@ from shelfmark.download.network import get_proxies, get_ssl_verify
logger = setup_logger(__name__)
SELENIUMBASE_RUNTIME_ROOT = "/tmp/shelfmark/seleniumbase"
SELENIUMBASE_DOWNLOADS_DIR = os.path.join(SELENIUMBASE_RUNTIME_ROOT, "downloaded_files")
SELENIUMBASE_RUNTIME_ROOT = Path("/tmp/shelfmark/seleniumbase")
SELENIUMBASE_DOWNLOADS_DIR = SELENIUMBASE_RUNTIME_ROOT / "downloaded_files"
# Challenge detection indicators
CLOUDFLARE_INDICATORS = [
@@ -54,13 +56,14 @@ DISPLAY = {
LOCKED = threading.Lock()
def _describe_runtime_path(path: str) -> str:
def _describe_runtime_path(path: str | Path) -> str:
"""Return compact ownership/mode info for a runtime path."""
try:
path = Path(path)
link_target = ""
if os.path.islink(path):
link_target = f" -> {os.readlink(path)}"
st = os.stat(path)
if path.is_symlink():
link_target = f" -> {path.readlink()}"
st = path.stat()
mode = stat.S_IMODE(st.st_mode)
return f"{path}{link_target} exists uid={st.st_uid} gid={st.st_gid} mode={oct(mode)}"
except FileNotFoundError:
@@ -71,8 +74,8 @@ def _describe_runtime_path(path: str) -> str:
class _CdpWorker:
def __init__(self) -> None:
self._thread: Optional[threading.Thread] = None
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._thread: threading.Thread | None = None
self._loop: asyncio.AbstractEventLoop | None = None
self._ready = threading.Event()
self._lock = threading.Lock()
@@ -107,7 +110,7 @@ class _CdpWorker:
if not self._ready.wait(timeout=10):
raise RuntimeError("CDP worker loop failed to start")
def run(self, coro: Any, timeout: Optional[float] = None) -> Any:
def run(self, coro: Any, timeout: float | None = None) -> Any:
self.start()
if not self._loop or self._loop.is_closed():
raise RuntimeError("CDP worker loop not available")
@@ -126,24 +129,34 @@ _cf_cookies_lock = threading.Lock()
_cf_user_agents: dict[str, str] = {}
# Protection cookie names we care about (Cloudflare and DDoS-Guard)
CF_COOKIE_NAMES = {'cf_clearance', '__cf_bm', 'cf_chl_2', 'cf_chl_prog'}
DDG_COOKIE_NAMES = {'__ddg1_', '__ddg2_', '__ddg5_', '__ddg8_', '__ddg9_', '__ddg10_', '__ddgid_', '__ddgmark_', 'ddg_last_challenge'}
CF_COOKIE_NAMES = {"cf_clearance", "__cf_bm", "cf_chl_2", "cf_chl_prog"}
DDG_COOKIE_NAMES = {
"__ddg1_",
"__ddg2_",
"__ddg5_",
"__ddg8_",
"__ddg9_",
"__ddg10_",
"__ddgid_",
"__ddgmark_",
"ddg_last_challenge",
}
# Domains requiring full session cookies (not just protection cookies)
FULL_COOKIE_DOMAINS = {'z-lib.fm', 'z-lib.gs', 'z-lib.id', 'z-library.sk', 'zlibrary-global.se'}
FULL_COOKIE_DOMAINS = {"z-lib.fm", "z-lib.gs", "z-lib.id", "z-library.sk", "zlibrary-global.se"}
def _get_base_domain(domain: str) -> str:
"""Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
return '.'.join(domain.split('.')[-2:]) if '.' in domain else domain
return ".".join(domain.split(".")[-2:]) if "." in domain else domain
def _should_extract_cookie(name: str, extract_all: bool) -> bool:
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
"""Determine if a cookie should be extracted based on its name."""
if extract_all:
return True
is_cf = name in CF_COOKIE_NAMES or name.startswith('cf_')
is_ddg = name in DDG_COOKIE_NAMES or name.startswith('__ddg')
is_cf = name in CF_COOKIE_NAMES or name.startswith("cf_")
is_ddg = name in DDG_COOKIE_NAMES or name.startswith("__ddg")
return is_cf or is_ddg
@@ -151,7 +164,7 @@ def _store_extracted_cookies(
*,
url: str,
cookies: list[Any],
user_agent: Optional[str] = None,
user_agent: str | None = None,
) -> None:
"""Store filtered bypass cookies (and optional UA) for a URL domain."""
parsed = urlparse(url)
@@ -165,7 +178,7 @@ def _store_extracted_cookies(
cookies_found: dict[str, dict[str, Any]] = {}
for cookie in cookies:
name = getattr(cookie, "name", "") or ""
if not _should_extract_cookie(name, extract_all):
if not _should_extract_cookie(name, extract_all=extract_all):
continue
expires = getattr(cookie, "expires", None)
if expires is not None and expires <= 0:
@@ -186,21 +199,21 @@ def _store_extracted_cookies(
_cf_cookies[base_domain] = cookies_found
if user_agent:
_cf_user_agents[base_domain] = user_agent
logger.debug(f"Stored UA for {base_domain}: {str(user_agent)[:60]}...")
logger.debug("Stored UA for %s: %s...", base_domain, str(user_agent)[:60])
else:
logger.debug(f"No UA captured for {base_domain}")
logger.debug("No UA captured for %s", base_domain)
cookie_type = "all" if extract_all else "protection"
logger.debug(f"Extracted {len(cookies_found)} {cookie_type} cookies for {base_domain}")
logger.debug("Extracted %s %s cookies for %s", len(cookies_found), cookie_type, base_domain)
async def _extract_cookies_from_cdp(driver, page, url: str) -> None:
async def _extract_cookies_from_cdp(driver: Any, page: Any, url: str) -> None:
"""Extract cookies from a CDP browser after successful bypass."""
try:
try:
all_cookies = await driver.cookies.get_all(requests_cookie_format=True)
except Exception as e:
logger.debug(f"Failed to get cookies via CDP: {e}")
logger.debug("Failed to get cookies via CDP: %s", e)
return
try:
@@ -211,7 +224,8 @@ async def _extract_cookies_from_cdp(driver, page, url: str) -> None:
_store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
except Exception as e:
logger.debug(f"Failed to extract cookies: {e}")
logger.debug("Failed to extract cookies: %s", e)
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
"""Get stored cookies for a domain. Returns empty dict if none available."""
@@ -225,17 +239,17 @@ def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
if not cookies:
return {}
cf_clearance = cookies.get('cf_clearance', {})
cf_clearance = cookies.get("cf_clearance", {})
if cf_clearance:
expiry = cf_clearance.get('expiry')
expiry = cf_clearance.get("expiry")
if expiry is None:
expiry = cf_clearance.get('expires')
expiry = cf_clearance.get("expires")
if expiry and expiry > 0 and time.time() > expiry:
logger.debug(f"CF cookies expired for {base_domain}")
logger.debug("CF cookies expired for %s", base_domain)
_cf_cookies.pop(base_domain, None)
return {}
return {name: c['value'] for name, c in cookies.items()}
return {name: c["value"] for name, c in cookies.items()}
def has_valid_cf_cookies(domain: str) -> bool:
@@ -243,7 +257,7 @@ def has_valid_cf_cookies(domain: str) -> bool:
return bool(get_cf_cookies_for_domain(domain))
def get_cf_user_agent_for_domain(domain: str) -> Optional[str]:
def get_cf_user_agent_for_domain(domain: str) -> str | None:
"""Get the User-Agent that was used during bypass for a domain."""
if not domain:
return None
@@ -251,7 +265,7 @@ def get_cf_user_agent_for_domain(domain: str) -> Optional[str]:
return _cf_user_agents.get(_get_base_domain(domain))
def clear_cf_cookies(domain: str = None) -> None:
def clear_cf_cookies(domain: str | None = None) -> None:
"""Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
with _cf_cookies_lock:
if domain:
@@ -279,43 +293,39 @@ def _cleanup_orphan_processes() -> int:
for proc_name in processes_to_kill:
try:
result = subprocess.run(
["pgrep", "-f", proc_name],
capture_output=True,
text=True,
timeout=5
["pgrep", "-f", proc_name], capture_output=True, text=True, timeout=5
)
if result.returncode != 0 or not result.stdout.strip():
continue
pids = result.stdout.strip().split('\n')
pids = result.stdout.strip().split("\n")
count = len(pids)
logger.info(f"Found {count} orphan {proc_name} process(es), killing...")
logger.info("Found %s orphan %s process(es), killing...", count, proc_name)
kill_result = subprocess.run(
["pkill", "-9", "-f", proc_name],
capture_output=True,
timeout=5
["pkill", "-9", "-f", proc_name], capture_output=True, timeout=5
)
if kill_result.returncode == 0:
total_killed += count
else:
logger.warning(f"pkill for {proc_name} returned {kill_result.returncode}")
logger.warning("pkill for %s returned %s", proc_name, kill_result.returncode)
except subprocess.TimeoutExpired:
logger.warning(f"Timeout while checking for {proc_name} processes")
logger.warning("Timeout while checking for %s processes", proc_name)
except Exception as e:
logger.debug(f"Error checking for {proc_name} processes: {e}")
logger.debug("Error checking for %s processes: %s", proc_name, e)
if total_killed > 0:
time.sleep(1)
logger.info(f"Cleaned up {total_killed} orphan process(es)")
logger.info("Cleaned up %s orphan process(es)", total_killed)
logger.log_resource_usage()
else:
logger.debug("No orphan processes found")
return total_killed
async def _get_page_info(page) -> tuple[str, str, str]:
async def _get_page_info(page: Any) -> tuple[str, str, str]:
"""Extract page title, body text, and current URL safely."""
try:
title = (await page.get_title() or "").lower()
@@ -333,30 +343,35 @@ async def _get_page_info(page) -> tuple[str, str, str]:
return title, body, current_url
def _check_indicators(title: str, body: str, indicators: list[str]) -> Optional[str]:
def _check_indicators(title: str, body: str, indicators: list[str]) -> str | None:
"""Check if any indicator is present in title or body. Returns the found indicator or None."""
for indicator in indicators:
if indicator in title or indicator in body:
return indicator
return None
def _has_cloudflare_patterns(body: str, url: str) -> bool:
"""Check for Cloudflare-specific patterns in body or URL."""
return "cf-" in body or "cloudflare" in url.lower() or "/cdn-cgi/" in url
async def _detect_challenge_type(page) -> str:
async def _detect_challenge_type(page: Any) -> str:
"""Detect challenge type: 'cloudflare', 'ddos_guard', or 'none'."""
try:
title, body, current_url = await _get_page_info(page)
except Exception as e:
logger.warning("Error detecting challenge type: %s", e)
return "none"
else:
# DDOS-Guard indicators
if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS):
logger.debug(f"DDOS-Guard indicator found: '{found}'")
logger.debug("DDOS-Guard indicator found: '%s'", found)
return "ddos_guard"
# Cloudflare indicators
if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS):
logger.debug(f"Cloudflare indicator found: '{found}'")
logger.debug("Cloudflare indicator found: '%s'", found)
return "cloudflare"
# Check URL patterns
@@ -364,24 +379,28 @@ async def _detect_challenge_type(page) -> str:
return "cloudflare"
return "none"
except Exception as e:
logger.warning(f"Error detecting challenge type: {e}")
return "none"
async def _is_bypassed(page, escape_emojis: bool = True) -> bool:
async def _is_bypassed(page: Any, *, escape_emojis: bool = True) -> bool:
"""Check if the protection has been bypassed."""
try:
title, body, current_url = await _get_page_info(page)
except Exception as e:
logger.warning("Error checking bypass status: %s", e)
return False
else:
body_len = len(body.strip())
# Long page content = probably bypassed
if body_len > 100000:
logger.debug(f"Page content too long, probably bypassed (len: {body_len})")
logger.debug("Page content too long, probably bypassed (len: %s)", body_len)
return True
# Multiple emojis = probably real content
if escape_emojis:
import emoji
if len(emoji.emoji_list(body)) >= 3:
logger.debug("Detected emojis in page, probably bypassed")
return True
@@ -400,14 +419,11 @@ async def _is_bypassed(page, escape_emojis: bool = True) -> bool:
logger.debug("Page content too short, might still be loading")
return False
logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {body_len}")
logger.debug("Bypass check passed - Title: '%s', Body length: %s", title[:100], body_len)
return True
except Exception as e:
logger.warning(f"Error checking bypass status: {e}")
return False
async def _bypass_method_humanlike(page) -> bool:
async def _bypass_method_humanlike(page: Any) -> bool:
"""Human-like behavior with scroll, wait, and reload."""
try:
logger.debug("Attempting bypass: human-like interaction")
@@ -421,7 +437,7 @@ async def _bypass_method_humanlike(page) -> bool:
await page.wait()
await asyncio.sleep(random.uniform(2, 3))
except Exception as e:
logger.debug(f"Scroll behavior failed: {e}")
logger.debug("Scroll behavior failed: %s", e)
if await _is_bypassed(page):
return True
@@ -437,15 +453,15 @@ async def _bypass_method_humanlike(page) -> bool:
await page.solve_captcha()
await asyncio.sleep(random.uniform(3, 5))
except Exception as e:
logger.debug(f"Final captcha click failed: {e}")
logger.debug("Final captcha click failed: %s", e)
return await _is_bypassed(page)
except Exception as e:
logger.debug(f"Human-like method failed: {e}")
logger.debug("Human-like method failed: %s", e)
return False
async def _bypass_method_cdp_solve(page) -> bool:
async def _bypass_method_cdp_solve(page: Any) -> bool:
"""CDP Mode with solve_captcha() - auto-detects challenge type."""
try:
logger.debug("Attempting bypass: CDP solve_captcha")
@@ -453,21 +469,21 @@ async def _bypass_method_cdp_solve(page) -> bool:
await asyncio.sleep(random.uniform(3, 5))
return await _is_bypassed(page)
except Exception as e:
logger.debug(f"CDP solve_captcha failed: {e}")
logger.debug("CDP solve_captcha failed: %s", e)
return False
CDP_CLICK_SELECTORS = [
"#turnstile-widget div", # Cloudflare Turnstile
"#cf-turnstile div", # Alternative CF Turnstile
"#turnstile-widget div", # Cloudflare Turnstile
"#cf-turnstile div", # Alternative CF Turnstile
"iframe[src*='challenges']", # CF challenge iframe
"input[type='checkbox']", # Generic checkbox (DDOS-Guard)
"[class*='checkbox']", # Class-based checkbox
"#challenge-running", # CF challenge indicator
"input[type='checkbox']", # Generic checkbox (DDOS-Guard)
"[class*='checkbox']", # Class-based checkbox
"#challenge-running", # CF challenge indicator
]
async def _bypass_method_cdp_click(page) -> bool:
async def _bypass_method_cdp_click(page: Any) -> bool:
"""CDP Mode with native clicking - no PyAutoGUI dependency."""
try:
logger.debug("Attempting bypass: CDP native click")
@@ -477,31 +493,31 @@ async def _bypass_method_cdp_click(page) -> bool:
if not await page.is_element_visible(selector):
continue
logger.debug(f"CDP clicking: {selector}")
logger.debug("CDP clicking: %s", selector)
await page.click(selector)
await asyncio.sleep(random.uniform(2, 4))
if await _is_bypassed(page):
return True
except Exception as e:
logger.debug(f"CDP click on '{selector}' failed: {e}")
logger.debug("CDP click on '%s' failed: %s", selector, e)
return await _is_bypassed(page)
except Exception as e:
logger.debug(f"CDP Mode click failed: {e}")
logger.debug("CDP Mode click failed: %s", e)
return False
CDP_GUI_CLICK_SELECTORS = [
"#turnstile-widget div", # Cloudflare Turnstile
"#cf-turnstile div", # Alternative CF Turnstile
"#challenge-stage div", # CF challenge stage
"input[type='checkbox']", # Generic checkbox
"[class*='cb-i']", # DDOS-Guard checkbox
"#turnstile-widget div", # Cloudflare Turnstile
"#cf-turnstile div", # Alternative CF Turnstile
"#challenge-stage div", # CF challenge stage
"input[type='checkbox']", # Generic checkbox
"[class*='cb-i']", # DDOS-Guard checkbox
]
async def _bypass_method_cdp_gui_click(page) -> bool:
async def _bypass_method_cdp_gui_click(page: Any) -> bool:
"""CDP Mode with gui_click-style behavior."""
try:
logger.debug("Attempting bypass: CDP gui_click (mouse-based)")
@@ -514,25 +530,25 @@ async def _bypass_method_cdp_gui_click(page) -> bool:
if await _is_bypassed(page):
return True
except Exception as e:
logger.debug(f"solve_captcha() failed: {e}")
logger.debug("solve_captcha() failed: %s", e)
for selector in CDP_GUI_CLICK_SELECTORS:
try:
if not await page.is_element_visible(selector):
continue
logger.debug(f"CDP click_with_offset: {selector}")
logger.debug("CDP click_with_offset: %s", selector)
await page.click_with_offset(selector, 0, 0, center=True)
await asyncio.sleep(random.uniform(3, 5))
if await _is_bypassed(page):
return True
except Exception as e:
logger.debug(f"CDP gui_click on '{selector}' failed: {e}")
logger.debug("CDP gui_click on '%s' failed: %s", selector, e)
return await _is_bypassed(page)
except Exception as e:
logger.debug(f"CDP Mode gui_click failed: {e}")
logger.debug("CDP Mode gui_click failed: %s", e)
return False
@@ -546,14 +562,16 @@ BYPASS_METHODS = [
MAX_CONSECUTIVE_SAME_CHALLENGE = 3
def _check_cancellation(cancel_flag: Optional[Event], message: str) -> None:
def _check_cancellation(cancel_flag: Event | None, message: str) -> None:
"""Check if cancellation was requested and raise if so."""
if cancel_flag and cancel_flag.is_set():
logger.info(message)
raise BypassCancelledException("Bypass cancelled")
async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional[Event] = None) -> bool:
async def _bypass(
page: Any, max_retries: int | None = None, cancel_flag: Event | None = None
) -> bool:
"""Attempt to bypass Cloudflare/DDOS-Guard protection using multiple methods."""
max_retries = max_retries if max_retries is not None else app_config.MAX_RETRY
@@ -571,7 +589,7 @@ async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional
return True
challenge_type = await _detect_challenge_type(page)
logger.debug(f"Challenge detected: {challenge_type}")
logger.debug("Challenge detected: %s", challenge_type)
# No challenge detected but page doesn't look bypassed - wait and retry
if challenge_type == "none":
@@ -587,14 +605,16 @@ async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional
logger.info("Bypass successful after refresh")
return True
except Exception as e:
logger.debug(f"Refresh during no-challenge wait failed: {e}")
logger.debug("Refresh during no-challenge wait failed: %s", e)
continue
if challenge_type == last_challenge_type:
consecutive_same_challenge += 1
if consecutive_same_challenge >= min_same_challenge_before_abort:
logger.warning(
f"Same challenge ({challenge_type}) detected {consecutive_same_challenge} times - aborting"
"Same challenge (%s) detected %s times - aborting",
challenge_type,
consecutive_same_challenge,
)
return False
else:
@@ -602,11 +622,11 @@ async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional
last_challenge_type = challenge_type
method = BYPASS_METHODS[try_count % len(BYPASS_METHODS)]
logger.info(f"Bypass attempt {try_count + 1}/{max_retries} using {method.__name__}")
logger.info("Bypass attempt %s/%s using %s", try_count + 1, max_retries, method.__name__)
if try_count > 0:
wait_time = min(random.uniform(2, 4) * try_count, 12)
logger.info(f"Waiting {wait_time:.1f}s before trying...")
logger.info("Waiting %0.1fs before trying...", wait_time)
for _ in range(int(wait_time)):
_check_cancellation(cancel_flag, "Bypass cancelled during wait")
await asyncio.sleep(1)
@@ -614,18 +634,19 @@ async def _bypass(page, max_retries: Optional[int] = None, cancel_flag: Optional
try:
if await method(page):
logger.info(f"Bypass successful using {method.__name__}")
logger.info("Bypass successful using %s", method.__name__)
return True
except BypassCancelledException:
raise
except Exception as e:
logger.warning(f"Exception in {method.__name__}: {e}")
logger.warning("Exception in %s: %s", method.__name__, e)
logger.info(f"Bypass method {method.__name__} failed.")
logger.info("Bypass method %s failed.", method.__name__)
logger.warning("Exceeded maximum retries. Bypass failed.")
return False
def _get_browser_args() -> list[str]:
"""Build extra Chrome arguments, pre-resolving hostnames via patched DNS.
@@ -645,16 +666,14 @@ def _get_browser_args() -> list[str]:
]
if app_config.get("DEBUG", False):
arguments.extend([
"--enable-logging",
"--v=1",
"--log-file=" + str(LOG_DIR / "chrome_browser.log")
])
arguments.extend(
["--enable-logging", "--v=1", "--log-file=" + str(LOG_DIR / "chrome_browser.log")]
)
host_rules = _build_host_resolver_rules()
if host_rules:
arguments.append(f'--host-resolver-rules={", ".join(host_rules)}')
logger.debug(f"Chrome: Using host resolver rules for {len(host_rules)} hosts")
arguments.append(f"--host-resolver-rules={', '.join(host_rules)}")
logger.debug("Chrome: Using host resolver rules for %s hosts", len(host_rules))
else:
logger.warning("Chrome: No hosts could be pre-resolved")
@@ -676,40 +695,39 @@ def _build_host_resolver_rules() -> list[str]:
if results:
ip = results[0][4][0]
host_rules.append(f"MAP {hostname} {ip}")
logger.debug(f"Chrome: Pre-resolved {hostname} -> {ip}")
logger.debug("Chrome: Pre-resolved %s -> %s", hostname, ip)
else:
logger.warning(f"Chrome: No addresses returned for {hostname}")
logger.warning("Chrome: No addresses returned for %s", hostname)
except socket.gaierror as e:
logger.warning(f"Chrome: Could not pre-resolve {hostname}: {e}")
logger.warning("Chrome: Could not pre-resolve %s: %s", hostname, e)
except Exception as e:
logger.error_trace(f"Error pre-resolving hostnames for Chrome: {e}")
return host_rules
DRIVER_RESET_ERRORS = {"ProtocolException", "RuntimeError", "TimeoutError"}
async def _get(url: str, driver, cancel_flag: Optional[Event] = None) -> str:
async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str:
"""Fetch URL with Cloudflare bypass using a CDP browser."""
_check_cancellation(cancel_flag, "Bypass cancelled before starting")
logger.debug(f"CDP_GET: {url}")
logger.debug("CDP_GET: %s", url)
logger.debug("Opening URL with SeleniumBase CDP...")
page = await driver.get(url)
try:
with suppress(Exception):
await page.wait()
except Exception:
pass
_check_cancellation(cancel_flag, "Bypass cancelled after page load")
try:
current_url = await page.get_current_url()
title = await page.get_title()
logger.debug(f"Page loaded - URL: {current_url}, Title: {title}")
logger.debug("Page loaded - URL: %s, Title: %s", current_url, title)
except Exception as e:
logger.debug(f"Could not get page info: {e}")
logger.debug("Could not get page info: %s", e)
logger.debug("Starting bypass process...")
if await _bypass(page, cancel_flag=cancel_flag):
@@ -727,7 +745,7 @@ async def _get(url: str, driver, cancel_flag: Optional[Event] = None) -> str:
return ""
def get(url: str, retry: Optional[int] = None, cancel_flag: Optional[Event] = None) -> str:
def get(url: str, retry: int | None = None, cancel_flag: Event | None = None) -> str:
"""Fetch a URL with protection bypass. Creates fresh Chrome instance for each bypass."""
retry = retry if retry is not None else app_config.MAX_RETRY
@@ -753,8 +771,10 @@ def get(url: str, retry: Optional[int] = None, cancel_flag: Optional[Event] = No
raise
except Exception as e:
error_details = f"{type(e).__name__}: {e}"
logger.warning(f"Bypass failed (attempt {attempt + 1}/{retry}): {error_details}")
logger.debug(f"Stack trace: {traceback.format_exc()}")
logger.warning(
"Bypass failed (attempt %s/%s): %s", attempt + 1, retry, error_details
)
logger.debug("Stack trace: %s", traceback.format_exc())
# On CDP errors, quit and create a fresh browser
if type(e).__name__ in DRIVER_RESET_ERRORS:
@@ -762,7 +782,7 @@ def get(url: str, retry: Optional[int] = None, cancel_flag: Optional[Event] = No
await _close_cdp_driver(driver)
driver = await _create_cdp_browser(url)
logger.error(f"Bypass failed after {retry} attempts")
logger.error("Bypass failed after %s attempts", retry)
return ""
finally:
if driver:
@@ -770,7 +790,8 @@ def get(url: str, retry: Optional[int] = None, cancel_flag: Optional[Event] = No
return _CDP_WORKER.run(_run_bypass())
def _get_proxy_string(url: str) -> Optional[str]:
def _get_proxy_string(url: str) -> str | None:
"""Return a single proxy string for CDP, honoring NO_PROXY."""
proxies = get_proxies(url)
if not proxies:
@@ -787,8 +808,8 @@ async def _create_cdp_browser(url: str) -> Any:
display_height = screen_height + 150
proxy = _get_proxy_string(url)
logger.debug(f"Creating Pure CDP browser with args: {browser_args}")
logger.debug(f"Browser screen size: {screen_width}x{screen_height}")
logger.debug("Creating Pure CDP browser with args: %s", browser_args)
logger.debug("Browser screen size: %sx%s", screen_width, screen_height)
try:
driver = await cdp_driver.start_async(
@@ -804,21 +825,21 @@ async def _create_cdp_browser(url: str) -> Any:
browser_args=browser_args,
)
except Exception as e:
logger.warning(f"Pure CDP browser startup failed: {type(e).__name__}: {e}")
logger.warning("Pure CDP browser startup failed: %s: %s", type(e).__name__, e)
logger.warning(
"SeleniumBase runtime paths: "
f"cwd={os.getcwd()}; "
f"{_describe_runtime_path(SELENIUMBASE_DOWNLOADS_DIR)}; "
f"{_describe_runtime_path('/app/downloaded_files')}; "
f"{_describe_runtime_path('downloaded_files')}; "
f"{_describe_runtime_path('/tmp')}"
"SeleniumBase runtime paths: cwd=%s; %s; %s; %s; %s",
Path.cwd(),
_describe_runtime_path(SELENIUMBASE_DOWNLOADS_DIR),
_describe_runtime_path("/app/downloaded_files"),
_describe_runtime_path("downloaded_files"),
_describe_runtime_path("/tmp"),
)
raise
try:
await driver.page.set_window_rect(0, 0, screen_width, screen_height)
except Exception as e:
logger.debug(f"Failed to set window size: {e}")
logger.debug("Failed to set window size: %s", e)
# Start FFmpeg recording if debug mode (record each bypass session)
if app_config.get("DEBUG", False) and not DISPLAY.get("ffmpeg"):
@@ -830,7 +851,7 @@ async def _create_cdp_browser(url: str) -> Any:
return driver
async def _close_cdp_driver(driver) -> None:
async def _close_cdp_driver(driver: Any) -> None:
"""Close CDP connections and stop the browser."""
if not driver:
return
@@ -846,18 +867,15 @@ async def _close_cdp_driver(driver) -> None:
if hasattr(driver, "targets") and driver.targets:
connections.extend(driver.targets)
for conn in connections:
try:
await conn.aclose()
except Exception as e:
logger.debug(f"Failed to close websocket connection: {e}")
await _close_websocket_connection(conn)
except Exception as e:
logger.debug(f"Error during connection cleanup: {e}")
logger.debug("Error during connection cleanup: %s", e)
try:
driver.stop()
logger.debug("Stopped CDP browser")
except Exception as e:
logger.debug(f"CDP stop: {e}")
logger.debug("CDP stop: %s", e)
if env.DOCKERMODE:
await asyncio.sleep(0.3)
@@ -879,15 +897,23 @@ async def _close_cdp_driver(driver) -> None:
await asyncio.sleep(0.1)
if _pid_alive(pid):
os.kill(pid, signal.SIGKILL)
logger.debug(f"Killed Chrome pid {pid}")
logger.debug("Killed Chrome pid %s", pid)
except Exception as e:
logger.debug(f"Failed to kill Chrome pid {pid}: {e}")
logger.debug("Failed to kill Chrome pid %s: %s", pid, e)
except Exception as e:
logger.debug(f"Process cleanup failed: {e}")
logger.debug("Process cleanup failed: %s", e)
logger.log_resource_usage()
async def _close_websocket_connection(conn: Any) -> None:
"""Close one websocket-like connection, ignoring best-effort failures."""
try:
await conn.aclose()
except Exception as e:
logger.debug("Failed to close websocket connection: %s", e)
def _start_ffmpeg_recording(display: str) -> None:
"""Start FFmpeg screen recording for debug mode."""
global DISPLAY
@@ -900,16 +926,37 @@ def _start_ffmpeg_recording(display: str) -> None:
display_height = screen_height + 150
ffmpeg_cmd = [
"ffmpeg", "-y", "-f", "x11grab",
"-video_size", f"{display_width}x{display_height}",
"-i", display,
"-c:v", "libx264", "-preset", "ultrafast",
"-maxrate", "700k", "-bufsize", "1400k", "-crf", "36",
"-pix_fmt", "yuv420p", "-tune", "animation",
"-x264-params", "bframes=0:deblock=-1,-1",
"-r", "15", "-an",
"ffmpeg",
"-y",
"-f",
"x11grab",
"-video_size",
f"{display_width}x{display_height}",
"-i",
display,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-maxrate",
"700k",
"-bufsize",
"1400k",
"-crf",
"36",
"-pix_fmt",
"yuv420p",
"-tune",
"animation",
"-x264-params",
"bframes=0:deblock=-1,-1",
"-r",
"15",
"-an",
output_file.as_posix(),
"-nostats", "-loglevel", "0"
"-nostats",
"-loglevel",
"0",
]
logger.debug("Starting FFmpeg recording to %s", output_file)
logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
@@ -920,9 +967,9 @@ def _start_ffmpeg_recording(display: str) -> None:
def _stop_ffmpeg_recording() -> None:
"""Stop FFmpeg screen recording if running."""
import signal
global DISPLAY
proc = DISPLAY.get("ffmpeg")
output_file = DISPLAY.get("ffmpeg_output")
if not proc:
return
if proc.poll() is not None:
@@ -935,21 +982,17 @@ def _stop_ffmpeg_recording() -> None:
proc.wait(timeout=5)
logger.debug("Stopped ffmpeg recording")
except Exception as e:
logger.debug(f"ffmpeg stop: {e}")
try:
logger.debug("ffmpeg stop: %s", e)
with suppress(Exception):
proc.terminate()
proc.wait(timeout=2)
except Exception:
pass
try:
with suppress(Exception):
proc.kill()
except Exception:
pass
DISPLAY["ffmpeg"] = None
DISPLAY["ffmpeg_output"] = None
def _try_with_cached_cookies(url: str, hostname: str) -> Optional[str]:
def _try_with_cached_cookies(url: str, hostname: str) -> str | None:
"""Attempt request with cached cookies before using Chrome."""
cookies = get_cf_cookies_for_domain(hostname)
if not cookies:
@@ -959,10 +1002,17 @@ def _try_with_cached_cookies(url: str, hostname: str) -> Optional[str]:
headers = {}
stored_ua = get_cf_user_agent_for_domain(hostname)
if stored_ua:
headers['User-Agent'] = stored_ua
headers["User-Agent"] = stored_ua
logger.debug(f"Trying request with cached cookies: {url}")
response = requests.get(url, cookies=cookies, headers=headers, proxies=get_proxies(url), timeout=(5, 10), verify=get_ssl_verify(url))
logger.debug("Trying request with cached cookies: %s", url)
response = requests.get(
url,
cookies=cookies,
headers=headers,
proxies=get_proxies(url),
timeout=(5, 10),
verify=get_ssl_verify(url),
)
if response.status_code == 200:
logger.debug("Cached cookies worked, skipped Chrome bypass")
return response.text
@@ -973,10 +1023,8 @@ def _try_with_cached_cookies(url: str, hostname: str) -> Optional[str]:
def get_bypassed_page(
url: str,
selector: Optional[network.AAMirrorSelector] = None,
cancel_flag: Optional[Event] = None
) -> Optional[str]:
url: str, selector: network.AAMirrorSelector | None = None, cancel_flag: Event | None = None
) -> str | None:
"""Fetch HTML content from a URL using the internal Cloudflare Bypasser."""
sel = selector or network.AAMirrorSelector()
attempt_url = sel.rewrite(url)
shelfmark/config/booklore_settings.py
+15 -11
View File
@@ -130,12 +130,13 @@ def get_booklore_library_options() -> list[dict[str, Any]]:
try:
library_options, _ = _get_booklore_cached_options(base_url, username, password)
return library_options
except Exception as exc:
logger.error(f"Failed to fetch Booklore libraries: {exc}")
except Exception:
logger.exception("Failed to fetch Booklore libraries")
if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
return []
else:
return library_options
def get_booklore_path_options() -> list[dict[str, Any]]:
@@ -154,19 +155,22 @@ def get_booklore_path_options() -> list[dict[str, Any]]:
try:
_, path_options = _get_booklore_cached_options(base_url, username, password)
return path_options
except Exception as exc:
logger.error(f"Failed to fetch Booklore paths: {exc}")
except Exception:
logger.exception("Failed to fetch Booklore paths")
if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
return []
else:
return path_options
def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
def test_booklore_connection(
current_values: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Test the Booklore connection using current form values."""
current_values = current_values or {}
def _get_value(key: str, default: Any = None) -> Any:
def _get_value(key: str, default: object = None) -> object:
value = current_values.get(key)
if value not in (None, ""):
return value
@@ -187,11 +191,11 @@ def test_booklore_connection(current_values: dict[str, Any] | None = None) -> di
try:
library_options, _ = _get_booklore_select_options(base_url, username, password)
except BookloreError as exc:
return {"success": False, "message": str(exc)}
else:
message = "Connected to Grimmory"
if library_options:
message = f"Connected to Grimmory ({len(library_options)} libraries)"
return {"success": True, "message": message}
except BookloreError as exc:
return {"success": False, "message": str(exc)}
shelfmark/config/email_settings.py
+12 -7
View File
@@ -3,15 +3,20 @@ from __future__ import annotations
from typing import Any
from shelfmark.core.config import config
from shelfmark.download.outputs.email import EmailOutputError, build_email_smtp_config, test_smtp_connection
from shelfmark.download.outputs.email import (
EmailOutputError,
build_email_smtp_config,
test_smtp_connection,
)
def test_email_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
def test_email_connection(
current_values: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Test SMTP connectivity using current form values (including unsaved changes)."""
current_values = current_values or {}
def _get_value(key: str, default: Any = None) -> Any:
def _get_value(key: str, default: object = None) -> object:
value = current_values.get(key)
if value not in (None, ""):
return value
@@ -28,15 +33,15 @@ def test_email_connection(current_values: dict[str, Any] | None = None) -> dict[
"EMAIL_FROM": _get_value("EMAIL_FROM", ""),
"EMAIL_SUBJECT_TEMPLATE": _get_value("EMAIL_SUBJECT_TEMPLATE", "{Title}"),
"EMAIL_SMTP_TIMEOUT_SECONDS": _get_value("EMAIL_SMTP_TIMEOUT_SECONDS", 60),
"EMAIL_ALLOW_UNVERIFIED_TLS": _get_value("EMAIL_ALLOW_UNVERIFIED_TLS", False),
"EMAIL_ALLOW_UNVERIFIED_TLS": _get_value("EMAIL_ALLOW_UNVERIFIED_TLS", default=False),
}
try:
smtp_config = build_email_smtp_config(settings)
test_smtp_connection(smtp_config)
return {"success": True, "message": "Connected to SMTP server"}
except EmailOutputError as exc:
return {"success": False, "message": str(exc)}
except Exception as exc:
return {"success": False, "message": f"SMTP test failed: {exc}"}
else:
return {"success": True, "message": "Connected to SMTP server"}
shelfmark/config/env.py
+9 -7
View File
@@ -23,11 +23,11 @@ def _read_debug_from_config() -> bool:
if config_file.exists():
try:
with open(config_file, "r") as f:
with config_file.open() as f:
config = json.load(f)
if "DEBUG" in config:
return bool(config["DEBUG"])
except (json.JSONDecodeError, OSError):
except json.JSONDecodeError, OSError:
pass
return False
@@ -36,10 +36,10 @@ def _read_debug_from_config() -> bool:
def _is_sqlite_file(path: Path) -> bool:
"""Check if a file is a valid SQLite database by reading magic bytes."""
try:
with open(path, "rb") as f:
with path.open("rb") as f:
header = f.read(16)
return header[:16] == b"SQLite format 3\x00"
except (OSError, PermissionError):
except OSError, PermissionError:
return False
@@ -67,14 +67,16 @@ def _is_config_dir_writable() -> bool:
test_file = CONFIG_DIR / ".write_test"
test_file.touch()
test_file.unlink()
return True
except (OSError, PermissionError):
except OSError, PermissionError:
return False
else:
return True
def is_covers_cache_enabled() -> bool:
"""Check if cover caching is enabled (requires setting + writable config dir)."""
from shelfmark.core.config import config
setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
return setting_enabled and _is_config_dir_writable()
@@ -151,7 +153,7 @@ ONBOARDING = string_to_bool(os.getenv("ONBOARDING", "true"))
# Debug: skip specific download sources for testing fallback chains
# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
DEBUG_SKIP_SOURCES = {s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip()}
# =============================================================================
shelfmark/config/migrations.py
+34 -33
View File
@@ -1,8 +1,11 @@
"""Configuration migration helpers."""
import json
from typing import Any, Callable
from pathlib import Path
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from collections.abc import Callable
_DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
"PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
@@ -11,7 +14,7 @@ _DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
)
def _as_bool(value: Any) -> bool:
def _as_bool(value: object) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
@@ -23,10 +26,7 @@ def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None:
"""Pick the best legacy admin-restriction value to migrate."""
auth_method = str(config.get("AUTH_METHOD", "")).strip().lower()
if (
auth_method == "proxy"
and "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config
):
if auth_method == "proxy" and "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN" in config:
return _as_bool(config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN"))
if auth_method == "cwa" and "CWA_RESTRICT_SETTINGS_TO_ADMIN" in config:
@@ -50,9 +50,9 @@ def migrate_security_settings(
load_users_config: Callable[[], dict[str, Any]],
save_users_config: Callable[[dict[str, Any]], None],
ensure_config_dir: Callable[[], None],
get_config_path: Callable[[], Any],
get_config_path: Callable[[], object],
sync_builtin_admin_user: Callable[[str, str], None],
logger: Any,
logger: object,
) -> None:
"""Migrate legacy security keys and sync builtin admin credentials."""
try:
@@ -67,13 +67,12 @@ def migrate_security_settings(
if old_value:
config["AUTH_METHOD"] = "cwa"
logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
elif config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
config["AUTH_METHOD"] = "builtin"
logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
else:
if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
config["AUTH_METHOD"] = "builtin"
logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
else:
config["AUTH_METHOD"] = "none"
logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
config["AUTH_METHOD"] = "none"
logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
migrated_security = True
else:
logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
@@ -82,14 +81,17 @@ def migrate_security_settings(
# Backfill AUTH_METHOD for configs that have builtin credentials but
# were never migrated from USE_CWA_AUTH (e.g. dev builds that predated
# the AUTH_METHOD field).
if "AUTH_METHOD" not in config:
if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
config["AUTH_METHOD"] = "builtin"
migrated_security = True
logger.info(
"Backfilled AUTH_METHOD='builtin' from legacy "
"BUILTIN_USERNAME/BUILTIN_PASSWORD_HASH credentials"
)
if (
"AUTH_METHOD" not in config
and config.get("BUILTIN_USERNAME")
and config.get("BUILTIN_PASSWORD_HASH")
):
config["AUTH_METHOD"] = "builtin"
migrated_security = True
logger.info(
"Backfilled AUTH_METHOD='builtin' from legacy "
"BUILTIN_USERNAME/BUILTIN_PASSWORD_HASH credentials"
)
if "RESTRICT_SETTINGS_TO_ADMIN" not in users_config:
legacy_restrict = _pick_legacy_settings_restriction(config)
@@ -97,31 +99,30 @@ def migrate_security_settings(
save_users_config({"RESTRICT_SETTINGS_TO_ADMIN": legacy_restrict})
migrated_users = True
logger.info(
"Migrated legacy settings-admin restriction to users.RESTRICT_SETTINGS_TO_ADMIN="
f"{legacy_restrict}"
"Migrated legacy settings-admin restriction to users.RESTRICT_SETTINGS_TO_ADMIN=%s",
legacy_restrict,
)
for deprecated_key in _DEPRECATED_SETTINGS_RESTRICTION_KEYS:
if deprecated_key in config:
config.pop(deprecated_key, None)
migrated_security = True
logger.info(f"Removed deprecated security setting: {deprecated_key}")
logger.info("Removed deprecated security setting: %s", deprecated_key)
try:
sync_builtin_admin_user(
config.get("BUILTIN_USERNAME", ""),
config.get("BUILTIN_PASSWORD_HASH", ""),
)
except Exception as exc:
logger.error(
"Failed to sync builtin credentials to users database during migration: "
f"{exc}"
except Exception:
logger.exception(
"Failed to sync builtin credentials to users database during migration"
)
if migrated_security:
ensure_config_dir()
config_path = get_config_path()
with open(config_path, "w") as f:
config_path = Path(get_config_path())
with config_path.open("w") as f:
json.dump(config, f, indent=2)
logger.info("Security settings migration completed successfully")
elif migrated_users:
@@ -131,5 +132,5 @@ def migrate_security_settings(
except FileNotFoundError:
logger.debug("No existing security config file found - nothing to migrate")
except Exception as exc:
logger.error(f"Failed to migrate security settings: {exc}")
except Exception:
logger.exception("Failed to migrate security settings")
shelfmark/config/notifications_settings.py
+3 -2
View File
@@ -11,6 +11,7 @@ from shelfmark.core.notifications import NotificationEvent, send_test_notificati
from shelfmark.core.settings_registry import (
ActionButton,
HeadingField,
SettingsField,
TableField,
load_config_file,
register_on_save,
@@ -124,7 +125,7 @@ def _count_invalid_route_urls(routes: list[dict[str, Any]]) -> int:
def _ensure_default_route_row(routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
return routes if routes else [dict(row) for row in _DEFAULT_ROUTE_ROWS]
return routes or [dict(row) for row in _DEFAULT_ROUTE_ROWS]
def _extract_unique_route_urls(routes: list[dict[str, Any]]) -> list[str]:
@@ -260,7 +261,7 @@ register_on_save("notifications", _on_save_notifications)
@register_settings("notifications", "Notifications", icon="bell", order=7)
def notifications_settings():
def notifications_settings() -> list[SettingsField]:
"""Global notifications settings."""
return [
HeadingField(
shelfmark/config/security.py
+32 -25
View File
@@ -1,6 +1,6 @@
"""Authentication settings registration."""
from typing import Any, Dict, Callable
from typing import TYPE_CHECKING, Any
from shelfmark.config.migrations import migrate_security_settings
from shelfmark.config.security_handlers import (
@@ -10,19 +10,23 @@ from shelfmark.config.security_handlers import (
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import (
register_settings,
register_on_save,
load_config_file,
TextField,
SelectField,
PasswordField,
CheckboxField,
ActionButton,
TagListField,
CheckboxField,
CustomComponentField,
PasswordField,
SelectField,
SettingsField,
TagListField,
TextField,
load_config_file,
register_on_save,
register_settings,
)
from shelfmark.core.user_db import sync_builtin_admin_user
if TYPE_CHECKING:
from collections.abc import Callable
logger = setup_logger(__name__)
@@ -36,8 +40,8 @@ def _auth_field(factory: Callable[..., Any], auth_method: str, **kwargs: Any) ->
def _migrate_security_settings() -> None:
from shelfmark.core.settings_registry import (
_get_config_file_path,
_ensure_config_dir,
_get_config_file_path,
save_config_file,
)
@@ -52,12 +56,11 @@ def _migrate_security_settings() -> None:
)
def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
def _on_save_security(values: dict[str, Any]) -> dict[str, Any]:
return on_save_security(values)
def _test_oidc_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
def _test_oidc_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
return test_oidc_connection(
load_security_config=lambda: {
"OIDC_DISCOVERY_URL": app_config.get("OIDC_DISCOVERY_URL", ""),
@@ -68,7 +71,7 @@ def _test_oidc_connection(current_values: Dict[str, Any] = None) -> Dict[str, An
@register_settings("security", "Security", icon="shield", order=5)
def security_settings():
def security_settings() -> list[SettingsField]:
"""Security and authentication settings."""
from shelfmark.config.env import CWA_DB_PATH
@@ -105,18 +108,22 @@ def security_settings():
label="A local admin account is required before OIDC can be enabled.",
show_when=_auth_condition("oidc"),
),
*([] if cwa_db_available else [
CustomComponentField(
key="cwa_db_missing",
component="oidc_admin_hint",
label=(
"Calibre-Web database not detected. Mount your app.db to "
"/auth/app.db to enable this method. Authentication will fall "
"back to none until the database is available."
*(
[]
if cwa_db_available
else [
CustomComponentField(
key="cwa_db_missing",
component="oidc_admin_hint",
label=(
"Calibre-Web database not detected. Mount your app.db to "
"/auth/app.db to enable this method. Authentication will fall "
"back to none until the database is available."
),
show_when=_auth_condition("cwa"),
),
show_when=_auth_condition("cwa"),
),
]),
]
),
ActionButton(
key="open_users_tab",
label="Go to Users",
shelfmark/config/security_handlers.py
+18 -8
View File
@@ -1,21 +1,26 @@
"""Operational handlers for security settings (save/actions)."""
import os
from typing import Any, Callable
from pathlib import Path
from typing import TYPE_CHECKING, Any
from shelfmark.core.utils import normalize_http_url
from shelfmark.core.user_db import UserDB
from shelfmark.core.utils import normalize_http_url
from shelfmark.download.network import get_ssl_verify
if TYPE_CHECKING:
from collections.abc import Callable
_OIDC_LOCKOUT_MESSAGE = "A local admin account with a password is required before enabling OIDC. Use the 'Go to Users' button above to create one. This ensures you can still sign in if your identity provider is unavailable."
def _has_local_password_admin() -> bool:
root = os.environ.get("CONFIG_DIR", "/config")
user_db = UserDB(os.path.join(root, "users.db"))
user_db = UserDB(str(Path(root) / "users.db"))
user_db.initialize()
return any(user.get("password_hash") and user.get("role") == "admin" for user in user_db.list_users())
return any(
user.get("password_hash") and user.get("role") == "admin" for user in user_db.list_users()
)
def on_save_security(
@@ -56,7 +61,9 @@ def test_oidc_connection(
try:
# Prefer the current (unsaved) form value over the saved config
discovery_url = (current_values or {}).get("OIDC_DISCOVERY_URL") or load_security_config().get("OIDC_DISCOVERY_URL", "")
discovery_url = (current_values or {}).get(
"OIDC_DISCOVERY_URL"
) or load_security_config().get("OIDC_DISCOVERY_URL", "")
if not discovery_url:
return {"success": False, "message": "Discovery URL is not configured."}
@@ -67,9 +74,12 @@ def test_oidc_connection(
required_fields = ["issuer", "authorization_endpoint", "token_endpoint"]
missing_fields = [field for field in required_fields if field not in document]
if missing_fields:
return {"success": False, "message": f"Discovery document missing fields: {', '.join(missing_fields)}"}
return {
"success": False,
"message": f"Discovery document missing fields: {', '.join(missing_fields)}",
}
return {"success": True, "message": f"Connected to {document['issuer']}"}
except Exception as exc:
logger.error(f"OIDC connection test failed: {exc}")
return {"success": False, "message": f"Connection failed: {str(exc)}"}
logger.exception("OIDC connection test failed")
return {"success": False, "message": f"Connection failed: {exc!s}"}
shelfmark/config/settings.py
+151 -105
View File
@@ -1,12 +1,11 @@
"""Core settings registration and derived configuration values."""
import os
from pathlib import Path
import json
from typing import Any, Dict
from pathlib import Path
from typing import Any
def _on_save_advanced(values: Dict[str, Any]) -> Dict[str, Any]:
def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
"""Validate advanced settings before persisting."""
from shelfmark.core.logger import setup_logger
@@ -37,7 +36,7 @@ def _on_save_advanced(values: Dict[str, Any]) -> Dict[str, Any]:
if not host or not remote_path or not local_path:
logger.debug(
"Skipping entry %d: missing field(s) - host=%r, remotePath=%r, localPath=%r",
"Skipping entry %d: missing field(s) - host=%s, remotePath=%s, localPath=%s",
i,
host,
remote_path,
@@ -57,7 +56,9 @@ def _on_save_advanced(values: Dict[str, Any]) -> Dict[str, Any]:
logger.info("Saved %d remote path mapping(s)", len(cleaned))
if cleaned:
for m in cleaned:
logger.debug(" Mapping: %s -> %s (client: %s)", m["remotePath"], m["localPath"], m["host"])
logger.debug(
" Mapping: %s -> %s (client: %s)", m["remotePath"], m["localPath"], m["host"]
)
values["PROWLARR_REMOTE_PATH_MAPPINGS"] = cleaned
return {"error": False, "values": values}
@@ -76,19 +77,19 @@ logger = setup_logger(__name__)
# Log bootstrap configuration values at DEBUG level
logger.debug("Bootstrap configuration:")
for key in ['CONFIG_DIR', 'LOG_DIR', 'TMP_DIR', 'INGEST_DIR', 'DEBUG', 'DOCKERMODE']:
for key in ["CONFIG_DIR", "LOG_DIR", "TMP_DIR", "INGEST_DIR", "DEBUG", "DOCKERMODE"]:
if hasattr(env, key):
logger.debug(f" {key}: {getattr(env, key)}")
logger.debug(" %s: %s", key, getattr(env, key))
# Load supported book languages from data file
# Path is relative to the package root, not this file
_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
with open(_DATA_DIR / "book-languages.json") as file:
with (_DATA_DIR / "book-languages.json").open() as file:
_SUPPORTED_BOOK_LANGUAGE = json.load(file)
# Directory settings
BASE_DIR = Path(__file__).resolve().parent.parent.parent
logger.debug(f"BASE_DIR: {BASE_DIR}")
logger.debug("BASE_DIR: %s", BASE_DIR)
if env.ENABLE_LOGGING:
env.LOG_DIR.mkdir(exist_ok=True)
@@ -106,6 +107,7 @@ RECORDING_DIR = env.LOG_DIR / "recording"
def _log_external_bypasser_warning() -> None:
"""Log warning about external bypasser DNS limitations (called after config is available)."""
from shelfmark.core.config import config
if config.get("USING_EXTERNAL_BYPASSER", False) and config.get("USE_CF_BYPASS", True):
logger.warning(
"Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
@@ -116,36 +118,31 @@ def _log_external_bypasser_warning() -> None:
from shelfmark.core.settings_registry import (
register_settings,
ActionButton,
CheckboxField,
HeadingField,
MultiSelectField,
NumberField,
OrderableListField,
PasswordField,
SelectField,
SettingsField,
TableField,
TagListField,
TextField,
load_config_file,
register_group,
register_on_save,
load_config_file,
TextField,
PasswordField,
NumberField,
CheckboxField,
SelectField,
MultiSelectField,
TagListField,
OrderableListField,
TableField,
HeadingField,
ActionButton,
register_settings,
)
register_group(
"direct_download",
"Direct Download",
icon="download",
order=20
)
register_group("direct_download", "Direct Download", icon="download", order=20)
register_group(
"metadata_providers",
"Metadata Providers",
icon="book",
order=12 # Between Network (10) and Advanced (15)
order=12, # Between Network (10) and Advanced (15)
)
@@ -203,15 +200,15 @@ _DOWNLOAD_TO_BROWSER_CONTENT_TYPE_VALUES = {
}
def _get_metadata_provider_options():
def _get_metadata_provider_options() -> list[dict[str, str]]:
"""Build metadata provider options dynamically from enabled providers only."""
from shelfmark.metadata_providers import list_providers, is_provider_enabled
from shelfmark.metadata_providers import is_provider_enabled, list_providers
options = []
for provider in list_providers():
# Only show providers that are enabled
if is_provider_enabled(provider["name"]):
options.append({"value": provider["name"], "label": provider["display_name"]})
options = [
{"value": provider["name"], "label": provider["display_name"]}
for provider in list_providers()
if is_provider_enabled(provider["name"])
]
# If no providers enabled, show a placeholder option
if not options:
@@ -222,12 +219,12 @@ def _get_metadata_provider_options():
return options
def _get_metadata_provider_options_with_none():
def _get_metadata_provider_options_with_none() -> list[dict[str, str]]:
"""Build metadata provider options with a 'Use main provider' option first."""
return [{"value": "", "label": "Use book provider"}] + _get_metadata_provider_options()
return [{"value": "", "label": "Use book provider"}, *_get_metadata_provider_options()]
def _get_release_source_options_for_content_type(content_type: str):
def _get_release_source_options_for_content_type(content_type: str) -> list[dict[str, str]]:
"""Build release source options dynamically for a specific content type."""
from shelfmark.release_sources import list_available_sources
@@ -239,25 +236,28 @@ def _get_release_source_options_for_content_type(content_type: str):
]
def _get_book_release_source_options():
def _get_book_release_source_options() -> list[dict[str, str]]:
"""Build default release source options for book searches."""
return _get_release_source_options_for_content_type("ebook")
def _get_audiobook_release_source_options():
def _get_audiobook_release_source_options() -> list[dict[str, str]]:
"""Build default release source options for audiobook searches."""
return [{"value": "", "label": "Use book release source"}] + _get_release_source_options_for_content_type(
"audiobook"
)
return [
{"value": "", "label": "Use book release source"},
*_get_release_source_options_for_content_type("audiobook"),
]
_LANGUAGE_OPTIONS = [
{"value": lang["code"], "label": lang["language"]} for lang in _SUPPORTED_BOOK_LANGUAGE
]
_LANGUAGE_OPTIONS = [{"value": lang["code"], "label": lang["language"]} for lang in _SUPPORTED_BOOK_LANGUAGE]
def _get_aa_base_url_options():
def _get_aa_base_url_options() -> list[dict[str, str]]:
"""Build AA URL options dynamically, including additional mirrors from config."""
from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS, get_aa_mirrors
from shelfmark.core.config import config
from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS, get_aa_mirrors
from shelfmark.core.utils import normalize_http_url
options = [{"value": "auto", "label": "Auto (Recommended)"}]
@@ -273,7 +273,7 @@ def _get_aa_base_url_options():
allow_special=("auto",),
)
if configured_url and configured_url != "auto" and configured_url not in all_mirrors:
all_mirrors = [configured_url] + all_mirrors
all_mirrors = [configured_url, *all_mirrors]
for url in all_mirrors:
domain = url.replace("https://", "").replace("http://", "")
@@ -286,10 +286,10 @@ def _get_aa_base_url_options():
return options
def _get_zlib_mirror_options():
def _get_zlib_mirror_options() -> list[dict[str, str]]:
"""Build Z-Library mirror options for SelectField."""
from shelfmark.core.mirrors import DEFAULT_ZLIB_MIRRORS
from shelfmark.core.config import config
from shelfmark.core.mirrors import DEFAULT_ZLIB_MIRRORS
options = []
@@ -310,10 +310,10 @@ def _get_zlib_mirror_options():
return options
def _get_welib_mirror_options():
def _get_welib_mirror_options() -> list[dict[str, str]]:
"""Build Welib mirror options for SelectField."""
from shelfmark.core.mirrors import DEFAULT_WELIB_MIRRORS
from shelfmark.core.config import config
from shelfmark.core.mirrors import DEFAULT_WELIB_MIRRORS
options = []
@@ -345,16 +345,17 @@ def _clear_covers_cache(current_values: dict) -> dict:
# Reset the singleton so it reinitializes with fresh state
reset_image_cache()
except Exception as e:
logger.exception("Failed to clear cover cache")
return {
"success": False,
"message": f"Failed to clear cache: {e!s}",
}
else:
return {
"success": True,
"message": f"Cleared {count} cached cover images.",
}
except Exception as e:
logger.error(f"Failed to clear cover cache: {e}")
return {
"success": False,
"message": f"Failed to clear cache: {str(e)}",
}
def _clear_metadata_cache(current_values: dict) -> dict:
@@ -371,15 +372,15 @@ def _clear_metadata_cache(current_values: dict) -> dict:
"message": f"Cleared {stats_before['size']} cached entries.",
}
except Exception as e:
logger.error(f"Failed to clear metadata cache: {e}")
logger.exception("Failed to clear metadata cache")
return {
"success": False,
"message": f"Failed to clear cache: {str(e)}",
"message": f"Failed to clear cache: {e!s}",
}
@register_settings("general", "General", icon="settings", order=0)
def general_settings():
def general_settings() -> list[SettingsField]:
"""Core application settings."""
return [
TextField(
@@ -424,7 +425,7 @@ def general_settings():
@register_settings("search_mode", "Search Mode", icon="search", order=1)
def search_mode_settings():
def search_mode_settings() -> list[SettingsField]:
"""Configure how you search for and download books."""
return [
HeadingField(
@@ -531,10 +532,9 @@ def search_mode_settings():
@register_settings("network", "Network", icon="globe", order=10)
def network_settings():
def network_settings() -> list[SettingsField]:
"""Network and connectivity settings."""
# Check if Tor variant is available and if Tor is currently enabled
tor_available = env.TOR_VARIANT_AVAILABLE
# Check if Tor is currently enabled.
tor_enabled = env.USING_TOR
# When Tor is enabled, DNS/proxy settings are overridden by iptables rules
@@ -595,7 +595,10 @@ def network_settings():
disabled=tor_overrides_network,
disabled_reason="DNS over HTTPS is not used when Tor routing is enabled.",
# Hide for manual and system (no DoH endpoint available for custom IPs or system DNS)
show_when={"field": "CUSTOM_DNS", "value": ["auto", "google", "cloudflare", "quad9", "opendns"]},
show_when={
"field": "CUSTOM_DNS",
"value": ["auto", "google", "cloudflare", "quad9", "opendns"],
},
# Disable for auto (always uses DoH)
disabled_when={
"field": "CUSTOM_DNS",
@@ -690,9 +693,7 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
normalized_content_types: list[str] = []
elif isinstance(raw_content_types, list):
normalized_content_types = [
str(value).strip().lower()
for value in raw_content_types
if str(value).strip()
str(value).strip().lower() for value in raw_content_types if str(value).strip()
]
else:
return {
@@ -773,28 +774,44 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
try:
port = int(effective.get("EMAIL_SMTP_PORT", 587))
except (TypeError, ValueError):
except TypeError, ValueError:
return {"error": True, "message": "SMTP port must be a number", "values": values}
if port < 1 or port > 65535:
return {"error": True, "message": "SMTP port must be between 1 and 65535", "values": values}
return {
"error": True,
"message": "SMTP port must be between 1 and 65535",
"values": values,
}
try:
timeout_seconds = int(effective.get("EMAIL_SMTP_TIMEOUT_SECONDS", 60))
except (TypeError, ValueError):
return {"error": True, "message": "SMTP timeout (seconds) must be a number", "values": values}
except TypeError, ValueError:
return {
"error": True,
"message": "SMTP timeout (seconds) must be a number",
"values": values,
}
if timeout_seconds < 1:
return {"error": True, "message": "SMTP timeout (seconds) must be >= 1", "values": values}
return {
"error": True,
"message": "SMTP timeout (seconds) must be >= 1",
"values": values,
}
username = str(effective.get("EMAIL_SMTP_USERNAME", "") or "").strip()
password = effective.get("EMAIL_SMTP_PASSWORD", "") or ""
if username and not password:
return {"error": True, "message": "SMTP password is required when username is set", "values": values}
return {
"error": True,
"message": "SMTP password is required when username is set",
"values": values,
}
try:
attachment_limit_mb = int(effective.get("EMAIL_ATTACHMENT_SIZE_LIMIT_MB", 25))
except (TypeError, ValueError):
except TypeError, ValueError:
return {
"error": True,
"message": "Attachment size limit (MB) must be a number",
@@ -846,7 +863,7 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
@register_settings("downloads", "Downloads", icon="folder", order=5)
def download_settings():
def download_settings() -> list[SettingsField]:
"""Configure download behavior and file locations."""
return [
# === BOOKS SECTION ===
@@ -901,17 +918,17 @@ def download_settings():
{
"value": "none",
"label": "None",
"description": "Keep original filename from source"
"description": "Keep original filename from source",
},
{
"value": "rename",
"label": "Rename Only",
"description": "Rename single-file downloads; multi-file keeps original names."
"description": "Rename single-file downloads; multi-file keeps original names.",
},
{
"value": "organize",
"label": "Rename and Organize",
"description": "Create folders and rename files using a template. Do not use with ingest folders."
"description": "Create folders and rename files using a template. Do not use with ingest folders.",
},
],
default="rename",
@@ -1081,7 +1098,11 @@ def download_settings():
description="Transport security mode for SMTP.",
options=[
{"value": "none", "label": "None", "description": "No TLS (not recommended)."},
{"value": "starttls", "label": "STARTTLS", "description": "Upgrade to TLS after connecting (recommended)."},
{
"value": "starttls",
"label": "STARTTLS",
"description": "Upgrade to TLS after connecting (recommended).",
},
{"value": "ssl", "label": "SSL/TLS", "description": "Connect using TLS (SMTPS)."},
],
default="starttls",
@@ -1139,7 +1160,6 @@ def download_settings():
callback=test_email_connection,
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "email"},
),
# === AUDIOBOOKS SECTION ===
# Universal mode only
HeadingField(
@@ -1160,9 +1180,21 @@ def download_settings():
label="File Organization",
description="Choose how downloaded audiobook files are named and organized.",
options=[
{"value": "none", "label": "None", "description": "Keep original filename from source"},
{"value": "rename", "label": "Rename Only", "description": "Rename single-file downloads; multi-file keeps original names."},
{"value": "organize", "label": "Rename and Organize", "description": "Create folders and rename files using a template. Recommended for Audiobookshelf. Do not use with ingest folders."},
{
"value": "none",
"label": "None",
"description": "Keep original filename from source",
},
{
"value": "rename",
"label": "Rename Only",
"description": "Rename single-file downloads; multi-file keeps original names.",
},
{
"value": "organize",
"label": "Rename and Organize",
"description": "Create folders and rename files using a template. Recommended for Audiobookshelf. Do not use with ingest folders.",
},
],
default="rename",
universal_only=True,
@@ -1194,7 +1226,6 @@ def download_settings():
default=True,
universal_only=True,
),
# === OPTIONS SECTION ===
HeadingField(
key="options_heading",
@@ -1239,7 +1270,7 @@ def download_settings():
register_on_save("downloads", _on_save_downloads)
def _get_fast_source_options():
def _get_fast_source_options() -> list[dict[str, str | bool | int | None]]:
"""Fast download sources - configurable list shown in settings."""
from shelfmark.core.config import config
@@ -1263,7 +1294,7 @@ def _get_fast_source_options():
]
def _get_fast_source_defaults():
def _get_fast_source_defaults() -> list[dict[str, str | bool]]:
"""Default values for fast sources display."""
return [
{"id": "aa-fast", "enabled": True},
@@ -1271,7 +1302,7 @@ def _get_fast_source_defaults():
]
def _get_slow_source_options():
def _get_slow_source_options() -> list[dict[str, str | bool | None]]:
"""Slow download sources - configurable order. All require bypasser."""
from shelfmark.core.config import config
@@ -1311,7 +1342,7 @@ def _get_slow_source_options():
]
def _get_slow_source_defaults():
def _get_slow_source_defaults() -> list[dict[str, str | bool]]:
"""Default source priority order for slow sources."""
from shelfmark.config.env import _LEGACY_ALLOW_USE_WELIB
@@ -1323,8 +1354,10 @@ def _get_slow_source_defaults():
]
@register_settings("download_sources", "Download Sources", icon="download", order=21, group="direct_download")
def download_source_settings():
@register_settings(
"download_sources", "Download Sources", icon="download", order=21, group="direct_download"
)
def download_source_settings() -> list[SettingsField]:
"""Settings for download source behavior."""
return [
PasswordField(
@@ -1429,8 +1462,10 @@ def download_source_settings():
]
@register_settings("cloudflare_bypass", "Cloudflare Bypass", icon="shield", order=22, group="direct_download")
def cloudflare_bypass_settings():
@register_settings(
"cloudflare_bypass", "Cloudflare Bypass", icon="shield", order=22, group="direct_download"
)
def cloudflare_bypass_settings() -> list[SettingsField]:
"""Settings for Cloudflare bypass behavior."""
return [
CheckboxField(
@@ -1477,7 +1512,8 @@ def cloudflare_bypass_settings():
),
]
def _on_save_mirrors(values: Dict[str, Any]) -> Dict[str, Any]:
def _on_save_mirrors(values: dict[str, Any]) -> dict[str, Any]:
"""Normalize mirror list settings before persisting."""
from shelfmark.core.logger import setup_logger
from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS
@@ -1512,14 +1548,19 @@ def _on_save_mirrors(values: Dict[str, Any]) -> Dict[str, Any]:
values["AA_MIRROR_URLS"] = normalized
return {"error": False, "values": values}
# Register the on_save handler for this tab
register_on_save("mirrors", _on_save_mirrors)
@register_settings("mirrors", "Mirrors", icon="globe", order=23, group="direct_download")
def mirror_settings():
def mirror_settings() -> list[SettingsField]:
"""Configure download source mirrors."""
from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS, DEFAULT_ZLIB_MIRRORS, DEFAULT_WELIB_MIRRORS
from shelfmark.core.mirrors import (
DEFAULT_AA_MIRRORS,
DEFAULT_WELIB_MIRRORS,
DEFAULT_ZLIB_MIRRORS,
)
return [
# === PRIMARY SOURCE ===
@@ -1548,7 +1589,6 @@ def mirror_settings():
description="Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables.",
show_when={"field": "AA_ADDITIONAL_URLS", "notEmpty": True},
),
# === LIBGEN ===
HeadingField(
key="libgen_mirrors_heading",
@@ -1560,7 +1600,6 @@ def mirror_settings():
label="Additional Mirrors",
description="Comma-separated list of custom LibGen mirrors to add to the defaults.",
),
# === Z-LIBRARY ===
HeadingField(
key="zlib_mirrors_heading",
@@ -1579,7 +1618,6 @@ def mirror_settings():
label="Additional Mirrors",
description="Comma-separated list of custom Z-Library mirror URLs.",
),
# === WELIB ===
HeadingField(
key="welib_mirrors_heading",
@@ -1602,7 +1640,7 @@ def mirror_settings():
@register_settings("advanced", "Advanced", icon="cog", order=15)
def advanced_settings():
def advanced_settings() -> list[SettingsField]:
"""Advanced settings for power users."""
return [
TextField(
@@ -1648,8 +1686,16 @@ def advanced_settings():
label="Custom Script Path Mode",
description="Pass the path to the custom script as an absolute path or relative to the destination folder.",
options=[
{"value": "absolute", "label": "Absolute", "description": "Pass the full destination path (default)."},
{"value": "relative", "label": "Relative", "description": "Pass the path relative to the destination folder."},
{
"value": "absolute",
"label": "Absolute",
"description": "Pass the full destination path (default).",
},
{
"value": "relative",
"label": "Relative",
"description": "Pass the path relative to the destination folder.",
},
],
default="absolute",
),
shelfmark/config/users_settings.py
+54 -52
View File
@@ -7,6 +7,11 @@ that talks to /api/admin/users endpoints.
from typing import Any
from shelfmark.core.request_policy import (
get_source_content_type_capabilities,
parse_policy_mode,
validate_policy_rules,
)
from shelfmark.core.settings_registry import (
CheckboxField,
CustomComponentField,
@@ -14,16 +19,11 @@ from shelfmark.core.settings_registry import (
MultiSelectField,
NumberField,
SelectField,
SettingsField,
TableField,
register_on_save,
register_settings,
)
from shelfmark.core.request_policy import (
get_source_content_type_capabilities,
parse_policy_mode,
validate_policy_rules,
)
_REQUEST_DEFAULT_MODE_OPTIONS = [
{
@@ -72,7 +72,11 @@ _SELF_SETTINGS_SECTION_OPTIONS = [
_SELF_SETTINGS_SECTION_VALUES = {option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS}
_SELF_SETTINGS_SECTION_DEFAULTS = [option["value"] for option in _SELF_SETTINGS_SECTION_OPTIONS]
_SEARCH_MODE_VALUES = {"direct", "universal"}
_SEARCH_PREFERENCE_PROVIDER_KEYS = {"METADATA_PROVIDER", "METADATA_PROVIDER_AUDIOBOOK", "METADATA_PROVIDER_COMBINED"}
_SEARCH_PREFERENCE_PROVIDER_KEYS = {
"METADATA_PROVIDER",
"METADATA_PROVIDER_AUDIOBOOK",
"METADATA_PROVIDER_COMBINED",
}
_SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
"SEARCH_MODE",
"DEFAULT_RELEASE_SOURCE",
@@ -104,19 +108,17 @@ _USERS_HEADING_DESCRIPTION_BY_AUTH_MODE = {
}
def _get_request_source_options():
def _get_request_source_options() -> list[dict[str, str]]:
"""Build request-policy source options from registered release sources."""
from shelfmark.release_sources import list_available_sources
options = []
for source in list_available_sources():
options.append(
{
"value": source["name"],
"label": source["display_name"],
}
)
return options
return [
{
"value": source["name"],
"label": source["display_name"],
}
for source in list_available_sources()
]
def _get_valid_release_source_names_for_content_type(content_type: str) -> set[str]:
@@ -131,20 +133,20 @@ def _get_valid_release_source_names_for_content_type(content_type: str) -> set[s
return valid_sources
def _get_request_policy_rule_columns():
def _get_request_policy_rule_columns() -> list[dict[str, object]]:
source_capabilities = get_source_content_type_capabilities()
content_type_options = []
for source_name, supported_types in source_capabilities.items():
normalized_types = [t for t in ("ebook", "audiobook") if t in supported_types]
for content_type in normalized_types:
content_type_options.append(
{
"value": content_type,
"label": "Ebook" if content_type == "ebook" else "Audiobook",
"childOf": source_name,
}
)
content_type_options.extend(
{
"value": content_type,
"label": "Ebook" if content_type == "ebook" else "Audiobook",
"childOf": source_name,
}
for content_type in normalized_types
)
return [
{
@@ -224,7 +226,7 @@ def validate_search_preference_value(key: str, value: Any) -> tuple[Any, str | N
return value, None
def _on_save_users(values):
def _on_save_users(values: dict[str, object]) -> dict[str, object]:
"""Validate users/request-policy settings before persistence."""
if "VISIBLE_SELF_SETTINGS_SECTIONS" in values:
raw_sections = values["VISIBLE_SELF_SETTINGS_SECTIONS"]
@@ -233,7 +235,9 @@ def _on_save_users(values):
elif isinstance(raw_sections, str):
candidate_sections = [s.strip() for s in raw_sections.split(",") if s.strip()]
elif isinstance(raw_sections, (list, tuple, set)):
candidate_sections = [str(section).strip() for section in raw_sections if str(section).strip()]
candidate_sections = [
str(section).strip() for section in raw_sections if str(section).strip()
]
else:
return {
"error": True,
@@ -258,21 +262,25 @@ def _on_save_users(values):
values["VISIBLE_SELF_SETTINGS_SECTIONS"] = normalized_sections
if "REQUEST_POLICY_DEFAULT_EBOOK" in values:
if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_EBOOK"]) is None:
return {
"error": True,
"message": "REQUEST_POLICY_DEFAULT_EBOOK must be a valid policy mode",
"values": values,
}
if (
"REQUEST_POLICY_DEFAULT_EBOOK" in values
and parse_policy_mode(values["REQUEST_POLICY_DEFAULT_EBOOK"]) is None
):
return {
"error": True,
"message": "REQUEST_POLICY_DEFAULT_EBOOK must be a valid policy mode",
"values": values,
}
if "REQUEST_POLICY_DEFAULT_AUDIOBOOK" in values:
if parse_policy_mode(values["REQUEST_POLICY_DEFAULT_AUDIOBOOK"]) is None:
return {
"error": True,
"message": "REQUEST_POLICY_DEFAULT_AUDIOBOOK must be a valid policy mode",
"values": values,
}
if (
"REQUEST_POLICY_DEFAULT_AUDIOBOOK" in values
and parse_policy_mode(values["REQUEST_POLICY_DEFAULT_AUDIOBOOK"]) is None
):
return {
"error": True,
"message": "REQUEST_POLICY_DEFAULT_AUDIOBOOK must be a valid policy mode",
"values": values,
}
if "REQUEST_POLICY_RULES" in values:
normalized_rules, errors = validate_policy_rules(values["REQUEST_POLICY_RULES"])
@@ -303,7 +311,7 @@ register_on_save("users", _on_save_users)
@register_settings("users", "Users & Requests", icon="users", order=6)
def users_settings():
def users_settings() -> list[SettingsField]:
"""User management tab - rendered as a custom component on the frontend."""
return [
HeadingField(
@@ -330,9 +338,7 @@ def users_settings():
HeadingField(
key="requests_heading",
title="Requests",
description=(
"Choose what users can download directly and what needs approval first."
),
description=("Choose what users can download directly and what needs approval first."),
),
CheckboxField(
key="REQUESTS_ENABLED",
@@ -356,9 +362,7 @@ def users_settings():
SelectField(
key="REQUEST_POLICY_DEFAULT_EBOOK",
label="Default Ebook Mode",
description=(
"Sets the baseline for all ebook sources."
),
description=("Sets the baseline for all ebook sources."),
options=_REQUEST_DEFAULT_MODE_OPTIONS,
default="download",
user_overridable=True,
@@ -366,9 +370,7 @@ def users_settings():
SelectField(
key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
label="Default Audiobook Mode",
description=(
"Sets the baseline for all audiobook sources."
),
description=("Sets the baseline for all audiobook sources."),
options=_REQUEST_DEFAULT_MODE_OPTIONS,
default="download",
user_overridable=True,
shelfmark/core/__init__.py
+11 -2
View File
@@ -1,5 +1,14 @@
"""Core module - shared models, queue, and utilities."""
from shelfmark.core.models import QueueItem, SearchFilters, QueueStatus
from shelfmark.core.queue import BookQueue, book_queue
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import QueueItem, QueueStatus, SearchFilters
from shelfmark.core.queue import BookQueue, book_queue
__all__ = [
"BookQueue",
"QueueItem",
"QueueStatus",
"SearchFilters",
"book_queue",
"setup_logger",
]
shelfmark/core/activity_routes.py
+97 -54
View File
@@ -2,9 +2,9 @@
from __future__ import annotations
from typing import Any, Callable, NamedTuple
from typing import TYPE_CHECKING, Any, NamedTuple
from flask import Flask, jsonify, request, session
from flask import Flask, Response, jsonify, request, session
from shelfmark.core.activity_view_state_service import (
ADMIN_VIEWER_SCOPE,
@@ -12,22 +12,34 @@ from shelfmark.core.activity_view_state_service import (
ActivityViewStateService,
user_viewer_scope,
)
from shelfmark.core.download_history_service import ACTIVE_DOWNLOAD_STATUS, DownloadHistoryService, VALID_TERMINAL_STATUSES
from shelfmark.core.download_history_service import (
ACTIVE_DOWNLOAD_STATUS,
VALID_TERMINAL_STATUSES,
DownloadHistoryService,
)
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import ACTIVE_QUEUE_STATUSES, QueueStatus, TERMINAL_QUEUE_STATUSES
from shelfmark.core.request_validation import RequestStatus
from shelfmark.core.models import (
ACTIVE_QUEUE_STATUSES,
TERMINAL_QUEUE_STATUSES,
QueueStatus,
)
from shelfmark.core.request_helpers import (
emit_ws_event,
extract_release_source_id,
normalize_positive_int,
populate_request_usernames,
)
from shelfmark.core.user_db import UserDB
from shelfmark.core.request_validation import RequestStatus
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
def _normalize_log_field(value: Any) -> str:
def _normalize_log_field(value: object) -> str:
if value is None:
return "-"
text = str(value).strip()
@@ -39,15 +51,15 @@ def _log_activity_rejection(
*,
status_code: int,
reason: str,
auth_mode: Any = None,
viewer_scope: Any = None,
item_type: Any = None,
item_key: Any = None,
auth_mode: object = None,
viewer_scope: object = None,
item_type: object = None,
item_key: object = None,
item_count: int | None = None,
missing_item_keys: list[str] | None = None,
owner_user_id: Any = None,
final_status: Any = None,
request_id: Any = None,
owner_user_id: object = None,
final_status: object = None,
request_id: object = None,
) -> None:
parts = [
f"Activity {action} rejected",
@@ -86,16 +98,16 @@ def _activity_error_response(
status_code: int,
error: str,
code: str | None = None,
auth_mode: Any = None,
viewer_scope: Any = None,
item_type: Any = None,
item_key: Any = None,
auth_mode: object = None,
viewer_scope: object = None,
item_type: object = None,
item_key: object = None,
item_count: int | None = None,
missing_item_keys: list[str] | None = None,
owner_user_id: Any = None,
final_status: Any = None,
request_id: Any = None,
):
owner_user_id: object = None,
final_status: object = None,
request_id: object = None,
) -> tuple[Response, int]:
_log_activity_rejection(
action,
status_code=status_code,
@@ -119,7 +131,9 @@ def _activity_error_response(
return jsonify(payload), status_code
def _require_authenticated(resolve_auth_mode: Callable[[], str], *, action: str):
def _require_authenticated(
resolve_auth_mode: Callable[[], str], *, action: str
) -> tuple[Response, int] | None:
auth_mode = resolve_auth_mode()
if auth_mode == "none":
return None
@@ -134,12 +148,12 @@ def _require_authenticated(resolve_auth_mode: Callable[[], str], *, action: str)
def _resolve_db_user_id(
require_in_auth_mode: bool = True,
*,
require_in_auth_mode: bool = True,
user_db: UserDB | None = None,
action: str | None = None,
auth_mode: str | None = None,
):
) -> tuple[int | None, tuple[Response, int] | None]:
raw_db_user_id = session.get("db_user_id")
if raw_db_user_id is None:
if not require_in_auth_mode:
@@ -153,7 +167,7 @@ def _resolve_db_user_id(
)
try:
parsed_db_user_id = int(raw_db_user_id)
except (TypeError, ValueError):
except TypeError, ValueError:
if not require_in_auth_mode:
return None, None
return None, _activity_error_response(
@@ -208,7 +222,7 @@ def _resolve_activity_actor(
user_db: UserDB,
resolve_auth_mode: Callable[[], str],
action: str,
) -> tuple[_ActorContext | None, Any | None]:
) -> tuple[_ActorContext | None, object | None]:
"""Resolve acting user identity for activity mutations.
Returns (actor, error_response). On success actor is non-None.
@@ -251,7 +265,7 @@ def _activity_ws_room(actor: _ActorContext) -> str:
return "admins"
def _check_item_ownership(actor: _ActorContext, row: dict[str, Any]) -> Any | None:
def _check_item_ownership(actor: _ActorContext, row: dict[str, Any]) -> object | None:
"""Return an error string if the actor doesn't own the item, else None."""
if actor.is_admin:
return None
@@ -261,14 +275,14 @@ def _check_item_ownership(actor: _ActorContext, row: dict[str, Any]) -> Any | No
return None
def _check_terminal_download(row: dict[str, Any]) -> Any | None:
def _check_terminal_download(row: dict[str, Any]) -> object | None:
final_status = str(row.get("final_status") or "").strip().lower()
if final_status not in VALID_TERMINAL_STATUSES:
return "Only terminal downloads can be dismissed"
return None
def _check_terminal_request(row: dict[str, Any]) -> Any | None:
def _check_terminal_request(row: dict[str, Any]) -> object | None:
if _request_terminal_status(row) is None:
return "Only terminal requests can be dismissed"
return None
@@ -289,7 +303,9 @@ def _request_row_log_context(row: dict[str, Any]) -> dict[str, Any]:
}
def _list_visible_requests(user_db: UserDB, *, is_admin: bool, db_user_id: int | None) -> list[dict[str, Any]]:
def _list_visible_requests(
user_db: UserDB, *, is_admin: bool, db_user_id: int | None
) -> list[dict[str, Any]]:
if is_admin:
request_rows = user_db.list_requests()
populate_request_usernames(request_rows, user_db)
@@ -300,7 +316,7 @@ def _list_visible_requests(user_db: UserDB, *, is_admin: bool, db_user_id: int |
return user_db.list_requests(user_id=db_user_id)
def _parse_item_key(item_key: Any, prefix: str) -> str | None:
def _parse_item_key(item_key: object, prefix: str) -> str | None:
"""Extract the value after 'prefix:' from an item_key string."""
if not isinstance(item_key, str) or not item_key.startswith(f"{prefix}:"):
return None
@@ -311,7 +327,9 @@ def _parse_item_key(item_key: Any, prefix: str) -> str | None:
_ALL_BUCKET_KEYS = (*ACTIVE_QUEUE_STATUSES, *TERMINAL_QUEUE_STATUSES)
def _build_queue_index(queue_status: dict[str, dict[str, Any]]) -> dict[str, tuple[str, dict[str, Any]]]:
def _build_queue_index(
queue_status: dict[str, dict[str, Any]],
) -> dict[str, tuple[str, dict[str, Any]]]:
"""Index live queue entries by task id for fast activity lookups."""
queue_index: dict[str, tuple[str, dict[str, Any]]] = {}
for bucket_key in _ALL_BUCKET_KEYS:
@@ -319,7 +337,9 @@ def _build_queue_index(queue_status: dict[str, dict[str, Any]]) -> dict[str, tup
if not isinstance(bucket, dict):
continue
for task_id, payload in bucket.items():
normalized_bucket_key = bucket_key.value if isinstance(bucket_key, QueueStatus) else str(bucket_key)
normalized_bucket_key = (
bucket_key.value if isinstance(bucket_key, QueueStatus) else str(bucket_key)
)
queue_index[str(task_id)] = (normalized_bucket_key, payload)
return queue_index
@@ -474,12 +494,12 @@ def register_activity_routes(
queue_status: Callable[..., dict[str, dict[str, Any]]],
sync_request_delivery_states: Callable[..., list[dict[str, Any]]],
emit_request_updates: Callable[[list[dict[str, Any]]], None],
ws_manager: Any | None = None,
ws_manager: object | None = None,
) -> None:
"""Register activity routes."""
@app.route("/api/activity/snapshot", methods=["GET"])
def api_activity_snapshot():
def api_activity_snapshot() -> Response | tuple[Response, int]:
auth_gate = _require_authenticated(resolve_auth_mode, action="snapshot")
if auth_gate is not None:
return auth_gate
@@ -548,7 +568,7 @@ def register_activity_routes(
)
@app.route("/api/activity/dismiss", methods=["POST"])
def api_activity_dismiss():
def api_activity_dismiss() -> Response | tuple[Response, int]:
auth_gate = _require_authenticated(resolve_auth_mode, action="dismiss")
if auth_gate is not None:
return auth_gate
@@ -630,7 +650,10 @@ def register_activity_routes(
item_type="download",
item_key=f"download:{task_id}",
)
dismissal_item = {"item_type": "download", "item_key": f"download:{task_id}"}
dismissal_item = {
"item_type": "download",
"item_key": f"download:{task_id}",
}
elif item_type == "request":
request_id = normalize_positive_int(_parse_item_key(item_key, "request"))
@@ -688,7 +711,10 @@ def register_activity_routes(
item_type="request",
item_key=f"request:{request_id}",
)
dismissal_item = {"item_type": "request", "item_key": f"request:{request_id}"}
dismissal_item = {
"item_type": "request",
"item_key": f"request:{request_id}",
}
else:
return _activity_error_response(
"dismiss",
@@ -715,7 +741,7 @@ def register_activity_routes(
return jsonify({"status": "dismissed", "item": dismissal_item})
@app.route("/api/activity/dismiss-many", methods=["POST"])
def api_activity_dismiss_many():
def api_activity_dismiss_many() -> Response | tuple[Response, int]:
auth_gate = _require_authenticated(resolve_auth_mode, action="dismiss_many")
if auth_gate is not None:
return auth_gate
@@ -860,7 +886,9 @@ def register_activity_routes(
item_count=len(items),
**_request_row_log_context(request_row),
)
dismissal_items.append({"item_type": "request", "item_key": f"request:{request_id}"})
dismissal_items.append(
{"item_type": "request", "item_key": f"request:{request_id}"}
)
continue
return _activity_error_response(
@@ -904,7 +932,7 @@ def register_activity_routes(
return jsonify({"status": "dismissed", "count": dismissed_count})
@app.route("/api/activity/history", methods=["GET"])
def api_activity_history():
def api_activity_history() -> Response | tuple[Response, int]:
auth_gate = _require_authenticated(resolve_auth_mode, action="history")
if auth_gate is not None:
return auth_gate
@@ -924,9 +952,15 @@ def register_activity_routes(
if offset is None:
offset = 0
if limit < 1:
return _activity_error_response("history", status_code=400, error="limit must be a positive integer")
return _activity_error_response(
"history", status_code=400, error="limit must be a positive integer"
)
if offset < 0:
return _activity_error_response("history", status_code=400, error="offset must be a non-negative integer")
return _activity_error_response(
"history",
status_code=400,
error="offset must be a non-negative integer",
)
history_rows = activity_view_state_service.list_history(
viewer_scope=actor.viewer_scope,
@@ -942,21 +976,25 @@ def register_activity_routes(
dismissed_at = history_row.get("dismissed_at")
if not isinstance(dismissed_at, str) or not dismissed_at.strip():
raise RuntimeError(f"Activity history state missing dismissed_at for {item_key}")
msg = f"Activity history state missing dismissed_at for {item_key}"
raise RuntimeError(msg)
if item_type == "download":
task_id = _parse_item_key(item_key, "download")
if task_id is None:
raise RuntimeError(f"Invalid activity history item_key: {item_key}")
msg = f"Invalid activity history item_key: {item_key}"
raise RuntimeError(msg)
download_row = download_history_service.get_by_task_id(task_id)
if download_row is None:
raise RuntimeError(f"Download history row not found for {item_key}")
msg = f"Download history row not found for {item_key}"
raise RuntimeError(msg)
if not actor.is_admin:
owner_user_id = normalize_positive_int(download_row.get("user_id"))
if owner_user_id != actor.db_user_id:
raise RuntimeError(f"Viewer state out of scope for {item_key}")
msg = f"Viewer state out of scope for {item_key}"
raise RuntimeError(msg)
effective_download_row = _effective_download_row_for_activity(
download_row,
@@ -973,16 +1011,19 @@ def register_activity_routes(
if item_type == "request":
request_id = normalize_positive_int(_parse_item_key(item_key, "request"))
if request_id is None:
raise RuntimeError(f"Invalid activity history item_key: {item_key}")
msg = f"Invalid activity history item_key: {item_key}"
raise RuntimeError(msg)
request_row = user_db.get_request(request_id)
if request_row is None:
raise RuntimeError(f"Request row not found for {item_key}")
msg = f"Request row not found for {item_key}"
raise RuntimeError(msg)
if not actor.is_admin:
owner_user_id = normalize_positive_int(request_row.get("user_id"))
if owner_user_id != actor.db_user_id:
raise RuntimeError(f"Viewer state out of scope for {item_key}")
msg = f"Viewer state out of scope for {item_key}"
raise RuntimeError(msg)
populate_request_usernames([request_row], user_db)
entry = _request_history_entry(
@@ -990,16 +1031,18 @@ def register_activity_routes(
dismissed_at=dismissed_at,
)
if entry is None:
raise RuntimeError(f"Failed to build request history entry for {item_key}")
msg = f"Failed to build request history entry for {item_key}"
raise RuntimeError(msg)
payload.append(entry)
continue
raise RuntimeError(f"Unknown activity history item_type: {item_type}")
msg = f"Unknown activity history item_type: {item_type}"
raise RuntimeError(msg)
return jsonify(payload)
@app.route("/api/activity/history", methods=["DELETE"])
def api_activity_history_clear():
def api_activity_history_clear() -> Response | tuple[Response, int]:
auth_gate = _require_authenticated(resolve_auth_mode, action="history_clear")
if auth_gate is not None:
return auth_gate
shelfmark/core/activity_view_state_service.py
+30 -20
View File
@@ -8,7 +8,6 @@ from typing import Any
from shelfmark.core.request_helpers import now_utc_iso
VALID_ACTIVITY_ITEM_TYPES = frozenset({"download", "request"})
ADMIN_VIEWER_SCOPE = "admin:shared"
NOAUTH_VIEWER_SCOPE = "noauth:shared"
@@ -17,58 +16,65 @@ USER_VIEWER_SCOPE_PREFIX = "user:"
def user_viewer_scope(user_id: int) -> str:
if not isinstance(user_id, int) or user_id < 1:
raise ValueError("user_id must be a positive integer")
msg = "user_id must be a positive integer"
raise ValueError(msg)
return f"{USER_VIEWER_SCOPE_PREFIX}{user_id}"
def normalize_viewer_scope(viewer_scope: Any) -> str:
def normalize_viewer_scope(viewer_scope: object) -> str:
if not isinstance(viewer_scope, str) or not viewer_scope.strip():
raise ValueError("viewer_scope must be a non-empty string")
msg = "viewer_scope must be a non-empty string"
raise ValueError(msg)
normalized = viewer_scope.strip()
if normalized in {ADMIN_VIEWER_SCOPE, NOAUTH_VIEWER_SCOPE}:
return normalized
if not normalized.startswith(USER_VIEWER_SCOPE_PREFIX):
raise ValueError(
"viewer_scope must be one of: admin:shared, noauth:shared, or user:<id>"
)
msg = "viewer_scope must be one of: admin:shared, noauth:shared, or user:<id>"
raise ValueError(msg)
raw_user_id = normalized[len(USER_VIEWER_SCOPE_PREFIX):].strip()
raw_user_id = normalized[len(USER_VIEWER_SCOPE_PREFIX) :].strip()
try:
parsed_user_id = int(raw_user_id)
except (TypeError, ValueError) as exc:
raise ValueError("viewer_scope user id must be a positive integer") from exc
msg = "viewer_scope user id must be a positive integer"
raise ValueError(msg) from exc
return user_viewer_scope(parsed_user_id)
def _normalize_item_type(item_type: Any) -> str:
def _normalize_item_type(item_type: object) -> str:
if not isinstance(item_type, str) or not item_type.strip():
raise ValueError("item_type must be a non-empty string")
msg = "item_type must be a non-empty string"
raise ValueError(msg)
normalized = item_type.strip().lower()
if normalized not in VALID_ACTIVITY_ITEM_TYPES:
raise ValueError("item_type must be one of: download, request")
msg = "item_type must be one of: download, request"
raise ValueError(msg)
return normalized
def _normalize_item_key(item_key: Any, *, item_type: str) -> str:
def _normalize_item_key(item_key: object, *, item_type: str) -> str:
if not isinstance(item_key, str) or not item_key.strip():
raise ValueError("item_key must be a non-empty string")
msg = "item_key must be a non-empty string"
raise ValueError(msg)
normalized = item_key.strip()
expected_prefix = f"{item_type}:"
if not normalized.startswith(expected_prefix):
raise ValueError(f"item_key must be in the format {expected_prefix}<id>")
msg_0 = f"item_key must be in the format {expected_prefix}<id>"
raise ValueError(msg_0)
if not normalized.split(":", 1)[1].strip():
raise ValueError(f"item_key must be in the format {expected_prefix}<id>")
msg_0 = f"item_key must be in the format {expected_prefix}<id>"
raise ValueError(msg_0)
return normalized
class ActivityViewStateService:
"""Service for per-viewer activity dismissal and history visibility."""
def __init__(self, db_path: str):
def __init__(self, db_path: str) -> None:
self._db_path = db_path
self._lock = threading.Lock()
@@ -215,7 +221,12 @@ class ActivityViewStateService:
dismissed_at = excluded.dismissed_at,
cleared_at = NULL
""",
(normalized_scope, normalized_type, normalized_key, dismissed_at),
(
normalized_scope,
normalized_type,
normalized_key,
dismissed_at,
),
)
rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
total += max(rowcount, 0)
@@ -286,8 +297,7 @@ class ActivityViewStateService:
def delete_items(self, *, item_type: str, item_keys: list[str]) -> int:
normalized_type = _normalize_item_type(item_type)
normalized_keys = [
_normalize_item_key(item_key, item_type=normalized_type)
for item_key in item_keys
_normalize_item_key(item_key, item_type=normalized_type) for item_key in item_keys
]
if not normalized_keys:
return 0
shelfmark/core/admin_routes.py
+123 -86
View File
@@ -4,12 +4,12 @@ Registers /api/admin/users CRUD endpoints for managing users.
All endpoints require admin session.
"""
from functools import wraps
import os
import sqlite3
from typing import Any
from functools import wraps
from typing import TYPE_CHECKING, Any
from flask import Flask, g, jsonify, request, session
from flask import Flask, Response, g, jsonify, request, session
from werkzeug.security import generate_password_hash
from shelfmark.config.booklore_settings import (
@@ -17,7 +17,6 @@ from shelfmark.config.booklore_settings import (
get_booklore_path_options,
)
from shelfmark.config.env import CWA_DB_PATH
from shelfmark.core.config import config as app_config
from shelfmark.core.admin_settings_routes import (
register_admin_settings_routes,
validate_user_settings,
@@ -31,12 +30,24 @@ from shelfmark.core.auth_modes import (
load_active_auth_mode,
normalize_auth_source,
)
from shelfmark.core.config import config as app_config
from shelfmark.core.cwa_user_sync import sync_cwa_users_from_rows
from shelfmark.core.logger import setup_logger
from shelfmark.core.user_db import UserDB
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
__all__ = [
"get_booklore_library_options",
"get_booklore_path_options",
"register_admin_routes",
"validate_user_settings",
]
def _get_user_edit_capabilities(
user: dict[str, Any],
@@ -85,8 +96,7 @@ def _oidc_role_management_message(security_config: dict[str, Any] | None = None)
f"'{admin_group}' group in your identity provider"
)
return (
"Disable 'Use Admin Group for Authorization' in security settings "
"to manage roles manually"
"Disable 'Use Admin Group for Authorization' in security settings to manage roles manually"
)
@@ -109,8 +119,6 @@ def _serialize_user(
return payload
def _sync_all_cwa_users(user_db: UserDB) -> dict[str, int]:
"""Sync all users from the Calibre-Web database into users.db."""
if not CWA_DB_PATH or not CWA_DB_PATH.exists():
@@ -132,15 +140,18 @@ def _sync_all_cwa_users(user_db: UserDB) -> dict[str, int]:
def register_admin_routes(app: Flask, user_db: UserDB) -> None:
"""Register admin user management routes on the Flask app."""
def _require_admin(f):
def _require_admin(
f: Callable[..., Response | tuple[Response, int]],
) -> Callable[..., Response | tuple[Response, int]]:
"""Decorator to require admin session for admin routes.
In no-auth mode, everyone has access (is_admin defaults True).
In auth-required modes, requires an authenticated session with admin role.
Caches the resolved auth_mode in ``g.auth_mode`` for the request.
"""
@wraps(f)
def decorated(*args, **kwargs):
def decorated(*args, **kwargs) -> Response | tuple[Response, int]:
auth_mode = load_active_auth_mode(CWA_DB_PATH, user_db=user_db)
g.auth_mode = auth_mode
if auth_mode != "none":
@@ -149,22 +160,20 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
if not session.get("is_admin", False):
return jsonify({"error": "Admin access required"}), 403
return f(*args, **kwargs)
return decorated
@app.route("/api/admin/users", methods=["GET"])
@_require_admin
def admin_list_users():
def admin_list_users() -> Response | tuple[Response, int]:
"""List all users."""
users = user_db.list_users()
auth_mode = g.auth_mode
return jsonify([
_serialize_user(u, auth_mode)
for u in users
])
return jsonify([_serialize_user(u, auth_mode) for u in users])
@app.route("/api/admin/users", methods=["POST"])
@_require_admin
def admin_create_user():
def admin_create_user() -> Response | tuple[Response, int]:
"""Create a new user with password authentication."""
data = request.get_json() or {}
auth_mode = g.auth_mode
@@ -176,13 +185,15 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
role = data.get("role", "user")
if auth_mode in {AUTH_SOURCE_PROXY, AUTH_SOURCE_CWA}:
return jsonify({
"error": "Local user creation is disabled in this authentication mode",
"message": (
"Users are provisioned by your external authentication source. "
"Switch to builtin or OIDC mode to create local users."
),
}), 400
return jsonify(
{
"error": "Local user creation is disabled in this authentication mode",
"message": (
"Users are provisioned by your external authentication source. "
"Switch to builtin or OIDC mode to create local users."
),
}
), 400
if not username:
return jsonify({"error": "Username is required"}), 400
@@ -213,9 +224,11 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
except ValueError:
return jsonify({"error": "Username already exists"}), 409
logger.info(
"Shelfmark user created "
f"(source=manual_admin_create, created_by={session.get('user_id', 'unknown')}, "
f"username={username}, role={role}, auth_source={AUTH_SOURCE_BUILTIN})"
"Shelfmark user created (source=manual_admin_create, created_by=%s, username=%s, role=%s, auth_source=%s)",
session.get("user_id", "unknown"),
username,
role,
AUTH_SOURCE_BUILTIN,
)
return jsonify(
_serialize_user(
@@ -226,7 +239,7 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
@app.route("/api/admin/users/<int:user_id>", methods=["GET"])
@_require_admin
def admin_get_user(user_id):
def admin_get_user(user_id: int) -> Response | tuple[Response, int]:
"""Get a user by ID with their settings."""
user = user_db.get_user(user_id=user_id)
if not user:
@@ -241,7 +254,7 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
@app.route("/api/admin/users/<int:user_id>", methods=["PUT"])
@_require_admin
def admin_update_user(user_id):
def admin_update_user(user_id: int) -> Response | tuple[Response, int]:
"""Update user fields and/or settings."""
user = user_db.get_user(user_id=user_id)
if not user:
@@ -258,10 +271,12 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
password = data.get("password", "")
if password:
if not capabilities["canSetPassword"]:
return jsonify({
"error": f"Cannot set password for {auth_source.upper()} users",
"message": "Password authentication is only available for local users.",
}), 400
return jsonify(
{
"error": f"Cannot set password for {auth_source.upper()} users",
"message": "Password authentication is only available for local users.",
}
), 400
if len(password) < 4:
return jsonify({"error": "Password must be at least 4 characters"}), 400
user_db.update_user(user_id, password_hash=generate_password_hash(password))
@@ -277,40 +292,49 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
role_changed = "role" in user_fields and user_fields["role"] != user.get("role")
email_changed = "email" in user_fields and user_fields["email"] != user.get("email")
display_name_changed = (
"display_name" in user_fields
and user_fields["display_name"] != user.get("display_name")
)
display_name_changed = "display_name" in user_fields and user_fields[
"display_name"
] != user.get("display_name")
if role_changed and not capabilities["canEditRole"]:
if auth_source == AUTH_SOURCE_OIDC:
return jsonify({
"error": "Cannot change role for OIDC user when group-based authorization is enabled",
"message": _oidc_role_management_message(),
}), 400
return jsonify(
{
"error": "Cannot change role for OIDC user when group-based authorization is enabled",
"message": _oidc_role_management_message(),
}
), 400
return jsonify({
"error": f"Cannot change role for {auth_source.upper()} users",
"message": "Role is managed by the external authentication source.",
}), 400
return jsonify(
{
"error": f"Cannot change role for {auth_source.upper()} users",
"message": "Role is managed by the external authentication source.",
}
), 400
if email_changed and not capabilities["canEditEmail"]:
if auth_source == AUTH_SOURCE_CWA:
return jsonify({
"error": "Cannot change email for CWA users",
"message": "Email is synced from Calibre-Web.",
}), 400
return jsonify(
{
"error": "Cannot change email for CWA users",
"message": "Email is synced from Calibre-Web.",
}
), 400
return jsonify({
"error": "Cannot change email for OIDC users",
"message": "Email is managed by your identity provider.",
}), 400
return jsonify(
{
"error": "Cannot change email for OIDC users",
"message": "Email is managed by your identity provider.",
}
), 400
if display_name_changed and not capabilities["canEditDisplayName"]:
return jsonify({
"error": "Cannot change display name for OIDC users",
"message": "Display name is managed by your identity provider.",
}), 400
return jsonify(
{
"error": "Cannot change display name for OIDC users",
"message": "Display name is managed by your identity provider.",
}
), 400
# Allow demoting the last admin account.
# Auth mode resolution automatically falls back to "none" when no
@@ -331,15 +355,18 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
validated_settings, validation_errors = validate_user_settings(data["settings"])
if validation_errors:
return jsonify({
"error": "Invalid settings payload",
"details": validation_errors,
}), 400
return jsonify(
{
"error": "Invalid settings payload",
"details": validation_errors,
}
), 400
user_db.set_user_settings(user_id, validated_settings)
# Ensure runtime reads see updated per-user overrides immediately.
try:
from shelfmark.core.config import config as app_config
app_config.refresh(force=True)
except Exception:
pass
@@ -350,30 +377,36 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
g.auth_mode,
)
result["settings"] = user_db.get_user_settings(user_id)
logger.info(f"Admin updated user {user_id}")
logger.info("Admin updated user %s", user_id)
return jsonify(result)
@app.route("/api/admin/users/sync-cwa", methods=["POST"])
@_require_admin
def admin_sync_cwa_users():
def admin_sync_cwa_users() -> Response | tuple[Response, int]:
"""Manually sync users from Calibre-Web into users.db."""
if g.auth_mode != AUTH_SOURCE_CWA:
return jsonify({
"error": "CWA sync is only available when CWA authentication is enabled",
}), 400
return jsonify(
{
"error": "CWA sync is only available when CWA authentication is enabled",
}
), 400
try:
summary = _sync_all_cwa_users(user_db)
except FileNotFoundError:
return jsonify({
"error": "Calibre-Web database is not available",
"message": "Verify app.db is mounted and readable at /auth/app.db.",
}), 503
except Exception as exc:
logger.error(f"Failed to sync CWA users: {exc}")
return jsonify({
"error": "Failed to sync users from Calibre-Web",
}), 500
return jsonify(
{
"error": "Calibre-Web database is not available",
"message": "Verify app.db is mounted and readable at /auth/app.db.",
}
), 503
except Exception:
logger.exception("Failed to sync CWA users")
return jsonify(
{
"error": "Failed to sync users from Calibre-Web",
}
), 500
message = (
f"Synced {summary['total']} CWA users "
@@ -381,17 +414,19 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
f"{summary.get('deleted', 0)} deleted)."
)
logger.info(message)
return jsonify({
"success": True,
"message": message,
**summary,
})
return jsonify(
{
"success": True,
"message": message,
**summary,
}
)
register_admin_settings_routes(app, user_db, _require_admin)
@app.route("/api/admin/users/<int:user_id>", methods=["DELETE"])
@_require_admin
def admin_delete_user(user_id):
def admin_delete_user(user_id: int) -> Response | tuple[Response, int]:
"""Delete a user."""
# Prevent self-deletion
if session.get("db_user_id") == user_id:
@@ -406,15 +441,17 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
user.get("oidc_subject"),
)
if auth_source == AUTH_SOURCE_CWA and auth_source == g.auth_mode:
return jsonify({
"error": f"Cannot delete active {auth_source.upper()} users",
"message": f"{auth_source.upper()} users are automatically re-provisioned on login.",
}), 400
return jsonify(
{
"error": f"Cannot delete active {auth_source.upper()} users",
"message": f"{auth_source.upper()} users are automatically re-provisioned on login.",
}
), 400
# Allow deleting the last local admin account.
# Auth mode resolution automatically falls back to "none" when no
# local password admin remains.
user_db.delete_user(user_id)
logger.info(f"Admin deleted user {user_id}: {user['username']}")
logger.info("Admin deleted user %s: %s", user_id, user["username"])
return jsonify({"success": True})
shelfmark/core/admin_settings_routes.py
+51 -37
View File
@@ -1,27 +1,37 @@
"""Admin settings-introspection routes and settings validation helpers."""
from typing import Any, Callable
from typing import TYPE_CHECKING, Any
from flask import Flask, jsonify, request
from flask import Flask, Response, jsonify, request
from shelfmark.core.config import config as app_config
from shelfmark.config.notifications_settings import (
build_notification_test_result,
is_valid_notification_url,
normalize_notification_routes,
)
from shelfmark.config.users_settings import validate_search_preference_value
from shelfmark.core.config import config as app_config
from shelfmark.core.request_policy import parse_policy_mode, validate_policy_rules
from shelfmark.core.settings_registry import load_config_file
from shelfmark.core.user_settings_overrides import (
build_user_preferences_payload as _build_user_preferences_payload,
)
from shelfmark.core.user_settings_overrides import (
get_ordered_user_overridable_fields as _get_ordered_user_overridable_fields,
)
from shelfmark.core.user_settings_overrides import (
get_settings_registry as _get_settings_registry,
)
from shelfmark.core.user_db import UserDB
from shelfmark.core.request_policy import parse_policy_mode, validate_policy_rules
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.user_db import UserDB
def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
def validate_user_settings(
settings: dict[str, Any],
) -> tuple[dict[str, Any], list[str]]:
settings_registry = _get_settings_registry()
field_map = settings_registry.get_settings_field_map()
overridable_map = settings_registry.get_user_overridable_fields()
@@ -39,10 +49,12 @@ def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], li
valid[key] = None
continue
if key in {"REQUEST_POLICY_DEFAULT_EBOOK", "REQUEST_POLICY_DEFAULT_AUDIOBOOK"}:
if parse_policy_mode(value) is None:
errors.append(f"Invalid policy mode for {key}: {value}")
continue
if (
key in {"REQUEST_POLICY_DEFAULT_EBOOK", "REQUEST_POLICY_DEFAULT_AUDIOBOOK"}
and parse_policy_mode(value) is None
):
errors.append(f"Invalid policy mode for {key}: {value}")
continue
if key == "REQUEST_POLICY_RULES":
normalized_rules, rule_errors = validate_policy_rules(value)
@@ -61,16 +73,16 @@ def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], li
)
if invalid_count:
errors.append(
(
f"Invalid value for {key}: found {invalid_count} invalid URL(s). "
"Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
)
f"Invalid value for {key}: found {invalid_count} invalid URL(s). "
"Use URL values with a valid scheme, e.g. discord://... or ntfys://..."
)
continue
valid[key] = normalized_routes
continue
normalized_search_value, search_validation_error = validate_search_preference_value(key, value)
normalized_search_value, search_validation_error = validate_search_preference_value(
key, value
)
if search_validation_error:
errors.append(search_validation_error)
continue
@@ -90,9 +102,7 @@ def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], li
continue
candidate_values = [
str(entry).strip().lower()
for entry in value
if str(entry).strip()
str(entry).strip().lower() for entry in value if str(entry).strip()
]
normalized_values: list[str] = []
has_invalid_value = False
@@ -120,7 +130,7 @@ def validate_user_settings(settings: dict[str, Any]) -> tuple[dict[str, Any], li
def build_user_notification_test_response(
*,
user_id: int,
payload: Any,
payload: object,
) -> tuple[dict[str, Any], int]:
from shelfmark.core.config import config as app_config
@@ -139,11 +149,11 @@ def build_user_notification_test_response(
def register_admin_settings_routes(
app: Flask,
user_db: UserDB,
require_admin: Callable[[Callable[..., Any]], Callable[..., Any]],
require_admin: Callable[[Callable[..., object]], Callable[..., object]],
) -> None:
@app.route("/api/admin/download-defaults", methods=["GET"])
@require_admin
def admin_download_defaults():
def admin_download_defaults() -> Response | tuple[Response, int]:
defaults = {
key: ("" if (value := app_config.get(key, field.default)) is None else value)
for key, field in _get_ordered_user_overridable_fields("downloads")
@@ -156,17 +166,19 @@ def register_admin_settings_routes(
@app.route("/api/admin/booklore-options", methods=["GET"])
@require_admin
def admin_booklore_options():
def admin_booklore_options() -> Response | tuple[Response, int]:
from shelfmark.core import admin_routes
return jsonify({
"libraries": admin_routes.get_booklore_library_options(),
"paths": admin_routes.get_booklore_path_options(),
})
return jsonify(
{
"libraries": admin_routes.get_booklore_library_options(),
"paths": admin_routes.get_booklore_path_options(),
}
)
@app.route("/api/admin/users/<int:user_id>/delivery-preferences", methods=["GET"])
@require_admin
def admin_get_delivery_preferences(user_id):
def admin_get_delivery_preferences(user_id: int) -> Response | tuple[Response, int]:
user = user_db.get_user(user_id=user_id)
if not user:
return jsonify({"error": "User not found"}), 404
@@ -180,7 +192,7 @@ def register_admin_settings_routes(
@app.route("/api/admin/users/<int:user_id>/search-preferences", methods=["GET"])
@require_admin
def admin_get_search_preferences(user_id):
def admin_get_search_preferences(user_id: int) -> Response | tuple[Response, int]:
user = user_db.get_user(user_id=user_id)
if not user:
return jsonify({"error": "User not found"}), 404
@@ -194,7 +206,7 @@ def register_admin_settings_routes(
@app.route("/api/admin/users/<int:user_id>/notification-preferences", methods=["GET"])
@require_admin
def admin_get_notification_preferences(user_id):
def admin_get_notification_preferences(user_id: int) -> Response | tuple[Response, int]:
user = user_db.get_user(user_id=user_id)
if not user:
return jsonify({"error": "User not found"}), 404
@@ -208,7 +220,7 @@ def register_admin_settings_routes(
@app.route("/api/admin/users/<int:user_id>/notification-preferences/test", methods=["POST"])
@require_admin
def admin_test_notification_preferences(user_id):
def admin_test_notification_preferences(user_id: int) -> Response | tuple[Response, int]:
user = user_db.get_user(user_id=user_id)
if not user:
return jsonify({"error": "User not found"}), 404
@@ -222,7 +234,7 @@ def register_admin_settings_routes(
@app.route("/api/admin/settings/overrides-summary", methods=["GET"])
@require_admin
def admin_settings_overrides_summary():
def admin_settings_overrides_summary() -> Response | tuple[Response, int]:
settings_registry = _get_settings_registry()
tab_name = (request.args.get("tab") or "downloads").strip()
@@ -241,11 +253,13 @@ def register_admin_settings_routes(
if key not in user_settings or user_settings[key] is None:
continue
entry = keys_payload.setdefault(key, {"count": 0, "users": []})
entry["users"].append({
"userId": user_record["id"],
"username": user_record["username"],
"value": user_settings[key],
})
entry["users"].append(
{
"userId": user_record["id"],
"username": user_record["username"],
"value": user_settings[key],
}
)
for summary in keys_payload.values():
summary["count"] = len(summary["users"])
@@ -254,7 +268,7 @@ def register_admin_settings_routes(
@app.route("/api/admin/users/<int:user_id>/effective-settings", methods=["GET"])
@require_admin
def admin_get_effective_settings(user_id):
def admin_get_effective_settings(user_id: int) -> Response | tuple[Response, int]:
user = user_db.get_user(user_id=user_id)
if not user:
return jsonify({"error": "User not found"}), 404
shelfmark/core/auth_modes.py
+14 -10
View File
@@ -1,7 +1,11 @@
"""Authentication mode, auth-source normalization, and admin access policy helpers."""
import os
from typing import Any, Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from collections.abc import Mapping
AUTH_SOURCE_BUILTIN = "builtin"
AUTH_SOURCE_OIDC = "oidc"
@@ -17,7 +21,7 @@ AUTH_SOURCE_SET = frozenset(AUTH_SOURCES)
_ALWAYS_ADMIN_SETTINGS_TABS = frozenset({"security", "users"})
def has_local_password_admin(user_db: Any | None = None) -> bool:
def has_local_password_admin(user_db: object | None = None) -> bool:
"""Return True when at least one local admin with a password exists."""
try:
db = user_db
@@ -25,7 +29,7 @@ def has_local_password_admin(user_db: Any | None = None) -> bool:
from shelfmark.core.user_db import UserDB
config_root = os.environ.get("CONFIG_DIR", "/config")
db = UserDB(os.path.join(config_root, "users.db"))
db = UserDB(str(Path(config_root) / "users.db"))
db.initialize()
return db.has_admin_with_password()
@@ -34,8 +38,8 @@ def has_local_password_admin(user_db: Any | None = None) -> bool:
def normalize_auth_source(
source: Any,
oidc_subject: Any = None,
source: object,
oidc_subject: object = None,
) -> str:
"""Resolve a stable auth source value from persisted fields."""
normalized = str(source or "").strip().lower()
@@ -48,7 +52,7 @@ def normalize_auth_source(
def determine_auth_mode(
security_config: Mapping[str, Any],
cwa_db_path: Any | None,
cwa_db_path: object | None,
*,
has_local_admin: bool = True,
) -> str:
@@ -76,9 +80,9 @@ def determine_auth_mode(
def load_active_auth_mode(
cwa_db_path: Any | None,
cwa_db_path: object | None,
*,
user_db: Any | None = None,
user_db: object | None = None,
) -> str:
"""Resolve active auth mode using current security config and runtime prerequisites."""
try:
@@ -109,7 +113,7 @@ def is_user_active_for_auth_mode(user: Mapping[str, Any], auth_mode: str) -> boo
def is_settings_or_onboarding_path(path: str) -> bool:
"""Return True when request path targets protected admin settings routes."""
return path.startswith("/api/settings") or path.startswith("/api/onboarding")
return path.startswith(("/api/settings", "/api/onboarding"))
def get_settings_tab_from_path(path: str) -> str | None:
@@ -117,7 +121,7 @@ def get_settings_tab_from_path(path: str) -> str | None:
if not path.startswith("/api/settings/"):
return None
suffix = path[len("/api/settings/"):]
suffix = path[len("/api/settings/") :]
if not suffix:
return None
shelfmark/core/cache.py
+27 -36
View File
@@ -4,32 +4,37 @@ import threading
import time
from dataclasses import dataclass
from functools import wraps
from typing import Any, Callable, Dict, Optional, TypeVar
from typing import TYPE_CHECKING, ParamSpec, TypeVar, cast
from shelfmark.core.logger import setup_logger
if TYPE_CHECKING:
from collections.abc import Callable
logger = setup_logger(__name__)
T = TypeVar("T")
P = ParamSpec("P")
R = TypeVar("R")
@dataclass
class CacheEntry:
"""A cached value with expiration time."""
value: Any
value: object
expires_at: float
class CacheService:
"""Thread-safe in-memory cache with TTL support."""
def __init__(self, max_size: int = 1000):
def __init__(self, max_size: int = 1000) -> None:
"""Initialize cache with max_size entries before eviction."""
self._cache: Dict[str, CacheEntry] = {}
self._cache: dict[str, CacheEntry] = {}
self._lock = threading.Lock()
self._max_size = max_size
def get(self, key: str) -> Optional[Any]:
def get(self, key: str) -> object | None:
"""Get cached value if not expired."""
with self._lock:
entry = self._cache.get(key)
@@ -42,17 +47,14 @@ class CacheService:
return entry.value
def set(self, key: str, value: Any, ttl: int) -> None:
def set(self, key: str, value: object, ttl: int) -> None:
"""Cache value with TTL in seconds."""
with self._lock:
# Evict oldest entries if at capacity
if len(self._cache) >= self._max_size:
self._evict_oldest()
self._cache[key] = CacheEntry(
value=value,
expires_at=time.time() + ttl
)
self._cache[key] = CacheEntry(value=value, expires_at=time.time() + ttl)
def invalidate(self, key: str) -> bool:
"""Remove specific cache entry. Returns True if found."""
@@ -79,10 +81,7 @@ class CacheService:
"""Remove all expired entries. Returns count removed."""
with self._lock:
now = time.time()
expired_keys = [
key for key, entry in self._cache.items()
if entry.expires_at < now
]
expired_keys = [key for key, entry in self._cache.items() if entry.expires_at < now]
for key in expired_keys:
del self._cache[key]
return len(expired_keys)
@@ -94,21 +93,15 @@ class CacheService:
# Remove ~10% of entries, oldest first
entries_to_remove = max(1, len(self._cache) // 10)
sorted_entries = sorted(
self._cache.items(),
key=lambda x: x[1].expires_at
)
sorted_entries = sorted(self._cache.items(), key=lambda x: x[1].expires_at)
for key, _ in sorted_entries[:entries_to_remove]:
del self._cache[key]
def stats(self) -> Dict[str, int]:
def stats(self) -> dict[str, int]:
"""Get cache statistics (size, max_size)."""
with self._lock:
return {
"size": len(self._cache),
"max_size": self._max_size
}
return {"size": len(self._cache), "max_size": self._max_size}
# Global cache instance for metadata providers
@@ -128,15 +121,16 @@ def cache_key(*args, **kwargs) -> str:
def cacheable(
ttl: Optional[int] = None,
ttl_key: Optional[str] = None,
ttl: int | None = None,
ttl_key: str | None = None,
ttl_default: int = 300,
key_prefix: str = ""
):
key_prefix: str = "",
) -> Callable[[Callable[P, R]], Callable[P, R]]:
"""Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
def decorator(func: Callable[..., T]) -> Callable[..., T]:
def decorator(func: Callable[P, R]) -> Callable[P, R]:
@wraps(func)
def wrapper(*args, **kwargs) -> T:
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
# Check if metadata caching is enabled
from shelfmark.core.config import config
@@ -156,16 +150,12 @@ def cacheable(
# Skip 'self' argument if present (first arg of method)
cache_args = args[1:] if args and hasattr(args[0], func.__name__) else args
key = cache_key(
key_prefix or func.__name__,
*cache_args,
**kwargs
)
key = cache_key(key_prefix or func.__name__, *cache_args, **kwargs)
# Check cache
cached = _metadata_cache.get(key)
if cached is not None:
return cached
return cast("R", cached)
# Execute function and cache result
result = func(*args, **kwargs)
@@ -177,4 +167,5 @@ def cacheable(
return result
return wrapper
return decorator
shelfmark/core/config.py
+57 -44
View File
@@ -3,53 +3,65 @@
import os
import sqlite3
import time
from importlib import import_module
from pathlib import Path
from threading import Lock
from typing import Any, Dict, Optional
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from types import ModuleType
from shelfmark.core.user_db import UserDB
# Import lazily to avoid circular imports
_registry_module = None
_env_module = None
_user_db_module = None
_SETTINGS_REFRESH_COOLDOWN_SECONDS = 0.05
def _get_registry():
def _get_registry() -> ModuleType:
"""Lazy import of settings registry to avoid circular imports."""
global _registry_module
if _registry_module is None:
from shelfmark.core import settings_registry
_registry_module = settings_registry
return _registry_module
def _get_env():
def _get_env() -> ModuleType:
"""Lazy import of env module for fallback values."""
global _env_module
if _env_module is None:
from shelfmark.config import env
_env_module = env
return _env_module
def _get_user_db_module():
def _get_user_db_module() -> type[UserDB]:
"""Lazy import of user DB module to avoid optional dependency loops."""
global _user_db_module
if _user_db_module is None:
from shelfmark.core.user_db import UserDB
_user_db_module = UserDB
return _user_db_module
class Config:
"""
Dynamic configuration singleton that provides live settings access.
"""Dynamic configuration singleton that provides live settings access.
Settings are resolved with priority: ENV var > config file > default.
Values are cached for performance and can be refreshed when settings change.
"""
_instance: Optional['Config'] = None
_instance: Config | None = None
_lock = Lock()
def __new__(cls) -> 'Config':
def __new__(cls) -> Config:
if cls._instance is None:
with cls._lock:
if cls._instance is None:
@@ -57,13 +69,13 @@ class Config:
cls._instance._initialized = False
return cls._instance
def __init__(self):
def __init__(self) -> None:
if self._initialized:
return
self._cache: Dict[str, Any] = {}
self._field_map: Dict[str, tuple] = {} # key -> (field, tab_name)
self._cache: dict[str, Any] = {}
self._field_map: dict[str, tuple] = {} # key -> (field, tab_name)
self._cache_lock = Lock()
self._user_settings_cache: Dict[int, Dict[str, Any]] = {}
self._user_settings_cache: dict[int, dict[str, Any]] = {}
self._user_settings_cache_lock = Lock()
self._user_db = None
self._user_db_load_attempted = False
@@ -85,12 +97,12 @@ class Config:
# Ensure all settings modules are imported before loading
# This handles cases where config is accessed before settings are registered
try:
import shelfmark.config.settings # noqa: F401 - main app settings
import shelfmark.config.security # noqa: F401 - security/auth settings
import shelfmark.config.notifications_settings # noqa: F401 - notifications settings
import shelfmark.config.users_settings # noqa: F401 - users/request settings
import shelfmark.release_sources # noqa: F401 - plugin settings
import shelfmark.metadata_providers # noqa: F401 - plugin settings
import_module("shelfmark.config.notifications_settings")
import_module("shelfmark.config.security")
import_module("shelfmark.config.settings")
import_module("shelfmark.config.users_settings")
import_module("shelfmark.metadata_providers")
import_module("shelfmark.release_sources")
except ImportError:
pass
@@ -98,7 +110,7 @@ class Config:
# On first load, sync ENV values to config files
# This ensures ENV values persist even if ENV vars are later removed
if not hasattr(self, '_env_synced'):
if not hasattr(self, "_env_synced"):
registry.sync_env_to_config()
self._env_synced = True
@@ -112,9 +124,8 @@ class Config:
self._loaded = True
def refresh(self, force: bool = False) -> None:
"""
Refresh all cached settings from config files.
def refresh(self, *, force: bool = False) -> None:
"""Refresh all cached settings from config files.
Call this after settings are updated via the UI to ensure
the config singleton reflects the new values.
@@ -125,7 +136,7 @@ class Config:
(e.g. after a settings write).
"""
now = time.monotonic()
if not force and (now - self._last_refresh_time) < 0.05:
if not force and (now - self._last_refresh_time) < _SETTINGS_REFRESH_COOLDOWN_SECONDS:
return
with self._cache_lock:
@@ -137,7 +148,7 @@ class Config:
self._user_db_load_attempted = False
self._last_refresh_time = time.monotonic()
def _get_user_db(self):
def _get_user_db(self) -> UserDB | None:
"""Get or initialize a UserDB handle if available."""
if self._user_db is not None:
return self._user_db
@@ -147,16 +158,17 @@ class Config:
self._user_db_load_attempted = True
try:
user_db_cls = _get_user_db_module()
db_path = os.path.join(os.environ.get("CONFIG_DIR", "/config"), "users.db")
db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db")
user_db = user_db_cls(db_path)
user_db.initialize()
self._user_db = user_db
return self._user_db
except Exception:
# Multi-user support is optional; fall back to global config when unavailable.
return None
else:
self._user_db = user_db
return self._user_db
def _get_user_settings(self, user_id: int) -> Dict[str, Any]:
def _get_user_settings(self, user_id: int) -> dict[str, Any]:
"""Get cached per-user settings from user DB."""
with self._user_settings_cache_lock:
if user_id in self._user_settings_cache:
@@ -168,7 +180,7 @@ class Config:
try:
settings = user_db.get_user_settings(user_id)
except (sqlite3.OperationalError, OSError, ValueError, TypeError):
except sqlite3.OperationalError, OSError, ValueError, TypeError:
return {}
if not isinstance(settings, dict):
@@ -178,14 +190,13 @@ class Config:
self._user_settings_cache[user_id] = settings
return settings
def _get_user_override(self, user_id: int, key: str) -> Any:
def _get_user_override(self, user_id: int, key: str) -> object:
"""Get a user override for a specific key."""
user_settings = self._get_user_settings(user_id)
return user_settings.get(key)
def get(self, key: str, default: Any = None, user_id: Optional[int] = None) -> Any:
"""
Get a setting value by key.
def get(self, key: str, default: object = None, user_id: int | None = None) -> object:
"""Get a setting value by key.
Args:
key: The setting key (e.g., 'MAX_RETRY')
@@ -194,6 +205,7 @@ class Config:
Returns:
The setting value, or default if not found
"""
self._ensure_loaded()
@@ -213,15 +225,15 @@ class Config:
return self._cache.get(key, default)
def __getattr__(self, name: str) -> Any:
"""
Allow attribute-style access to settings.
def __getattr__(self, name: str) -> object:
"""Allow attribute-style access to settings.
Example: config.MAX_RETRY instead of config.get('MAX_RETRY')
"""
# Avoid recursion for internal attributes
if name.startswith('_'):
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
if name.startswith("_"):
msg = f"'{type(self).__name__}' object has no attribute '{name}'"
raise AttributeError(msg)
self._ensure_loaded()
@@ -234,17 +246,18 @@ class Config:
if hasattr(env, name):
return getattr(env, name)
raise AttributeError(f"Setting '{name}' not found in config or env")
msg = f"Setting '{name}' not found in config or env"
raise AttributeError(msg)
def is_from_env(self, key: str) -> bool:
"""
Check if a setting's value comes from an environment variable.
"""Check if a setting's value comes from an environment variable.
Args:
key: The setting key
Returns:
True if the value is set via ENV var, False otherwise
"""
self._ensure_loaded()
@@ -255,12 +268,12 @@ class Config:
registry = _get_registry()
return registry.is_value_from_env(field)
def get_all(self) -> Dict[str, Any]:
"""
Get all cached settings as a dictionary.
def get_all(self) -> dict[str, Any]:
"""Get all cached settings as a dictionary.
Returns:
Dict of all setting keys to their current values
"""
self._ensure_loaded()
return dict(self._cache)
shelfmark/core/cwa_user_sync.py
+16 -8
View File
@@ -2,16 +2,20 @@
from __future__ import annotations
from typing import Any, Iterable
from typing import TYPE_CHECKING, Any
from shelfmark.core.auth_modes import AUTH_SOURCE_CWA, normalize_auth_source
from shelfmark.core.external_user_linking import upsert_external_user
from shelfmark.core.user_db import UserDB
if TYPE_CHECKING:
from collections.abc import Iterable
from shelfmark.core.user_db import UserDB
_CWA_ALIAS_SUFFIX = "__cwa"
def _normalize_email(value: Any) -> str | None:
def _normalize_email(value: object) -> str | None:
if value is None:
return None
email = str(value).strip()
@@ -40,7 +44,8 @@ def upsert_cwa_user(
context=context,
)
if user is None:
raise RuntimeError("Unexpected CWA user sync result: no user returned")
msg = "Unexpected CWA user sync result: no user returned"
raise RuntimeError(msg)
return user, action
@@ -73,10 +78,13 @@ def sync_cwa_users_from_rows(
deleted = 0
for existing_user in user_db.list_users():
if normalize_auth_source(
existing_user.get("auth_source"),
existing_user.get("oidc_subject"),
) != AUTH_SOURCE_CWA:
if (
normalize_auth_source(
existing_user.get("auth_source"),
existing_user.get("oidc_subject"),
)
!= AUTH_SOURCE_CWA
):
continue
existing_id = int(existing_user.get("id") or 0)
shelfmark/core/download_history_service.py
+46 -30
View File
@@ -3,15 +3,19 @@
from __future__ import annotations
import json
import os
import sqlite3
import threading
from datetime import datetime, timezone
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import TERMINAL_QUEUE_STATUSES
from shelfmark.core.request_helpers import normalize_optional_positive_int, normalize_optional_text, now_utc_iso
from shelfmark.core.request_helpers import (
normalize_optional_positive_int,
normalize_optional_text,
now_utc_iso,
)
logger = setup_logger(__name__)
@@ -21,40 +25,45 @@ ACTIVE_DOWNLOAD_STATUS = "active"
VALID_ORIGINS = frozenset({"direct", "requested"})
def _normalize_task_id(task_id: Any) -> str:
def _normalize_task_id(task_id: object) -> str:
normalized = normalize_optional_text(task_id)
if normalized is None:
raise ValueError("task_id must be a non-empty string")
msg = "task_id must be a non-empty string"
raise ValueError(msg)
return normalized
def _normalize_origin(origin: Any) -> str:
def _normalize_origin(origin: object) -> str:
normalized = normalize_optional_text(origin)
if normalized is None:
return "direct"
lowered = normalized.lower()
if lowered not in VALID_ORIGINS:
raise ValueError("origin must be one of: direct, requested")
msg = "origin must be one of: direct, requested"
raise ValueError(msg)
return lowered
def _normalize_final_status(final_status: Any) -> str:
def _normalize_final_status(final_status: object) -> str:
normalized = normalize_optional_text(final_status)
if normalized is None:
raise ValueError("final_status must be a non-empty string")
msg = "final_status must be a non-empty string"
raise ValueError(msg)
lowered = normalized.lower()
if lowered not in VALID_TERMINAL_STATUSES:
raise ValueError("final_status must be one of: complete, error, cancelled")
msg = "final_status must be one of: complete, error, cancelled"
raise ValueError(msg)
return lowered
def _normalize_limit(value: Any, *, default: int, minimum: int, maximum: int) -> int:
def _normalize_limit(value: object, *, default: int, minimum: int, maximum: int) -> int:
if value is None:
return default
try:
parsed = int(value)
except (TypeError, ValueError) as exc:
raise ValueError("limit must be an integer") from exc
msg = "limit must be an integer"
raise ValueError(msg) from exc
if parsed < minimum:
return minimum
if parsed > maximum:
@@ -65,7 +74,7 @@ def _normalize_limit(value: Any, *, default: int, minimum: int, maximum: int) ->
class DownloadHistoryService:
"""Service for persisted canonical download activity rows."""
def __init__(self, db_path: str):
def __init__(self, db_path: str) -> None:
self._db_path = db_path
self._lock = threading.Lock()
@@ -80,7 +89,9 @@ class DownloadHistoryService:
if row is None:
return None
normalized = dict(row)
normalized["retry_payload"] = cls._deserialize_retry_payload(normalized.get("retry_payload"))
normalized["retry_payload"] = cls._deserialize_retry_payload(
normalized.get("retry_payload")
)
return normalized
@classmethod
@@ -92,23 +103,24 @@ class DownloadHistoryService:
return f"download:{task_id}"
@staticmethod
def _resolve_existing_download_path(value: Any) -> str | None:
def _resolve_existing_download_path(value: object) -> str | None:
normalized = normalize_optional_text(value)
if normalized is None:
return None
return normalized if os.path.exists(normalized) else None
return normalized if Path(normalized).exists() else None
@staticmethod
def _serialize_retry_payload(payload: Any) -> str | None:
def _serialize_retry_payload(payload: object) -> str | None:
if payload is None:
return None
try:
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
except (TypeError, ValueError) as exc:
raise ValueError("retry_payload must be JSON-serializable") from exc
msg = "retry_payload must be JSON-serializable"
raise ValueError(msg) from exc
@staticmethod
def _deserialize_retry_payload(value: Any) -> dict[str, Any] | None:
def _deserialize_retry_payload(value: object) -> dict[str, Any] | None:
if isinstance(value, dict):
return dict(value)
normalized = normalize_optional_text(value)
@@ -126,7 +138,7 @@ class DownloadHistoryService:
normalized_staged_path = normalize_optional_text(staged_path)
if normalized_staged_path is None:
return False
return os.path.exists(normalized_staged_path)
return Path(normalized_staged_path).exists()
@staticmethod
def _can_retry_without_staged_source(retry_payload: dict[str, Any]) -> bool:
@@ -134,16 +146,16 @@ class DownloadHistoryService:
@staticmethod
def is_retry_available(row: dict[str, Any]) -> bool:
final_status = str(
row.get("retry_final_status") or row.get("final_status") or ""
).strip().lower()
final_status = (
str(row.get("retry_final_status") or row.get("final_status") or "").strip().lower()
)
retry_payload = DownloadHistoryService._deserialize_retry_payload(row.get("retry_payload"))
if retry_payload is None:
return False
has_staged_retry_source = DownloadHistoryService._has_staged_retry_source(retry_payload)
can_retry_without_staged_source = (
DownloadHistoryService._can_retry_without_staged_source(retry_payload)
can_retry_without_staged_source = DownloadHistoryService._can_retry_without_staged_source(
retry_payload
)
request_id = normalize_optional_positive_int(row.get("request_id"), "request_id")
if request_id is None:
@@ -174,7 +186,9 @@ class DownloadHistoryService:
"source": row.get("source"),
"source_display_name": row.get("source_display_name"),
"status_message": row.get("status_message"),
"download_path": DownloadHistoryService._resolve_existing_download_path(row.get("download_path")),
"download_path": DownloadHistoryService._resolve_existing_download_path(
row.get("download_path")
),
"added_time": DownloadHistoryService._iso_to_epoch(row.get("queued_at")),
"user_id": row.get("user_id"),
"username": row.get("username"),
@@ -183,7 +197,7 @@ class DownloadHistoryService:
}
@staticmethod
def _iso_to_epoch(value: Any) -> float | None:
def _iso_to_epoch(value: object) -> float | None:
if not isinstance(value, str) or not value.strip():
return None
normalized = value.strip().replace("Z", "+00:00")
@@ -192,7 +206,7 @@ class DownloadHistoryService:
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
parsed = parsed.replace(tzinfo=UTC)
return parsed.timestamp()
@classmethod
@@ -249,10 +263,12 @@ class DownloadHistoryService:
normalized_request_id = normalize_optional_positive_int(request_id, "request_id")
normalized_source = normalize_optional_text(source)
if normalized_source is None:
raise ValueError("source must be a non-empty string")
msg = "source must be a non-empty string"
raise ValueError(msg)
normalized_title = normalize_optional_text(title)
if normalized_title is None:
raise ValueError("title must be a non-empty string")
msg = "title must be a non-empty string"
raise ValueError(msg)
normalized_origin = _normalize_origin(origin)
normalized_retry_payload = self._serialize_retry_payload(retry_payload)
recorded_at = now_utc_iso()
shelfmark/core/external_user_linking.py
+53 -29
View File
@@ -3,11 +3,13 @@
from __future__ import annotations
import re
from typing import Any, Literal
from typing import TYPE_CHECKING, Any, Literal
from shelfmark.core.auth_modes import normalize_auth_source
from shelfmark.core.logger import setup_logger
from shelfmark.core.user_db import UserDB
if TYPE_CHECKING:
from shelfmark.core.user_db import UserDB
UNSET = object()
@@ -21,18 +23,18 @@ MatchReason = Literal[
logger = setup_logger(__name__)
def _normalize_username(value: Any) -> str:
def _normalize_username(value: object) -> str:
return str(value or "").strip()
def _normalize_email(value: Any) -> str | None:
def _normalize_email(value: object) -> str | None:
if value is None:
return None
email = str(value).strip()
return email or None
def _normalize_display_name(value: Any) -> str | None:
def _normalize_display_name(value: object) -> str | None:
if value is None:
return None
name = str(value).strip()
@@ -43,11 +45,13 @@ def _email_key(value: str | None) -> str:
return (value or "").strip().lower()
def _normalize_role(value: Any) -> str:
def _normalize_role(value: object) -> str:
return "admin" if str(value or "").strip().lower() == "admin" else "user"
def _get_by_subject(user_db: UserDB, subject_field: str | None, subject: str | None) -> dict[str, Any] | None:
def _get_by_subject(
user_db: UserDB, subject_field: str | None, subject: str | None
) -> dict[str, Any] | None:
if not subject_field or not subject:
return None
if subject_field == "oidc_subject":
@@ -83,10 +87,14 @@ def find_external_user_match(
return by_subject, "subject_match"
by_username = user_db.get_user(username=normalized_username)
if by_username and normalize_auth_source(
by_username.get("auth_source"),
by_username.get("oidc_subject"),
) == auth_source:
if (
by_username
and normalize_auth_source(
by_username.get("auth_source"),
by_username.get("oidc_subject"),
)
== auth_source
):
return by_username, "existing_source_username_match"
if allow_email_link:
@@ -133,7 +141,8 @@ def _find_existing_alias_user(
) -> dict[str, Any] | None:
pattern = re.compile(rf"^{re.escape(alias_base)}(?:_\d+)?$")
candidates = [
user for user in user_db.list_users()
user
for user in user_db.list_users()
if pattern.match(str(user.get("username") or ""))
and normalize_auth_source(user.get("auth_source"), user.get("oidc_subject")) == auth_source
]
@@ -158,7 +167,11 @@ def _resolve_create_username(
return None, existing, "username_collision_takeover"
if strategy == "suffix":
return _next_suffix_username(user_db, requested_username), None, "username_collision_suffix"
return (
_next_suffix_username(user_db, requested_username),
None,
"username_collision_suffix",
)
alias_base = f"{requested_username}{alias_suffix}"
alias_existing = _find_existing_alias_user(
@@ -197,7 +210,8 @@ def upsert_external_user(
"""
normalized_username = _normalize_username(username)
if not normalized_username:
raise ValueError("External username is required")
msg = "External username is required"
raise ValueError(msg)
normalized_email = _normalize_email(email) if email is not UNSET else None
normalized_display_name = (
@@ -227,18 +241,22 @@ def upsert_external_user(
user_db.update_user(matched["id"], **updates)
mapped = user_db.get_user(user_id=matched["id"]) or matched
logger.info(
"External user mapped to existing Shelfmark user "
f"(source={auth_source}, context={context or 'unspecified'}, reason={match_reason}, "
f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
f"shelfmark_username={mapped['username']})"
"External user mapped to existing Shelfmark user (source=%s, context=%s, reason=%s, external_username=%s, shelfmark_user_id=%s, shelfmark_username=%s)",
auth_source,
context or "unspecified",
match_reason,
normalized_username,
mapped["id"],
mapped["username"],
)
return mapped, "updated"
if not allow_create:
logger.info(
"External user could not be mapped and creation is disabled "
f"(source={auth_source}, context={context or 'unspecified'}, "
f"external_username={normalized_username})"
"External user could not be mapped and creation is disabled (source=%s, context=%s, external_username=%s)",
auth_source,
context or "unspecified",
normalized_username,
)
return None, "not_found"
@@ -254,10 +272,13 @@ def upsert_external_user(
user_db.update_user(takeover_target["id"], **updates)
mapped = user_db.get_user(user_id=takeover_target["id"]) or takeover_target
logger.info(
"External user mapped to existing Shelfmark user "
f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
f"external_username={normalized_username}, shelfmark_user_id={mapped['id']}, "
f"shelfmark_username={mapped['username']})"
"External user mapped to existing Shelfmark user (source=%s, context=%s, reason=%s, external_username=%s, shelfmark_user_id=%s, shelfmark_username=%s)",
auth_source,
context or "unspecified",
create_reason,
normalized_username,
mapped["id"],
mapped["username"],
)
return mapped, "updated"
@@ -275,9 +296,12 @@ def upsert_external_user(
created = user_db.create_user(**create_kwargs)
logger.info(
"External user created Shelfmark user "
f"(source={auth_source}, context={context or 'unspecified'}, reason={create_reason}, "
f"external_username={normalized_username}, shelfmark_user_id={created['id']}, "
f"shelfmark_username={created['username']})"
"External user created Shelfmark user (source=%s, context=%s, reason=%s, external_username=%s, shelfmark_user_id=%s, shelfmark_username=%s)",
auth_source,
context or "unspecified",
create_reason,
normalized_username,
created["id"],
created["username"],
)
return created, "created"
shelfmark/core/image_cache.py
+130 -112
View File
@@ -2,13 +2,12 @@
import ipaddress
import json
import os
import socket
import threading
import time
from http import HTTPStatus
from io import BytesIO
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
import requests
@@ -16,22 +15,25 @@ import requests
from shelfmark.core.logger import setup_logger
from shelfmark.download.network import get_ssl_verify
if TYPE_CHECKING:
from pathlib import Path
logger = setup_logger(__name__)
# Image type detection via magic bytes
IMAGE_SIGNATURES = {
b'\xff\xd8\xff': ('image/jpeg', 'jpg'),
b'\x89PNG\r\n\x1a\n': ('image/png', 'png'),
b'GIF87a': ('image/gif', 'gif'),
b'GIF89a': ('image/gif', 'gif'),
b'RIFF': ('image/webp', 'webp'), # WebP starts with RIFF
b"\xff\xd8\xff": ("image/jpeg", "jpg"),
b"\x89PNG\r\n\x1a\n": ("image/png", "png"),
b"GIF87a": ("image/gif", "gif"),
b"GIF89a": ("image/gif", "gif"),
b"RIFF": ("image/webp", "webp"), # WebP starts with RIFF
}
# HTTP headers for image fetching
FETCH_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36',
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36",
"Accept": "image/webp,image/apng,image/*,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
}
# Maximum image size to fetch (5 MB)
@@ -44,8 +46,11 @@ NEGATIVE_CACHE_TTL = 3600
# Short enough to retry soon, long enough to prevent spam during one page view
TRANSIENT_CACHE_TTL = 60
_MIN_WEBP_HEADER_LENGTH = 12
HTTP_NOT_FOUND = HTTPStatus.NOT_FOUND
def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]:
def _detect_image_type(data: bytes) -> tuple[str, str] | None:
"""Detect image type from magic bytes.
Args:
@@ -53,14 +58,15 @@ def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]:
Returns:
Tuple of (content_type, extension) or None if not recognized
"""
for signature, (content_type, ext) in IMAGE_SIGNATURES.items():
if data.startswith(signature):
return content_type, ext
# Special case for WebP - check for WEBP after RIFF
if data.startswith(b'RIFF') and len(data) > 12 and data[8:12] == b'WEBP':
return 'image/webp', 'webp'
if data.startswith(b"RIFF") and len(data) > _MIN_WEBP_HEADER_LENGTH and data[8:12] == b"WEBP":
return "image/webp", "webp"
return None
@@ -68,20 +74,21 @@ def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]:
class ImageCacheService:
"""Persistent image cache with LRU eviction and TTL support."""
def __init__(self, cache_dir: Path, max_size_mb: int = 500, ttl_seconds: int = 0):
def __init__(self, cache_dir: Path, max_size_mb: int = 500, ttl_seconds: int = 0) -> None:
"""Initialize the image cache.
Args:
cache_dir: Directory to store cached images
max_size_mb: Maximum cache size in megabytes
ttl_seconds: Time-to-live in seconds (0 = forever)
"""
self.cache_dir = cache_dir
self.max_size_bytes = max_size_mb * 1024 * 1024
self.ttl_seconds = ttl_seconds
self.index_path = cache_dir / "cache_index.json"
self._lock = threading.RLock()
self._index: Dict[str, Dict[str, Any]] = {}
self._index: dict[str, dict[str, Any]] = {}
# Stats tracking
self._hits = 0
@@ -101,9 +108,9 @@ class ImageCacheService:
return
try:
with open(self.index_path, 'r') as f:
with self.index_path.open() as f:
self._index = json.load(f)
except (json.JSONDecodeError, IOError):
except OSError, json.JSONDecodeError:
self._index = {}
def _sync_index_with_files(self) -> None:
@@ -113,12 +120,12 @@ class ImageCacheService:
- Removes entries for files that no longer exist (non-negative only)
- Preserves negative cache entries (they have no files)
"""
image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.webp'}
image_extensions = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
added_count = 0
removed_count = 0
# Build set of files that exist on disk
existing_files: Dict[str, Path] = {}
existing_files: dict[str, Path] = {}
for file_path in self.cache_dir.iterdir():
if not file_path.is_file():
continue
@@ -131,31 +138,31 @@ class ImageCacheService:
if cache_id in self._index:
continue
ext = file_path.suffix.lstrip('.')
ext = file_path.suffix.lstrip(".")
stat = file_path.stat()
# Detect content type
try:
with open(file_path, 'rb') as f:
with file_path.open("rb") as f:
header = f.read(16)
detected = _detect_image_type(header)
content_type = detected[0] if detected else f'image/{ext}'
except IOError:
content_type = f'image/{ext}'
content_type = detected[0] if detected else f"image/{ext}"
except OSError:
content_type = f"image/{ext}"
self._index[cache_id] = {
'ext': ext,
'content_type': content_type,
'size': stat.st_size,
'cached_at': stat.st_mtime,
'accessed_at': stat.st_mtime,
"ext": ext,
"content_type": content_type,
"size": stat.st_size,
"cached_at": stat.st_mtime,
"accessed_at": stat.st_mtime,
}
added_count += 1
# Remove index entries for missing files (skip negative cache entries)
stale_entries = []
for cache_id, entry in self._index.items():
if entry.get('negative', False):
if entry.get("negative", False):
continue # Negative entries don't have files
if cache_id not in existing_files:
stale_entries.append(cache_id)
@@ -171,39 +178,39 @@ class ImageCacheService:
"""Save cache index to disk."""
try:
# Write to temp file first, then rename for atomicity
temp_path = self.index_path.with_suffix('.tmp')
with open(temp_path, 'w') as f:
temp_path = self.index_path.with_suffix(".tmp")
with temp_path.open("w") as f:
json.dump(self._index, f)
temp_path.rename(self.index_path)
except IOError:
except OSError:
pass
def _get_image_path(self, cache_id: str, ext: str) -> Path:
"""Get the file path for a cached image."""
return self.cache_dir / f"{cache_id}.{ext}"
def _is_expired(self, entry: Dict[str, Any]) -> bool:
def _is_expired(self, entry: dict[str, Any]) -> bool:
"""Check if a cache entry is expired."""
if self.ttl_seconds == 0:
return False
return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
return (time.time() - entry.get("cached_at", 0)) > self.ttl_seconds
def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
def _is_negative_expired(self, entry: dict[str, Any]) -> bool:
"""Check if a negative cache entry is expired.
Transient failures (timeouts) expire after TRANSIENT_CACHE_TTL (60s).
Permanent failures (404s) expire after NEGATIVE_CACHE_TTL (1 hour).
"""
if not entry.get('negative', False):
if not entry.get("negative", False):
return False
cached_at = entry.get('cached_at', 0)
ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
cached_at = entry.get("cached_at", 0)
ttl = TRANSIENT_CACHE_TTL if entry.get("transient", False) else NEGATIVE_CACHE_TTL
return (time.time() - cached_at) > ttl
def _calculate_total_size(self) -> int:
"""Calculate total size of cached images."""
return sum(entry.get('size', 0) for entry in self._index.values())
return sum(entry.get("size", 0) for entry in self._index.values())
def _evict_if_needed(self, required_space: int = 0) -> None:
"""Evict old entries if cache is over size limit.
@@ -217,10 +224,7 @@ class ImageCacheService:
return
# Sort entries by accessed_at (oldest first)
sorted_entries = sorted(
self._index.items(),
key=lambda x: x[1].get('accessed_at', 0)
)
sorted_entries = sorted(self._index.items(), key=lambda x: x[1].get("accessed_at", 0))
evicted_count = 0
for cache_id, entry in sorted_entries:
@@ -228,23 +232,23 @@ class ImageCacheService:
break
# Delete the image file
ext = entry.get('ext', 'jpg')
ext = entry.get("ext", "jpg")
image_path = self._get_image_path(cache_id, ext)
try:
if image_path.exists():
image_path.unlink()
except IOError:
except OSError:
pass
# Update tracking
current_size -= entry.get('size', 0)
current_size -= entry.get("size", 0)
del self._index[cache_id]
evicted_count += 1
if evicted_count > 0:
self._save_index()
def get(self, cache_id: str) -> Optional[Tuple[bytes, str]]:
def get(self, cache_id: str) -> tuple[bytes, str] | None:
"""Get a cached image.
Args:
@@ -252,6 +256,7 @@ class ImageCacheService:
Returns:
Tuple of (image_data, content_type) or None if not cached/expired
"""
with self._lock:
entry = self._index.get(cache_id)
@@ -265,7 +270,7 @@ class ImageCacheService:
return None
# Check for negative cache (failed fetch)
if entry.get('negative', False):
if entry.get("negative", False):
if self._is_negative_expired(entry):
# Negative cache expired, allow retry
del self._index[cache_id]
@@ -278,12 +283,12 @@ class ImageCacheService:
# Check for expired entry
if self._is_expired(entry):
# Remove expired entry
ext = entry.get('ext', 'jpg')
ext = entry.get("ext", "jpg")
image_path = self._get_image_path(cache_id, ext)
try:
if image_path.exists():
image_path.unlink()
except IOError:
except OSError:
pass
del self._index[cache_id]
self._save_index()
@@ -291,9 +296,10 @@ class ImageCacheService:
return None
# Try to read the cached image
ext = entry.get('ext', 'jpg')
content_type = entry.get('content_type', 'image/jpeg')
ext = entry.get("ext", "jpg")
content_type = entry.get("content_type", "image/jpeg")
image_path = self._get_image_path(cache_id, ext)
result: tuple[bytes, str] | None = None
try:
if not image_path.exists():
@@ -303,19 +309,20 @@ class ImageCacheService:
self._misses += 1
return None
with open(image_path, 'rb') as f:
with image_path.open("rb") as f:
data = f.read()
# Update accessed time
entry['accessed_at'] = time.time()
entry["accessed_at"] = time.time()
self._save_index()
result = data, content_type
self._hits += 1
return data, content_type
except IOError:
except OSError:
self._misses += 1
return None
else:
self._hits += 1
return result
def put(self, cache_id: str, data: bytes, content_type: str) -> bool:
"""Store an image in the cache.
@@ -327,24 +334,24 @@ class ImageCacheService:
Returns:
True if stored successfully
"""
with self._lock:
# Detect image type for extension
detected = _detect_image_type(data)
if detected:
content_type, ext = detected
# Fall back to content-type header
elif "jpeg" in content_type or "jpg" in content_type:
ext = "jpg"
elif "png" in content_type:
ext = "png"
elif "gif" in content_type:
ext = "gif"
elif "webp" in content_type:
ext = "webp"
else:
# Fall back to content-type header
if 'jpeg' in content_type or 'jpg' in content_type:
ext = 'jpg'
elif 'png' in content_type:
ext = 'png'
elif 'gif' in content_type:
ext = 'gif'
elif 'webp' in content_type:
ext = 'webp'
else:
ext = 'jpg' # Default
ext = "jpg" # Default
image_size = len(data)
@@ -354,36 +361,37 @@ class ImageCacheService:
# Write image to disk
image_path = self._get_image_path(cache_id, ext)
try:
with open(image_path, 'wb') as f:
with image_path.open("wb") as f:
f.write(data)
except IOError:
except OSError:
return False
# Update index
now = time.time()
self._index[cache_id] = {
'ext': ext,
'content_type': content_type,
'size': image_size,
'cached_at': now,
'accessed_at': now,
'negative': False,
"ext": ext,
"content_type": content_type,
"size": image_size,
"cached_at": now,
"accessed_at": now,
"negative": False,
}
self._save_index()
return True
def put_negative(self, cache_id: str, transient: bool = False) -> None:
def put_negative(self, cache_id: str, *, transient: bool = False) -> None:
"""Store a negative cache entry (failed fetch).
Args:
cache_id: Cache key
transient: If True, uses shorter TTL (for timeouts/connection errors)
"""
with self._lock:
self._index[cache_id] = {
'negative': True,
'transient': transient,
'cached_at': time.time(),
"negative": True,
"transient": transient,
"cached_at": time.time(),
}
self._save_index()
@@ -395,6 +403,7 @@ class ImageCacheService:
Returns:
True if entry existed and was deleted
"""
with self._lock:
entry = self._index.get(cache_id)
@@ -402,13 +411,13 @@ class ImageCacheService:
return False
# Delete file if it exists
if not entry.get('negative', False):
ext = entry.get('ext', 'jpg')
if not entry.get("negative", False):
ext = entry.get("ext", "jpg")
image_path = self._get_image_path(cache_id, ext)
try:
if image_path.exists():
image_path.unlink()
except IOError:
except OSError:
pass
del self._index[cache_id]
@@ -420,19 +429,20 @@ class ImageCacheService:
Returns:
Number of entries cleared
"""
with self._lock:
count = len(self._index)
# Delete all image files
for cache_id, entry in self._index.items():
if not entry.get('negative', False):
ext = entry.get('ext', 'jpg')
if not entry.get("negative", False):
ext = entry.get("ext", "jpg")
image_path = self._get_image_path(cache_id, ext)
try:
if image_path.exists():
image_path.unlink()
except IOError:
except OSError:
pass
# Clear index
@@ -445,28 +455,29 @@ class ImageCacheService:
return count
def stats(self) -> Dict[str, Any]:
def stats(self) -> dict[str, Any]:
"""Get cache statistics.
Returns:
Dict with size, count, hit rate, etc.
"""
with self._lock:
total_size = self._calculate_total_size()
entry_count = len(self._index)
negative_count = sum(1 for e in self._index.values() if e.get('negative', False))
negative_count = sum(1 for e in self._index.values() if e.get("negative", False))
total_requests = self._hits + self._misses
hit_rate = (self._hits / total_requests * 100) if total_requests > 0 else 0
return {
'entry_count': entry_count,
'negative_count': negative_count,
'total_size_bytes': total_size,
'total_size_mb': round(total_size / (1024 * 1024), 2),
'max_size_mb': self.max_size_bytes / (1024 * 1024),
'hits': self._hits,
'misses': self._misses,
'hit_rate': round(hit_rate, 1),
"entry_count": entry_count,
"negative_count": negative_count,
"total_size_bytes": total_size,
"total_size_mb": round(total_size / (1024 * 1024), 2),
"max_size_mb": self.max_size_bytes / (1024 * 1024),
"hits": self._hits,
"misses": self._misses,
"hit_rate": round(hit_rate, 1),
}
@staticmethod
@@ -477,7 +488,7 @@ class ImageCacheService:
except Exception:
return False
if parsed.scheme not in ('http', 'https'):
if parsed.scheme not in ("http", "https"):
return False
hostname = parsed.hostname
@@ -490,12 +501,12 @@ class ImageCacheService:
ip = ipaddress.ip_address(sockaddr[0])
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
return False
except (socket.gaierror, ValueError):
except socket.gaierror, ValueError:
return False
return True
def fetch_and_cache(self, cache_id: str, url: str) -> Optional[Tuple[bytes, str]]:
def fetch_and_cache(self, cache_id: str, url: str) -> tuple[bytes, str] | None:
"""Fetch an image from URL and cache it.
Args:
@@ -504,10 +515,12 @@ class ImageCacheService:
Returns:
Tuple of (image_data, content_type) or None on failure
"""
cached_data: tuple[bytes, str] | None = None
try:
if not self._is_safe_url(url):
logger.warning(f"Blocked request to disallowed URL: {url}")
logger.warning("Blocked request to disallowed URL: %s", url)
return None
response = requests.get(
@@ -520,8 +533,8 @@ class ImageCacheService:
response.raise_for_status()
# Validate content type
content_type = response.headers.get('content-type', '')
if not content_type.startswith('image/'):
content_type = response.headers.get("content-type", "")
if not content_type.startswith("image/"):
self.put_negative(cache_id)
return None
@@ -545,9 +558,7 @@ class ImageCacheService:
detected = _detect_image_type(image_data)
if detected:
content_type = detected[0]
return image_data, content_type
return None
cached_data = image_data, content_type
except requests.exceptions.Timeout:
self.put_negative(cache_id, transient=True)
@@ -556,15 +567,17 @@ class ImageCacheService:
self.put_negative(cache_id, transient=True)
return None
except requests.exceptions.HTTPError as e:
is_404 = e.response is not None and e.response.status_code == 404
is_404 = e.response is not None and e.response.status_code == HTTP_NOT_FOUND
self.put_negative(cache_id, transient=not is_404)
return None
except Exception:
return None
else:
return cached_data
# Singleton instance (initialized lazily when config is available)
_instance: Optional[ImageCacheService] = None
_instance: ImageCacheService | None = None
_instance_lock = threading.Lock()
@@ -578,8 +591,8 @@ def get_image_cache() -> ImageCacheService:
if _instance is None:
with _instance_lock:
if _instance is None:
from shelfmark.core.config import config
from shelfmark.config.env import CONFIG_DIR
from shelfmark.core.config import config
cache_dir = CONFIG_DIR / "covers"
max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500)
@@ -591,7 +604,12 @@ def get_image_cache() -> ImageCacheService:
max_size_mb=max_size_mb,
ttl_seconds=ttl_seconds,
)
logger.debug(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)")
logger.debug(
"Initialized image cache: %s (max %sMB, TTL %s days)",
cache_dir,
max_size_mb,
ttl_days,
)
return _instance
shelfmark/core/logger.py
+36 -33
View File
@@ -2,59 +2,59 @@
import logging
import sys
from pathlib import Path
from logging.handlers import RotatingFileHandler
from typing import Any
from typing import TYPE_CHECKING
from shelfmark.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL
from shelfmark.config.env import ENABLE_LOGGING, LOG_FILE, LOG_LEVEL
if TYPE_CHECKING:
from pathlib import Path
class CustomLogger(logging.Logger):
"""Custom logger class with additional error_trace method."""
def error_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
def error_trace(self, msg: object, *args: object, **kwargs: object) -> None:
"""Log an error message with full stack trace."""
self.log_resource_usage()
kwargs.pop('exc_info', None)
kwargs.pop("exc_info", None)
self.error(msg, *args, exc_info=True, **kwargs)
def warning_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
"""Log a warning message with full stack trace."""
self.log_resource_usage()
kwargs.pop('exc_info', None)
self.warning(msg, *args, exc_info=True, **kwargs)
def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
"""Log an info message (stack trace only if exception active)."""
kwargs.pop('exc_info', None)
# Only include exc_info if there's actually an exception
has_exception = sys.exc_info()[0] is not None
self.info(msg, *args, exc_info=has_exception, **kwargs)
def debug_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
def debug_trace(self, msg: object, *args: object, **kwargs: object) -> None:
"""Log a debug message (stack trace only if exception active)."""
kwargs.pop('exc_info', None)
kwargs.pop("exc_info", None)
# Only include exc_info if there's actually an exception
has_exception = sys.exc_info()[0] is not None
self.debug(msg, *args, exc_info=has_exception, **kwargs)
def log_resource_usage(self):
def log_resource_usage(self) -> None:
# Best-effort only; this should never raise during exception logging.
try:
import psutil
def _get_process_rss_mb(proc: object) -> float | None:
try:
mem = proc.info.get("memory_info")
if mem:
return mem.rss / (1024 * 1024)
except (
psutil.NoSuchProcess,
psutil.AccessDenied,
KeyError,
AttributeError,
):
return None
return None
# Sum RSS of all processes for actual app memory (container-friendly),
# but fall back gracefully on platforms that restrict process enumeration.
app_memory_mb = 0.0
try:
for proc in psutil.process_iter(['memory_info']):
try:
mem = proc.info.get('memory_info')
if mem:
app_memory_mb += mem.rss / (1024 * 1024)
except (psutil.NoSuchProcess, psutil.AccessDenied, KeyError, AttributeError):
continue
except (PermissionError, psutil.AccessDenied, OSError):
for proc in psutil.process_iter(["memory_info"]):
proc_rss_mb = _get_process_rss_mb(proc)
if proc_rss_mb is not None:
app_memory_mb += proc_rss_mb
except PermissionError, psutil.AccessDenied, OSError:
try:
app_memory_mb = psutil.Process().memory_info().rss / (1024 * 1024)
except Exception:
@@ -82,6 +82,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
Returns:
CustomLogger: Configured logger instance with error_trace method
"""
# Register our custom logger class
logging.setLoggerClass(CustomLogger)
@@ -92,19 +93,21 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
logger.setLevel(log_level)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
"%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s"
)
# Console handler for Docker output
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(formatter)
console_handler.setLevel(log_level)
console_handler.addFilter(lambda record: record.levelno < logging.ERROR) # Only allow logs below ERROR to stdout
console_handler.addFilter(
lambda record: record.levelno < logging.ERROR
) # Only allow logs below ERROR to stdout
logger.addHandler(console_handler)
# Error handler for stderr
error_handler = logging.StreamHandler(sys.stderr)
error_handler.setLevel(logging.ERROR) # Error and above go to stderr
error_handler.setLevel(logging.ERROR) # Error and above go to stderr
error_handler.setFormatter(formatter)
logger.addHandler(error_handler)
@@ -117,7 +120,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
file_handler = RotatingFileHandler(
log_file,
maxBytes=10485760, # 10MB
backupCount=5
backupCount=5,
)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
shelfmark/core/mirrors.py
+31 -26
View File
@@ -1,18 +1,23 @@
"""Centralized mirror configuration for all download sources."""
from typing import List
# Lazy import to avoid circular imports
from typing import TYPE_CHECKING
from shelfmark.core.utils import normalize_http_url
# Lazy import to avoid circular imports
if TYPE_CHECKING:
from types import ModuleType
_config_module = None
def _get_config():
def _get_config() -> ModuleType:
"""Lazy import of config module to avoid circular imports."""
global _config_module
if _config_module is None:
from shelfmark.core.config import config
_config_module = config
return _config_module
@@ -50,9 +55,8 @@ def _normalize_mirror_url(url: str) -> str:
return normalize_http_url(url, default_scheme="https")
def get_aa_mirrors() -> List[str]:
"""
Get Anna's Archive mirrors.
def get_aa_mirrors() -> list[str]:
"""Get Anna's Archive mirrors.
Returns:
Ordered list of AA mirror URLs.
@@ -60,9 +64,10 @@ def get_aa_mirrors() -> List[str]:
If AA_MIRROR_URLS is configured, it is treated as the full list.
Otherwise, defaults are used and AA_ADDITIONAL_URLS (legacy) is appended.
Notes:
Notes:
- The list is used to populate the AA mirror dropdown in Settings.
- When AA_BASE_URL is set to 'auto', mirrors are tried in the order listed.
"""
config = _get_config()
@@ -96,12 +101,12 @@ def get_aa_mirrors() -> List[str]:
return mirrors
def get_libgen_mirrors() -> List[str]:
"""
Get LibGen mirrors: defaults + any additional from config.
def get_libgen_mirrors() -> list[str]:
"""Get LibGen mirrors: defaults + any additional from config.
Returns:
List of LibGen mirror URLs (defaults first, then custom additions).
"""
mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
mirrors = [url for url in mirrors if url]
@@ -117,12 +122,12 @@ def get_libgen_mirrors() -> List[str]:
return mirrors
def get_zlib_mirrors() -> List[str]:
"""
Get Z-Library mirrors, with primary first.
def get_zlib_mirrors() -> list[str]:
"""Get Z-Library mirrors, with primary first.
Returns:
List of Z-Library mirror URLs, primary first.
"""
config = _get_config()
@@ -149,11 +154,11 @@ def get_zlib_mirrors() -> List[str]:
def get_zlib_primary_url() -> str:
"""
Get the primary Z-Library mirror URL.
"""Get the primary Z-Library mirror URL.
Returns:
Primary Z-Library mirror URL.
"""
config = _get_config()
primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
@@ -161,22 +166,22 @@ def get_zlib_primary_url() -> str:
def get_zlib_url_template() -> str:
"""
Get Z-Library URL template using configured primary mirror.
"""Get Z-Library URL template using configured primary mirror.
Returns:
URL template with {md5} placeholder.
"""
primary = get_zlib_primary_url()
return f"{primary}/md5/{{md5}}"
def get_welib_mirrors() -> List[str]:
"""
Get Welib mirrors, with primary first.
def get_welib_mirrors() -> list[str]:
"""Get Welib mirrors, with primary first.
Returns:
List of Welib mirror URLs, primary first.
"""
config = _get_config()
@@ -203,11 +208,11 @@ def get_welib_mirrors() -> List[str]:
def get_welib_primary_url() -> str:
"""
Get the primary Welib mirror URL.
"""Get the primary Welib mirror URL.
Returns:
Primary Welib mirror URL.
"""
config = _get_config()
primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
@@ -215,24 +220,24 @@ def get_welib_primary_url() -> str:
def get_welib_url_template() -> str:
"""
Get Welib URL template using configured primary mirror.
"""Get Welib URL template using configured primary mirror.
Returns:
URL template with {md5} placeholder.
"""
primary = get_welib_primary_url()
return f"{primary}/md5/{{md5}}"
def get_zlib_cookie_domains() -> set:
"""
Get set of Z-Library domains that need full cookie handling.
"""Get set of Z-Library domains that need full cookie handling.
Used by internal_bypasser for CF bypass cookie management.
Returns:
Set of domain strings (without protocol).
"""
domains = set()
shelfmark/core/models.py
+76 -57
View File
@@ -1,18 +1,18 @@
"""Data structures and models used across the application."""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional
from enum import Enum
import re
import time
from dataclasses import dataclass, field
from enum import StrEnum
from pathlib import Path
from typing import Any
def build_filename(
title: str,
author: Optional[str] = None,
year: Optional[str] = None,
fmt: Optional[str] = None,
author: str | None = None,
year: str | None = None,
fmt: str | None = None,
) -> str:
parts = []
if author:
@@ -23,7 +23,7 @@ def build_filename(
parts.append(f" ({year})")
filename = "".join(parts)
filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245]
filename = re.sub(r'[\\/:*?"<>|]', "_", filename.strip())[:245]
if fmt:
filename = f"{filename}.{fmt}"
@@ -31,8 +31,9 @@ def build_filename(
return filename
class QueueStatus(str, Enum):
class QueueStatus(StrEnum):
"""Enum for possible book queue statuses."""
QUEUED = "queued"
RESOLVING = "resolving"
LOCATING = "locating"
@@ -42,16 +43,25 @@ class QueueStatus(str, Enum):
CANCELLED = "cancelled"
TERMINAL_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset({
QueueStatus.COMPLETE, QueueStatus.ERROR, QueueStatus.CANCELLED,
})
TERMINAL_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset(
{
QueueStatus.COMPLETE,
QueueStatus.ERROR,
QueueStatus.CANCELLED,
}
)
ACTIVE_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset({
QueueStatus.QUEUED, QueueStatus.RESOLVING, QueueStatus.LOCATING, QueueStatus.DOWNLOADING,
})
ACTIVE_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset(
{
QueueStatus.QUEUED,
QueueStatus.RESOLVING,
QueueStatus.LOCATING,
QueueStatus.DOWNLOADING,
}
)
class SearchMode(str, Enum):
class SearchMode(StrEnum):
DIRECT = "direct"
UNIVERSAL = "universal"
@@ -59,11 +69,12 @@ class SearchMode(str, Enum):
@dataclass
class QueueItem:
"""Queue item with priority and metadata."""
book_id: str
priority: int
added_time: float
def __lt__(self, other):
def __lt__(self, other: QueueItem) -> bool:
"""Compare items for priority queue (lower priority number = higher precedence)."""
if self.priority != other.priority:
return self.priority < other.priority
@@ -72,60 +83,67 @@ class QueueItem:
@dataclass
class DownloadTask:
task_id: str # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
source: str # Handler name ("direct_download", "prowlarr")
title: str # Display title for queue sidebar
task_id: str # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
source: str # Handler name ("direct_download", "prowlarr")
title: str # Display title for queue sidebar
# Display info for queue sidebar
author: Optional[str] = None
year: Optional[str] = None
format: Optional[str] = None
size: Optional[str] = None
preview: Optional[str] = None
content_type: Optional[str] = None # "book (fiction)", "audiobook", "magazine", etc.
source_url: Optional[str] = None # Original release URL used by source-specific handlers
retry_download_url: Optional[str] = None # Resolved download URL for restart-safe retries
retry_download_protocol: Optional[str] = None # Protocol for retry_download_url (e.g. torrent, usenet)
retry_release_name: Optional[str] = None # Display name to send back to external download clients
retry_expected_hash: Optional[str] = None # Optional torrent hash used to match client downloads
retry_ratio_limit: Optional[float] = None # Optional post-download seeding ratio
retry_seeding_time_limit_minutes: Optional[int] = None # Optional post-download seeding time limit
can_retry_without_staged_source: bool = True # Whether the source can restart without a preserved staged file
author: str | None = None
year: str | None = None
format: str | None = None
size: str | None = None
preview: str | None = None
content_type: str | None = None # "book (fiction)", "audiobook", "magazine", etc.
source_url: str | None = None # Original release URL used by source-specific handlers
retry_download_url: str | None = None # Resolved download URL for restart-safe retries
retry_download_protocol: str | None = (
None # Protocol for retry_download_url (e.g. torrent, usenet)
)
retry_release_name: str | None = None # Display name to send back to external download clients
retry_expected_hash: str | None = None # Optional torrent hash used to match client downloads
retry_ratio_limit: float | None = None # Optional post-download seeding ratio
retry_seeding_time_limit_minutes: int | None = None # Optional post-download seeding time limit
can_retry_without_staged_source: bool = (
True # Whether the source can restart without a preserved staged file
)
# Series info (for library naming templates)
series_name: Optional[str] = None
series_position: Optional[float] = None # Float for novellas (e.g., 1.5)
subtitle: Optional[str] = None # Book subtitle for naming templates
series_name: str | None = None
series_position: float | None = None # Float for novellas (e.g., 1.5)
subtitle: str | None = None # Book subtitle for naming templates
# Hardlinking support
original_download_path: Optional[str] = None # Path in download client (for hardlinking)
original_download_path: str | None = None # Path in download client (for hardlinking)
# Search mode - determines post-download processing behavior
# See SearchMode enum for behavioral differences
search_mode: Optional[SearchMode] = None
search_mode: SearchMode | None = None
# Output selection for post-processing.
# This is captured at queue time so in-flight tasks are not affected if the user changes settings later.
output_mode: Optional[str] = None # e.g. "folder", "booklore", "email"
output_args: Dict[str, Any] = field(default_factory=dict) # Per-output parameters (e.g. email recipient)
output_mode: str | None = None
output_args: dict[str, Any] = field(
default_factory=dict
) # Per-output parameters (e.g. email recipient)
# User association (multi-user support)
user_id: Optional[int] = None # DB user ID who queued this download
username: Optional[str] = None # Username for {User} template variable
request_id: Optional[int] = None # Origin request ID when queued from request fulfilment
user_id: int | None = None # DB user ID who queued this download
username: str | None = None # Username for {User} template variable
request_id: int | None = None # Origin request ID when queued from request fulfilment
# Runtime state
priority: int = 0
added_time: float = field(default_factory=time.time)
progress: float = 0.0
status: QueueStatus = QueueStatus.QUEUED
status_message: Optional[str] = None
download_path: Optional[str] = None
last_error_message: Optional[str] = None
last_error_type: Optional[str] = None
staged_path: Optional[str] = None
status_message: str | None = None
download_path: str | None = None
last_error_message: str | None = None
last_error_type: str | None = None
staged_path: str | None = None
def __lt__(self, other):
def __lt__(self, other: DownloadTask) -> bool:
"""Compare tasks for priority queue (lower priority number = higher precedence)."""
if self.priority != other.priority:
return self.priority < other.priority
@@ -141,10 +159,11 @@ class DownloadTask:
@dataclass
class SearchFilters:
"""Filters for book search queries."""
isbn: Optional[List[str]] = None
author: Optional[List[str]] = None
title: Optional[List[str]] = None
lang: Optional[List[str]] = None
sort: Optional[str] = None
content: Optional[List[str]] = None
format: Optional[List[str]] = None
isbn: list[str] | None = None
author: list[str] | None = None
title: list[str] | None = None
lang: list[str] | None = None
sort: str | None = None
content: list[str] | None = None
format: list[str] | None = None
shelfmark/core/naming.py
+51 -51
View File
@@ -1,48 +1,50 @@
"""Template-based naming for library organization."""
import os
import re
from pathlib import Path
from typing import Dict, Optional, Union, Mapping
from typing import TYPE_CHECKING
from shelfmark.core.logger import setup_logger
if TYPE_CHECKING:
from collections.abc import Mapping
logger = setup_logger(__name__)
# Known variable tokens, sorted longest-first to avoid partial matches
# e.g., "SeriesPosition" must match before "Series"
KNOWN_TOKENS = [
'seriesposition',
'originalname',
'partnumber',
'subtitle',
'author',
'series',
'title',
'year',
'user',
"seriesposition",
"originalname",
"partnumber",
"subtitle",
"author",
"series",
"title",
"year",
"user",
]
# Match any {...} block for template parsing
BRACE_PATTERN = re.compile(r'\{([^}]+)\}')
BRACE_PATTERN = re.compile(r"\{([^}]+)\}")
# Characters that are invalid in filenames on various filesystems
INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
def _sanitize(name: Optional[str], max_length: int = 245) -> str:
def _sanitize(name: str | None, max_length: int = 245) -> str:
"""Sanitize a string for filesystem use."""
if not name:
return ""
sanitized = INVALID_CHARS.sub('_', name)
sanitized = re.sub(r'^[\s.]+|[\s.]+$', '', sanitized) # Strip whitespace and dots
sanitized = re.sub(r'_+', '_', sanitized) # Collapse underscores
sanitized = INVALID_CHARS.sub("_", name)
sanitized = re.sub(r"^[\s.]+|[\s.]+$", "", sanitized) # Strip whitespace and dots
sanitized = re.sub(r"_+", "_", sanitized) # Collapse underscores
return sanitized[:max_length]
def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
def sanitize_filename(name: str | None, max_length: int = 245) -> str:
"""Sanitize a string for use as a filename or path component."""
return _sanitize(name, max_length)
@@ -51,7 +53,7 @@ def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
sanitize_path_component = sanitize_filename
def format_series_position(position: Optional[Union[str, int, float]]) -> str:
def format_series_position(position: str | float | None) -> str:
if position is None:
return ""
@@ -63,10 +65,10 @@ def format_series_position(position: Optional[Union[str, int, float]]) -> str:
# Pads numbers to 9 digits for natural sorting (e.g., "Part 2" -> "Part 000000002")
PAD_NUMBERS_PATTERN = re.compile(r'\d+')
PAD_NUMBERS_PATTERN = re.compile(r"\d+")
def natural_sort_key(path: Union[str, Path]) -> str:
def natural_sort_key(path: str | Path) -> str:
"""Generate a sort key with padded numbers for natural sorting."""
filename = Path(path).name.lower()
return PAD_NUMBERS_PATTERN.sub(lambda m: m.group().zfill(9), filename)
@@ -89,7 +91,7 @@ def assign_part_numbers(
def parse_naming_template(
template: str,
metadata: Mapping[str, Optional[Union[str, int, float]]],
metadata: Mapping[str, str | int | float | None],
*,
allow_path_separators: bool = True,
) -> str:
@@ -99,7 +101,7 @@ def parse_naming_template(
# Normalize metadata keys to lowercase for case-insensitive matching
normalized = {k.lower(): v for k, v in metadata.items()}
def find_token(content: str) -> tuple[Optional[str], int]:
def find_token(content: str) -> tuple[str | None, int]:
content_lower = content.lower()
for token in KNOWN_TOKENS:
idx = content_lower.find(token)
@@ -109,19 +111,19 @@ def parse_naming_template(
def token_value(token: str) -> str:
value = normalized.get(token)
if token == 'seriesposition':
if token == "seriesposition":
value = format_series_position(value)
if value is None:
return ""
return str(value).strip()
def render_block(content: str) -> Optional[str]:
def render_block(content: str) -> str | None:
token, idx = find_token(content)
if token is None:
return None
prefix = content[:idx]
suffix = content[idx + len(token):]
suffix = content[idx + len(token) :]
value = token_value(token)
if not value:
return ""
@@ -140,7 +142,7 @@ def parse_naming_template(
parts: list[str] = []
cursor = 0
for idx, match in enumerate(matches):
parts.append(template[cursor:match.start()])
parts.append(template[cursor : match.start()])
content = match.group(1)
rendered = render_block(content)
@@ -157,11 +159,10 @@ def parse_naming_template(
include_literal = bool(token_value(next_token))
if include_literal:
parts.append(content)
elif not conditional_literal:
elif not conditional_literal and re.search(r"\s", content):
# Preserve blocks that look like literal text, but treat bare unknown
# placeholders as missing variables.
if re.search(r"\s", content):
parts.append(match.group(0))
parts.append(match.group(0))
cursor = match.end()
@@ -169,41 +170,39 @@ def parse_naming_template(
result = "".join(parts)
# Clean up any double slashes that might result from empty tokens
result = re.sub(r'/+', '/', result)
result = re.sub(r"/+", "/", result)
# Remove leading/trailing slashes
result = result.strip('/')
result = result.strip("/")
# Clean up any orphaned separators (e.g., " - " at start/end, or " - - ")
result = re.sub(r'^[\s\-_.]+', '', result)
result = re.sub(r'[\s\-_.]+$', '', result)
result = re.sub(r'(\s*-\s*){2,}', ' - ', result)
result = re.sub(r"^[\s\-_.]+", "", result)
result = re.sub(r"[\s\-_.]+$", "", result)
result = re.sub(r"(\s*-\s*){2,}", " - ", result)
# Clean up empty parentheses/brackets
result = re.sub(r'\(\s*\)', '', result)
result = re.sub(r'\[\s*\]', '', result)
result = re.sub(r"\(\s*\)", "", result)
result = re.sub(r"\[\s*\]", "", result)
# Final trim of any trailing separators left after cleanup
result = re.sub(r'[\s\-_.]+$', '', result)
return result
return re.sub(r"[\s\-_.]+$", "", result)
def build_library_path(
base_path: str,
template: str,
metadata: Mapping[str, Optional[Union[str, int, float]]],
extension: Optional[str] = None,
metadata: Mapping[str, str | int | float | None],
extension: str | None = None,
) -> Path:
relative = parse_naming_template(template, metadata, allow_path_separators=True)
if not relative:
# Fallback to title if template produces empty result
title = metadata.get('Title') or metadata.get('title') or 'Unknown'
title = metadata.get("Title") or metadata.get("title") or "Unknown"
relative = sanitize_filename(str(title))
# Remove any path traversal attempts
relative = relative.replace('..', '')
relative = relative.replace("..", "")
base = Path(base_path).resolve()
full_path = (base / relative).resolve()
@@ -211,11 +210,12 @@ def build_library_path(
# Verify the path is within the base directory
try:
full_path.relative_to(base)
except ValueError:
raise ValueError(f"Path traversal detected: template would escape library directory")
except ValueError as exc:
msg = "Path traversal detected: template would escape library directory"
raise ValueError(msg) from exc
if extension:
ext = extension.lstrip('.')
ext = extension.lstrip(".")
# Don't use with_suffix() - it replaces everything after the first dot
# e.g., "2.5 - Title" would become "2.epub" instead of "2.5 - Title.epub"
full_path = Path(f"{full_path}.{ext}")
@@ -223,27 +223,27 @@ def build_library_path(
return full_path
def same_filesystem(path1: Union[str, Path], path2: Union[str, Path]) -> bool:
def same_filesystem(path1: str | Path, path2: str | Path) -> bool:
"""Check if two paths are on the same filesystem."""
path1 = Path(path1)
path2 = Path(path2)
def get_device(p: Path) -> Optional[int]:
def get_device(p: Path) -> int | None:
try:
while not p.exists():
p = p.parent
if p == p.parent:
break
return os.stat(p).st_dev
return p.stat().st_dev
except (OSError, PermissionError) as e:
logger.debug(f"Cannot stat {p}: {e}")
logger.debug("Cannot stat %s: %s", p, e)
return None
dev1 = get_device(path1)
dev2 = get_device(path2)
if dev1 is None or dev2 is None:
logger.warning(f"Cannot determine filesystem for hardlink check, falling back to copy")
logger.warning("Cannot determine filesystem for hardlink check, falling back to copy")
return False
return dev1 == dev2
shelfmark/core/notifications.py
+42 -28
View File
@@ -5,20 +5,23 @@ from __future__ import annotations
import logging
import threading
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from enum import Enum
from typing import Any, Iterable, Iterator
from enum import StrEnum
from typing import TYPE_CHECKING, Any
from urllib.parse import urlsplit
try:
import apprise
except Exception: # pragma: no cover - exercised in tests via monkeypatch
except ImportError: # pragma: no cover - exercised in tests via monkeypatch
apprise = None # type: ignore[assignment]
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
logger = setup_logger(__name__)
# Small pool for non-blocking dispatch. Notification sends are I/O bound and infrequent.
@@ -32,7 +35,7 @@ _APPRISE_LOGO_URL = (
_APPRISE_LOGGER_NAME = "apprise"
class NotificationEvent(str, Enum):
class NotificationEvent(StrEnum):
"""Global notification event identifiers."""
REQUEST_CREATED = "request_created"
@@ -57,7 +60,7 @@ class NotificationContext:
error_message: str | None = None
def _normalize_urls(value: Any) -> list[str]:
def _normalize_urls(value: object) -> list[str]:
if value is None:
return []
@@ -103,7 +106,7 @@ def _extract_url_schemes(urls: Iterable[str]) -> list[str]:
class _AppriseLogCapture(logging.Handler):
def __init__(self, *, thread_id: int):
def __init__(self, *, thread_id: int) -> None:
super().__init__(level=logging.INFO)
self.records: list[tuple[int, str, str, str]] = []
self._thread_id = thread_id
@@ -126,7 +129,9 @@ class _AppriseLogCapture(logging.Handler):
@contextmanager
def _capture_apprise_logs(*, min_level: int = logging.INFO) -> Iterator[list[tuple[int, str, str, str]]]:
def _capture_apprise_logs(
*, min_level: int = logging.INFO
) -> Iterator[list[tuple[int, str, str, str]]]:
apprise_logger = logging.getLogger(_APPRISE_LOGGER_NAME)
previous_level = apprise_logger.level
handler = _AppriseLogCapture(thread_id=threading.get_ident())
@@ -170,7 +175,7 @@ def _log_apprise_exception_debug(*, action: str, scheme: str, exc: Exception) ->
type(exc).__name__,
scheme,
exc,
exc_info=True,
exc_info=(type(exc), exc, exc.__traceback__),
)
@@ -197,7 +202,7 @@ def _build_apprise_warning_detail(
return None
def _normalize_routes(value: Any) -> list[dict[str, str]]:
def _normalize_routes(value: object) -> list[dict[str, str]]:
if not isinstance(value, list):
return []
@@ -248,10 +253,10 @@ def _resolve_admin_routes() -> list[dict[str, str]]:
return _normalize_routes(app_config.get("ADMIN_NOTIFICATION_ROUTES", []))
def _normalize_user_id(value: Any) -> int | None:
def _normalize_user_id(value: object) -> int | None:
try:
user_id = int(value)
except (TypeError, ValueError):
except TypeError, ValueError:
return None
if user_id < 1:
return None
@@ -291,7 +296,7 @@ def _resolve_route_urls_for_event(
return selected
def _resolve_notify_type(event: NotificationEvent) -> Any:
def _resolve_notify_type(event: NotificationEvent) -> object:
if apprise is None:
fallback = {
NotificationEvent.REQUEST_CREATED: "info",
@@ -312,7 +317,7 @@ def _resolve_notify_type(event: NotificationEvent) -> Any:
return mapping[event]
def _clean_text(value: Any, fallback: str) -> str:
def _clean_text(value: object, fallback: str) -> str:
text = str(value or "").strip()
return text or fallback
@@ -330,7 +335,10 @@ def _render_message(context: NotificationContext) -> tuple[str, str]:
if event == NotificationEvent.REQUEST_REJECTED:
note = _clean_text(context.admin_note, "")
note_line = f"\nNote: {note}" if note else ""
return "Request Rejected", f'Request for "{title}" by {author} was rejected.{note_line}'
return (
"Request Rejected",
f'Request for "{title}" by {author} was rejected.{note_line}',
)
if event == NotificationEvent.DOWNLOAD_COMPLETE:
return "Download Complete", f'"{title}" by {author} downloaded successfully.'
@@ -339,7 +347,7 @@ def _render_message(context: NotificationContext) -> tuple[str, str]:
return "Download Failed", f'Failed to download "{title}" by {author}.{error_line}'
def _plugin_label(plugin: Any, fallback_scheme: str) -> str:
def _plugin_label(plugin: object, fallback_scheme: str) -> str:
"""Build a human-readable label from a validated Apprise plugin.
Combines the URL scheme with the plugin's service name (app_id) and
@@ -351,10 +359,8 @@ def _plugin_label(plugin: Any, fallback_scheme: str) -> str:
app_id = getattr(plugin, "app_id", None)
if app_id and str(app_id) != fallback_scheme:
privacy_url: str | None = None
try:
with suppress(Exception):
privacy_url = plugin.url(privacy=True)
except Exception:
pass
suffix = str(app_id)
if privacy_url:
@@ -369,7 +375,7 @@ def _dispatch_to_apprise(
*,
title: str,
body: str,
notify_type: Any,
notify_type: object,
) -> dict[str, Any]:
normalized_urls = _normalize_urls(list(urls))
url_schemes = _extract_url_schemes(normalized_urls)
@@ -443,9 +449,7 @@ def _dispatch_to_apprise(
if warning_detail:
failure_details.append(warning_detail)
else:
failure_details.append(
f"{scheme}: notify raised {type(exc).__name__}: {exc}"
)
failure_details.append(f"{scheme}: notify raised {type(exc).__name__}: {exc}")
continue
_log_apprise_records(apprise_records)
@@ -502,7 +506,7 @@ def _dispatch_to_apprise(
return result
def _create_apprise_client() -> Any:
def _create_apprise_client() -> object:
if apprise is None:
return None
@@ -535,7 +539,9 @@ def _create_apprise_client() -> Any:
return apprise_cls()
def _send_admin_event(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> dict[str, Any]:
def _send_admin_event(
event: NotificationEvent, context: NotificationContext, urls: list[str]
) -> dict[str, Any]:
title, body = _render_message(context)
notify_type = _resolve_notify_type(event)
return _dispatch_to_apprise(urls, title=title, body=body, notify_type=notify_type)
@@ -554,7 +560,9 @@ def notify_admin(event: NotificationEvent, context: NotificationContext) -> None
logger.warning("Failed to queue admin notification '%s': %s", event.value, exc)
def notify_user(user_id: int | None, event: NotificationEvent, context: NotificationContext) -> None:
def notify_user(
user_id: int | None, event: NotificationEvent, context: NotificationContext
) -> None:
"""Send a per-user notification for an event if subscribed."""
normalized_user_id = _normalize_user_id(user_id)
if normalized_user_id is None:
@@ -576,10 +584,16 @@ def notify_user(user_id: int | None, event: NotificationEvent, context: Notifica
)
def _dispatch_admin_async(event: NotificationEvent, context: NotificationContext, urls: list[str]) -> None:
def _dispatch_admin_async(
event: NotificationEvent, context: NotificationContext, urls: list[str]
) -> None:
result = _send_admin_event(event, context, urls)
if not result.get("success", False):
logger.warning("Admin notification failed for event '%s': %s", event.value, result.get("message"))
logger.warning(
"Admin notification failed for event '%s': %s",
event.value,
result.get("message"),
)
def _dispatch_user_async(
shelfmark/core/oidc_auth.py
+11 -7
View File
@@ -4,12 +4,15 @@ Handles group claim parsing, user info extraction, and user provisioning.
Flask route handlers are registered separately in main.py.
"""
from typing import Any, Dict, List, Optional
from typing import TYPE_CHECKING, Any
from shelfmark.core.external_user_linking import upsert_external_user
from shelfmark.core.user_db import UserDB
def parse_group_claims(id_token: Dict[str, Any], group_claim: str) -> List[str]:
if TYPE_CHECKING:
from shelfmark.core.user_db import UserDB
def parse_group_claims(id_token: dict[str, Any], group_claim: str) -> list[str]:
"""Extract group list from an ID token claim.
Supports list, comma-separated string, or pipe-separated string.
@@ -26,7 +29,7 @@ def parse_group_claims(id_token: Dict[str, Any], group_claim: str) -> List[str]:
return []
def extract_user_info(id_token: Dict[str, Any]) -> Dict[str, Any]:
def extract_user_info(id_token: dict[str, Any]) -> dict[str, Any]:
"""Extract user info from OIDC ID token claims.
Returns a dict with keys: oidc_subject, username, email, display_name.
@@ -47,11 +50,12 @@ def extract_user_info(id_token: Dict[str, Any]) -> Dict[str, Any]:
def provision_oidc_user(
db: UserDB,
user_info: Dict[str, Any],
is_admin: Optional[bool] = None,
user_info: dict[str, Any],
*,
is_admin: bool | None = None,
allow_email_link: bool = False,
allow_create: bool = True,
) -> Optional[Dict[str, Any]]:
) -> dict[str, Any] | None:
"""Create or update a user from OIDC claims.
Matching and collision handling use the shared external user linker:
shelfmark/core/oidc_routes.py
+42 -38
View File
@@ -4,12 +4,12 @@ Registers /api/auth/oidc/login and /api/auth/oidc/callback endpoints.
Business logic remains in oidc_auth.py.
"""
from typing import Any
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode, urlsplit, urlunsplit
from authlib.jose.errors import InvalidClaimError
from authlib.integrations.flask_client import OAuth
from flask import Flask, jsonify, redirect, request, session
from authlib.jose.errors import InvalidClaimError
from flask import Flask, Response, jsonify, redirect, request, session
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
@@ -18,15 +18,17 @@ from shelfmark.core.oidc_auth import (
parse_group_claims,
provision_oidc_user,
)
from shelfmark.core.user_db import UserDB
from shelfmark.download.network import get_ssl_verify
if TYPE_CHECKING:
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
oauth = OAuth()
_RETURN_TO_SESSION_KEY = "oidc_return_to"
def _normalize_claims(raw_claims: Any) -> dict[str, Any]:
def _normalize_claims(raw_claims: object) -> dict[str, Any]:
"""Return a plain dict for claims from Authlib token/userinfo payloads."""
if raw_claims is None:
return {}
@@ -60,7 +62,7 @@ def _login_error_url(message: str) -> str:
return f"{login_url}?{urlencode(params)}"
def _normalize_return_to(raw_return_to: Any) -> str | None:
def _normalize_return_to(raw_return_to: object) -> str | None:
"""Return a safe app-relative post-login target."""
if not isinstance(raw_return_to, str):
return None
@@ -79,14 +81,9 @@ def _normalize_return_to(raw_return_to: Any) -> str | None:
if path == script_root:
path = "/"
elif path.startswith(f"{script_root}/"):
path = path[len(script_root):] or "/"
path = path[len(script_root) :] or "/"
if (
path == "/login"
or path.startswith("/login/")
or path == "/api"
or path.startswith("/api/")
):
if path in {"/login", "/api"} or path.startswith(("/login/", "/api/")):
return None
return urlunsplit(("", "", path, parsed.query, parsed.fragment))
@@ -95,9 +92,7 @@ def _normalize_return_to(raw_return_to: Any) -> str | None:
def _get_pending_return_to(*, clear: bool = False) -> str | None:
"""Read the pending post-login target from the session."""
raw_return_to = (
session.pop(_RETURN_TO_SESSION_KEY, None)
if clear
else session.get(_RETURN_TO_SESSION_KEY)
session.pop(_RETURN_TO_SESSION_KEY, None) if clear else session.get(_RETURN_TO_SESSION_KEY)
)
normalized = _normalize_return_to(raw_return_to)
if normalized is None and not clear:
@@ -122,18 +117,21 @@ def _get_oidc_client() -> tuple[Any, dict[str, Any]]:
client_id = str(app_config.get("OIDC_CLIENT_ID", "") or "")
if not discovery_url or not client_id:
raise ValueError("OIDC not configured")
msg = "OIDC not configured"
raise ValueError(msg)
configured_scopes = app_config.get("OIDC_SCOPES", ["openid", "email", "profile"])
if isinstance(configured_scopes, list):
scope_values = [str(scope).strip() for scope in configured_scopes if str(scope).strip()]
elif isinstance(configured_scopes, str):
delimiter = "," if "," in configured_scopes else " "
scope_values = [scope.strip() for scope in configured_scopes.split(delimiter) if scope.strip()]
scope_values = [
scope.strip() for scope in configured_scopes.split(delimiter) if scope.strip()
]
else:
scope_values = []
scopes = list(dict.fromkeys(["openid"] + scope_values))
scopes = list(dict.fromkeys(["openid", *scope_values]))
admin_group = app_config.get("OIDC_ADMIN_GROUP", "")
group_claim = app_config.get("OIDC_GROUP_CLAIM", "groups")
@@ -141,7 +139,7 @@ def _get_oidc_client() -> tuple[Any, dict[str, Any]]:
if admin_group and use_admin_group and group_claim and group_claim not in scopes:
scopes.append(group_claim)
def _ssl_compliance_fix(session, **kwargs):
def _ssl_compliance_fix(session: Any, **kwargs: Any) -> Any:
"""Set session.verify based on the Certificate Validation setting."""
session.verify = get_ssl_verify(discovery_url)
return session
@@ -162,7 +160,8 @@ def _get_oidc_client() -> tuple[Any, dict[str, Any]]:
client = oauth.create_client("shelfmark_idp")
if client is None:
raise RuntimeError("OIDC client initialization failed")
msg = "OIDC client initialization failed"
raise RuntimeError(msg)
return client, {
"OIDC_DISCOVERY_URL": discovery_url,
@@ -178,7 +177,7 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
oauth.init_app(app)
@app.route("/api/auth/oidc/login", methods=["GET"])
def oidc_login():
def oidc_login() -> Response | tuple[Response, int]:
"""Initiate OIDC login flow and redirect to the provider."""
try:
client, _ = _get_oidc_client()
@@ -191,17 +190,17 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
return client.authorize_redirect(redirect_uri)
except ValueError:
return jsonify({"error": "OIDC not configured"}), 500
except Exception as e:
logger.error(f"OIDC login error: {e}")
except Exception:
logger.exception("OIDC login error")
return jsonify({"error": "OIDC login failed"}), 500
@app.route("/api/auth/oidc/callback", methods=["GET"])
def oidc_callback():
def oidc_callback() -> Response | tuple[Response, int]:
"""Handle OIDC callback from identity provider."""
try:
error = request.args.get("error")
if error:
logger.warning(f"OIDC callback error from IdP: {error}")
logger.warning("OIDC callback error from IdP: %s", error)
return redirect(_login_error_url("Authentication failed"))
client, config = _get_oidc_client()
@@ -216,12 +215,14 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
if isinstance(metadata, dict):
provider_issuer = str(metadata.get("issuer", ""))
except Exception as metadata_error:
logger.debug(f"OIDC metadata lookup failed during claim diagnostics: {metadata_error}")
logger.debug(
"OIDC metadata lookup failed during claim diagnostics: %s",
metadata_error,
)
logger.error(
"OIDC callback claim validation failed: claim=%s error=%s discovery_url=%s provider_issuer=%s",
logger.exception(
"OIDC callback claim validation failed: claim=%s discovery_url=%s provider_issuer=%s",
claim_name,
e,
discovery_url or "<unset>",
provider_issuer or "<unknown>",
)
@@ -232,7 +233,9 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
)
return redirect(_login_error_url(msg))
return redirect(_login_error_url(f"OIDC token claim validation failed: {claim_name}"))
return redirect(
_login_error_url(f"OIDC token claim validation failed: {claim_name}")
)
claims = _normalize_claims(token.get("userinfo"))
# If userinfo is missing or claims are too sparse, request it explicitly.
@@ -242,8 +245,8 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
fetched_claims = _normalize_claims(client.userinfo(token=token))
except TypeError:
fetched_claims = _normalize_claims(client.userinfo())
except Exception as e:
logger.error(f"Failed to fetch OIDC userinfo: {e}")
except Exception:
logger.exception("Failed to fetch OIDC userinfo")
if fetched_claims:
claims = {**claims, **fetched_claims}
@@ -274,7 +277,8 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
)
if user is None:
logger.warning(
f"OIDC login rejected: auto-provision disabled for {user_info['username']}"
"OIDC login rejected: auto-provision disabled for %s",
user_info["username"],
)
return redirect(_login_error_url("Account not found. Contact your administrator."))
@@ -283,12 +287,12 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
session["db_user_id"] = user["id"]
session.permanent = True
logger.info(f"OIDC login successful: {user['username']} (admin={is_admin})")
logger.info("OIDC login successful: %s (admin=%s)", user["username"], is_admin)
return redirect(_post_login_redirect_target(_get_pending_return_to(clear=True)))
except ValueError as e:
logger.error(f"OIDC callback error: {e}")
logger.exception("OIDC callback error")
return redirect(_login_error_url(str(e)))
except Exception as e:
logger.error(f"OIDC callback error: {e}")
except Exception:
logger.exception("OIDC callback error")
return redirect(_login_error_url("Authentication failed"))
shelfmark/core/onboarding.py
+63 -55
View File
@@ -1,5 +1,4 @@
"""
Onboarding wizard configuration.
"""Onboarding wizard configuration.
Defines the steps and fields for the first-run onboarding experience.
Reuses field definitions from the settings registry where possible.
@@ -8,16 +7,16 @@ Reuses field definitions from the settings registry where possible.
import json
from dataclasses import replace
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import (
HeadingField,
SettingsField,
get_settings_tab,
serialize_field,
save_config_file,
get_setting_value,
get_settings_tab,
save_config_file,
serialize_field,
)
logger = setup_logger(__name__)
@@ -29,6 +28,7 @@ ONBOARDING_STORAGE_KEY = "onboarding_complete"
def _get_config_dir() -> Path:
"""Get the config directory path."""
from shelfmark.config.env import CONFIG_DIR
return Path(CONFIG_DIR)
@@ -45,11 +45,11 @@ def is_onboarding_complete() -> bool:
return False
try:
with open(config_file, 'r') as f:
with config_file.open() as f:
config = json.load(f)
return config.get(ONBOARDING_STORAGE_KEY, False)
except (json.JSONDecodeError, OSError) as e:
logger.warning(f"Could not read onboarding status from settings.json: {e}")
logger.warning("Could not read onboarding status from settings.json: %s", e)
return False
@@ -57,14 +57,13 @@ def mark_onboarding_complete() -> bool:
"""Mark onboarding as complete."""
try:
return save_config_file("general", {ONBOARDING_STORAGE_KEY: True})
except Exception as e:
logger.error(f"Failed to mark onboarding complete: {e}")
except Exception:
logger.exception("Failed to mark onboarding complete")
return False
def _get_field_from_tab(tab_name: str, field_key: str) -> Optional[SettingsField]:
"""
Extract a specific field from a registered settings tab.
def _get_field_from_tab(tab_name: str, field_key: str) -> SettingsField | None:
"""Extract a specific field from a registered settings tab.
Args:
tab_name: Name of the settings tab (e.g., 'search_mode', 'hardcover')
@@ -72,23 +71,23 @@ def _get_field_from_tab(tab_name: str, field_key: str) -> Optional[SettingsField
Returns:
The field if found, None otherwise
"""
tab = get_settings_tab(tab_name)
if not tab:
logger.warning(f"Settings tab not found: {tab_name}")
logger.warning("Settings tab not found: %s", tab_name)
return None
for field in tab.fields:
if hasattr(field, 'key') and field.key == field_key:
if hasattr(field, "key") and field.key == field_key:
return field
logger.warning(f"Field {field_key} not found in tab {tab_name}")
logger.warning("Field %s not found in tab %s", field_key, tab_name)
return None
def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField:
"""
Clone a field with optional attribute overrides.
"""Clone a field with optional attribute overrides.
Useful for customizing labels, descriptions, or defaults for onboarding context.
"""
@@ -100,9 +99,9 @@ def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsFi
# =============================================================================
def get_search_mode_fields() -> List[SettingsField]:
def get_search_mode_fields() -> list[SettingsField]:
"""Step 1: Choose search mode - uses actual SEARCH_MODE field from settings."""
fields: List[SettingsField] = [
fields: list[SettingsField] = [
HeadingField(
key="welcome_heading",
title="Welcome to Shelfmark",
@@ -114,17 +113,19 @@ def get_search_mode_fields() -> List[SettingsField]:
search_mode_field = _get_field_from_tab("search_mode", "SEARCH_MODE")
if search_mode_field:
# Clone with onboarding-specific description
fields.append(_clone_field_with_overrides(
search_mode_field,
description="Choose how you want to find books.",
))
fields.append(
_clone_field_with_overrides(
search_mode_field,
description="Choose how you want to find books.",
)
)
return fields
def get_metadata_provider_fields() -> List[SettingsField]:
def get_metadata_provider_fields() -> list[SettingsField]:
"""Step 2: Choose metadata provider - uses actual METADATA_PROVIDER field."""
fields: List[SettingsField] = [
fields: list[SettingsField] = [
HeadingField(
key="metadata_heading",
title="Metadata Provider",
@@ -155,18 +156,20 @@ def get_metadata_provider_fields() -> List[SettingsField]:
]
# Clone with onboarding-specific options and default
fields.append(_clone_field_with_overrides(
provider_field,
default="hardcover",
options=onboarding_options,
))
fields.append(
_clone_field_with_overrides(
provider_field,
default="hardcover",
options=onboarding_options,
)
)
return fields
def get_hardcover_setup_fields() -> List[SettingsField]:
def get_hardcover_setup_fields() -> list[SettingsField]:
"""Step 3a: Configure Hardcover - uses actual API key and test connection fields."""
fields: List[SettingsField] = [
fields: list[SettingsField] = [
HeadingField(
key="hardcover_setup_heading",
title="Hardcover Setup",
@@ -189,9 +192,9 @@ def get_hardcover_setup_fields() -> List[SettingsField]:
return fields
def get_googlebooks_setup_fields() -> List[SettingsField]:
def get_googlebooks_setup_fields() -> list[SettingsField]:
"""Step 3b: Configure Google Books - uses actual API key and test connection fields."""
fields: List[SettingsField] = [
fields: list[SettingsField] = [
HeadingField(
key="googlebooks_setup_heading",
title="Google Books Setup",
@@ -214,9 +217,9 @@ def get_googlebooks_setup_fields() -> List[SettingsField]:
return fields
def get_prowlarr_fields() -> List[SettingsField]:
def get_prowlarr_fields() -> list[SettingsField]:
"""Step 4: Configure Prowlarr connection - uses actual Prowlarr fields."""
fields: List[SettingsField] = [
fields: list[SettingsField] = [
HeadingField(
key="prowlarr_heading",
title="Prowlarr Integration (Optional)",
@@ -234,9 +237,9 @@ def get_prowlarr_fields() -> List[SettingsField]:
return fields
def get_prowlarr_indexers_fields() -> List[SettingsField]:
def get_prowlarr_indexers_fields() -> list[SettingsField]:
"""Step 5: Select Prowlarr indexers to search."""
fields: List[SettingsField] = [
fields: list[SettingsField] = [
HeadingField(
key="prowlarr_indexers_heading",
title="Select Indexers",
@@ -316,10 +319,8 @@ ONBOARDING_STEPS = [
]
def get_onboarding_config() -> Dict[str, Any]:
"""
Get the full onboarding configuration including steps and current values.
"""
def get_onboarding_config() -> dict[str, Any]:
"""Get the full onboarding configuration including steps and current values."""
steps = []
all_values = {}
@@ -334,9 +335,11 @@ def get_onboarding_config() -> Dict[str, Any]:
serialized_fields.append(serialized)
# Collect values (skip HeadingFields)
if hasattr(field, 'key') and field.key and not isinstance(field, HeadingField):
if hasattr(field, "key") and field.key and not isinstance(field, HeadingField):
value = get_setting_value(field, tab_name)
all_values[field.key] = value if value is not None else getattr(field, 'default', '')
all_values[field.key] = (
value if value is not None else getattr(field, "default", "")
)
step = {
"id": step_config["id"],
@@ -359,19 +362,19 @@ def get_onboarding_config() -> Dict[str, Any]:
}
def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
"""
Save onboarding settings and mark as complete.
def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]:
"""Save onboarding settings and mark as complete.
Args:
values: Dict of field key -> value
Returns:
Dict with success status and message
"""
try:
# Group values by their target tab
tab_values: Dict[str, Dict[str, Any]] = {}
tab_values: dict[str, dict[str, Any]] = {}
for step_config in ONBOARDING_STEPS:
tab_name = step_config["tab"]
@@ -391,7 +394,7 @@ def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
for tab_name, tab_data in tab_values.items():
if tab_data:
save_config_file(tab_name, tab_data)
logger.info(f"Saved onboarding settings to {tab_name}: {list(tab_data.keys())}")
logger.info("Saved onboarding settings to %s: %s", tab_name, list(tab_data.keys()))
# Enable the selected metadata provider
search_mode = values.get("SEARCH_MODE", "direct")
@@ -416,7 +419,11 @@ def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
provider_config["GOOGLEBOOKS_API_KEY"] = values["GOOGLEBOOKS_API_KEY"]
save_config_file(provider, provider_config)
logger.info(f"Enabled metadata provider: {provider} with keys: {list(provider_config.keys())}")
logger.info(
"Enabled metadata provider: %s with keys: %s",
provider,
list(provider_config.keys()),
)
# Mark onboarding as complete
mark_onboarding_complete()
@@ -424,12 +431,13 @@ def save_onboarding_settings(values: Dict[str, Any]) -> Dict[str, Any]:
# Refresh config
try:
from shelfmark.core.config import config
config.refresh()
except ImportError as e:
logger.debug(f"Could not refresh config after onboarding: {e}")
return {"success": True, "message": "Onboarding complete!"}
logger.debug("Could not refresh config after onboarding: %s", e)
except Exception as e:
logger.error(f"Failed to save onboarding settings: {e}")
logger.exception("Failed to save onboarding settings")
return {"success": False, "message": str(e)}
else:
return {"success": True, "message": "Onboarding complete!"}
shelfmark/core/path_mappings.py
+20 -11
View File
@@ -11,7 +11,12 @@ from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable, Optional
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Iterable
_WINDOWS_DRIVE_PREFIX_LENGTH = 2
@dataclass(frozen=True)
@@ -36,14 +41,14 @@ def _normalize_prefix(path: str) -> str:
def _is_windows_path(path: str) -> bool:
"""Check if a path looks like a Windows path (has a drive letter like C:/)."""
return len(path) >= 2 and path[1] == ":" and path[0].isalpha()
return len(path) >= _WINDOWS_DRIVE_PREFIX_LENGTH and path[1] == ":" and path[0].isalpha()
def _normalize_host(host: str) -> str:
return str(host or "").strip().lower()
def parse_remote_path_mappings(value: Any) -> list[RemotePathMapping]:
def parse_remote_path_mappings(value: object) -> list[RemotePathMapping]:
if not value or not isinstance(value, list):
return []
@@ -60,7 +65,9 @@ def parse_remote_path_mappings(value: Any) -> list[RemotePathMapping]:
if not host or not remote_path or not local_path:
continue
mappings.append(RemotePathMapping(host=host, remote_path=remote_path, local_path=local_path))
mappings.append(
RemotePathMapping(host=host, remote_path=remote_path, local_path=local_path)
)
mappings.sort(key=lambda m: len(m.remote_path), reverse=True)
return mappings
@@ -96,16 +103,17 @@ def remap_remote_to_local_with_match(
prefix_lower = remote_prefix.lower()
matches = remote_lower == prefix_lower or remote_lower.startswith(prefix_lower + "/")
else:
matches = remote_normalized == remote_prefix or remote_normalized.startswith(remote_prefix + "/")
matches = remote_normalized == remote_prefix or remote_normalized.startswith(
remote_prefix + "/"
)
if matches:
# Use the length of the original prefix to extract remainder
# This preserves the original case in folder names
remainder = remote_normalized[len(remote_prefix):]
remainder = remote_normalized[len(remote_prefix) :]
local_prefix = _normalize_prefix(mapping.local_path)
if remainder.startswith("/"):
remainder = remainder[1:]
remainder = remainder.removeprefix("/")
remapped = Path(local_prefix) / remainder if remainder else Path(local_prefix)
return remapped, True
@@ -113,7 +121,9 @@ def remap_remote_to_local_with_match(
return Path(remote_normalized), False
def remap_remote_to_local(*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path) -> Path:
def remap_remote_to_local(
*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path
) -> Path:
remapped, _ = remap_remote_to_local_with_match(
mappings=mappings,
host=host,
@@ -122,13 +132,12 @@ def remap_remote_to_local(*, mappings: Iterable[RemotePathMapping], host: str, r
return remapped
def get_client_host_identifier(client: Any) -> Optional[str]:
def get_client_host_identifier(client: object) -> str | None:
"""Return a stable identifier used by the mapping UI.
Sonarr uses the download client's configured host. Shelfmark currently uses
the download client 'name' (e.g. qbittorrent, sabnzbd).
"""
name = getattr(client, "name", None)
if isinstance(name, str) and name.strip():
return name.strip().lower()
shelfmark/core/prefix_middleware.py
+12 -4
View File
@@ -2,18 +2,26 @@
from __future__ import annotations
from typing import Iterable, Optional
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
class PrefixMiddleware:
"""Strip a configured URL prefix from PATH_INFO before routing."""
def __init__(self, app, prefix: str, bypass_paths: Optional[Iterable[str]] = None) -> None:
def __init__(
self,
app: Callable[[dict[str, object], Callable[..., object]], object],
prefix: str,
bypass_paths: Iterable[str] | None = None,
) -> None:
self.app = app
self.prefix = prefix.rstrip("/")
self.bypass_paths = set(bypass_paths or [])
def __call__(self, environ, start_response):
def __call__(self, environ: dict[str, object], start_response: Callable[..., object]) -> object:
path = environ.get("PATH_INFO", "") or ""
if path in self.bypass_paths:
@@ -24,7 +32,7 @@ class PrefixMiddleware:
if path == self.prefix or path.startswith(self.prefix + "/"):
environ["SCRIPT_NAME"] = self.prefix
environ["PATH_INFO"] = path[len(self.prefix):] or "/"
environ["PATH_INFO"] = path[len(self.prefix) :] or "/"
return self.app(environ, start_response)
start_response("404 Not Found", [("Content-Type", "text/plain")])
shelfmark/core/queue.py
+89 -79
View File
@@ -4,12 +4,20 @@ import queue
import time
from datetime import datetime, timedelta
from pathlib import Path
from threading import Lock, Event
from typing import Dict, List, Optional, Tuple, Any, Callable
from threading import Event, Lock
from typing import TYPE_CHECKING, Any
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import QueueStatus, QueueItem, DownloadTask, TERMINAL_QUEUE_STATUSES
from shelfmark.core.models import (
TERMINAL_QUEUE_STATUSES,
DownloadTask,
QueueItem,
QueueStatus,
)
if TYPE_CHECKING:
from collections.abc import Callable
logger = setup_logger(__name__)
@@ -25,10 +33,8 @@ class BookQueue:
self._status_timestamps: dict[str, datetime] = {} # Track when each status was last updated
self._cancel_flags: dict[str, Event] = {} # Cancellation flags for active downloads
self._active_downloads: dict[str, bool] = {} # Track currently downloading tasks
self._terminal_status_hook: Optional[
Callable[[str, QueueStatus, DownloadTask], None]
] = None
self._queue_hook: Optional[Callable[[str, DownloadTask], None]] = None
self._terminal_status_hook: Callable[[str, QueueStatus, DownloadTask], None] | None = None
self._queue_hook: Callable[[str, DownloadTask], None] | None = None
@property
def _status_timeout(self) -> timedelta:
@@ -37,12 +43,15 @@ class BookQueue:
def add(self, task: DownloadTask) -> bool:
"""Add a download task to the queue. Returns False if already exists."""
hook: Optional[Callable[[str, DownloadTask], None]] = None
hook: Callable[[str, DownloadTask], None] | None = None
with self._lock:
task_id = task.task_id
# Don't add if already exists and not in error/cancelled state
if task_id in self._status and self._status[task_id] not in [QueueStatus.ERROR, QueueStatus.CANCELLED]:
if task_id in self._status and self._status[task_id] not in [
QueueStatus.ERROR,
QueueStatus.CANCELLED,
]:
return False
# Ensure added_time is set
@@ -62,7 +71,7 @@ class BookQueue:
logger.warning("Queue hook failed while adding task %s: %s", task_id, exc)
return True
def get_next(self) -> Optional[Tuple[str, Event]]:
def get_next(self) -> tuple[str, Event] | None:
"""Get next task ID from queue with cancellation flag."""
# Use iterative approach to avoid stack overflow if many items are cancelled
while True:
@@ -79,17 +88,17 @@ class BookQueue:
cancel_flag = Event()
self._cancel_flags[task_id] = cancel_flag
self._active_downloads[task_id] = True
return task_id, cancel_flag
except queue.Empty:
return None
else:
return task_id, cancel_flag
def get_task(self, task_id: str) -> Optional[DownloadTask]:
def get_task(self, task_id: str) -> DownloadTask | None:
"""Get a task by its ID."""
with self._lock:
return self._task_data.get(task_id)
def get_task_status(self, task_id: str) -> Optional[QueueStatus]:
def get_task_status(self, task_id: str) -> QueueStatus | None:
"""Get queue status for a task id."""
with self._lock:
return self._status.get(task_id)
@@ -101,7 +110,7 @@ class BookQueue:
def set_terminal_status_hook(
self,
hook: Optional[Callable[[str, QueueStatus, DownloadTask], None]],
hook: Callable[[str, QueueStatus, DownloadTask], None] | None,
) -> None:
"""Register a callback invoked when a task first enters a terminal status."""
with self._lock:
@@ -109,7 +118,7 @@ class BookQueue:
def set_queue_hook(
self,
hook: Optional[Callable[[str, DownloadTask], None]],
hook: Callable[[str, DownloadTask], None] | None,
) -> None:
"""Register a callback invoked when a task is added to the queue."""
with self._lock:
@@ -117,8 +126,8 @@ class BookQueue:
def update_status(self, book_id: str, status: QueueStatus) -> None:
"""Update status of a book in the queue."""
hook: Optional[Callable[[str, QueueStatus, DownloadTask], None]] = None
hook_task: Optional[DownloadTask] = None
hook: Callable[[str, QueueStatus, DownloadTask], None] | None = None
hook_task: DownloadTask | None = None
with self._lock:
previous_status = self._status.get(book_id)
self._update_status(book_id, status)
@@ -159,16 +168,19 @@ class BookQueue:
if task_id in self._task_data:
self._task_data[task_id].status_message = message
def get_status(self, user_id: Optional[int] = None) -> Dict[QueueStatus, Dict[str, DownloadTask]]:
def get_status(self, user_id: int | None = None) -> dict[QueueStatus, dict[str, DownloadTask]]:
"""Get current queue status grouped by status.
Args:
user_id: If provided, only return tasks belonging to this user.
If None, return all.
"""
self.refresh()
with self._lock:
result: Dict[QueueStatus, Dict[str, DownloadTask]] = {status: {} for status in QueueStatus}
result: dict[QueueStatus, dict[str, DownloadTask]] = {
status: {} for status in QueueStatus
}
for task_id, status in self._status.items():
if task_id in self._task_data:
task = self._task_data[task_id]
@@ -177,7 +189,7 @@ class BookQueue:
result[status][task_id] = task
return result
def get_queue_order(self) -> List[Dict[str, Any]]:
def get_queue_order(self) -> list[dict[str, Any]]:
"""Get current queue order for display."""
with self._lock:
queue_items = []
@@ -185,39 +197,42 @@ class BookQueue:
# Get items from priority queue without removing them
temp_items = []
while not self._queue.empty():
try:
item = self._queue.get_nowait()
temp_items.append(item)
task_id = item.book_id # QueueItem uses book_id as the ID field
if task_id in self._task_data:
task = self._task_data[task_id]
queue_items.append({
'id': task_id,
'title': task.title,
'author': task.author,
'priority': item.priority,
'added_time': item.added_time,
'status': self._status.get(task_id, QueueStatus.QUEUED)
})
except queue.Empty:
break
item = self._queue.get_nowait()
temp_items.append(item)
task_id = item.book_id # QueueItem uses book_id as the ID field
if task_id in self._task_data:
task = self._task_data[task_id]
queue_items.append(
{
"id": task_id,
"title": task.title,
"author": task.author,
"priority": item.priority,
"added_time": item.added_time,
"status": self._status.get(task_id, QueueStatus.QUEUED),
}
)
# Put items back in queue
for item in temp_items:
self._queue.put(item)
return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
return sorted(queue_items, key=lambda x: (x["priority"], x["added_time"]))
def cancel_download(self, task_id: str) -> bool:
"""Cancel an active or queued download."""
with self._lock:
current_status = self._status.get(task_id)
if current_status in [QueueStatus.RESOLVING, QueueStatus.LOCATING, QueueStatus.DOWNLOADING]:
if current_status in [
QueueStatus.RESOLVING,
QueueStatus.LOCATING,
QueueStatus.DOWNLOADING,
]:
# Signal active download to stop
if task_id in self._cancel_flags:
self._cancel_flags[task_id].set()
elif current_status not in [QueueStatus.QUEUED]:
elif current_status != QueueStatus.QUEUED:
# Not in a cancellable state
return False
@@ -235,20 +250,17 @@ class BookQueue:
found = False
while not self._queue.empty():
try:
item = self._queue.get_nowait()
if item.book_id == task_id: # QueueItem uses book_id as the ID field
# Create new item with updated priority
new_item = QueueItem(task_id, new_priority, item.added_time)
temp_items.append(new_item)
found = True
# Update task data priority
if task_id in self._task_data:
self._task_data[task_id].priority = new_priority
else:
temp_items.append(item)
except queue.Empty:
break
item = self._queue.get_nowait()
if item.book_id == task_id: # QueueItem uses book_id as the ID field
# Create new item with updated priority
new_item = QueueItem(task_id, new_priority, item.added_time)
temp_items.append(new_item)
found = True
# Update task data priority
if task_id in self._task_data:
self._task_data[task_id].priority = new_priority
else:
temp_items.append(item)
# Put all items back
for item in temp_items:
@@ -256,13 +268,13 @@ class BookQueue:
return found
def enqueue_existing(self, task_id: str, *, priority: Optional[int] = None) -> bool:
def enqueue_existing(self, task_id: str, *, priority: int | None = None) -> bool:
"""Requeue an existing task regardless of current status.
This is used for retries where task metadata should be preserved.
"""
hook: Optional[Callable[[str, DownloadTask], None]] = None
hook_task: Optional[DownloadTask] = None
hook: Callable[[str, DownloadTask], None] | None = None
hook_task: DownloadTask | None = None
with self._lock:
task = self._task_data.get(task_id)
if task is None:
@@ -278,10 +290,7 @@ class BookQueue:
# De-duplicate queue entries for this task id.
temp_items: list[QueueItem] = []
while not self._queue.empty():
try:
item = self._queue.get_nowait()
except queue.Empty:
break
item = self._queue.get_nowait()
if item.book_id != task_id:
temp_items.append(item)
@@ -301,25 +310,22 @@ class BookQueue:
logger.warning("Queue hook failed while requeueing task %s: %s", task_id, exc)
return True
def reorder_queue(self, task_priorities: Dict[str, int]) -> bool:
def reorder_queue(self, task_priorities: dict[str, int]) -> bool:
"""Bulk reorder queue by mapping task_id to new priority."""
with self._lock:
# Extract all items from queue
all_items = []
while not self._queue.empty():
try:
item = self._queue.get_nowait()
task_id = item.book_id # QueueItem uses book_id as the ID field
# Update priority if specified
if task_id in task_priorities:
new_priority = task_priorities[task_id]
item = QueueItem(task_id, new_priority, item.added_time)
# Update task data priority
if task_id in self._task_data:
self._task_data[task_id].priority = new_priority
all_items.append(item)
except queue.Empty:
break
item = self._queue.get_nowait()
task_id = item.book_id # QueueItem uses book_id as the ID field
# Update priority if specified
if task_id in task_priorities:
new_priority = task_priorities[task_id]
item = QueueItem(task_id, new_priority, item.added_time)
# Update task data priority
if task_id in self._task_data:
self._task_data[task_id].priority = new_priority
all_items.append(item)
# Put all items back with updated priorities
for item in all_items:
@@ -327,7 +333,7 @@ class BookQueue:
return True
def get_active_downloads(self) -> List[str]:
def get_active_downloads(self) -> list[str]:
"""Get list of currently active download task IDs."""
with self._lock:
return list(self._active_downloads.keys())
@@ -357,9 +363,12 @@ class BookQueue:
# Check for stale status entries
last_update = self._status_timestamps.get(task_id)
if last_update and (current_time - last_update) > self._status_timeout:
if status in terminal_statuses:
to_remove.append(task_id)
if (
last_update
and (current_time - last_update) > self._status_timeout
and status in terminal_statuses
):
to_remove.append(task_id)
# Remove stale entries
for task_id in to_remove:
@@ -367,5 +376,6 @@ class BookQueue:
self._status_timestamps.pop(task_id, None)
self._task_data.pop(task_id, None)
# Global instance of BookQueue
book_queue = BookQueue()
shelfmark/core/request_helpers.py
+25 -21
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from typing import Any
from shelfmark.core.config import config as app_config
@@ -13,11 +13,11 @@ _logger = setup_logger(__name__)
def now_utc_iso() -> str:
"""Return the current UTC time as a seconds-precision ISO 8601 string."""
return datetime.now(timezone.utc).isoformat(timespec="seconds")
return datetime.now(UTC).isoformat(timespec="seconds")
def emit_ws_event(
ws_manager: Any,
ws_manager: object,
*,
event_name: str,
payload: dict[str, Any],
@@ -33,20 +33,22 @@ def emit_ws_event(
return
socketio.emit(event_name, payload, to=room)
except Exception as exc:
_logger.warning("Failed to emit WebSocket event '%s' to room '%s': %s", event_name, room, exc)
_logger.warning(
"Failed to emit WebSocket event '%s' to room '%s': %s",
event_name,
room,
exc,
)
def load_users_request_policy_settings() -> dict[str, Any]:
"""Load global request-policy settings from the users config file."""
from shelfmark.core.request_policy import REQUEST_POLICY_KEYS
return {
key: app_config.get(key)
for key in REQUEST_POLICY_KEYS
}
return {key: app_config.get(key) for key in REQUEST_POLICY_KEYS}
def coerce_bool(value: Any, default: bool = False) -> bool:
def coerce_bool(value: object, *, default: bool = False) -> bool:
"""Coerce arbitrary values into booleans with string-friendly semantics."""
if isinstance(value, bool):
return value
@@ -61,24 +63,24 @@ def coerce_bool(value: Any, default: bool = False) -> bool:
return bool(value)
def get_session_db_user_id(session_obj: Any) -> int | None:
def get_session_db_user_id(session_obj: object) -> int | None:
"""Extract and coerce `db_user_id` from a Flask session to ``int | None``."""
raw = session_obj.get("db_user_id") if session_obj is not None else None
try:
return int(raw) if raw is not None else None
except (TypeError, ValueError):
except TypeError, ValueError:
return None
def coerce_int(value: Any, default: int) -> int:
def coerce_int(value: object, default: int) -> int:
"""Best-effort integer coercion with fallback to default."""
try:
return int(value)
except (TypeError, ValueError):
except TypeError, ValueError:
return default
def normalize_optional_text(value: Any) -> str | None:
def normalize_optional_text(value: object) -> str | None:
"""Return a trimmed string or None for empty/non-string input."""
if not isinstance(value, str):
return None
@@ -86,16 +88,16 @@ def normalize_optional_text(value: Any) -> str | None:
return normalized or None
def normalize_positive_int(value: Any) -> int | None:
def normalize_positive_int(value: object) -> int | None:
"""Parse *value* as a positive integer, returning ``None`` on failure."""
try:
parsed = int(value)
except (TypeError, ValueError):
except TypeError, ValueError:
return None
return parsed if parsed > 0 else None
def normalize_optional_positive_int(value: Any, field_name: str = "value") -> int | None:
def normalize_optional_positive_int(value: object, field_name: str = "value") -> int | None:
"""Parse *value* as a positive integer or ``None``.
Raises ``ValueError`` when *value* is present but not a valid
@@ -106,13 +108,15 @@ def normalize_optional_positive_int(value: Any, field_name: str = "value") -> in
try:
parsed = int(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"{field_name} must be a positive integer when provided") from exc
msg = f"{field_name} must be a positive integer when provided"
raise ValueError(msg) from exc
if parsed < 1:
raise ValueError(f"{field_name} must be a positive integer when provided")
msg = f"{field_name} must be a positive integer when provided"
raise ValueError(msg)
return parsed
def populate_request_usernames(rows: list[dict[str, Any]], user_db: Any) -> None:
def populate_request_usernames(rows: list[dict[str, Any]], user_db: object) -> None:
"""Add 'username' to each request row by looking up user_id."""
cache: dict[int, str] = {}
for row in rows:
@@ -123,7 +127,7 @@ def populate_request_usernames(rows: list[dict[str, Any]], user_db: Any) -> None
row["username"] = cache[requester_id]
def extract_release_source_id(release_data: Any) -> str | None:
def extract_release_source_id(release_data: object) -> str | None:
"""Extract and normalize release_data.source_id."""
if not isinstance(release_data, dict):
return None
shelfmark/core/request_policy.py
+26 -17
View File
@@ -6,11 +6,12 @@ routes/services and tested independently.
from __future__ import annotations
from enum import Enum
from typing import Any, Iterable, Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from enum import StrEnum
from typing import Any
class PolicyMode(str, Enum):
class PolicyMode(StrEnum):
"""Allowed request-policy modes.
Ordered from most to least permissive. The content-type default acts as a
@@ -33,7 +34,9 @@ _MODE_PERMISSIVENESS: dict[PolicyMode, int] = {
}
# Modes allowed in REQUEST_POLICY_RULES matrix rows.
MATRIX_ALLOWED_MODES = frozenset({PolicyMode.DOWNLOAD, PolicyMode.REQUEST_RELEASE, PolicyMode.BLOCKED})
MATRIX_ALLOWED_MODES = frozenset(
{PolicyMode.DOWNLOAD, PolicyMode.REQUEST_RELEASE, PolicyMode.BLOCKED}
)
def cap_mode(mode: PolicyMode, ceiling: PolicyMode) -> PolicyMode:
@@ -48,6 +51,7 @@ def _source_results_are_releases(source: Any) -> bool:
if normalized_source in {"", "*"}:
return False
from shelfmark.release_sources import source_results_are_releases
return source_results_are_releases(normalized_source)
@@ -105,7 +109,9 @@ def merge_request_policy_settings(
(source, content_type): (source, content_type, mode)
for source, content_type, mode in global_rules
}
for source, content_type, mode in _iter_rules(user_filtered.get("REQUEST_POLICY_RULES", [])):
for source, content_type, mode in _iter_rules(
user_filtered.get("REQUEST_POLICY_RULES", [])
):
merged_rules[(source, content_type)] = (source, content_type, mode)
merged["REQUEST_POLICY_RULES"] = [
{"source": source, "content_type": content_type, "mode": mode.value}
@@ -181,7 +187,7 @@ def get_source_content_type_capabilities() -> dict[str, set[str]]:
"""Return source -> supported content type map from registered sources."""
try:
from shelfmark.release_sources import list_available_sources
except Exception:
except ImportError:
return {}
capabilities: dict[str, set[str]] = {}
@@ -219,9 +225,15 @@ def validate_policy_rules(
- known source names
- source/content-type compatibility from source declarations
"""
capabilities = source_capabilities if source_capabilities is not None else get_source_content_type_capabilities()
capabilities = (
source_capabilities
if source_capabilities is not None
else get_source_content_type_capabilities()
)
normalized_capabilities = {
normalize_source(source): {normalize_content_type(content_type) for content_type in content_types}
normalize_source(source): {
normalize_content_type(content_type) for content_type in content_types
}
for source, content_types in capabilities.items()
}
@@ -248,26 +260,24 @@ def validate_policy_rules(
if source is None:
errors.append(f"{row_label}: source is required")
continue
if (
raw_content_type is None
or (isinstance(raw_content_type, str) and not raw_content_type.strip())
if raw_content_type is None or (
isinstance(raw_content_type, str) and not raw_content_type.strip()
):
errors.append(f"{row_label}: content_type is required")
continue
if content_type is None:
errors.append(f"{row_label}: invalid content_type '{rule.get('content_type')}'")
continue
if (
raw_mode is None
or (isinstance(raw_mode, str) and not raw_mode.strip())
):
if raw_mode is None or (isinstance(raw_mode, str) and not raw_mode.strip()):
errors.append(f"{row_label}: mode is required")
continue
if mode is None:
errors.append(f"{row_label}: invalid mode '{rule.get('mode')}'")
continue
if mode not in MATRIX_ALLOWED_MODES:
errors.append(f"{row_label}: mode '{mode.value}' is not allowed in matrix rules (use content-type defaults instead)")
errors.append(
f"{row_label}: mode '{mode.value}' is not allowed in matrix rules (use content-type defaults instead)"
)
continue
if source != "*" and source not in normalized_capabilities:
@@ -340,7 +350,6 @@ def resolve_policy_mode(
- sources whose browse results are already concrete releases normalize
request_book to request_release.
"""
effective = merge_request_policy_settings(global_settings, user_settings)
normalized_source = normalize_source(source)
normalized_content_type = normalize_content_type(content_type)
shelfmark/core/request_routes.py
+106 -81
View File
@@ -2,14 +2,29 @@
from __future__ import annotations
from typing import Any, Callable
from typing import TYPE_CHECKING, Any
from flask import Flask, jsonify, request, session
from flask import Flask, Response, jsonify, request, session
from shelfmark.core.logger import setup_logger
from shelfmark.core.notifications import (
NotificationContext,
NotificationEvent,
notify_admin,
notify_user,
)
from shelfmark.core.request_helpers import (
coerce_bool,
coerce_int,
emit_ws_event,
load_users_request_policy_settings,
normalize_optional_text,
normalize_positive_int,
populate_request_usernames,
)
from shelfmark.core.request_policy import (
PolicyMode,
REQUEST_POLICY_DEFAULT_FALLBACK_MODE,
PolicyMode,
get_source_content_type_capabilities,
merge_request_policy_settings,
normalize_content_type,
@@ -26,22 +41,11 @@ from shelfmark.core.requests_service import (
fulfil_request,
reject_request,
)
from shelfmark.core.notifications import (
NotificationContext,
NotificationEvent,
notify_admin,
notify_user,
)
from shelfmark.core.request_helpers import (
coerce_bool,
coerce_int,
emit_ws_event,
load_users_request_policy_settings,
normalize_optional_text,
normalize_positive_int,
populate_request_usernames,
)
from shelfmark.core.user_db import UserDB
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
@@ -52,7 +56,7 @@ def _error_response(
*,
code: str | None = None,
required_mode: str | None = None,
):
) -> tuple[Response, int]:
payload: dict[str, Any] = {"error": message}
if code is not None:
payload["code"] = code
@@ -61,7 +65,9 @@ def _error_response(
return jsonify(payload), status_code
def _require_request_endpoints_available(resolve_auth_mode: Callable[[], str]):
def _require_request_endpoints_available(
resolve_auth_mode: Callable[[], str],
) -> tuple[Response, int] | None:
auth_mode = resolve_auth_mode()
if auth_mode == "none":
return _error_response(
@@ -74,7 +80,7 @@ def _require_request_endpoints_available(resolve_auth_mode: Callable[[], str]):
return None
def _require_db_user_id() -> tuple[int | None, Any | None]:
def _require_db_user_id() -> tuple[int | None, object | None]:
raw_user_id = session.get("db_user_id")
if raw_user_id is None:
return None, _error_response(
@@ -84,7 +90,7 @@ def _require_db_user_id() -> tuple[int | None, Any | None]:
)
try:
return int(raw_user_id), None
except (TypeError, ValueError):
except TypeError, ValueError:
return None, _error_response(
"User identity is unavailable for request workflow",
403,
@@ -92,7 +98,7 @@ def _require_db_user_id() -> tuple[int | None, Any | None]:
)
def _require_admin_user_id() -> tuple[int | None, Any | None]:
def _require_admin_user_id() -> tuple[int | None, object | None]:
if not session.get("is_admin", False):
return None, (jsonify({"error": "Admin access required"}), 403)
raw_admin_id = session.get("db_user_id")
@@ -100,7 +106,7 @@ def _require_admin_user_id() -> tuple[int | None, Any | None]:
return None, (jsonify({"error": "Admin user identity unavailable"}), 403)
try:
return int(raw_admin_id), None
except (TypeError, ValueError):
except TypeError, ValueError:
return None, (jsonify({"error": "Admin user identity unavailable"}), 403)
@@ -112,11 +118,11 @@ def _resolve_effective_policy(
global_settings = load_users_request_policy_settings()
user_settings = user_db.get_user_settings(db_user_id) if db_user_id is not None else {}
effective = merge_request_policy_settings(global_settings, user_settings)
requests_enabled = coerce_bool(effective.get("REQUESTS_ENABLED"), False)
requests_enabled = coerce_bool(effective.get("REQUESTS_ENABLED"), default=False)
return global_settings, user_settings, effective, requests_enabled
def _resolve_title_from_book_data(book_data: Any) -> str:
def _resolve_title_from_book_data(book_data: object) -> str:
if isinstance(book_data, dict):
title = normalize_optional_text(book_data.get("title"))
if title is not None:
@@ -124,7 +130,7 @@ def _resolve_title_from_book_data(book_data: Any) -> str:
return "Unknown title"
def _normalize_optional_source_id(value: Any) -> str | None:
def _normalize_optional_source_id(value: object) -> str | None:
"""Normalize source identifiers while allowing integer provider ids."""
if isinstance(value, bool) or value is None:
return None
@@ -140,9 +146,9 @@ def _build_release_result_data_from_book_data(
content_type: str,
) -> dict[str, Any]:
"""Build release-level payload fields for sources whose browse results are releases."""
source_id = _normalize_optional_source_id(book_data.get("provider_id")) or _normalize_optional_source_id(
book_data.get("id")
)
source_id = _normalize_optional_source_id(
book_data.get("provider_id")
) or _normalize_optional_source_id(book_data.get("id"))
payload: dict[str, Any] = {
"source": source,
"source_id": source_id,
@@ -164,15 +170,16 @@ def _source_results_are_releases(source: str) -> bool:
if normalized_source in {"", "*"}:
return False
from shelfmark.release_sources import source_results_are_releases
return source_results_are_releases(normalized_source)
def _normalize_release_result_request_payload(
*,
source: str,
request_level: Any,
book_data: Any,
release_data: Any,
request_level: object,
book_data: object,
release_data: object,
content_type: str,
) -> tuple[Any, Any]:
"""Concrete-release browse results are always handled as release-level requests."""
@@ -194,13 +201,15 @@ def _normalize_release_result_request_payload(
if normalized_release_data.get("content_type") is None:
normalized_release_data["content_type"] = content_type
normalized_source_id = _normalize_optional_source_id(normalized_release_data.get("source_id"))
normalized_source_id = _normalize_optional_source_id(
normalized_release_data.get("source_id")
)
if normalized_source_id is not None:
normalized_release_data["source_id"] = normalized_source_id
elif isinstance(book_data, dict):
fallback_source_id = _normalize_optional_source_id(book_data.get("provider_id")) or _normalize_optional_source_id(
book_data.get("id")
)
fallback_source_id = _normalize_optional_source_id(
book_data.get("provider_id")
) or _normalize_optional_source_id(book_data.get("id"))
if fallback_source_id is not None:
normalized_release_data["source_id"] = fallback_source_id
@@ -237,26 +246,30 @@ def _resolve_request_user_context(
*,
actor_user_id: int,
actor_username: str | None,
on_behalf_of_user_id: Any,
on_behalf_of_user_id: object,
) -> tuple[int, str | None, str]:
if on_behalf_of_user_id in (None, ""):
actor_label = _format_user_label(actor_username, actor_user_id)
return actor_user_id, actor_username, actor_label
if not session.get("is_admin", False):
raise RequestServiceError("Admin required", status_code=403)
msg = "Admin required"
raise RequestServiceError(msg, status_code=403)
try:
target_user_id = int(on_behalf_of_user_id)
except (TypeError, ValueError) as exc:
raise RequestServiceError("Invalid on_behalf_of_user_id", status_code=400) from exc
msg = "Invalid on_behalf_of_user_id"
raise RequestServiceError(msg, status_code=400) from exc
if target_user_id <= 0:
raise RequestServiceError("Invalid on_behalf_of_user_id", status_code=400)
msg = "Invalid on_behalf_of_user_id"
raise RequestServiceError(msg, status_code=400)
target_user = user_db.get_user(user_id=target_user_id)
if not target_user:
raise RequestServiceError("User not found", status_code=404)
msg = "User not found"
raise RequestServiceError(msg, status_code=404)
target_username = normalize_optional_text(target_user.get("username"))
actor_label = _format_user_label(actor_username, actor_user_id)
@@ -270,8 +283,9 @@ def _prepare_request_create_arguments(
) -> dict[str, Any]:
db_user_id, db_gate = _require_db_user_id()
if db_gate is not None or db_user_id is None:
msg = "User identity is unavailable for request workflow"
raise RequestServiceError(
"User identity is unavailable for request workflow",
msg,
status_code=403,
code="user_identity_unavailable",
)
@@ -286,7 +300,8 @@ def _prepare_request_create_arguments(
context = data.get("context") or {}
if not isinstance(context, dict):
raise RequestServiceError("context must be an object", status_code=400)
msg = "context must be an object"
raise RequestServiceError(msg, status_code=400)
source = normalize_source(context.get("source"))
release_data = data.get("release_data")
@@ -296,13 +311,12 @@ def _prepare_request_create_arguments(
book_data = data.get("book_data")
if not isinstance(book_data, dict):
raise RequestServiceError("book_data must be an object", status_code=400)
msg = "book_data must be an object"
raise RequestServiceError(msg, status_code=400)
request_title = _resolve_title_from_book_data(book_data)
content_type = normalize_content_type(
context.get("content_type")
or data.get("content_type")
or book_data.get("content_type")
context.get("content_type") or data.get("content_type") or book_data.get("content_type")
)
request_level, release_data = _normalize_release_result_request_payload(
source=source,
@@ -317,8 +331,9 @@ def _prepare_request_create_arguments(
db_user_id=target_user_id,
)
if not requests_enabled:
msg = "Request workflow is disabled by policy"
raise RequestServiceError(
"Request workflow is disabled by policy",
msg,
status_code=403,
code="requests_unavailable",
)
@@ -327,10 +342,8 @@ def _prepare_request_create_arguments(
effective.get("MAX_PENDING_REQUESTS_PER_USER"),
default=20,
)
if max_pending < 1:
max_pending = 1
if max_pending > 1000:
max_pending = 1000
max_pending = max(max_pending, 1)
max_pending = min(max_pending, 1000)
allow_notes = coerce_bool(effective.get("REQUESTS_ALLOW_NOTES"), default=True)
note_value = data.get("note") if allow_notes else None
@@ -342,7 +355,7 @@ def _prepare_request_create_arguments(
)
logger.debug(
"request create policy actor=%s target_user_id=%s source=%s content_type=%s request_level=%s resolved_mode=%s",
session.get("user_id"),
actor_label,
target_user_id,
source,
content_type,
@@ -351,8 +364,9 @@ def _prepare_request_create_arguments(
)
if resolved_mode == PolicyMode.BLOCKED:
msg = "Requesting is blocked by policy"
raise RequestServiceError(
"Requesting is blocked by policy",
msg,
status_code=403,
code="policy_blocked",
required_mode=PolicyMode.BLOCKED.value,
@@ -360,8 +374,9 @@ def _prepare_request_create_arguments(
requested_level = str(request_level).strip().lower() if isinstance(request_level, str) else ""
if resolved_mode == PolicyMode.REQUEST_BOOK and requested_level != "book":
msg = "Policy requires book-level requests"
raise RequestServiceError(
"Policy requires book-level requests",
msg,
status_code=403,
code="policy_requires_request",
required_mode=PolicyMode.REQUEST_BOOK.value,
@@ -385,7 +400,9 @@ def _prepare_request_create_arguments(
}
def _resolve_request_source_and_format(request_row: dict[str, Any]) -> tuple[str, str | None]:
def _resolve_request_source_and_format(
request_row: dict[str, Any],
) -> tuple[str, str | None]:
release_data = request_row.get("release_data")
if isinstance(release_data, dict):
source = normalize_source(release_data.get("source") or request_row.get("source_hint"))
@@ -431,8 +448,9 @@ def _queue_prepared_download_submission(
) -> dict[str, Any]:
release_data = create_args.get("release_data")
if not isinstance(release_data, dict):
msg = "Download policy requires a concrete release"
raise RequestServiceError(
"Download policy requires a concrete release",
msg,
status_code=400,
code="policy_requires_download",
required_mode=PolicyMode.DOWNLOAD.value,
@@ -440,7 +458,8 @@ def _queue_prepared_download_submission(
requester = user_db.get_user(user_id=create_args["user_id"])
if requester is None:
raise RequestServiceError("Requesting user not found", status_code=404)
msg = "Requesting user not found"
raise RequestServiceError(msg, status_code=404)
success, error = queue_release(
dict(release_data),
@@ -461,8 +480,6 @@ def _queue_prepared_download_submission(
)
def _notify_admin_for_request_event(
user_db: UserDB,
*,
@@ -516,12 +533,12 @@ def register_request_routes(
*,
resolve_auth_mode: Callable[[], str],
queue_release: Callable[..., tuple[bool, str | None]],
ws_manager: Any | None = None,
ws_manager: object | None = None,
) -> None:
"""Register request policy and request lifecycle routes."""
@app.route("/api/request-policy", methods=["GET"])
def api_request_policy():
def api_request_policy() -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -537,7 +554,7 @@ def register_request_routes(
if raw_id is not None:
try:
db_user_id = int(raw_id)
except (TypeError, ValueError):
except TypeError, ValueError:
db_user_id = None
global_settings, user_settings, effective, requests_enabled = _resolve_effective_policy(
@@ -550,6 +567,7 @@ def register_request_routes(
source_capabilities = get_source_content_type_capabilities()
from shelfmark.release_sources import source_results_are_releases
source_modes = []
for source_name in sorted(source_capabilities):
supported_types = sorted(
@@ -597,7 +615,7 @@ def register_request_routes(
)
@app.route("/api/requests", methods=["POST"])
def api_create_request():
def api_create_request() -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -663,7 +681,7 @@ def register_request_routes(
return jsonify(created), 201
@app.route("/api/requests/batch", methods=["POST"])
def api_create_requests_batch():
def api_create_requests_batch() -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -716,7 +734,11 @@ def register_request_routes(
results_by_index: dict[int, dict[str, Any]] = {}
for (index, prepared), created in zip(request_prepared_items, created_rows):
for (index, prepared), created in zip(
request_prepared_items,
created_rows,
strict=True,
):
event_payload = {
"request_id": created["id"],
"status": created["status"],
@@ -773,7 +795,7 @@ def register_request_routes(
return jsonify(ordered_results), status_code
@app.route("/api/requests", methods=["GET"])
def api_list_requests():
def api_list_requests() -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -798,7 +820,7 @@ def register_request_routes(
return jsonify(rows)
@app.route("/api/requests/<int:request_id>", methods=["DELETE"])
def api_cancel_request(request_id: int):
def api_cancel_request(request_id: int) -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -821,7 +843,9 @@ def register_request_routes(
"status": updated["status"],
"title": _resolve_request_title(updated),
}
actor_label = _format_user_label(normalize_optional_text(session.get("user_id")), db_user_id)
actor_label = _format_user_label(
normalize_optional_text(session.get("user_id")), db_user_id
)
logger.info(
"Request cancelled #%s for '%s' by %s",
updated["id"],
@@ -844,7 +868,7 @@ def register_request_routes(
return jsonify(updated)
@app.route("/api/admin/requests", methods=["GET"])
def api_admin_list_requests():
def api_admin_list_requests() -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -865,17 +889,14 @@ def register_request_routes(
return jsonify(rows)
@app.route("/api/admin/requests/count", methods=["GET"])
def api_admin_request_counts():
def api_admin_request_counts() -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
if not session.get("is_admin", False):
return jsonify({"error": "Admin access required"}), 403
by_status = {
status: len(user_db.list_requests(status=status))
for status in RequestStatus
}
by_status = {status: len(user_db.list_requests(status=status)) for status in RequestStatus}
return jsonify(
{
"pending": by_status[RequestStatus.PENDING],
@@ -885,7 +906,7 @@ def register_request_routes(
)
@app.route("/api/admin/requests/<int:request_id>/fulfil", methods=["POST"])
def api_admin_fulfil_request(request_id: int):
def api_admin_fulfil_request(request_id: int) -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -916,7 +937,9 @@ def register_request_routes(
"status": updated["status"],
"title": _resolve_request_title(updated),
}
admin_label = _format_user_label(normalize_optional_text(session.get("user_id")), admin_user_id)
admin_label = _format_user_label(
normalize_optional_text(session.get("user_id")), admin_user_id
)
requester_label = _format_requester_label(user_db, updated)
logger.info(
"Request fulfilled #%s for '%s' by %s (requested by %s)",
@@ -947,7 +970,7 @@ def register_request_routes(
return jsonify(updated)
@app.route("/api/admin/requests/<int:request_id>/reject", methods=["POST"])
def api_admin_reject_request(request_id: int):
def api_admin_reject_request(request_id: int) -> Response | tuple[Response, int]:
auth_gate = _require_request_endpoints_available(resolve_auth_mode)
if auth_gate is not None:
return auth_gate
@@ -975,7 +998,9 @@ def register_request_routes(
"status": updated["status"],
"title": _resolve_request_title(updated),
}
admin_label = _format_user_label(normalize_optional_text(session.get("user_id")), admin_user_id)
admin_label = _format_user_label(
normalize_optional_text(session.get("user_id")), admin_user_id
)
requester_label = _format_requester_label(user_db, updated)
logger.info(
"Request rejected #%s for '%s' by %s (requested by %s)",
shelfmark/core/request_validation.py
+36 -22
View File
@@ -2,15 +2,15 @@
from __future__ import annotations
from enum import Enum
from typing import Any
from enum import StrEnum
from shelfmark.core.models import QueueStatus
from shelfmark.core.request_policy import parse_policy_mode
class RequestStatus(str, Enum):
class RequestStatus(StrEnum):
"""Enum for request lifecycle statuses."""
PENDING = "pending"
FULFILLED = "fulfilled"
REJECTED = "rejected"
@@ -20,65 +20,79 @@ class RequestStatus(str, Enum):
DELIVERY_STATE_NONE = "none"
VALID_REQUEST_STATUSES = frozenset(RequestStatus)
TERMINAL_REQUEST_STATUSES = frozenset({
RequestStatus.FULFILLED, RequestStatus.REJECTED, RequestStatus.CANCELLED,
})
TERMINAL_REQUEST_STATUSES = frozenset(
{
RequestStatus.FULFILLED,
RequestStatus.REJECTED,
RequestStatus.CANCELLED,
}
)
VALID_REQUEST_LEVELS = frozenset({"book", "release"})
VALID_DELIVERY_STATES = frozenset({DELIVERY_STATE_NONE} | set(QueueStatus))
def normalize_request_status(status: Any) -> str:
def normalize_request_status(status: object) -> str:
"""Validate and normalize request status values."""
if not isinstance(status, str):
raise ValueError(f"Invalid request status: {status}")
msg = f"Invalid request status: {status}"
raise TypeError(msg)
normalized = status.strip().lower()
if normalized not in VALID_REQUEST_STATUSES:
raise ValueError(f"Invalid request status: {status}")
msg = f"Invalid request status: {status}"
raise ValueError(msg)
return normalized
def normalize_policy_mode(mode: Any) -> str:
def normalize_policy_mode(mode: object) -> str:
"""Validate and normalize policy mode values."""
parsed = parse_policy_mode(mode)
if parsed is None:
raise ValueError(f"Invalid policy_mode: {mode}")
msg = f"Invalid policy_mode: {mode}"
raise ValueError(msg)
return parsed.value
def normalize_request_level(request_level: Any) -> str:
def normalize_request_level(request_level: object) -> str:
"""Validate and normalize request level values."""
if not isinstance(request_level, str):
raise ValueError(f"Invalid request_level: {request_level}")
msg = f"Invalid request_level: {request_level}"
raise TypeError(msg)
normalized = request_level.strip().lower()
if normalized not in VALID_REQUEST_LEVELS:
raise ValueError(f"Invalid request_level: {request_level}")
msg = f"Invalid request_level: {request_level}"
raise ValueError(msg)
return normalized
def normalize_delivery_state(state: Any) -> str:
def normalize_delivery_state(state: object) -> str:
"""Validate and normalize delivery-state values."""
if not isinstance(state, str):
raise ValueError(f"Invalid delivery_state: {state}")
msg = f"Invalid delivery_state: {state}"
raise TypeError(msg)
normalized = state.strip().lower()
if normalized not in VALID_DELIVERY_STATES:
raise ValueError(f"Invalid delivery_state: {state}")
msg = f"Invalid delivery_state: {state}"
raise ValueError(msg)
return normalized
def validate_request_level_payload(request_level: Any, release_data: Any) -> str:
def validate_request_level_payload(request_level: object, release_data: object) -> str:
"""Validate request_level and release_data shape coupling."""
normalized_level = normalize_request_level(request_level)
if normalized_level == "release" and release_data is None:
raise ValueError("request_level=release requires non-null release_data")
msg = "request_level=release requires non-null release_data"
raise ValueError(msg)
if normalized_level == "book" and release_data is not None:
raise ValueError("request_level=book requires null release_data")
msg = "request_level=book requires null release_data"
raise ValueError(msg)
return normalized_level
def validate_status_transition(current_status: Any, new_status: Any) -> tuple[str, str]:
def validate_status_transition(current_status: object, new_status: object) -> tuple[str, str]:
"""Validate request status transitions and terminal immutability."""
current = normalize_request_status(current_status)
new = normalize_request_status(new_status)
if current in TERMINAL_REQUEST_STATUSES and new != current:
raise ValueError("Terminal request statuses are immutable")
msg = "Terminal request statuses are immutable"
raise ValueError(msg)
return current, new
shelfmark/core/requests_service.py
+100 -67
View File
@@ -2,29 +2,30 @@
from __future__ import annotations
from datetime import datetime, timezone
import json
from typing import Any, Callable, TYPE_CHECKING
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from shelfmark.core.request_policy import normalize_content_type
from shelfmark.core.models import QueueStatus
from shelfmark.core.request_helpers import (
extract_release_source_id,
normalize_positive_int,
)
from shelfmark.core.request_policy import normalize_content_type
from shelfmark.core.request_validation import (
DELIVERY_STATE_NONE,
RequestStatus,
normalize_policy_mode,
normalize_request_level,
normalize_request_status,
validate_request_level_payload,
validate_status_transition,
)
from shelfmark.core.request_helpers import extract_release_source_id, normalize_positive_int
MAX_REQUEST_NOTE_LENGTH = 1000
MAX_REQUEST_JSON_BLOB_BYTES = 10 * 1024
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.user_db import UserDB
@@ -38,68 +39,76 @@ class RequestServiceError(ValueError):
status_code: int = 400,
code: str | None = None,
required_mode: str | None = None,
):
) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.required_mode = required_mode
def _normalize_match_text(value: Any) -> str:
def _normalize_match_text(value: object) -> str:
if not isinstance(value, str):
return ""
return value.strip().lower()
def normalize_note(note: Any) -> str | None:
def normalize_note(note: object) -> str | None:
"""Validate request notes and normalize empty strings to None."""
if note is None:
return None
if not isinstance(note, str):
raise RequestServiceError("note must be a string", status_code=400)
msg = "note must be a string"
raise RequestServiceError(msg, status_code=400)
normalized = note.strip()
if len(normalized) > MAX_REQUEST_NOTE_LENGTH:
msg_0 = f"note must be <= {MAX_REQUEST_NOTE_LENGTH} characters"
raise RequestServiceError(
f"note must be <= {MAX_REQUEST_NOTE_LENGTH} characters",
msg_0,
status_code=400,
)
return normalized or None
def _validate_book_data(book_data: Any) -> dict[str, Any]:
def _validate_book_data(book_data: object) -> dict[str, Any]:
if not isinstance(book_data, dict):
raise RequestServiceError("book_data must be an object", status_code=400)
msg = "book_data must be an object"
raise RequestServiceError(msg, status_code=400)
required_fields = ("title", "author", "provider", "provider_id")
missing = [field for field in required_fields if not _normalize_match_text(book_data.get(field))]
missing = [
field for field in required_fields if not _normalize_match_text(book_data.get(field))
]
if missing:
msg_0 = f"book_data missing required field(s): {', '.join(missing)}"
raise RequestServiceError(
f"book_data missing required field(s): {', '.join(missing)}",
msg_0,
status_code=400,
)
return dict(book_data)
def _validate_json_blob_size(field: str, payload: Any) -> None:
def _validate_json_blob_size(field: str, payload: object) -> None:
if payload is None:
return
try:
serialized = json.dumps(payload, separators=(",", ":"), ensure_ascii=False)
except (TypeError, ValueError) as exc:
raise RequestServiceError(f"{field} must be JSON-serializable", status_code=400) from exc
msg = f"{field} must be JSON-serializable"
raise RequestServiceError(msg, status_code=400) from exc
payload_size = len(serialized.encode("utf-8"))
if payload_size > MAX_REQUEST_JSON_BLOB_BYTES:
msg = f"{field} must be <= {MAX_REQUEST_JSON_BLOB_BYTES} bytes"
raise RequestServiceError(
f"{field} must be <= {MAX_REQUEST_JSON_BLOB_BYTES} bytes",
msg,
status_code=400,
code="request_payload_too_large",
)
def _find_duplicate_pending_request(
user_db: "UserDB",
user_db: UserDB,
*,
user_id: int,
title: str,
@@ -123,14 +132,15 @@ def _find_duplicate_pending_request(
def _now_timestamp() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds")
return datetime.now(UTC).isoformat(timespec="seconds")
def _normalize_admin_note(admin_note: Any) -> str | None:
def _normalize_admin_note(admin_note: object) -> str | None:
if admin_note is None:
return None
if not isinstance(admin_note, str):
raise RequestServiceError("admin_note must be a string", status_code=400)
msg = "admin_note must be a string"
raise RequestServiceError(msg, status_code=400)
return admin_note.strip() or None
@@ -138,12 +148,12 @@ def _prepare_request_create(
*,
user_id: int,
source_hint: str | None,
content_type: Any,
request_level: Any,
policy_mode: Any,
book_data: Any,
release_data: Any = None,
note: Any = None,
content_type: object,
request_level: object,
policy_mode: object,
book_data: object,
release_data: object = None,
note: object = None,
) -> dict[str, Any]:
validated_book_data = _validate_book_data(book_data)
normalized_note = normalize_note(note)
@@ -155,7 +165,7 @@ def _prepare_request_create(
try:
normalized_request_level = validate_request_level_payload(request_level, release_data)
normalized_policy_mode = normalize_policy_mode(policy_mode)
except ValueError as exc:
except (ValueError, TypeError) as exc:
raise RequestServiceError(str(exc), status_code=400) from exc
_validate_json_blob_size("book_data", validated_book_data)
@@ -174,7 +184,7 @@ def _prepare_request_create(
def sync_delivery_states_from_queue_status(
user_db: "UserDB",
user_db: UserDB,
*,
queue_status: dict[str, dict[str, Any]],
user_id: int | None = None,
@@ -257,16 +267,16 @@ def sync_delivery_states_from_queue_status(
def create_request(
user_db: "UserDB",
user_db: UserDB,
*,
user_id: int,
source_hint: str | None,
content_type: Any,
request_level: Any,
policy_mode: Any,
book_data: Any,
release_data: Any = None,
note: Any = None,
content_type: object,
request_level: object,
policy_mode: object,
book_data: object,
release_data: object = None,
note: object = None,
max_pending_per_user: int | None = None,
) -> dict[str, Any]:
"""Create a pending request after service-level validation."""
@@ -284,8 +294,9 @@ def create_request(
if max_pending_per_user is not None:
pending_count = user_db.count_user_pending_requests(user_id)
if pending_count >= max_pending_per_user:
msg = "Maximum pending requests reached for this user"
raise RequestServiceError(
"Maximum pending requests reached for this user",
msg,
status_code=409,
code="max_pending_reached",
)
@@ -298,26 +309,28 @@ def create_request(
content_type=prepared_request["content_type"],
)
if duplicate is not None:
msg = "Duplicate pending request exists for this title/author/content_type"
raise RequestServiceError(
"Duplicate pending request exists for this title/author/content_type",
msg,
status_code=409,
code="duplicate_pending_request",
)
try:
return user_db.create_request(**prepared_request)
except ValueError as exc:
except (ValueError, TypeError) as exc:
raise RequestServiceError(str(exc), status_code=400) from exc
def create_requests(
user_db: "UserDB",
user_db: UserDB,
*,
requests: list[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Create multiple pending requests atomically after validation."""
if not isinstance(requests, list) or len(requests) == 0:
raise RequestServiceError("requests must contain at least one request", status_code=400)
msg = "requests must contain at least one request"
raise RequestServiceError(msg, status_code=400)
prepared_requests: list[dict[str, Any]] = []
pending_counts_by_user: dict[int, int] = {}
@@ -325,7 +338,8 @@ def create_requests(
for request in requests:
if not isinstance(request, dict):
raise RequestServiceError("requests must contain objects", status_code=400)
msg = "requests must contain objects"
raise RequestServiceError(msg, status_code=400)
user_id = int(request["user_id"])
prepared_request = _prepare_request_create(
@@ -346,8 +360,9 @@ def create_requests(
prepared_request["content_type"],
)
if request_key in seen_request_keys:
msg = "Duplicate pending request exists for this title/author/content_type"
raise RequestServiceError(
"Duplicate pending request exists for this title/author/content_type",
msg,
status_code=409,
code="duplicate_pending_request",
)
@@ -359,8 +374,9 @@ def create_requests(
if existing_pending is None:
existing_pending = user_db.count_user_pending_requests(user_id)
if existing_pending >= max_pending_per_user:
msg = "Maximum pending requests reached for this user"
raise RequestServiceError(
"Maximum pending requests reached for this user",
msg,
status_code=409,
code="max_pending_reached",
)
@@ -374,8 +390,9 @@ def create_requests(
content_type=request_key[3],
)
if duplicate is not None:
msg = "Duplicate pending request exists for this title/author/content_type"
raise RequestServiceError(
"Duplicate pending request exists for this title/author/content_type",
msg,
status_code=409,
code="duplicate_pending_request",
)
@@ -389,7 +406,7 @@ def create_requests(
def ensure_request_access(
user_db: "UserDB",
user_db: UserDB,
*,
request_id: int,
actor_user_id: int | None,
@@ -398,26 +415,28 @@ def ensure_request_access(
"""Get request by ID and enforce ownership for non-admin actors."""
request_row = user_db.get_request(request_id)
if request_row is None:
raise RequestServiceError("Request not found", status_code=404)
msg = "Request not found"
raise RequestServiceError(msg, status_code=404)
if not is_admin:
if actor_user_id is None or request_row["user_id"] != actor_user_id:
raise RequestServiceError("Forbidden", status_code=403)
if not is_admin and (actor_user_id is None or request_row["user_id"] != actor_user_id):
msg = "Forbidden"
raise RequestServiceError(msg, status_code=403)
return request_row
def _require_pending(request_row: dict[str, Any]) -> None:
if request_row["status"] != RequestStatus.PENDING:
msg = "Request is already in a terminal state"
raise RequestServiceError(
"Request is already in a terminal state",
msg,
status_code=409,
code="stale_transition",
)
def cancel_request(
user_db: "UserDB",
user_db: UserDB,
*,
request_id: int,
actor_user_id: int,
@@ -437,16 +456,18 @@ def cancel_request(
expected_current_status=RequestStatus.PENDING,
status=RequestStatus.CANCELLED,
)
except TypeError as exc:
raise RequestServiceError(str(exc), status_code=400) from exc
except ValueError as exc:
raise RequestServiceError(str(exc), status_code=409, code="stale_transition") from exc
def reject_request(
user_db: "UserDB",
user_db: UserDB,
*,
request_id: int,
admin_user_id: int,
admin_note: Any = None,
admin_note: object = None,
) -> dict[str, Any]:
"""Reject a pending request as admin."""
request_row = ensure_request_access(
@@ -468,19 +489,21 @@ def reject_request(
reviewed_by=admin_user_id,
reviewed_at=_now_timestamp(),
)
except TypeError as exc:
raise RequestServiceError(str(exc), status_code=400) from exc
except ValueError as exc:
raise RequestServiceError(str(exc), status_code=409, code="stale_transition") from exc
def fulfil_request(
user_db: "UserDB",
user_db: UserDB,
*,
request_id: int,
admin_user_id: int,
queue_release: Callable[..., tuple[bool, str | None]],
release_data: Any = None,
admin_note: Any = None,
manual_approval: Any = False,
release_data: object = None,
admin_note: object = None,
manual_approval: object = False,
) -> dict[str, Any]:
"""Fulfil a pending request and queue the release under requesting-user identity."""
request_row = ensure_request_access(
@@ -494,11 +517,15 @@ def fulfil_request(
normalized_admin_note = _normalize_admin_note(admin_note)
if not isinstance(manual_approval, bool):
raise RequestServiceError("manual_approval must be a boolean", status_code=400)
msg = "manual_approval must be a boolean"
raise RequestServiceError(msg, status_code=400)
selected_release_data = release_data if release_data is not None else request_row.get("release_data")
selected_release_data = (
release_data if release_data is not None else request_row.get("release_data")
)
if selected_release_data is not None and not isinstance(selected_release_data, dict):
raise RequestServiceError("release_data must be an object", status_code=400)
msg = "release_data must be an object"
raise RequestServiceError(msg, status_code=400)
if selected_release_data is None and manual_approval:
try:
@@ -514,12 +541,15 @@ def fulfil_request(
reviewed_by=admin_user_id,
reviewed_at=_now_timestamp(),
)
except TypeError as exc:
raise RequestServiceError(str(exc), status_code=400) from exc
except ValueError as exc:
raise RequestServiceError(str(exc), status_code=409, code="stale_transition") from exc
if selected_release_data is None:
msg = "release_data is required to fulfil requests"
raise RequestServiceError(
"release_data is required to fulfil requests",
msg,
status_code=400,
)
@@ -527,7 +557,8 @@ def fulfil_request(
requester = user_db.get_user(user_id=request_row["user_id"])
if requester is None:
raise RequestServiceError("Requesting user not found", status_code=404)
msg = "Requesting user not found"
raise RequestServiceError(msg, status_code=404)
original_release_data = request_row.get("release_data")
try:
@@ -543,6 +574,8 @@ def fulfil_request(
reviewed_by=admin_user_id,
reviewed_at=_now_timestamp(),
)
except TypeError as exc:
raise RequestServiceError(str(exc), status_code=400) from exc
except ValueError as exc:
raise RequestServiceError(str(exc), status_code=409, code="stale_transition") from exc
@@ -579,7 +612,7 @@ def fulfil_request(
def reopen_failed_request(
user_db: "UserDB",
user_db: UserDB,
*,
request_id: int,
failure_reason: str | None = None,
shelfmark/core/search_plan.py
+23 -22
View File
@@ -1,18 +1,20 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import List, Optional
from typing import TYPE_CHECKING
MANUAL_QUERY_MAX_LEN = 256
from shelfmark.core.config import config
from shelfmark.core.models import SearchFilters
from shelfmark.metadata_providers import (
BookMetadata,
group_languages_by_localized_title,
build_localized_search_titles,
group_languages_by_localized_title,
)
if TYPE_CHECKING:
from shelfmark.core.models import SearchFilters
@dataclass(frozen=True)
class ReleaseSearchVariant:
@@ -20,7 +22,7 @@ class ReleaseSearchVariant:
title: str
author: str
languages: Optional[List[str]] = None
languages: list[str] | None = None
@property
def query(self) -> str:
@@ -31,28 +33,28 @@ class ReleaseSearchVariant:
class ReleaseSearchPlan:
"""Pre-computed search inputs shared across release sources."""
languages: Optional[List[str]]
isbn_candidates: List[str]
languages: list[str] | None
isbn_candidates: list[str]
author: str
title_variants: List[ReleaseSearchVariant]
grouped_title_variants: List[ReleaseSearchVariant]
manual_query: Optional[str] = None
indexers: Optional[List[str]] = None # Indexer names for Prowlarr (overrides settings)
source_filters: Optional[SearchFilters] = None
title_variants: list[ReleaseSearchVariant]
grouped_title_variants: list[ReleaseSearchVariant]
manual_query: str | None = None
indexers: list[str] | None = None # Indexer names for Prowlarr (overrides settings)
source_filters: SearchFilters | None = None
@property
def primary_query(self) -> str:
return self.title_variants[0].query if self.title_variants else ""
def _normalize_languages(languages: Optional[List[str]]) -> Optional[List[str]]:
def _normalize_languages(languages: list[str] | None) -> list[str] | None:
if not languages:
default = config.BOOK_LANGUAGE
if not default:
return None
return [str(lang).strip() for lang in default if str(lang).strip()]
normalized: List[str] = []
normalized: list[str] = []
for lang in languages:
if not lang:
continue
@@ -87,10 +89,10 @@ def _pick_search_title(book: BookMetadata) -> str:
def build_release_search_plan(
book: BookMetadata,
languages: Optional[List[str]] = None,
manual_query: Optional[str] = None,
indexers: Optional[List[str]] = None,
source_filters: Optional[SearchFilters] = None,
languages: list[str] | None = None,
manual_query: str | None = None,
indexers: list[str] | None = None,
source_filters: SearchFilters | None = None,
) -> ReleaseSearchPlan:
resolved_languages = _normalize_languages(languages)
@@ -115,7 +117,7 @@ def build_release_search_plan(
source_filters=source_filters,
)
isbn_candidates: List[str] = []
isbn_candidates: list[str] = []
if book.isbn_13:
isbn_candidates.append(book.isbn_13)
if book.isbn_10 and book.isbn_10 not in isbn_candidates:
@@ -135,7 +137,7 @@ def build_release_search_plan(
titles_by_language=titles_by_language,
)
grouped_variants: List[ReleaseSearchVariant] = [
grouped_variants: list[ReleaseSearchVariant] = [
ReleaseSearchVariant(title=title, author=author, languages=langs)
for title, langs in grouped
if title
@@ -148,7 +150,7 @@ def build_release_search_plan(
excluded_languages={"en", "eng", "english"},
)
title_variants: List[ReleaseSearchVariant] = [
title_variants: list[ReleaseSearchVariant] = [
ReleaseSearchVariant(title=title, author=author, languages=None)
for title in expanded_titles
if title
@@ -157,8 +159,7 @@ def build_release_search_plan(
# If no titles could be built, fall back to ISBN queries.
if not title_variants and isbn_candidates:
title_variants = [
ReleaseSearchVariant(title=isbn, author="", languages=None)
for isbn in isbn_candidates
ReleaseSearchVariant(title=isbn, author="", languages=None) for isbn in isbn_candidates
]
return ReleaseSearchPlan(
shelfmark/core/self_user_routes.py
+65 -32
View File
@@ -1,9 +1,9 @@
"""Self-service user account routes."""
from functools import wraps
from typing import Any, Callable, Mapping
from typing import TYPE_CHECKING, Any
from flask import Flask, g, jsonify, request, session
from flask import Flask, Response, g, jsonify, request, session
from werkzeug.security import generate_password_hash
from shelfmark.config.env import CWA_DB_PATH
@@ -24,9 +24,15 @@ from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import load_config_file
from shelfmark.core.user_settings_overrides import (
build_user_preferences_payload as _build_user_preferences_payload,
)
from shelfmark.core.user_settings_overrides import (
get_ordered_user_overridable_fields as _get_ordered_user_overridable_fields,
)
from shelfmark.core.user_db import UserDB
if TYPE_CHECKING:
from collections.abc import Callable, Mapping
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
@@ -43,11 +49,13 @@ _VALID_SELF_SETTINGS_SECTIONS = (
_DEFAULT_VISIBLE_SELF_SETTINGS_SECTIONS = list(_VALID_SELF_SETTINGS_SECTIONS)
def _get_current_user(user_db: UserDB) -> tuple[int | None, dict[str, Any] | None, tuple[Any, int] | None]:
def _get_current_user(
user_db: UserDB,
) -> tuple[int | None, dict[str, Any] | None, tuple[Any, int] | None]:
raw_user_id = session.get("db_user_id")
try:
user_id = int(raw_user_id)
except (TypeError, ValueError):
except TypeError, ValueError:
return None, None, (jsonify({"error": "Invalid user context"}), 400)
user = user_db.get_user(user_id=user_id)
@@ -83,7 +91,7 @@ def _serialize_self_user(user: Mapping[str, Any], auth_mode: str) -> dict[str, A
return payload
def _normalize_visible_self_settings_sections(raw_sections: Any) -> list[str]:
def _normalize_visible_self_settings_sections(raw_sections: object) -> list[str]:
"""Normalize users.VISIBLE_SELF_SETTINGS_SECTIONS to a safe ordered list."""
if raw_sections is None:
return list(_DEFAULT_VISIBLE_SELF_SETTINGS_SECTIONS)
@@ -91,7 +99,9 @@ def _normalize_visible_self_settings_sections(raw_sections: Any) -> list[str]:
if isinstance(raw_sections, str):
candidate_sections = [s.strip() for s in raw_sections.split(",") if s.strip()]
elif isinstance(raw_sections, (list, tuple, set)):
candidate_sections = [str(section).strip() for section in raw_sections if str(section).strip()]
candidate_sections = [
str(section).strip() for section in raw_sections if str(section).strip()
]
else:
return list(_DEFAULT_VISIBLE_SELF_SETTINGS_SECTIONS)
@@ -118,14 +128,10 @@ def _get_allowed_self_settings_keys(visible_sections: list[str]) -> set[str]:
visible_sections_set = set(visible_sections)
if _SELF_SETTINGS_SECTION_DELIVERY in visible_sections_set:
allowed_keys |= {
key for key, _field in _get_ordered_user_overridable_fields("downloads")
}
allowed_keys |= {key for key, _field in _get_ordered_user_overridable_fields("downloads")}
if _SELF_SETTINGS_SECTION_SEARCH in visible_sections_set:
allowed_keys |= {
key for key, _field in _get_ordered_user_overridable_fields("search_mode")
}
allowed_keys |= {key for key, _field in _get_ordered_user_overridable_fields("search_mode")}
if _SELF_SETTINGS_SECTION_NOTIFICATIONS in visible_sections_set:
allowed_keys |= {
@@ -138,25 +144,31 @@ def _get_allowed_self_settings_keys(visible_sections: list[str]) -> set[str]:
def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
"""Register self-service user endpoints."""
def _require_authenticated_user(f: Callable[..., Any]) -> Callable[..., Any]:
def _require_authenticated_user(
f: Callable[..., Response | tuple[Response, int]],
) -> Callable[..., Response | tuple[Response, int]]:
"""Decorator requiring an authenticated session linked to a local user row.
Caches the resolved auth_mode in ``g.auth_mode`` for the request.
"""
@wraps(f)
def decorated(*args, **kwargs):
def decorated(*args, **kwargs) -> Response | tuple[Response, int]:
auth_mode = load_active_auth_mode(CWA_DB_PATH, user_db=user_db)
g.auth_mode = auth_mode
if auth_mode != "none" and "user_id" not in session:
return jsonify({"error": "Authentication required"}), 401
if "db_user_id" not in session:
return jsonify({"error": "Authenticated session is missing local user context"}), 403
return jsonify(
{"error": "Authenticated session is missing local user context"}
), 403
return f(*args, **kwargs)
return decorated
@app.route("/api/users/me/edit-context", methods=["GET"])
@_require_authenticated_user
def users_me_edit_context():
def users_me_edit_context() -> Response | tuple[Response, int]:
user_id, user, user_error = _get_current_user(user_db)
if user_error:
return user_error
@@ -168,31 +180,49 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
delivery_preferences = None
if _SELF_SETTINGS_SECTION_DELIVERY in visible_self_settings_sections:
try:
delivery_preferences = _build_user_preferences_payload(user_db, user_id, "downloads")
delivery_preferences = _build_user_preferences_payload(
user_db, user_id, "downloads"
)
except ValueError:
return jsonify({"error": "Downloads settings tab not found"}), 500
except Exception as exc:
logger.warning(f"Failed to build user delivery preferences for user_id={user_id}: {exc}")
logger.warning(
"Failed to build user delivery preferences for user_id=%s: %s",
user_id,
exc,
)
delivery_preferences = None
search_preferences = None
if _SELF_SETTINGS_SECTION_SEARCH in visible_self_settings_sections:
try:
search_preferences = _build_user_preferences_payload(user_db, user_id, "search_mode")
search_preferences = _build_user_preferences_payload(
user_db, user_id, "search_mode"
)
except ValueError:
return jsonify({"error": "Search mode settings tab not found"}), 500
except Exception as exc:
logger.warning(f"Failed to build user search preferences for user_id={user_id}: {exc}")
logger.warning(
"Failed to build user search preferences for user_id=%s: %s",
user_id,
exc,
)
search_preferences = None
notification_preferences = None
if _SELF_SETTINGS_SECTION_NOTIFICATIONS in visible_self_settings_sections:
try:
notification_preferences = _build_user_preferences_payload(user_db, user_id, "notifications")
notification_preferences = _build_user_preferences_payload(
user_db, user_id, "notifications"
)
except ValueError:
return jsonify({"error": "Notifications settings tab not found"}), 500
except Exception as exc:
logger.warning(f"Failed to build user notification preferences for user_id={user_id}: {exc}")
logger.warning(
"Failed to build user notification preferences for user_id=%s: %s",
user_id,
exc,
)
notification_preferences = None
user_overridable_keys = sorted(
@@ -214,7 +244,7 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
@app.route("/api/users/me/notification-preferences/test", methods=["POST"])
@_require_authenticated_user
def users_me_test_notification_preferences():
def users_me_test_notification_preferences() -> Response | tuple[Response, int]:
user_id, _user, user_error = _get_current_user(user_db)
if user_error:
return user_error
@@ -230,7 +260,7 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
@app.route("/api/users/me", methods=["PUT"])
@_require_authenticated_user
def users_me_update():
def users_me_update() -> Response | tuple[Response, int]:
user_id, user, user_error = _get_current_user(user_db)
if user_error:
return user_error
@@ -252,7 +282,9 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
}
), 400
if len(password) < MIN_PASSWORD_LENGTH:
return jsonify({"error": f"Password must be at least {MIN_PASSWORD_LENGTH} characters"}), 400
return jsonify(
{"error": f"Password must be at least {MIN_PASSWORD_LENGTH} characters"}
), 400
user_db.update_user(user_id, password_hash=generate_password_hash(password))
user_fields: dict[str, Any] = {}
@@ -271,10 +303,9 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
)
email_changed = "email" in user_fields and user_fields["email"] != user.get("email")
display_name_changed = (
"display_name" in user_fields
and user_fields["display_name"] != user.get("display_name")
)
display_name_changed = "display_name" in user_fields and user_fields[
"display_name"
] != user.get("display_name")
if email_changed and not capabilities["canEditEmail"]:
if auth_source == AUTH_SOURCE_CWA:
@@ -312,7 +343,9 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
return jsonify({"error": "Settings must be an object"}), 400
visible_self_settings_sections = _get_visible_self_settings_sections()
allowed_user_settings_keys = _get_allowed_self_settings_keys(visible_self_settings_sections)
allowed_user_settings_keys = _get_allowed_self_settings_keys(
visible_self_settings_sections
)
disallowed_keys = sorted(
key for key in settings_payload if key not in allowed_user_settings_keys
)
@@ -349,5 +382,5 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
result = _serialize_self_user(updated, g.auth_mode)
result["settings"] = user_db.get_user_settings(user_id)
logger.info(f"User {user_id} updated their own account")
logger.info("User %s updated their own account", user_id)
return jsonify(result)
shelfmark/core/settings_registry.py
+350 -257
View File
File diff suppressed because it is too large Load Diff
shelfmark/core/user_db.py
+111 -108
View File
@@ -4,12 +4,12 @@ import json
import os
import sqlite3
import threading
from typing import Any, Dict, List, Optional
from pathlib import Path
from typing import Any, ClassVar
from shelfmark.core.auth_modes import AUTH_SOURCE_BUILTIN, AUTH_SOURCE_SET
from shelfmark.core.activity_view_state_service import user_viewer_scope
from shelfmark.core.auth_modes import AUTH_SOURCE_BUILTIN, AUTH_SOURCE_SET
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import normalize_optional_positive_int
from shelfmark.core.models import QueueStatus
from shelfmark.core.request_validation import (
DELIVERY_STATE_NONE,
@@ -116,16 +116,16 @@ WHERE dismissed_at IS NOT NULL;
"""
def get_users_db_path(config_dir: Optional[str] = None) -> str:
def get_users_db_path(config_dir: str | None = None) -> str:
"""Return the configured users database path."""
root = config_dir or os.environ.get("CONFIG_DIR", "/config")
return os.path.join(root, "users.db")
return str(Path(root) / "users.db")
def sync_builtin_admin_user(
username: str,
password_hash: str,
db_path: Optional[str] = None,
db_path: str | None = None,
) -> None:
"""Ensure a local admin user exists for configured builtin credentials."""
normalized_username = (username or "").strip()
@@ -138,7 +138,9 @@ def sync_builtin_admin_user(
existing = user_db.get_user(username=normalized_username)
if existing:
existing_auth_source = str(existing.get("auth_source") or AUTH_SOURCE_BUILTIN).strip().lower()
existing_auth_source = (
str(existing.get("auth_source") or AUTH_SOURCE_BUILTIN).strip().lower()
)
if existing_auth_source != AUTH_SOURCE_BUILTIN:
logger.warning(
"Skipped builtin admin sync for username '%s' because it belongs to auth_source='%s'",
@@ -155,7 +157,7 @@ def sync_builtin_admin_user(
updates["auth_source"] = AUTH_SOURCE_BUILTIN
if updates:
user_db.update_user(existing["id"], **updates)
logger.info(f"Updated local admin user '{normalized_username}' from builtin settings")
logger.info("Updated local admin user '%s' from builtin settings", normalized_username)
return
user_db.create_user(
@@ -164,15 +166,15 @@ def sync_builtin_admin_user(
auth_source=AUTH_SOURCE_BUILTIN,
role="admin",
)
logger.info(f"Created local admin user '{normalized_username}' from builtin settings")
logger.info("Created local admin user '%s' from builtin settings", normalized_username)
class UserDB:
"""Thread-safe SQLite user database."""
_VALID_AUTH_SOURCES = set(AUTH_SOURCE_SET)
_VALID_AUTH_SOURCES: ClassVar[frozenset[str]] = frozenset(AUTH_SOURCE_SET)
def __init__(self, db_path: str):
def __init__(self, db_path: str) -> None:
self._db_path = db_path
self._lock = threading.Lock()
@@ -204,14 +206,10 @@ class UserDB:
column_names = {str(col["name"]) for col in columns}
if "auth_source" not in column_names:
conn.execute(
"ALTER TABLE users ADD COLUMN auth_source TEXT NOT NULL DEFAULT 'builtin'"
)
conn.execute("ALTER TABLE users ADD COLUMN auth_source TEXT NOT NULL DEFAULT 'builtin'")
# Backfill OIDC-origin users created before auth_source existed.
conn.execute(
"UPDATE users SET auth_source = 'oidc' WHERE oidc_subject IS NOT NULL"
)
conn.execute("UPDATE users SET auth_source = 'oidc' WHERE oidc_subject IS NOT NULL")
# Defensive cleanup for any legacy null/blank values.
conn.execute(
"UPDATE users SET auth_source = 'builtin' WHERE auth_source IS NULL OR auth_source = ''"
@@ -266,13 +264,13 @@ class UserDB:
def create_user(
self,
username: str,
email: Optional[str] = None,
display_name: Optional[str] = None,
password_hash: Optional[str] = None,
oidc_subject: Optional[str] = None,
email: str | None = None,
display_name: str | None = None,
password_hash: str | None = None,
oidc_subject: str | None = None,
auth_source: str = "builtin",
role: str = "user",
) -> Dict[str, Any]:
) -> dict[str, Any]:
"""Create a new user. Raises ValueError if username or oidc_subject already exists."""
if auth_source not in self._VALID_AUTH_SOURCES:
raise ValueError(f"Invalid auth_source: {auth_source}")
@@ -298,25 +296,23 @@ class UserDB:
user_id = cursor.lastrowid
return self._get_user_by_id(conn, user_id)
except sqlite3.IntegrityError as e:
raise ValueError(f"User already exists: {e}")
raise ValueError(f"User already exists: {e}") from e
finally:
conn.close()
def get_user(
self,
user_id: Optional[int] = None,
username: Optional[str] = None,
oidc_subject: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
user_id: int | None = None,
username: str | None = None,
oidc_subject: str | None = None,
) -> dict[str, Any] | None:
"""Get a user by id, username, or oidc_subject. Returns None if not found."""
conn = self._connect()
try:
if user_id is not None:
return self._get_user_by_id(conn, user_id)
elif username is not None:
row = conn.execute(
"SELECT * FROM users WHERE username = ?", (username,)
).fetchone()
if username is not None:
row = conn.execute("SELECT * FROM users WHERE username = ?", (username,)).fetchone()
elif oidc_subject is not None:
row = conn.execute(
"SELECT * FROM users WHERE oidc_subject = ?", (oidc_subject,)
@@ -327,18 +323,20 @@ class UserDB:
finally:
conn.close()
def _get_user_by_id(self, conn: sqlite3.Connection, user_id: int) -> Optional[Dict[str, Any]]:
def _get_user_by_id(self, conn: sqlite3.Connection, user_id: int) -> dict[str, Any] | None:
row = conn.execute("SELECT * FROM users WHERE id = ?", (user_id,)).fetchone()
return dict(row) if row else None
_ALLOWED_UPDATE_COLUMNS = {
"email",
"display_name",
"password_hash",
"oidc_subject",
"auth_source",
"role",
}
_ALLOWED_UPDATE_COLUMNS: ClassVar[frozenset[str]] = frozenset(
{
"email",
"display_name",
"password_hash",
"oidc_subject",
"auth_source",
"role",
}
)
def update_user(self, user_id: int, **kwargs) -> None:
"""Update user fields. Raises ValueError if user not found or invalid column."""
@@ -356,7 +354,7 @@ class UserDB:
if not self._get_user_by_id(conn, user_id):
raise ValueError(f"User {user_id} not found")
sets = ", ".join(f"{k} = ?" for k in kwargs)
values = list(kwargs.values()) + [user_id]
values = [*list(kwargs.values()), user_id]
conn.execute(f"UPDATE users SET {sets} WHERE id = ?", values)
conn.commit()
finally:
@@ -386,13 +384,16 @@ class UserDB:
"DELETE FROM activity_view_state WHERE viewer_scope = ?",
(user_viewer_scope(user_id),),
)
conn.execute("UPDATE download_requests SET reviewed_by = NULL WHERE reviewed_by = ?", (user_id,))
conn.execute(
"UPDATE download_requests SET reviewed_by = NULL WHERE reviewed_by = ?",
(user_id,),
)
conn.execute("DELETE FROM users WHERE id = ?", (user_id,))
conn.commit()
finally:
conn.close()
def list_users(self) -> List[Dict[str, Any]]:
def list_users(self) -> list[dict[str, Any]]:
"""List all users."""
conn = self._connect()
try:
@@ -414,7 +415,7 @@ class UserDB:
finally:
conn.close()
def get_user_settings(self, user_id: int) -> Dict[str, Any]:
def get_user_settings(self, user_id: int) -> dict[str, Any]:
"""Get per-user settings. Returns empty dict if none set."""
conn = self._connect()
try:
@@ -427,7 +428,7 @@ class UserDB:
finally:
conn.close()
def set_user_settings(self, user_id: int, settings: Dict[str, Any]) -> None:
def set_user_settings(self, user_id: int, settings: dict[str, Any]) -> None:
"""Merge settings into user's existing settings."""
with self._lock:
conn = self._connect()
@@ -454,7 +455,7 @@ class UserDB:
conn.close()
@staticmethod
def _serialize_json(value: Any, field: str) -> Optional[str]:
def _serialize_json(value: Any, field: str) -> str | None:
if value is None:
return None
try:
@@ -463,7 +464,7 @@ class UserDB:
raise ValueError(f"{field} must be JSON-serializable") from exc
@staticmethod
def _parse_request_row(row: Optional[sqlite3.Row]) -> Optional[Dict[str, Any]]:
def _parse_request_row(row: sqlite3.Row | None) -> dict[str, Any] | None:
if row is None:
return None
@@ -475,7 +476,7 @@ class UserDB:
continue
try:
payload[key] = json.loads(raw_value)
except (ValueError, TypeError):
except ValueError, TypeError:
payload[key] = None
return payload
@@ -487,17 +488,17 @@ class UserDB:
content_type: str,
request_level: str,
policy_mode: str,
book_data: Dict[str, Any],
release_data: Optional[Dict[str, Any]] = None,
book_data: dict[str, Any],
release_data: dict[str, Any] | None = None,
status: str = RequestStatus.PENDING,
source_hint: Optional[str] = None,
note: Optional[str] = None,
admin_note: Optional[str] = None,
reviewed_by: Optional[int] = None,
reviewed_at: Optional[str] = None,
source_hint: str | None = None,
note: str | None = None,
admin_note: str | None = None,
reviewed_by: int | None = None,
reviewed_at: str | None = None,
delivery_state: str = DELIVERY_STATE_NONE,
delivery_updated_at: Optional[str] = None,
) -> Dict[str, Any]:
delivery_updated_at: str | None = None,
) -> dict[str, Any]:
cursor = conn.execute(
"""
INSERT INTO download_requests (
@@ -552,22 +553,22 @@ class UserDB:
content_type: str,
request_level: str,
policy_mode: str,
book_data: Dict[str, Any],
release_data: Optional[Dict[str, Any]] = None,
book_data: dict[str, Any],
release_data: dict[str, Any] | None = None,
status: str = RequestStatus.PENDING,
source_hint: Optional[str] = None,
note: Optional[str] = None,
admin_note: Optional[str] = None,
reviewed_by: Optional[int] = None,
reviewed_at: Optional[str] = None,
source_hint: str | None = None,
note: str | None = None,
admin_note: str | None = None,
reviewed_by: int | None = None,
reviewed_at: str | None = None,
delivery_state: str = DELIVERY_STATE_NONE,
delivery_updated_at: Optional[str] = None,
) -> Dict[str, Any]:
delivery_updated_at: str | None = None,
) -> dict[str, Any]:
"""Create a download request row and return the created record."""
if not isinstance(book_data, dict):
raise ValueError("book_data must be an object")
raise TypeError("book_data must be an object")
if release_data is not None and not isinstance(release_data, dict):
raise ValueError("release_data must be an object when provided")
raise TypeError("release_data must be an object when provided")
if not content_type:
raise ValueError("content_type is required")
@@ -601,20 +602,18 @@ class UserDB:
finally:
conn.close()
def create_requests(self, requests: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
def create_requests(self, requests: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Create multiple request rows atomically and return them in input order."""
with self._lock:
conn = self._connect()
try:
created: List[Dict[str, Any]] = []
for request in requests:
created.append(self._insert_request(conn, **request))
created = [self._insert_request(conn, **request) for request in requests]
conn.commit()
return created
finally:
conn.close()
def get_request(self, request_id: int) -> Optional[Dict[str, Any]]:
def get_request(self, request_id: int) -> dict[str, Any] | None:
"""Get a request row by ID."""
conn = self._connect()
try:
@@ -629,14 +628,14 @@ class UserDB:
def list_requests(
self,
*,
user_id: Optional[int] = None,
status: Optional[str] = None,
limit: Optional[int] = None,
user_id: int | None = None,
status: str | None = None,
limit: int | None = None,
offset: int = 0,
) -> List[Dict[str, Any]]:
) -> list[dict[str, Any]]:
"""List requests with optional user/status filters."""
where_clauses: List[str] = []
params: List[Any] = []
where_clauses: list[str] = []
params: list[Any] = []
if user_id is not None:
where_clauses.append("user_id = ?")
@@ -664,7 +663,7 @@ class UserDB:
conn = self._connect()
try:
rows = conn.execute(query, params).fetchall()
results: List[Dict[str, Any]] = []
results: list[dict[str, Any]] = []
for row in rows:
parsed = self._parse_request_row(row)
if parsed is not None:
@@ -673,29 +672,31 @@ class UserDB:
finally:
conn.close()
_ALLOWED_REQUEST_UPDATE_COLUMNS = {
"status",
"source_hint",
"content_type",
"request_level",
"policy_mode",
"book_data",
"release_data",
"note",
"admin_note",
"reviewed_by",
"reviewed_at",
"delivery_state",
"delivery_updated_at",
"last_failure_reason",
}
_ALLOWED_REQUEST_UPDATE_COLUMNS: ClassVar[frozenset[str]] = frozenset(
{
"status",
"source_hint",
"content_type",
"request_level",
"policy_mode",
"book_data",
"release_data",
"note",
"admin_note",
"reviewed_by",
"reviewed_at",
"delivery_state",
"delivery_updated_at",
"last_failure_reason",
}
)
def update_request(
self,
request_id: int,
expected_current_status: Optional[str] = None,
expected_current_status: str | None = None,
**kwargs,
) -> Dict[str, Any]:
) -> dict[str, Any]:
"""Update request fields and return the updated record."""
if not kwargs:
request = self.get_request(request_id)
@@ -745,7 +746,7 @@ class UserDB:
if "delivery_updated_at" in updates:
delivery_updated_at = updates["delivery_updated_at"]
if delivery_updated_at is not None and not isinstance(delivery_updated_at, str):
raise ValueError("delivery_updated_at must be a string when provided")
raise TypeError("delivery_updated_at must be a string when provided")
if "content_type" in updates and not updates["content_type"]:
raise ValueError("content_type is required")
@@ -755,19 +756,21 @@ class UserDB:
if "book_data" in updates:
if not isinstance(updates["book_data"], dict):
raise ValueError("book_data must be an object")
raise TypeError("book_data must be an object")
updates["book_data"] = self._serialize_json(updates["book_data"], "book_data")
if "release_data" in updates:
if updates["release_data"] is not None and not isinstance(updates["release_data"], dict):
raise ValueError("release_data must be an object when provided")
if updates["release_data"] is not None and not isinstance(
updates["release_data"], dict
):
raise TypeError("release_data must be an object when provided")
updates["release_data"] = self._serialize_json(
updates["release_data"],
"release_data",
)
set_clause = ", ".join(f"{column} = ?" for column in updates)
values = list(updates.values()) + [request_id]
values = [*list(updates.values()), request_id]
conn.execute(
f"UPDATE download_requests SET {set_clause} WHERE id = ?",
values,
@@ -789,8 +792,8 @@ class UserDB:
self,
request_id: int,
*,
failure_reason: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
failure_reason: str | None = None,
) -> dict[str, Any] | None:
"""Reopen a failed fulfilled request so admins can re-approve it."""
normalized_failure_reason = None
if isinstance(failure_reason, str):
@@ -849,9 +852,9 @@ class UserDB:
self,
request_id: int,
*,
release_data: Optional[Dict[str, Any]],
last_failure_reason: Optional[str] = None,
) -> Dict[str, Any]:
release_data: dict[str, Any] | None,
last_failure_reason: str | None = None,
) -> dict[str, Any]:
"""Restore a request to pending after fulfilment claimed it but queueing failed."""
with self._lock:
conn = self._connect()
shelfmark/core/user_settings_overrides.py
+17 -9
View File
@@ -1,17 +1,22 @@
"""Shared helpers for user-overridable settings metadata and payloads."""
from typing import Any
from importlib import import_module
from typing import TYPE_CHECKING, Any
from shelfmark.core.settings_registry import load_config_file
from shelfmark.core.user_db import UserDB
if TYPE_CHECKING:
from types import ModuleType
from shelfmark.core.user_db import UserDB
def get_settings_registry():
def get_settings_registry() -> ModuleType:
# Ensure settings modules are loaded before reading registry metadata.
import shelfmark.config.settings # noqa: F401
import shelfmark.config.security # noqa: F401
import shelfmark.config.notifications_settings # noqa: F401
import shelfmark.config.users_settings # noqa: F401
import_module("shelfmark.config.notifications_settings")
import_module("shelfmark.config.security")
import_module("shelfmark.config.settings")
import_module("shelfmark.config.users_settings")
from shelfmark.core import settings_registry
return settings_registry
@@ -33,7 +38,8 @@ def build_user_preferences_payload(user_db: UserDB, user_id: int, tab_name: str)
ordered_fields = get_ordered_user_overridable_fields(tab_name)
if not ordered_fields:
tab_label = tab_name.capitalize()
raise ValueError(f"{tab_label} settings tab not found")
msg = f"{tab_label} settings tab not found"
raise ValueError(msg)
tab_config = load_config_file(tab_name)
user_settings = user_db.get_user_settings(user_id)
@@ -45,7 +51,9 @@ def build_user_preferences_payload(user_db: UserDB, user_id: int, tab_name: str)
for key, field in ordered_fields:
serialized = settings_registry.serialize_field(field, tab_name, include_value=False)
serialized["fromEnv"] = bool(field.env_supported and settings_registry.is_value_from_env(field))
serialized["fromEnv"] = bool(
field.env_supported and settings_registry.is_value_from_env(field)
)
fields_payload.append(serialized)
global_values[key] = app_config.get(key, field.default)
shelfmark/core/utils.py
+29 -25
View File
@@ -4,15 +4,17 @@ import base64
import importlib
import os
import re
from threading import Lock
from types import ModuleType
from pathlib import Path
from typing import Optional
from threading import Lock
from typing import TYPE_CHECKING
from urllib.parse import urlparse
if TYPE_CHECKING:
from types import ModuleType
def normalize_http_url(
url: Optional[str],
url: str | None,
*,
default_scheme: str = "http",
strip_trailing_slash: bool = True,
@@ -26,7 +28,7 @@ def normalize_http_url(
if not normalized:
return ""
if (normalized.startswith("\"") and normalized.endswith("\"")) or (
if (normalized.startswith('"') and normalized.endswith('"')) or (
normalized.startswith("'") and normalized.endswith("'")
):
normalized = normalized[1:-1].strip()
@@ -34,11 +36,7 @@ def normalize_http_url(
return ""
if allow_special:
special_map = {
value.lower(): value
for value in allow_special
if isinstance(value, str)
}
special_map = {value.lower(): value for value in allow_special if isinstance(value, str)}
special_match = special_map.get(normalized.lower())
if special_match is not None:
return special_match
@@ -79,7 +77,7 @@ def get_hardened_xmlrpc_client() -> ModuleType:
return importlib.import_module("xmlrpc.client")
def normalize_base_path(value: Optional[str]) -> str:
def normalize_base_path(value: str | None) -> str:
"""Normalize a URL base path for reverse proxy subpath deployments."""
if not isinstance(value, str):
return ""
@@ -101,7 +99,7 @@ def normalize_base_path(value: Optional[str]) -> str:
return path.rstrip("/")
def is_audiobook(content_type: Optional[str]) -> bool:
def is_audiobook(content_type: str | None) -> bool:
"""Check if content type indicates an audiobook."""
return bool(content_type and "audiobook" in content_type.lower())
@@ -156,8 +154,8 @@ def _sanitize_user_for_path(username: str) -> str:
def _resolve_destination_username(
user_id: Optional[int] = None,
username: Optional[str] = None,
user_id: int | None = None,
username: str | None = None,
) -> str:
explicit = str(username or "").strip()
if explicit:
@@ -169,7 +167,7 @@ def _resolve_destination_username(
try:
from shelfmark.core.user_db import UserDB
user_db = UserDB(os.path.join(os.environ.get("CONFIG_DIR", "/config"), "users.db"))
user_db = UserDB(str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db"))
user_db.initialize()
user = user_db.get_user(user_id=user_id)
if not user:
@@ -181,8 +179,8 @@ def _resolve_destination_username(
def _expand_user_destination_placeholder(
path_value: str,
user_id: Optional[int] = None,
username: Optional[str] = None,
user_id: int | None = None,
username: str | None = None,
) -> str:
"""Expand `{User}` placeholders in destination paths."""
if not isinstance(path_value, str):
@@ -198,9 +196,10 @@ def _expand_user_destination_placeholder(
def get_destination(
*,
is_audiobook: bool = False,
user_id: Optional[int] = None,
username: Optional[str] = None,
user_id: int | None = None,
username: str | None = None,
) -> Path:
"""Get base destination directory. Audiobooks fall back to main destination."""
from shelfmark.core.config import config
@@ -219,7 +218,9 @@ def get_destination(
# Main destination (also fallback for audiobooks)
# Check new setting first, then legacy INGEST_DIR
destination = config.get("DESTINATION", "", user_id=user_id) or config.get("INGEST_DIR", "/books")
destination = config.get("DESTINATION", "", user_id=user_id) or config.get(
"INGEST_DIR", "/books"
)
return Path(
_expand_user_destination_placeholder(
str(destination),
@@ -229,12 +230,14 @@ def get_destination(
)
def get_aa_content_type_dir(content_type: Optional[str] = None) -> Optional[Path]:
def get_aa_content_type_dir(content_type: str | None = None) -> Path | None:
"""Get override directory for AA content-type routing if configured."""
from shelfmark.core.config import config
# Check if content-type routing is enabled (new or legacy setting)
if not config.get("AA_CONTENT_TYPE_ROUTING", False) and not config.get("USE_CONTENT_TYPE_DIRECTORIES", False):
if not config.get("AA_CONTENT_TYPE_ROUTING", False) and not config.get(
"USE_CONTENT_TYPE_DIRECTORIES", False
):
return None
if not content_type:
@@ -253,7 +256,7 @@ def get_aa_content_type_dir(content_type: Optional[str] = None) -> Optional[Path
return None
def get_ingest_dir(content_type: Optional[str] = None) -> Path:
def get_ingest_dir(content_type: str | None = None) -> Path:
"""DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead."""
from shelfmark.core.config import config
@@ -271,17 +274,18 @@ def get_ingest_dir(content_type: Optional[str] = None) -> Path:
return default_ingest_dir
def transform_cover_url(cover_url: Optional[str], cache_id: str) -> Optional[str]:
def transform_cover_url(cover_url: str | None, cache_id: str) -> str | None:
"""Transform external cover URL to local proxy URL when caching is enabled."""
if not cover_url:
return cover_url
# Skip if already a local URL (starts with /)
if cover_url.startswith('/'):
if cover_url.startswith("/"):
return cover_url
# Check if cover caching is enabled
from shelfmark.config.env import is_covers_cache_enabled
if not is_covers_cache_enabled():
return cover_url
shelfmark/download/archive.py
+68 -46
View File
@@ -1,21 +1,43 @@
"""Archive extraction utilities for downloaded book archives."""
import os
import shutil
import zipfile
from pathlib import Path
from typing import List, Optional, Tuple
from typing import TYPE_CHECKING
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.download.fs import atomic_write
from shelfmark.download.postprocess.policy import (
get_supported_audiobook_formats,
get_supported_formats,
)
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.download.fs import atomic_write
logger = setup_logger(__name__)
if TYPE_CHECKING:
import rarfile
ArchiveType = zipfile.ZipFile | rarfile.RarFile
else:
ArchiveType = zipfile.ZipFile
def _delete_file_with_logging(file_path: Path, file_type_label: str, *, rejected: bool) -> None:
"""Delete a file and log the outcome."""
try:
file_path.unlink()
if rejected:
logger.debug("Deleted rejected %s file: %s", file_type_label, file_path.name)
else:
logger.debug("Deleted non-%s file: %s", file_type_label, file_path.name)
except OSError as e:
if rejected:
logger.warning(
"Failed to delete rejected %s file %s: %s", file_type_label, file_path, e
)
else:
logger.warning("Failed to delete non-%s file %s: %s", file_type_label, file_path, e)
# Check for rarfile availability at module load
try:
@@ -30,20 +52,14 @@ except ImportError:
class ArchiveExtractionError(Exception):
"""Raised when archive extraction fails."""
pass
class PasswordProtectedError(ArchiveExtractionError):
"""Raised when archive requires a password."""
pass
class CorruptedArchiveError(ArchiveExtractionError):
"""Raised when archive is corrupted."""
pass
def is_archive(file_path: Path) -> bool:
"""Check if file is a supported archive format."""
@@ -51,7 +67,7 @@ def is_archive(file_path: Path) -> bool:
return suffix in ("zip", "rar")
def _is_supported_file(file_path: Path, content_type: Optional[str] = None) -> bool:
def _is_supported_file(file_path: Path, content_type: str | None = None) -> bool:
"""Check if file matches user's supported formats setting based on content type."""
ext = file_path.suffix.lower().lstrip(".")
if check_audiobook(content_type):
@@ -62,16 +78,30 @@ def _is_supported_file(file_path: Path, content_type: Optional[str] = None) -> b
# All known ebook extensions (superset of what user might enable)
ALL_EBOOK_EXTENSIONS = {'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr', '.doc', '.docx', '.rtf', '.txt'}
ALL_EBOOK_EXTENSIONS = {
".pdf",
".epub",
".mobi",
".azw",
".azw3",
".fb2",
".djvu",
".cbz",
".cbr",
".doc",
".docx",
".rtf",
".txt",
}
# All known audio extensions (superset of what user might enable for audiobooks)
ALL_AUDIO_EXTENSIONS = {'.m4b', '.mp3', '.m4a', '.aac', '.flac', '.ogg', '.wma', '.wav', '.opus'}
ALL_AUDIO_EXTENSIONS = {".m4b", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".wma", ".wav", ".opus"}
def _filter_files(
extracted_files: List[Path],
content_type: Optional[str] = None,
) -> Tuple[List[Path], List[Path], List[Path]]:
extracted_files: list[Path],
content_type: str | None = None,
) -> tuple[list[Path], list[Path], list[Path]]:
"""Filter files by content type. Returns (matched, rejected_format, other)."""
is_audiobook = check_audiobook(content_type)
known_extensions = ALL_AUDIO_EXTENSIONS if is_audiobook else ALL_EBOOK_EXTENSIONS
@@ -94,8 +124,8 @@ def _filter_files(
def extract_archive(
archive_path: Path,
output_dir: Path,
content_type: Optional[str] = None,
) -> Tuple[List[Path], List[str], List[Path]]:
content_type: str | None = None,
) -> tuple[list[Path], list[str], list[Path]]:
"""Extract archive and filter by content type. Returns (matched, warnings, rejected)."""
suffix = archive_path.suffix.lower().lstrip(".")
@@ -114,23 +144,17 @@ def extract_archive(
# Delete rejected files (valid formats but not enabled by user)
for rejected_file in rejected_files:
try:
rejected_file.unlink()
logger.debug(f"Deleted rejected {file_type_label} file: {rejected_file.name}")
except OSError as e:
logger.warning(f"Failed to delete rejected {file_type_label} file {rejected_file}: {e}")
_delete_file_with_logging(rejected_file, file_type_label, rejected=True)
if rejected_files:
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
warnings.append(f"Skipped {len(rejected_files)} {file_type_label}(s) with unsupported format: {', '.join(rejected_exts)}")
rejected_exts = sorted({f.suffix.lower() for f in rejected_files})
warnings.append(
f"Skipped {len(rejected_files)} {file_type_label}(s) with unsupported format: {', '.join(rejected_exts)}"
)
# Delete other files (images, html, etc)
for other_file in other_files:
try:
other_file.unlink()
logger.debug(f"Deleted non-{file_type_label} file: {other_file.name}")
except OSError as e:
logger.warning(f"Failed to delete non-{file_type_label} file {other_file}: {e}")
_delete_file_with_logging(other_file, file_type_label, rejected=False)
if other_files:
warnings.append(f"Skipped {len(other_files)} non-{file_type_label} file(s)")
@@ -141,7 +165,7 @@ def extract_archive(
def extract_archive_raw(
archive_path: Path,
output_dir: Path,
) -> Tuple[List[Path], List[str]]:
) -> tuple[list[Path], list[str]]:
"""Extract archive without filtering (returns all extracted files)."""
suffix = archive_path.suffix.lower().lstrip(".")
@@ -153,7 +177,7 @@ def extract_archive_raw(
raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
def _extract_files_from_archive(archive: ArchiveType, output_dir: Path) -> list[Path]:
"""Extract files from ZipFile or RarFile to output_dir with security checks."""
extracted_files = []
@@ -169,7 +193,7 @@ def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
# Security: reject filenames with null bytes or path separators
# Check both / and \ since archives may be created on different OSes
if "\x00" in filename or "/" in filename or "\\" in filename:
logger.warning(f"Skipping suspicious filename in archive: {info.filename!r}")
logger.warning("Skipping suspicious filename in archive: %r", info.filename)
continue
# Extract to output_dir with flat structure
@@ -179,19 +203,19 @@ def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
try:
target_path.resolve().relative_to(output_dir.resolve())
except ValueError:
logger.warning(f"Path traversal attempt blocked: {info.filename!r}")
logger.warning("Path traversal attempt blocked: %r", info.filename)
continue
with archive.open(info) as src:
data = src.read()
final_path = atomic_write(target_path, data)
extracted_files.append(final_path)
logger.debug(f"Extracted: {filename}")
logger.debug("Extracted: %s", filename)
return extracted_files
def _extract_zip(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
def _extract_zip(archive_path: Path, output_dir: Path) -> tuple[list[Path], list[str]]:
"""Extract files from a ZIP archive."""
try:
with zipfile.ZipFile(archive_path, "r") as zf:
@@ -208,12 +232,12 @@ def _extract_zip(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List
return _extract_files_from_archive(zf, output_dir), []
except zipfile.BadZipFile as e:
raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}")
raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}") from e
except PermissionError as e:
raise ArchiveExtractionError(f"Permission denied: {e}")
raise ArchiveExtractionError(f"Permission denied: {e}") from e
def _extract_rar(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
def _extract_rar(archive_path: Path, output_dir: Path) -> tuple[list[Path], list[str]]:
"""Extract files from a RAR archive."""
if not RAR_AVAILABLE:
raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
@@ -230,10 +254,8 @@ def _extract_rar(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List
return _extract_files_from_archive(rf, output_dir), []
except rarfile.BadRarFile as e:
raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}")
except rarfile.RarCannotExec:
raise ArchiveExtractionError("unrar binary not found - install unrar package")
raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}") from e
except rarfile.RarCannotExec as e:
raise ArchiveExtractionError("unrar binary not found - install unrar package") from e
except PermissionError as e:
raise ArchiveExtractionError(f"Permission denied: {e}")
raise ArchiveExtractionError(f"Permission denied: {e}") from e
shelfmark/download/clients/__init__.py
+100 -87
View File
@@ -1,5 +1,4 @@
"""
Shared download client infrastructure for external release sources.
"""Shared download client infrastructure for external release sources.
This module provides:
- DownloadState: Enum of valid download states
@@ -18,14 +17,18 @@ from abc import ABC, abstractmethod
from dataclasses import dataclass
from enum import Enum
from functools import wraps
from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast, Any
from pathlib import Path
from typing import TYPE_CHECKING, TypeVar, cast
import requests
if TYPE_CHECKING:
from collections.abc import Callable
_logger = logging.getLogger(__name__)
# Type variable for generic return type
T = TypeVar('T')
T = TypeVar("T")
# Exceptions that should trigger a retry
RETRYABLE_EXCEPTIONS = (
@@ -33,6 +36,9 @@ RETRYABLE_EXCEPTIONS = (
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
)
_MIN_RETRYABLE_STATUS = 500
_MIN_PROGRESS_PERCENT = 0
_MAX_PROGRESS_PERCENT = 100
def with_retry(
@@ -41,8 +47,7 @@ def with_retry(
max_delay: float = 10.0,
jitter: float = 0.5,
) -> Callable[[Callable[..., T]], Callable[..., T]]:
"""
Decorator for retrying API calls with exponential backoff.
"""Decorator for retrying API calls with exponential backoff.
Args:
max_attempts: Maximum number of attempts (default 3)
@@ -58,7 +63,9 @@ def with_retry(
Does NOT retry on:
- HTTP 4xx client errors (bad request, auth failures)
- Other exceptions (programming errors)
"""
def decorator(func: Callable[..., T]) -> Callable[..., T]:
@wraps(func)
def wrapper(*args, **kwargs) -> T:
@@ -69,7 +76,7 @@ def with_retry(
return func(*args, **kwargs)
except requests.exceptions.HTTPError as e:
# Only retry on server errors (5xx), not client errors (4xx)
if e.response is not None and e.response.status_code < 500:
if e.response is not None and e.response.status_code < _MIN_RETRYABLE_STATUS:
raise
last_exception = e
except RETRYABLE_EXCEPTIONS as e:
@@ -81,17 +88,23 @@ def with_retry(
# Add jitter to prevent thundering herd
delay += random.uniform(0, delay * jitter)
_logger.debug(
f"Retry {attempt}/{max_attempts} for {func.__name__} "
f"after {delay:.1f}s (error: {last_exception})"
"Retry %s/%s for %s after %.1fs (error: %s)",
attempt,
max_attempts,
func.__name__,
delay,
last_exception,
)
time.sleep(delay)
# All retries exhausted
if last_exception is None:
raise RuntimeError("Retry failed without exception")
raise cast(Exception, last_exception)
msg = "Retry failed without exception"
raise RuntimeError(msg)
raise cast("Exception", last_exception)
return wrapper
return decorator
@@ -114,15 +127,15 @@ class DownloadStatus:
"""Status of an external download (immutable)."""
progress: float # 0-100
state: Union[DownloadState, str] # Prefer DownloadState enum; strings auto-normalized
message: Optional[str] # Status message
state: DownloadState | str # Prefer DownloadState enum; strings auto-normalized
message: str | None # Status message
complete: bool # True when download finished
file_path: Optional[str] # Path in client's download dir (when complete)
download_speed: Optional[int] = None # Bytes per second
eta: Optional[int] = None # Seconds remaining
file_path: str | None # Path in client's download dir (when complete)
download_speed: int | None = None # Bytes per second
eta: int | None = None # Seconds remaining
@classmethod
def error(cls, message: str) -> "DownloadStatus":
def error(cls, message: str) -> DownloadStatus:
"""Create an error status."""
return cls(
progress=0,
@@ -132,21 +145,25 @@ class DownloadStatus:
file_path=None,
)
def __post_init__(self):
def __post_init__(self) -> None:
"""Validate and normalize state."""
# Normalize string states to enum
if isinstance(self.state, str):
try:
normalized_state = DownloadState(self.state)
object.__setattr__(self, 'state', normalized_state)
object.__setattr__(self, "state", normalized_state)
except ValueError:
# Unknown state string - keep as-is for backwards compatibility
_logger.warning(f"Unknown download state '{self.state}', keeping as string")
_logger.warning(
_logger.warning("Unknown download state '%s', keeping as string", self.state)
)
# Validate progress is in range
if not 0 <= self.progress <= 100:
_logger.debug(f"Progress {self.progress} out of range, clamping to [0, 100]")
object.__setattr__(self, 'progress', max(0, min(100, self.progress)))
if not _MIN_PROGRESS_PERCENT <= self.progress <= _MAX_PROGRESS_PERCENT:
_logger.debug(
_logger.debug("Progress %s out of range, clamping to [0, 100]", self.progress)
)
object.__setattr__(self, "progress", max(0, min(100, self.progress)))
@property
def state_value(self) -> str:
@@ -157,8 +174,7 @@ class DownloadStatus:
class DownloadClient(ABC):
"""
Base class for external download clients.
"""Base class for external download clients.
Subclasses implement protocol-specific download management:
- Torrent clients: qBittorrent, Transmission, Deluge
@@ -174,8 +190,7 @@ class DownloadClient(ABC):
name: str
def _log_error(self, method: str, e: Exception, level: str = "error") -> str:
"""
Log a client error with consistent formatting.
"""Log a client error with consistent formatting.
Args:
method: Name of the method that failed (e.g., "get_status")
@@ -184,6 +199,7 @@ class DownloadClient(ABC):
Returns:
Formatted error message string (for use in DownloadStatus.error())
"""
error_type = type(e).__name__
msg = f"{self.name} {method} failed ({error_type}): {e}"
@@ -198,15 +214,15 @@ class DownloadClient(ABC):
return f"{error_type}: {e}"
def _build_path(self, *components: str) -> Optional[str]:
"""
Safely build a file path from components.
def _build_path(self, *components: str) -> str | None:
"""Safely build a file path from components.
Args:
*components: Path components to join (e.g., save_path, name)
Returns:
Normalized path string, or None if any component is empty/None.
"""
# Filter out empty/None components
valid = [c for c in components if c]
@@ -214,9 +230,9 @@ class DownloadClient(ABC):
return None
# Join and normalize
return os.path.normpath(os.path.join(*valid))
return os.path.normpath(str(Path(valid[0]).joinpath(*valid[1:])))
def __init_subclass__(cls, **kwargs):
def __init_subclass__(cls, **kwargs) -> None:
"""Validate that subclasses define required class attributes."""
super().__init_subclass__(**kwargs)
@@ -225,48 +241,46 @@ class DownloadClient(ABC):
return
# Validate protocol attribute
if not hasattr(cls, 'protocol') or not cls.protocol:
raise TypeError(f"{cls.__name__} must define 'protocol' class attribute")
if cls.protocol not in ('torrent', 'usenet'):
raise TypeError(
f"{cls.__name__}.protocol must be 'torrent' or 'usenet', got '{cls.protocol}'"
)
if not hasattr(cls, "protocol") or not cls.protocol:
msg = f"{cls.__name__} must define 'protocol' class attribute"
raise TypeError(msg)
if cls.protocol not in ("torrent", "usenet"):
msg = f"{cls.__name__}.protocol must be 'torrent' or 'usenet', got '{cls.protocol}'"
raise TypeError(msg)
# Validate name attribute
if not hasattr(cls, 'name') or not cls.name:
raise TypeError(f"{cls.__name__} must define 'name' class attribute")
if not hasattr(cls, "name") or not cls.name:
msg = f"{cls.__name__} must define 'name' class attribute"
raise TypeError(msg)
@staticmethod
@abstractmethod
def is_configured() -> bool:
"""
Check if this client is configured.
"""Check if this client is configured.
Returns:
True if required settings (URL, etc.) are present.
"""
pass
@abstractmethod
def test_connection(self) -> Tuple[bool, str]:
"""
Test connectivity to the client.
def test_connection(self) -> tuple[bool, str]:
"""Test connectivity to the client.
Returns:
Tuple of (success, message).
"""
pass
@abstractmethod
def add_download(
self,
url: str,
name: str,
category: Optional[str] = None,
expected_hash: Optional[str] = None,
**kwargs: Any,
category: str | None = None,
expected_hash: str | None = None,
**kwargs: object,
) -> str:
"""Add a download to the client.
Args:
@@ -280,26 +294,24 @@ class DownloadClient(ABC):
Raises:
Exception: If adding fails.
"""
pass
@abstractmethod
def get_status(self, download_id: str) -> DownloadStatus:
"""
Get status of a download.
"""Get status of a download.
Args:
download_id: The ID returned by add_download()
Returns:
Current download status.
"""
pass
@abstractmethod
def remove(self, download_id: str, delete_files: bool = False) -> bool:
"""
Remove a download from the client.
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
"""Remove a download from the client.
Args:
download_id: The ID returned by add_download()
@@ -307,27 +319,25 @@ class DownloadClient(ABC):
Returns:
True if removal succeeded.
"""
pass
@abstractmethod
def get_download_path(self, download_id: str) -> Optional[str]:
"""
Get the path where files were downloaded.
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where files were downloaded.
Args:
download_id: The ID returned by add_download()
Returns:
File or directory path, or None if not available.
"""
pass
def find_existing(
self, url: str, category: Optional[str] = None
) -> Optional[Tuple[str, DownloadStatus]]:
"""
Check if a download for this URL already exists in the client.
self, url: str, category: str | None = None
) -> tuple[str, DownloadStatus] | None:
"""Check if a download for this URL already exists in the client.
This is useful for detecting already-completed downloads so we can
skip re-downloading and just copy the existing file.
@@ -339,17 +349,19 @@ class DownloadClient(ABC):
Returns:
Tuple of (download_id, status) if found, None if not found.
Default implementation returns None.
"""
return None
# Client registry: protocol -> list of client classes
_CLIENTS: Dict[str, List[Type[DownloadClient]]] = {}
_CLIENTS: dict[str, list[type[DownloadClient]]] = {}
def register_client(protocol: str):
"""
Decorator to register a download client for a protocol.
def register_client(
protocol: str,
) -> Callable[[type[DownloadClient]], type[DownloadClient]]:
"""Decorator to register a download client for a protocol.
Multiple clients can be registered for the same protocol.
The `is_configured()` method determines which one is active.
@@ -361,9 +373,10 @@ def register_client(protocol: str):
@register_client("torrent")
class QBittorrentClient(DownloadClient):
...
"""
def decorator(cls: Type[DownloadClient]) -> Type[DownloadClient]:
def decorator(cls: type[DownloadClient]) -> type[DownloadClient]:
if protocol not in _CLIENTS:
_CLIENTS[protocol] = []
_CLIENTS[protocol].append(cls)
@@ -372,9 +385,8 @@ def register_client(protocol: str):
return decorator
def get_client(protocol: str) -> Optional[DownloadClient]:
"""
Get a configured client instance for the given protocol.
def get_client(protocol: str) -> DownloadClient | None:
"""Get a configured client instance for the given protocol.
Iterates through all registered clients for the protocol and
returns the first one that is configured.
@@ -384,6 +396,7 @@ def get_client(protocol: str) -> Optional[DownloadClient]:
Returns:
Configured client instance, or None if not available/configured.
"""
if protocol not in _CLIENTS:
return None
@@ -395,12 +408,12 @@ def get_client(protocol: str) -> Optional[DownloadClient]:
return None
def list_configured_clients() -> List[str]:
"""
List protocols that have configured clients.
def list_configured_clients() -> list[str]:
"""List protocols that have configured clients.
Returns:
List of protocol names (e.g., ["torrent", "usenet"]).
"""
result = []
for protocol, client_classes in _CLIENTS.items():
@@ -411,21 +424,21 @@ def list_configured_clients() -> List[str]:
return result
def get_all_clients() -> Dict[str, List[Type[DownloadClient]]]:
"""
Get all registered client classes.
def get_all_clients() -> dict[str, list[type[DownloadClient]]]:
"""Get all registered client classes.
Returns:
Dict of protocol -> list of client classes.
"""
return dict(_CLIENTS)
# Import client implementations to trigger registration
# These imports are at the bottom to avoid circular imports
from shelfmark.download.clients import qbittorrent # noqa: F401, E402
from shelfmark.download.clients import nzbget # noqa: F401, E402
from shelfmark.download.clients import sabnzbd # noqa: F401, E402
from shelfmark.download.clients import transmission # noqa: F401, E402
from shelfmark.download.clients import deluge # noqa: F401, E402
from shelfmark.download.clients import rtorrent # noqa: F401, E402
from shelfmark.download.clients import deluge as deluge
from shelfmark.download.clients import nzbget as nzbget
from shelfmark.download.clients import qbittorrent as qbittorrent
from shelfmark.download.clients import rtorrent as rtorrent
from shelfmark.download.clients import sabnzbd as sabnzbd
from shelfmark.download.clients import transmission as transmission
shelfmark/download/clients/base_handler.py
+149 -75
View File
@@ -5,22 +5,27 @@ import time
from abc import ABC, abstractmethod
from dataclasses import dataclass
from pathlib import Path
from threading import Event
from typing import Callable, Optional
from typing import TYPE_CHECKING
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.core.utils import is_audiobook
from shelfmark.download.clients import (
DownloadClient,
DownloadState,
DownloadStatus,
get_client,
list_configured_clients,
)
from shelfmark.download.fs import run_blocking_io
from shelfmark.release_sources import DownloadHandler
if TYPE_CHECKING:
from collections.abc import Callable
from threading import Event
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
# How often to poll the download client for status (seconds)
@@ -37,23 +42,23 @@ class DownloadRequest:
url: str
protocol: str
release_name: str
expected_hash: Optional[str]
seeding_time_limit: Optional[int] = None # minutes
ratio_limit: Optional[float] = None
expected_hash: str | None
seeding_time_limit: int | None = None # minutes
ratio_limit: float | None = None
def _diagnose_path_issue(path: str) -> str:
"""
Analyze a path and return diagnostic hints for common issues.
"""Analyze a path and return diagnostic hints for common issues.
Args:
path: The path that failed to be accessed
Returns:
A hint string to help users diagnose the issue.
"""
# Detect Windows-style paths (won't work in Linux containers)
if len(path) >= 2 and path[1] == ':':
if len(path) >= 2 and path[1] == ":":
return (
f"Path '{path}' appears to be a Windows path. "
f"Shelfmark runs in Linux and cannot access Windows paths directly. "
@@ -79,7 +84,7 @@ def _diagnose_path_issue(path: str) -> str:
class ExternalClientHandler(DownloadHandler, ABC):
"""Shared lifecycle handler for sources that hand off to torrent/usenet clients."""
def __init__(self):
def __init__(self) -> None:
# Track downloads that may need client-side cleanup after Shelfmark completes import.
# task_id -> (client, download_id, protocol)
self._cleanup_refs: dict[str, tuple[DownloadClient, str, str]] = {}
@@ -88,15 +93,15 @@ class ExternalClientHandler(DownloadHandler, ABC):
def _resolve_download(
self,
task: DownloadTask,
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[DownloadRequest]:
status_callback: Callable[[str, str | None], None],
) -> DownloadRequest | None:
"""Resolve source-specific task metadata into a client download request."""
def _on_download_complete(self, task: DownloadTask) -> None:
"""Hook called after successful completion; override for source cleanup."""
return
def _get_client(self, protocol: str) -> Optional[DownloadClient]:
def _get_client(self, protocol: str) -> DownloadClient | None:
"""Resolve the active client for a protocol."""
return get_client(protocol)
@@ -116,7 +121,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
"""Maximum attempts when waiting for completed files."""
return COMPLETED_PATH_MAX_ATTEMPTS
def _get_category_for_task(self, client: DownloadClient, task: DownloadTask) -> Optional[str]:
def _get_category_for_task(self, client: DownloadClient, task: DownloadTask) -> str | None:
"""Get audiobook category if configured and applicable, else None for default."""
if not is_audiobook(task.content_type):
return None
@@ -132,7 +137,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
audiobook_key = audiobook_keys.get(client.name)
return config.get(audiobook_key, "") or None if audiobook_key else None
def post_process_cleanup(self, task: DownloadTask, success: bool) -> None:
def post_process_cleanup(self, task: DownloadTask, *, success: bool) -> None:
if not success:
self._cleanup_refs.pop(task.task_id, None)
return
@@ -152,7 +157,10 @@ class ExternalClientHandler(DownloadHandler, ABC):
self._remove_usenet_download(client, download_id, delete_files=True, archive=True)
except Exception as e:
logger.warning(
f"Failed to cleanup usenet download {download_id} in {getattr(client, 'name', 'client')}: {e}"
"Failed to cleanup usenet download %s in %s: %s",
download_id,
getattr(client, "name", "client"),
e,
)
elif protocol == "torrent":
@@ -162,7 +170,10 @@ class ExternalClientHandler(DownloadHandler, ABC):
client.remove(download_id, delete_files=False)
except Exception as e:
logger.warning(
f"Failed to remove torrent {download_id} from {getattr(client, 'name', 'client')}: {e}"
"Failed to remove torrent %s from %s: %s",
download_id,
getattr(client, "name", "client"),
e,
)
def _remove_usenet_download(
@@ -184,11 +195,13 @@ class ExternalClientHandler(DownloadHandler, ABC):
try:
raw_path = client.get_download_path(download_id)
except Exception as e:
logger.debug(f"Failed to resolve download path for {client.name} {download_id}: {e}")
logger.debug(
"Failed to resolve download path for %s %s: %s", client.name, download_id, e
)
return
if not raw_path:
logger.debug(f"No download path available for {client.name} {download_id}")
logger.debug("No download path available for %s %s", client.name, download_id)
return
from shelfmark.core.path_mappings import (
@@ -210,11 +223,16 @@ class ExternalClientHandler(DownloadHandler, ABC):
delete_path = remapped if matched_mapping else source_path_obj
if str(delete_path) in ("", "/"):
logger.warning(f"Refusing to delete unsafe path for {client.name} {download_id}: {delete_path}")
logger.warning(
"Refusing to delete unsafe path for %s %s: %s",
client.name,
download_id,
delete_path,
)
return
if not run_blocking_io(delete_path.exists):
logger.debug(f"Local download path does not exist for cleanup: {delete_path}")
logger.debug("Local download path does not exist for cleanup: %s", delete_path)
return
try:
@@ -222,18 +240,27 @@ class ExternalClientHandler(DownloadHandler, ABC):
run_blocking_io(shutil.rmtree, delete_path)
else:
run_blocking_io(delete_path.unlink)
logger.info(f"Deleted local download data for {client.name} {download_id}: {delete_path}")
logger.info(
"Deleted local download data for %s %s: %s", client.name, download_id, delete_path
)
except Exception as e:
logger.warning(f"Failed to delete local download data for {client.name} {download_id}: {e}")
logger.warning(
"Failed to delete local download data for %s %s: %s", client.name, download_id, e
)
def _safe_remove_download(self, client, download_id: str, protocol: str, reason: str) -> None:
def _safe_remove_download(
self,
client: DownloadClient,
download_id: str,
protocol: str,
reason: str,
) -> None:
"""Best-effort removal of a failed/cancelled download from the client.
Safety policy:
- torrents: never remove or delete client data (avoid breaking seeding)
- usenet: keep legacy behavior (delete client files on removal)
"""
if protocol != "usenet":
logger.info(
"Skipping download client cleanup for protocol=%s after %s (client=%s id=%s)",
@@ -250,7 +277,11 @@ class ExternalClientHandler(DownloadHandler, ABC):
self._remove_usenet_download(client, download_id, delete_files=True, archive=False)
except Exception as e:
logger.warning(
f"Failed to remove download {download_id} from {client.name} after {reason}: {e}"
"Failed to remove download %s from %s after %s: %s",
download_id,
client.name,
reason,
e,
)
def _handle_cancelled_download(
@@ -258,20 +289,26 @@ class ExternalClientHandler(DownloadHandler, ABC):
client: DownloadClient,
download_id: str,
protocol: str,
status_callback: Callable[[str, Optional[str]], None],
status_callback: Callable[[str, str | None], None],
) -> None:
if protocol == "usenet":
logger.info(f"Download cancelled, removing from {client.name}: {download_id}")
logger.info("Download cancelled, removing from %s: %s", client.name, download_id)
try:
self._delete_local_download_data(client, download_id)
self._remove_usenet_download(client, download_id, delete_files=True, archive=True)
except Exception as e:
logger.warning(
f"Failed to remove download {download_id} from {client.name} after cancellation: {e}"
"Failed to remove download %s from %s after cancellation: %s",
download_id,
client.name,
e,
)
else:
logger.info(
f"Download cancelled for protocol={protocol}; leaving in {client.name}: {download_id}"
"Download cancelled for protocol=%s; leaving in %s: %s",
protocol,
client.name,
download_id,
)
status_callback("cancelled", "Cancelled")
@@ -281,7 +318,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
download_id: str,
*,
log_details: bool,
) -> tuple[Optional[Path], Optional[str]]:
) -> tuple[Path | None, str | None]:
"""Resolve and validate the completed download path once."""
try:
raw_path = client.get_download_path(download_id)
@@ -291,12 +328,12 @@ class ExternalClientHandler(DownloadHandler, ABC):
f"Check volume mappings and category settings."
)
if log_details:
logger.error(
f"Failed to resolve download path for {client.name} {download_id}: {e}"
logger.exception(
"Failed to resolve download path for %s %s", client.name, download_id
)
else:
logger.debug(
f"Failed to resolve download path for {client.name} {download_id}: {e}"
"Failed to resolve download path for %s %s: %s", client.name, download_id, e
)
return None, message
@@ -306,9 +343,13 @@ class ExternalClientHandler(DownloadHandler, ABC):
f"Check volume mappings and category settings."
)
if log_details:
logger.error(f"Download client returned empty path for {client.name} {download_id}")
logger.error(
"Download client returned empty path for %s %s", client.name, download_id
)
else:
logger.debug(f"Download client returned empty path for {client.name} {download_id}")
logger.debug(
"Download client returned empty path for %s %s", client.name, download_id
)
return None, message
from shelfmark.core.path_mappings import (
@@ -365,13 +406,19 @@ class ExternalClientHandler(DownloadHandler, ABC):
)
if log_details:
logger.error(
f"Download path does not exist after remapping: {raw_path} -> {remapped}. "
f"Client: {client.name}, ID: {download_id}."
"Download path does not exist after remapping: %s -> %s. Client: %s, ID: %s.",
raw_path,
remapped,
client.name,
download_id,
)
else:
logger.debug(
f"Download path does not exist after remapping: {raw_path} -> {remapped}. "
f"Client: {client.name}, ID: {download_id}."
"Download path does not exist after remapping: %s -> %s. Client: %s, ID: %s.",
raw_path,
remapped,
client.name,
download_id,
)
return None, message
@@ -389,13 +436,19 @@ class ExternalClientHandler(DownloadHandler, ABC):
message = f"{hint} No remote path mapping matched for client '{client.name}'."
if log_details:
logger.error(
f"Download path does not exist and no remote path mapping matched for {client.name} "
f"({download_id}): {raw_path}. {hint}"
"Download path does not exist and no remote path mapping matched for %s (%s): %s. %s",
client.name,
download_id,
raw_path,
hint,
)
else:
logger.debug(
f"Download path does not exist and no remote path mapping matched for {client.name} "
f"({download_id}): {raw_path}. {hint}"
"Download path does not exist and no remote path mapping matched for %s (%s): %s. %s",
client.name,
download_id,
raw_path,
hint,
)
return None, message
@@ -404,13 +457,19 @@ class ExternalClientHandler(DownloadHandler, ABC):
message = hint
if log_details:
logger.error(
f"Download path does not exist: {raw_path}. "
f"Client: {client.name}, ID: {download_id}. {hint}"
"Download path does not exist: %s. Client: %s, ID: %s. %s",
raw_path,
client.name,
download_id,
hint,
)
else:
logger.debug(
f"Download path does not exist: {raw_path}. "
f"Client: {client.name}, ID: {download_id}. {hint}"
"Download path does not exist: %s. Client: %s, ID: %s. %s",
raw_path,
client.name,
download_id,
hint,
)
return None, message
@@ -421,11 +480,11 @@ class ExternalClientHandler(DownloadHandler, ABC):
client: DownloadClient,
download_id: str,
*,
cancel_flag: Optional[Event],
status_callback: Callable[[str, Optional[str]], None],
) -> tuple[Optional[Path], Optional[str]]:
cancel_flag: Event | None,
status_callback: Callable[[str, str | None], None],
) -> tuple[Path | None, str | None]:
"""Wait briefly for completed files to appear on disk."""
last_error: Optional[str] = None
last_error: str | None = None
max_attempts = self._completed_path_max_attempts()
retry_interval = self._completed_path_retry_interval()
@@ -462,7 +521,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
return None, last_error
def _build_progress_message(self, status) -> str:
def _build_progress_message(self, status: DownloadStatus) -> str:
"""Build a progress message from download status."""
msg = f"{status.progress:.0f}%"
@@ -485,8 +544,8 @@ class ExternalClientHandler(DownloadHandler, ABC):
task: DownloadTask,
cancel_flag: Event,
progress_callback: Callable[[float], None],
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[str]:
status_callback: Callable[[str, str | None], None],
) -> str | None:
"""Execute download via configured torrent/usenet client. Returns file path or None."""
try:
if cancel_flag.is_set():
@@ -516,7 +575,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
if existing:
download_id, existing_status = existing
logger.info(f"Found existing download in {client.name}: {download_id}")
logger.info("Found existing download in %s: %s", client.name, download_id)
# If already complete, skip straight to file handling
if existing_status.complete:
@@ -567,11 +626,13 @@ class ExternalClientHandler(DownloadHandler, ABC):
ratio_limit=request.ratio_limit,
)
except Exception as e:
logger.error(f"Failed to add to {client.name}: {e}")
logger.exception("Failed to add to %s", client.name)
status_callback("error", f"Failed to add to {client.name}: {e}")
return None
logger.info(f"Added to {client.name}: {download_id} for '{request.release_name}'")
logger.info(
"Added to %s: %s for '%s'", client.name, download_id, request.release_name
)
# Poll for progress
return self._poll_and_complete(
@@ -585,7 +646,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
)
except Exception as e:
logger.error(f"External client download error: {e}")
logger.exception("External client download error")
status_callback("error", str(e))
return None
@@ -597,8 +658,8 @@ class ExternalClientHandler(DownloadHandler, ABC):
task: DownloadTask,
cancel_flag: Event,
progress_callback: Callable[[float], None],
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[str]:
status_callback: Callable[[str, str | None], None],
) -> str | None:
"""Poll the download client for progress and handle completion."""
poll_interval = self._poll_interval()
# Track consecutive "not found" errors - torrents may take time to appear in client
@@ -606,7 +667,8 @@ class ExternalClientHandler(DownloadHandler, ABC):
max_not_found_retries = 15 # 15 retries * poll interval ~= 30s grace period
try:
logger.debug(f"Starting poll for {download_id} (content_type={task.content_type})")
result: str | None = None
logger.debug("Starting poll for %s (content_type=%s)", download_id, task.content_type)
while not cancel_flag.is_set():
status = client.get_status(download_id)
progress_callback(status.progress)
@@ -614,12 +676,18 @@ class ExternalClientHandler(DownloadHandler, ABC):
# Check for completion
if status.complete:
if status.state == DownloadState.ERROR:
logger.error(f"Download {download_id} completed with error: {status.message}")
logger.error(
"Download %s completed with error: %s", download_id, status.message
)
status_callback("error", status.message or "Download failed")
self._safe_remove_download(client, download_id, protocol, "completion error")
self._safe_remove_download(
client, download_id, protocol, "completion error"
)
return None
# Download complete - break to handle file
logger.debug(f"Download {download_id} complete, file_path={status.file_path}")
logger.debug(
"Download %s complete, file_path=%s", download_id, status.file_path
)
break
# Check for error state
@@ -653,8 +721,10 @@ class ExternalClientHandler(DownloadHandler, ABC):
not_found_count += 1
if not_found_count < max_not_found_retries:
logger.debug(
f"Download {download_id} not yet visible in client "
f"(attempt {not_found_count}/{max_not_found_retries})"
"Download %s not yet visible in client (attempt %s/%s)",
download_id,
not_found_count,
max_not_found_retries,
)
status_callback("resolving", "Waiting for download client...")
if cancel_flag.wait(timeout=poll_interval):
@@ -662,11 +732,13 @@ class ExternalClientHandler(DownloadHandler, ABC):
continue
logger.error(
f"Download {download_id} not found after {max_not_found_retries} attempts"
"Download %s not found after %s attempts",
download_id,
max_not_found_retries,
)
else:
# Fail fast on actionable errors (auth, connectivity, API issues)
logger.error(f"Download {download_id} error state: {status.message}")
logger.error("Download %s error state: %s", download_id, status.message)
status_callback("error", status.message or "Download failed")
self._safe_remove_download(client, download_id, protocol, "download error")
@@ -725,18 +797,20 @@ class ExternalClientHandler(DownloadHandler, ABC):
return result
except Exception as e:
logger.error(f"Error during download polling: {e}")
logger.exception("Error during download polling")
status_callback("error", str(e))
self._safe_remove_download(client, download_id, protocol, "polling exception")
return None
else:
return result
def _handle_completed_file(
self,
source_path: Path,
protocol: str,
task: DownloadTask,
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[str]:
status_callback: Callable[[str, str | None], None],
) -> str | None:
"""Handle a completed download and return its path.
For external download clients (torrents/usenet), staging large payloads into TMP_DIR
@@ -751,15 +825,15 @@ class ExternalClientHandler(DownloadHandler, ABC):
if protocol == "torrent":
task.original_download_path = str(source_path)
logger.debug(f"Download complete, returning original path: {source_path}")
logger.debug("Download complete, returning original path: %s", source_path)
return str(source_path)
except Exception as e:
logger.error(f"Failed to finalize completed download at {source_path}: {e}")
logger.exception("Failed to finalize completed download at %s", source_path)
status_callback("error", f"Failed to finalize completed download: {e}")
return None
def cancel(self, task_id: str) -> bool:
"""Default cancellation (primary cancellation happens via cancel_flag)."""
logger.debug(f"Cancel requested for external client task: {task_id}")
logger.debug("Cancel requested for external client task: %s", task_id)
return True
shelfmark/download/clients/deluge.py
+91 -53
View File
@@ -12,13 +12,13 @@ Requirements:
"""
import base64
from typing import Any, Optional, Tuple
from contextlib import suppress
from typing import Any, NoReturn
from urllib.parse import urlparse
import requests
from shelfmark.core.config import config
from shelfmark.download.network import get_ssl_verify
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url
from shelfmark.download.clients import (
@@ -29,22 +29,32 @@ from shelfmark.download.clients import (
from shelfmark.download.clients.torrent_utils import (
extract_torrent_info,
)
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
MIN_DAEMON_HOST_ENTRY_LENGTH = 2
MIN_DAEMON_HOST_STATUS_ENTRY_LENGTH = 4
DOWNLOAD_COMPLETE_PROGRESS = 100
ONE_WEEK_IN_SECONDS = 604800
class DelugeRpcError(RuntimeError):
def __init__(self, message: str, code: int | None = None):
def __init__(self, message: str, code: int | None = None) -> None:
super().__init__(message)
self.code = code
def _get_error_message(error: Any) -> Tuple[str, int | None]:
def _get_error_message(error: object) -> tuple[str, int | None]:
if isinstance(error, dict):
return str(error.get("message") or error), error.get("code")
return str(error), None
def _raise_runtime_error(message: str) -> NoReturn:
raise RuntimeError(message)
@register_client("torrent")
class DelugeClient(DownloadClient):
"""Deluge download client using Deluge Web UI JSON-RPC."""
@@ -52,15 +62,17 @@ class DelugeClient(DownloadClient):
protocol = "torrent"
name = "deluge"
def __init__(self):
def __init__(self) -> None:
raw_host = str(config.get("DELUGE_HOST", "localhost") or "")
raw_port = str(config.get("DELUGE_PORT", "8112") or "8112")
password = str(config.get("DELUGE_PASSWORD", "") or "")
if not raw_host:
raise ValueError("DELUGE_HOST is required")
msg = "DELUGE_HOST is required"
raise ValueError(msg)
if not password:
raise ValueError("DELUGE_PASSWORD is required")
msg = "DELUGE_PASSWORD is required"
raise ValueError(msg)
scheme = "http"
base_path = ""
@@ -69,7 +81,8 @@ class DelugeClient(DownloadClient):
# (useful when Deluge is behind a reverse proxy path).
raw_host = normalize_http_url(raw_host, strip_trailing_slash=False) if raw_host else ""
if not raw_host:
raise ValueError("DELUGE_HOST is invalid")
msg = "DELUGE_HOST is invalid"
raise ValueError(msg)
host = raw_host
port = int(raw_port)
@@ -81,13 +94,12 @@ class DelugeClient(DownloadClient):
if parsed.port is not None:
port = parsed.port
base_path = (parsed.path or "").rstrip("/")
else:
# Allow "host:port" in DELUGE_HOST for convenience.
if ":" in raw_host and raw_host.count(":") == 1:
host_part, port_part = raw_host.split(":", 1)
if host_part and port_part.isdigit():
host = host_part
port = int(port_part)
# Allow "host:port" in DELUGE_HOST for convenience.
elif ":" in raw_host and raw_host.count(":") == 1:
host_part, port_part = raw_host.split(":", 1)
if host_part and port_part.isdigit():
host = host_part
port = int(port_part)
self._rpc_url = f"{scheme}://{host}:{port}{base_path}/json"
self._password = password
@@ -104,14 +116,19 @@ class DelugeClient(DownloadClient):
self._rpc_id += 1
return self._rpc_id
def _rpc_call(self, method: str, *params: Any, timeout: int = 15) -> Any:
def _rpc_call(self, method: str, *params: object, timeout: int = 15) -> object:
payload = {
"id": self._next_rpc_id(),
"method": method,
"params": list(params),
}
response = self._session.post(self._rpc_url, json=payload, timeout=timeout, verify=get_ssl_verify(self._rpc_url))
response = self._session.post(
self._rpc_url,
json=payload,
timeout=timeout,
verify=get_ssl_verify(self._rpc_url),
)
response.raise_for_status()
data = response.json()
@@ -124,7 +141,8 @@ class DelugeClient(DownloadClient):
def _login(self) -> None:
result = self._rpc_call("auth.login", self._password)
if result is not True:
raise DelugeRpcError("Deluge Web UI authentication failed")
msg = "Deluge Web UI authentication failed"
raise DelugeRpcError(msg)
self._authenticated = True
def _select_daemon_host_id(self, hosts: list) -> str:
@@ -133,11 +151,19 @@ class DelugeClient(DownloadClient):
preferred_hosts = {"127.0.0.1", "localhost"}
for entry in hosts:
if isinstance(entry, list) and len(entry) >= 2 and entry[1] in preferred_hosts:
if (
isinstance(entry, list)
and len(entry) >= MIN_DAEMON_HOST_ENTRY_LENGTH
and entry[1] in preferred_hosts
):
return str(entry[0])
for entry in hosts:
if isinstance(entry, list) and len(entry) >= 4 and str(entry[3]).lower() == "online":
if (
isinstance(entry, list)
and len(entry) >= MIN_DAEMON_HOST_STATUS_ENTRY_LENGTH
and str(entry[3]).lower() == "online"
):
return str(entry[0])
return str(hosts[0][0])
@@ -155,23 +181,25 @@ class DelugeClient(DownloadClient):
hosts = self._rpc_call("web.get_hosts") or []
if not hosts:
raise DelugeRpcError(
msg = (
"Deluge Web UI isn't connected to Deluge core (no hosts configured). "
"Add/connect a daemon in Deluge Web UI → Connection Manager."
)
raise DelugeRpcError(msg)
host_id = self._select_daemon_host_id(hosts)
self._rpc_call("web.connect", host_id)
if self._rpc_call("web.connected") is not True:
raise DelugeRpcError(
msg = (
"Deluge Web UI couldn't connect to Deluge core. "
"Check daemon status in Deluge Web UI → Connection Manager."
)
raise DelugeRpcError(msg)
self._connected = True
def _get_daemon_version(self) -> Any:
def _get_daemon_version(self) -> object:
"""Fetch daemon version, preferring daemon.get_version when available."""
try:
methods = self._rpc_call("system.listMethods")
@@ -190,14 +218,12 @@ class DelugeClient(DownloadClient):
try:
# label.add will error if the plugin is unavailable or the label exists.
try:
with suppress(Exception):
self._rpc_call("label.add", label)
except Exception:
pass
self._rpc_call("label.set_torrent", torrent_id, label)
except Exception as e:
logger.debug(f"Could not set Deluge label '{label}' for {torrent_id}: {e}")
logger.debug("Could not set Deluge label '%s' for %s: %s", label, torrent_id, e)
@staticmethod
def is_configured() -> bool:
@@ -206,22 +232,23 @@ class DelugeClient(DownloadClient):
password = config.get("DELUGE_PASSWORD", "")
return client == "deluge" and bool(host) and bool(password)
def test_connection(self) -> Tuple[bool, str]:
def test_connection(self) -> tuple[bool, str]:
try:
self._ensure_connected()
version = self._get_daemon_version()
return True, f"Connected to Deluge {version}"
except Exception as e:
self._authenticated = False
self._connected = False
return False, f"Connection failed: {str(e)}"
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to Deluge {version}"
def add_download(
self,
url: str,
name: str,
category: Optional[str] = None,
expected_hash: Optional[str] = None,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
) -> str:
try:
@@ -231,7 +258,7 @@ class DelugeClient(DownloadClient):
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
if not torrent_info.is_magnet and not torrent_info.torrent_data:
raise Exception("Failed to fetch torrent file")
_raise_runtime_error("Failed to fetch torrent file")
options: dict[str, Any] = {}
if self._download_dir:
@@ -252,7 +279,7 @@ class DelugeClient(DownloadClient):
else:
torrent_data = torrent_info.torrent_data
if torrent_data is None:
raise Exception("Failed to fetch torrent file")
_raise_runtime_error("Failed to fetch torrent file")
torrent_data_bytes: bytes = torrent_data
filedump = base64.b64encode(torrent_data_bytes).decode("ascii")
@@ -264,19 +291,20 @@ class DelugeClient(DownloadClient):
)
if not torrent_id:
raise Exception("Deluge returned no torrent ID")
_raise_runtime_error("Deluge returned no torrent ID")
torrent_id = str(torrent_id).lower()
self._try_set_label(torrent_id, category_value)
logger.info(f"Added torrent to Deluge: {torrent_id}")
return torrent_id
logger.info("Added torrent to Deluge: %s", torrent_id)
except Exception as e:
except Exception:
self._authenticated = False
self._connected = False
logger.error(f"Deluge add failed: {e}")
logger.exception("Deluge add failed")
raise
else:
return torrent_id
def get_status(self, download_id: str) -> DownloadStatus:
try:
@@ -285,7 +313,14 @@ class DelugeClient(DownloadClient):
status = self._rpc_call(
"core.get_torrent_status",
download_id,
["state", "progress", "download_payload_rate", "eta", "save_path", "name"],
[
"state",
"progress",
"download_payload_rate",
"eta",
"save_path",
"name",
],
)
if not status:
@@ -308,7 +343,7 @@ class DelugeClient(DownloadClient):
progress = float(status.get("progress", 0))
# Don't mark complete while files are being moved
complete = progress >= 100 and deluge_state != "Moving"
complete = progress >= DOWNLOAD_COMPLETE_PROGRESS and deluge_state != "Moving"
if complete:
message = "Complete"
@@ -320,7 +355,7 @@ class DelugeClient(DownloadClient):
except Exception:
eta = None
if eta is not None and (eta < 0 or eta > 604800):
if eta is not None and (eta < 0 or eta > ONE_WEEK_IN_SECONDS):
eta = None
file_path = None
@@ -344,24 +379,26 @@ class DelugeClient(DownloadClient):
except Exception as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, delete_files: bool = False) -> bool:
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
try:
self._ensure_connected()
result = self._rpc_call("core.remove_torrent", download_id, delete_files)
if result:
logger.info(
f"Removed torrent from Deluge: {download_id}"
+ (" (with files)" if delete_files else "")
"Removed torrent from Deluge: %s%s",
download_id,
" (with files)" if delete_files else "",
)
return True
return False
except Exception as e:
self._log_error("remove", e)
return False
else:
return False
def get_download_path(self, download_id: str) -> Optional[str]:
def get_download_path(self, download_id: str) -> str | None:
try:
self._ensure_connected()
@@ -376,15 +413,16 @@ class DelugeClient(DownloadClient):
str(status.get("save_path", "")),
str(status.get("name", "")),
)
return None
except Exception as e:
self._log_error("get_download_path", e, level="debug")
return None
else:
return None
def find_existing(
self, url: str, category: Optional[str] = None
) -> Optional[Tuple[str, DownloadStatus]]:
self, url: str, category: str | None = None
) -> tuple[str, DownloadStatus] | None:
try:
self._ensure_connected()
@@ -402,10 +440,10 @@ class DelugeClient(DownloadClient):
full_status = self.get_status(torrent_info.info_hash)
return (torrent_info.info_hash, full_status)
return None
except Exception as e:
self._authenticated = False
self._connected = False
logger.debug(f"Error checking for existing torrent: {e}")
logger.debug("Error checking for existing torrent: %s", e)
return None
else:
return None
shelfmark/download/clients/nzbget.py
+79 -68
View File
@@ -1,24 +1,22 @@
"""
NZBGet download client for Prowlarr integration.
"""NZBGet download client for Prowlarr integration.
Uses NZBGet's JSON-RPC API directly via requests (no external dependency).
"""
import json
from typing import Any, Optional, Tuple
import requests
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url
from shelfmark.download.network import get_ssl_verify
from shelfmark.download.clients import (
DownloadClient,
DownloadStatus,
register_client,
with_retry,
)
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
@@ -30,15 +28,17 @@ class NZBGetClient(DownloadClient):
protocol = "usenet"
name = "nzbget"
def __init__(self):
def __init__(self) -> None:
"""Initialize NZBGet client with settings from config."""
raw_url = config.get("NZBGET_URL", "")
if not raw_url:
raise ValueError("NZBGET_URL is required")
msg = "NZBGET_URL is required"
raise ValueError(msg)
self.url = normalize_http_url(raw_url)
if not self.url:
raise ValueError("NZBGET_URL is invalid")
msg = "NZBGET_URL is invalid"
raise ValueError(msg)
self.username = config.get("NZBGET_USERNAME", "nzbget")
self.password = config.get("NZBGET_PASSWORD", "")
self._category = config.get("NZBGET_CATEGORY", "Books")
@@ -50,10 +50,22 @@ class NZBGetClient(DownloadClient):
url = normalize_http_url(config.get("NZBGET_URL", ""))
return client == "nzbget" and bool(url)
def _try_remove_command(
self, command: str, nzb_id: int, download_id: str
) -> tuple[bool, Exception | None]:
"""Try one NZBGet delete command and return any error."""
try:
result = self._rpc_call("editqueue", [command, 0, "", nzb_id])
if result:
logger.info("Removed NZB from NZBGet (%s): %s", command, download_id)
return True, None
except Exception as e:
return False, e
return False, None
@with_retry()
def _rpc_call(self, method: str, params: Optional[list] = None) -> Any:
"""
Make a JSON-RPC call to NZBGet.
def _rpc_call(self, method: str, params: list | None = None) -> object:
"""Make a JSON-RPC call to NZBGet.
Args:
method: RPC method name
@@ -64,15 +76,19 @@ class NZBGetClient(DownloadClient):
Raises:
Exception: If RPC call fails after retries.
"""
rpc_url = f"{self.url}/jsonrpc"
payload = json.dumps({
"jsonrpc": "2.0",
"id": 1,
"method": method,
"params": params or [],
}, separators=(',', ':'))
payload = json.dumps(
{
"jsonrpc": "2.0",
"id": 1,
"method": method,
"params": params or [],
},
separators=(",", ":"),
)
response = requests.post(
rpc_url,
@@ -85,34 +101,34 @@ class NZBGetClient(DownloadClient):
response.raise_for_status()
result = response.json()
if "error" in result and result["error"]:
raise Exception(result["error"].get("message", "RPC error"))
if result.get("error"):
raise RuntimeError(result["error"].get("message", "RPC error"))
return result.get("result")
def test_connection(self) -> Tuple[bool, str]:
def test_connection(self) -> tuple[bool, str]:
"""Test connection to NZBGet."""
try:
status = self._rpc_call("status")
version = status.get("Version", "unknown")
return True, f"Connected to NZBGet {version}"
except requests.exceptions.ConnectionError:
return False, "Could not connect to NZBGet"
except requests.exceptions.Timeout:
return False, "Connection timed out"
except Exception as e:
return False, f"Connection failed: {str(e)}"
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to NZBGet {version}"
def add_download(
self,
url: str,
name: str,
category: Optional[str] = None,
expected_hash: Optional[str] = None,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
) -> str:
"""
Add NZB by URL.
"""Add NZB by URL.
Fetches the NZB content from the URL (e.g., Prowlarr proxy) and sends
it base64-encoded to NZBGet, since NZBGet may not handle redirects well.
@@ -128,21 +144,26 @@ class NZBGetClient(DownloadClient):
Raises:
Exception: If adding fails.
"""
import base64
# Use configured category if not explicitly provided
category = category or self._category
def _raise_invalid_nzb_id() -> None:
msg = "NZBGet returned invalid ID"
raise RuntimeError(msg)
try:
# Fetch NZB content from the URL (handles Prowlarr proxy redirects)
logger.debug(f"Fetching NZB from: {url}")
logger.debug("Fetching NZB from: %s", url)
response = requests.get(url, timeout=30, verify=get_ssl_verify(url))
response.raise_for_status()
nzb_content = base64.b64encode(response.content).decode('ascii')
nzb_content = base64.b64encode(response.content).decode("ascii")
# Ensure filename has .nzb extension
nzb_filename = name if name.endswith('.nzb') else f"{name}.nzb"
nzb_filename = name if name.endswith(".nzb") else f"{name}.nzb"
# NZBGet append method parameters (all 10 required):
# NZBFilename, Content, Category, Priority, AddToTop, AddPaused,
@@ -164,26 +185,27 @@ class NZBGetClient(DownloadClient):
)
if nzb_id and nzb_id > 0:
logger.info(f"Added NZB to NZBGet: {nzb_id}")
logger.info("Added NZB to NZBGet: %s", nzb_id)
return str(nzb_id)
raise Exception("NZBGet returned invalid ID")
_raise_invalid_nzb_id()
except requests.RequestException as e:
logger.error(f"Failed to fetch NZB from URL: {e}")
raise Exception(f"Failed to fetch NZB: {e}")
except Exception as e:
logger.error(f"NZBGet add failed: {e}")
logger.exception("Failed to fetch NZB from URL")
msg = f"Failed to fetch NZB: {e}"
raise RuntimeError(msg) from e
except Exception:
logger.exception("NZBGet add failed")
raise
def get_status(self, download_id: str) -> DownloadStatus:
"""
Get NZB status by ID.
"""Get NZB status by ID.
Args:
download_id: NZBGet NZBID
Returns:
Current download status.
"""
try:
nzb_id = int(download_id)
@@ -195,18 +217,12 @@ class NZBGetClient(DownloadClient):
if group.get("NZBID") == nzb_id:
# Calculate progress
# NZBGet uses Hi/Lo for 64-bit values on 32-bit systems
file_size = (group.get("FileSizeHi", 0) << 32) + group.get(
"FileSizeLo", 0
)
file_size = (group.get("FileSizeHi", 0) << 32) + group.get("FileSizeLo", 0)
remaining = (group.get("RemainingSizeHi", 0) << 32) + group.get(
"RemainingSizeLo", 0
)
progress = (
((file_size - remaining) / file_size * 100)
if file_size > 0
else 0
)
progress = ((file_size - remaining) / file_size * 100) if file_size > 0 else 0
status = group.get("Status", "")
# Map NZBGet status to our states
@@ -229,9 +245,7 @@ class NZBGetClient(DownloadClient):
file_path=None,
download_speed=group.get("DownloadRate"),
eta=(
group.get("RemainingSec")
if group.get("RemainingSec", 0) > 0
else None
group.get("RemainingSec") if group.get("RemainingSec", 0) > 0 else None
),
)
@@ -254,7 +268,6 @@ class NZBGetClient(DownloadClient):
else:
file_path = None
if "SUCCESS" in status:
return DownloadStatus(
progress=100,
@@ -263,21 +276,20 @@ class NZBGetClient(DownloadClient):
complete=True,
file_path=file_path,
)
else:
return DownloadStatus(
progress=100,
state="error",
message=f"Download failed: {status}",
complete=True,
file_path=file_path,
)
return DownloadStatus(
progress=100,
state="error",
message=f"Download failed: {status}",
complete=True,
file_path=file_path,
)
# Not found in queue or history
return DownloadStatus.error("Download not found")
except Exception as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, delete_files: bool = False) -> bool:
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
"""Remove a download from NZBGet.
NZBGet can remove items from either the active queue (Group* commands) or from
@@ -289,6 +301,7 @@ class NZBGetClient(DownloadClient):
Returns:
True if successful.
"""
try:
nzb_id = int(download_id)
@@ -303,29 +316,27 @@ class NZBGetClient(DownloadClient):
else:
commands = ["GroupDelete", "HistoryDelete"]
last_error: Optional[Exception] = None
last_error: Exception | None = None
for command in commands:
try:
result = self._rpc_call("editqueue", [command, 0, "", nzb_id])
if result:
logger.info(f"Removed NZB from NZBGet ({command}): {download_id}")
return True
except Exception as e:
last_error = e
success, error = self._try_remove_command(command, nzb_id, download_id)
if success:
return True
if error is not None:
last_error = error
if last_error is not None:
self._log_error("remove", last_error)
return False
def get_download_path(self, download_id: str) -> Optional[str]:
"""
Get the path where NZB files are located.
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where NZB files are located.
Args:
download_id: NZBGet NZBID
Returns:
Destination directory, or None.
"""
status = self.get_status(download_id)
return status.file_path
shelfmark/download/clients/qbittorrent.py
+112 -80
View File
@@ -1,13 +1,14 @@
"""qBittorrent download client for Prowlarr integration."""
import time
from http import HTTPStatus
from pathlib import Path
from types import SimpleNamespace
from typing import Optional, Tuple
from typing import NoReturn
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url
from shelfmark.download.network import get_ssl_verify
from shelfmark.download.clients import (
DownloadClient,
DownloadStatus,
@@ -16,22 +17,33 @@ from shelfmark.download.clients import (
from shelfmark.download.clients.torrent_utils import (
extract_torrent_info,
)
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_HASH_LENGTH_40 = 40
_HASH_LENGTH_ED2K = 32
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
_HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
_ONE_WEEK_IN_SECONDS = 604800
def _hashes_match(hash1: str, hash2: str) -> bool:
"""Compare hashes, handling Amarr's 40-char zero-padded hashes vs 32-char ed2k hashes."""
h1, h2 = hash1.lower(), hash2.lower()
if h1 == h2:
return True
if len(h1) == 40 and len(h2) == 32 and h1.endswith("00000000"):
return h1[:32] == h2
if len(h2) == 40 and len(h1) == 32 and h2.endswith("00000000"):
return h2[:32] == h1
if len(h1) == _HASH_LENGTH_40 and len(h2) == _HASH_LENGTH_ED2K and h1.endswith("00000000"):
return h1[:_HASH_LENGTH_ED2K] == h2
if len(h2) == _HASH_LENGTH_40 and len(h1) == _HASH_LENGTH_ED2K and h2.endswith("00000000"):
return h2[:_HASH_LENGTH_ED2K] == h1
return False
def _raise_runtime_error(message: str) -> NoReturn:
raise RuntimeError(message)
def _normalize_tags(raw_tags: object) -> list[str]:
"""Normalize tag input to a clean, de-duplicated list of strings."""
if raw_tags is None:
@@ -82,7 +94,7 @@ def _is_explicit_add_failure(raw_result: object) -> bool:
class QBittorrentClient(DownloadClient):
"""qBittorrent download client."""
def _is_torrent_loaded(self, torrent_hash: str) -> tuple[bool, Optional[str]]:
def _is_torrent_loaded(self, torrent_hash: str) -> tuple[bool, str | None]:
"""Check whether qBittorrent has registered a torrent yet.
Uses `/api/v2/torrents/properties?hash=<hash>`.
@@ -92,6 +104,7 @@ class QBittorrentClient(DownloadClient):
Notes:
A false result with no error means "not loaded yet".
"""
import requests
@@ -103,23 +116,24 @@ class QBittorrentClient(DownloadClient):
response = self._client._session.get(url, params=params, timeout=10)
# Re-authenticate and retry once on 403
if response.status_code == 403:
logger.debug("qBittorrent returned 403 for properties; re-authenticating and retrying")
if response.status_code == _HTTP_STATUS_FORBIDDEN:
logger.debug(
"qBittorrent returned 403 for properties; re-authenticating and retrying"
)
self._client.auth_log_in()
response = self._client._session.get(url, params=params, timeout=10)
if response.status_code == 403:
if response.status_code == _HTTP_STATUS_FORBIDDEN:
return False, "qBittorrent authentication failed (HTTP 403)"
# qBittorrent returns 404/409-ish responses depending on version when missing.
if response.status_code == 404:
if response.status_code == _HTTP_STATUS_NOT_FOUND:
return False, None
response.raise_for_status()
return True, None
except requests.exceptions.HTTPError as e:
status = getattr(getattr(e, "response", None), "status_code", None)
if status == 404:
if status == _HTTP_STATUS_NOT_FOUND:
return False, None
if status:
return False, f"qBittorrent API request failed (HTTP {status})"
@@ -130,23 +144,27 @@ class QBittorrentClient(DownloadClient):
return False, f"qBittorrent request timed out at {self._base_url}"
except Exception as e:
return False, f"qBittorrent API error: {type(e).__name__}: {e}"
else:
return True, None
protocol = "torrent"
name = "qbittorrent"
def __init__(self):
def __init__(self) -> None:
"""Initialize qBittorrent client with settings from config."""
# Lazy import to avoid dependency issues if not using torrents
from qbittorrentapi import Client
raw_url = config.get("QBITTORRENT_URL", "")
if not raw_url:
raise ValueError("QBITTORRENT_URL is required")
msg = "QBITTORRENT_URL is required"
raise ValueError(msg)
# We use `_base_url` for direct HTTP calls, so it must be a fully-qualified URL.
self._base_url = normalize_http_url(raw_url)
if not self._base_url:
raise ValueError("QBITTORRENT_URL is invalid")
msg = "QBITTORRENT_URL is invalid"
raise ValueError(msg)
# qbittorrent-api accepts either a full URL or host:port; prefer the normalized URL
# for consistency.
@@ -160,10 +178,9 @@ class QBittorrentClient(DownloadClient):
self._download_dir = config.get("QBITTORRENT_DOWNLOAD_DIR", "")
self._tags = _normalize_tags(config.get("QBITTORRENT_TAG", []))
def _get_torrents_info(
self, torrent_hash: Optional[str] = None
) -> tuple[list[SimpleNamespace], Optional[str]]:
self, torrent_hash: str | None = None
) -> tuple[list[SimpleNamespace], str | None]:
"""Get torrent info using GET.
Behaviors:
@@ -174,6 +191,7 @@ class QBittorrentClient(DownloadClient):
Returns:
(torrents, error_message)
"""
import requests
@@ -188,13 +206,13 @@ class QBittorrentClient(DownloadClient):
response: requests.Response,
*,
request_params: dict[str, str],
) -> tuple[list[SimpleNamespace], Optional[str]]:
if response.status_code == 403:
) -> tuple[list[SimpleNamespace], str | None]:
if response.status_code == _HTTP_STATUS_FORBIDDEN:
logger.debug("qBittorrent returned 403; re-authenticating and retrying")
self._client.auth_log_in()
response = self._client._session.get(url, params=request_params, timeout=10)
if response.status_code == 403:
if response.status_code == _HTTP_STATUS_FORBIDDEN:
logger.warning("qBittorrent authentication failed (HTTP 403)")
return [], "qBittorrent authentication failed (HTTP 403)"
@@ -236,24 +254,22 @@ class QBittorrentClient(DownloadClient):
return all_torrents, None
return torrents, None
except requests.exceptions.HTTPError as e:
status = getattr(getattr(e, "response", None), "status_code", None)
if status:
logger.warning(f"qBittorrent API error (HTTP {status}): {e}")
logger.warning("qBittorrent API error (HTTP %s): %s", status, e)
return [], f"qBittorrent API request failed (HTTP {status})"
logger.warning(f"qBittorrent API error: {e}")
logger.warning("qBittorrent API error: %s", e)
return [], "qBittorrent API request failed"
except requests.exceptions.ConnectionError:
logger.warning(f"Cannot connect to qBittorrent at {self._base_url}")
logger.warning("Cannot connect to qBittorrent at %s", self._base_url)
return [], f"Cannot connect to qBittorrent at {self._base_url}"
except requests.exceptions.Timeout:
logger.warning(f"qBittorrent request timed out at {self._base_url}")
logger.warning("qBittorrent request timed out at %s", self._base_url)
return [], f"qBittorrent request timed out at {self._base_url}"
except Exception as e:
logger.debug(f"Failed to get torrents info: {e}")
logger.debug("Failed to get torrents info: %s", e)
# requests raises InvalidSchema when the base URL doesn't include http(s)
if type(e).__name__ == "InvalidSchema":
return (
@@ -262,6 +278,8 @@ class QBittorrentClient(DownloadClient):
f"Configured: {self._base_url}",
)
return [], f"qBittorrent API error: {type(e).__name__}: {e}"
else:
return torrents, None
@staticmethod
def is_configured() -> bool:
@@ -270,14 +288,15 @@ class QBittorrentClient(DownloadClient):
url = normalize_http_url(config.get("QBITTORRENT_URL", ""))
return client == "qbittorrent" and bool(url)
def test_connection(self) -> Tuple[bool, str]:
def test_connection(self) -> tuple[bool, str]:
"""Test connection to qBittorrent."""
try:
self._client.auth_log_in()
api_version = self._client.app.web_api_version
return True, f"Connected to qBittorrent (API v{api_version})"
except Exception as e:
return False, f"Connection failed: {str(e)}"
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to qBittorrent (API v{api_version})"
def add_download(
self,
@@ -287,8 +306,7 @@ class QBittorrentClient(DownloadClient):
expected_hash: str | None = None,
**kwargs,
) -> str:
"""
Add torrent by URL (magnet or .torrent).
"""Add torrent by URL (magnet or .torrent).
Args:
url: Magnet link or .torrent URL
@@ -301,6 +319,7 @@ class QBittorrentClient(DownloadClient):
Raises:
Exception: If adding fails.
"""
try:
# Use configured category if not explicitly provided
@@ -316,7 +335,10 @@ class QBittorrentClient(DownloadClient):
# Log other errors but continue since download may still work
if "Conflict" not in type(e).__name__ and "409" not in str(e):
logger.debug(
f"Could not create category '{category}': {type(e).__name__}: {e}"
"Could not create category '%s': %s: %s",
category,
type(e).__name__,
e,
)
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
@@ -356,13 +378,13 @@ class QBittorrentClient(DownloadClient):
)
result_text = _normalize_add_result(result)
logger.debug(f"qBittorrent add result: {result_text}")
logger.debug("qBittorrent add result: %s", result_text)
if not expected_hash:
raise Exception("Could not determine torrent hash from URL")
_raise_runtime_error("Could not determine torrent hash from URL")
if _is_explicit_add_failure(result):
raise Exception(f"Failed to add torrent: {result_text}")
_raise_runtime_error(f"Failed to add torrent: {result_text}")
# Some qBittorrent-compatible clients return HTTP 200 with an empty body
# instead of qBittorrent's literal "Ok." response. Prefer verifying that
@@ -370,30 +392,31 @@ class QBittorrentClient(DownloadClient):
for _ in range(10):
loaded, error = self._is_torrent_loaded(expected_hash)
if error:
logger.debug(f"qBittorrent add_download: {error}")
logger.debug("qBittorrent add_download: %s", error)
if loaded:
logger.info(f"Added torrent: {expected_hash}")
logger.info("Added torrent: %s", expected_hash)
return expected_hash.lower()
time.sleep(0.5)
logger.warning(
"Torrent add was not confirmed within the visibility grace period "
f"(response={result_text or '<empty>'}), returning expected hash"
"Torrent add was not confirmed within the visibility grace period (response=%s), returning expected hash",
result_text,
)
return expected_hash
except Exception as e:
logger.error(f"qBittorrent add failed: {e}")
except Exception:
logger.exception("qBittorrent add failed")
raise
else:
return expected_hash
def get_status(self, download_id: str) -> DownloadStatus:
"""
Get torrent status by hash.
"""Get torrent status by hash.
Args:
download_id: Torrent info_hash
Returns:
Current download status.
"""
try:
torrents, error = self._get_torrents_info(download_id)
@@ -405,7 +428,7 @@ class QBittorrentClient(DownloadClient):
t
for t in torrents
if isinstance(getattr(t, "hash", None), str)
and _hashes_match(getattr(t, "hash"), download_id)
and _hashes_match(t.hash, download_id)
),
None,
)
@@ -414,7 +437,10 @@ class QBittorrentClient(DownloadClient):
# Map qBittorrent states to our states and user-friendly messages
state_info = {
"downloading": ("downloading", None), # None = use default progress message
"downloading": (
"downloading",
None,
), # None = use default progress message
"stalledDL": ("downloading", "Stalled"),
"metaDL": ("downloading", "Fetching metadata"),
"forcedDL": ("downloading", None),
@@ -449,7 +475,11 @@ class QBittorrentClient(DownloadClient):
message = "Complete"
torrent_eta = getattr(torrent, "eta", 0)
eta = torrent_eta if isinstance(torrent_eta, int) and 0 < torrent_eta < 604800 else None
eta = (
torrent_eta
if isinstance(torrent_eta, int) and 0 < torrent_eta < _ONE_WEEK_IN_SECONDS
else None
)
# Get file path for completed downloads
file_path = None
@@ -471,9 +501,8 @@ class QBittorrentClient(DownloadClient):
except Exception as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, delete_files: bool = False) -> bool:
"""
Remove a torrent from qBittorrent.
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
"""Remove a torrent from qBittorrent.
Args:
download_id: Torrent info_hash
@@ -481,21 +510,22 @@ class QBittorrentClient(DownloadClient):
Returns:
True if successful.
"""
try:
self._client.torrents_delete(
torrent_hashes=download_id, delete_files=delete_files
)
self._client.torrents_delete(torrent_hashes=download_id, delete_files=delete_files)
logger.info(
f"Removed torrent from qBittorrent: {download_id}"
+ (" (with files)" if delete_files else "")
"Removed torrent from qBittorrent: %s%s",
download_id,
" (with files)" if delete_files else "",
)
return True
except Exception as e:
self._log_error("remove", e)
return False
else:
return True
def get_download_path(self, download_id: str) -> Optional[str]:
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where torrent files are located.
Prefer `content_path` when available.
@@ -506,12 +536,10 @@ class QBittorrentClient(DownloadClient):
- `/api/v2/torrents/files?hash=<hash>` for the first file name
- join `save_path` with the torrent's top-level directory
"""
import os
try:
torrents, error = self._get_torrents_info(download_id)
if error:
logger.debug(f"qBittorrent get_download_path: {error}")
logger.debug("qBittorrent get_download_path: %s", error)
return None
torrent = next(
@@ -519,7 +547,7 @@ class QBittorrentClient(DownloadClient):
t
for t in torrents
if isinstance(getattr(t, "hash", None), str)
and _hashes_match(getattr(t, "hash"), download_id)
and _hashes_match(t.hash, download_id)
),
None,
)
@@ -531,7 +559,7 @@ class QBittorrentClient(DownloadClient):
self._log_error("get_download_path", e, level="debug")
return None
def _resolve_completed_download_path(self, torrent: SimpleNamespace) -> Optional[str]:
def _resolve_completed_download_path(self, torrent: SimpleNamespace) -> str | None:
"""Resolve the completed path for a torrent.
Centralizes the logic shared by `get_status()` and `get_download_path()`:
@@ -539,7 +567,6 @@ class QBittorrentClient(DownloadClient):
- otherwise derive via properties+files
- finally fall back to `save_path + name`
"""
# Prefer content_path, but treat content_path == save_path as invalid.
content_path = getattr(torrent, "content_path", "")
save_path = getattr(torrent, "save_path", "")
@@ -558,19 +585,20 @@ class QBittorrentClient(DownloadClient):
getattr(torrent, "name", ""),
)
def _derive_download_path_from_files(self, download_id: str) -> Optional[str]:
def _derive_download_path_from_files(self, download_id: str) -> str | None:
"""Derive completed download path using `/torrents/properties` + `/torrents/files`.
This mirrors how common automation apps derive the path when
`content_path` isn't provided.
"""
import os
import requests
def get_with_auth(url: str, params: dict[str, str]) -> requests.Response:
self._client.auth_log_in()
resp = self._client._session.get(url, params=params, timeout=10)
if resp.status_code == 403:
if resp.status_code == _HTTP_STATUS_FORBIDDEN:
logger.debug("qBittorrent returned 403; re-authenticating and retrying")
self._client.auth_log_in()
resp = self._client._session.get(url, params=params, timeout=10)
@@ -581,7 +609,7 @@ class QBittorrentClient(DownloadClient):
files_url = f"{self._base_url}/api/v2/torrents/files"
props_resp = get_with_auth(properties_url, {"hash": download_id})
if props_resp.status_code == 404:
if props_resp.status_code == _HTTP_STATUS_NOT_FOUND:
return None
props_resp.raise_for_status()
props = props_resp.json() if isinstance(props_resp.json(), dict) else {}
@@ -591,7 +619,7 @@ class QBittorrentClient(DownloadClient):
return None
files_resp = get_with_auth(files_url, {"hash": download_id})
if files_resp.status_code == 404:
if files_resp.status_code == _HTTP_STATUS_NOT_FOUND:
return None
files_resp.raise_for_status()
files = files_resp.json() if isinstance(files_resp.json(), list) else []
@@ -608,14 +636,18 @@ class QBittorrentClient(DownloadClient):
if not top_level:
return None
return os.path.normpath(os.path.join(save_path, top_level))
return os.path.normpath(str(Path(save_path) / top_level))
except Exception as e:
logger.debug(f"qBittorrent could not derive path from files: {type(e).__name__}: {e}")
logger.debug(
"qBittorrent could not derive path from files: %s: %s",
type(e).__name__,
e,
)
return None
def find_existing(
self, url: str, category: Optional[str] = None
) -> Optional[Tuple[str, DownloadStatus]]:
self, url: str, category: str | None = None
) -> tuple[str, DownloadStatus] | None:
"""Check if a torrent for this URL already exists in qBittorrent."""
try:
torrent_info = extract_torrent_info(url)
@@ -624,7 +656,7 @@ class QBittorrentClient(DownloadClient):
torrents, error = self._get_torrents_info(torrent_info.info_hash)
if error:
logger.debug(f"qBittorrent find_existing: {error}")
logger.debug("qBittorrent find_existing: %s", error)
return None
torrent = next(
@@ -632,15 +664,15 @@ class QBittorrentClient(DownloadClient):
t
for t in torrents
if isinstance(getattr(t, "hash", None), str)
and _hashes_match(getattr(t, "hash"), torrent_info.info_hash)
and _hashes_match(t.hash, torrent_info.info_hash)
),
None,
)
if torrent and isinstance(getattr(torrent, "hash", None), str):
torrent_hash = getattr(torrent, "hash")
torrent_hash = torrent.hash
return (torrent_hash.lower(), self.get_status(torrent_hash.lower()))
return None
except Exception as e:
logger.debug(f"Error checking for existing torrent: {e}")
logger.debug("Error checking for existing torrent: %s", e)
return None
else:
return None
shelfmark/download/clients/rtorrent.py
+85 -66
View File
@@ -1,17 +1,15 @@
"""
rTorrent download client for Prowlarr integration.
"""rTorrent download client for Prowlarr integration.
Uses xmlrpc to communicate with rTorrent's RPC interface.
"""
import ssl
from typing import Any, Optional, Tuple
from typing import NoReturn
from urllib.parse import urlparse
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url, get_hardened_xmlrpc_client
from shelfmark.download.network import get_ssl_verify
from shelfmark.core.utils import get_hardened_xmlrpc_client, normalize_http_url
from shelfmark.download.clients import (
DownloadClient,
DownloadStatus,
@@ -20,11 +18,15 @@ from shelfmark.download.clients import (
from shelfmark.download.clients.torrent_utils import (
extract_torrent_info,
)
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
def _create_rtorrent_server_proxy(url: str) -> Any:
_ETA_MAX_SECONDS = 604800
def _create_rtorrent_server_proxy(url: str) -> object:
"""Create an XML-RPC ServerProxy honoring certificate validation mode."""
xmlrpc_client = get_hardened_xmlrpc_client()
@@ -39,6 +41,10 @@ def _create_rtorrent_server_proxy(url: str) -> Any:
return xmlrpc_client.ServerProxy(url)
def _raise_runtime_error(message: str) -> NoReturn:
raise RuntimeError(message)
@register_client("torrent")
class RTorrentClient(DownloadClient):
"""rTorrent download client using xmlrpc."""
@@ -46,24 +52,24 @@ class RTorrentClient(DownloadClient):
protocol = "torrent"
name = "rtorrent"
def __init__(self):
def __init__(self) -> None:
"""Initialize rTorrent client with settings from config."""
raw_url = config.get("RTORRENT_URL", "")
if not raw_url:
raise ValueError("RTORRENT_URL is required")
msg = "RTORRENT_URL is required"
raise ValueError(msg)
self._base_url = normalize_http_url(raw_url)
if not self._base_url:
raise ValueError("RTORRENT_URL is invalid")
msg = "RTORRENT_URL is invalid"
raise ValueError(msg)
username = config.get("RTORRENT_USERNAME", "")
password = config.get("RTORRENT_PASSWORD", "")
if username and password:
parsed = urlparse(self._base_url)
self._base_url = (
f"{parsed.scheme}://{username}:{password}@{parsed.netloc}{parsed.path}"
)
self._base_url = f"{parsed.scheme}://{username}:{password}@{parsed.netloc}{parsed.path}"
self._rpc = _create_rtorrent_server_proxy(self._base_url)
self._download_dir = config.get("RTORRENT_DOWNLOAD_DIR", "")
@@ -76,24 +82,24 @@ class RTorrentClient(DownloadClient):
url = normalize_http_url(config.get("RTORRENT_URL", ""))
return client == "rtorrent" and bool(url)
def test_connection(self) -> Tuple[bool, str]:
def test_connection(self) -> tuple[bool, str]:
"""Test connection to rTorrent."""
try:
version = self._rpc.system.client_version()
return True, f"Connected to rTorrent {version}"
except Exception as e:
return False, f"Connection failed: {str(e)}"
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to rTorrent {version}"
def add_download(
self,
url: str,
name: str,
category: Optional[str] = None,
expected_hash: Optional[str] = None,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
) -> str:
"""
Add torrent by URL (magnet or .torrent).
"""Add torrent by URL (magnet or .torrent).
Args:
url: Magnet link or .torrent URL
@@ -106,6 +112,7 @@ class RTorrentClient(DownloadClient):
Raises:
Exception: If adding fails.
"""
try:
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
@@ -114,44 +121,53 @@ class RTorrentClient(DownloadClient):
label = category or self._label
if label:
logger.debug(f"Setting rTorrent label: {label}")
logger.debug("Setting rTorrent label: %s", label)
commands.append(f"d.custom1.set={label}")
download_dir = self._download_dir or self._get_download_dir()
if download_dir:
logger.debug(f"Setting rTorrent download directory: {download_dir}")
logger.debug("Setting rTorrent download directory: %s", download_dir)
commands.append(f"d.directory.set={download_dir}")
if torrent_info.torrent_data:
logger.debug(f"Adding torrent data directly to rTorrent for: {name} with commands: {commands} with data size: {len(torrent_info.torrent_data)}")
self._rpc.load.raw_start(
"", torrent_info.torrent_data, ";".join(commands)
logger.debug(
"Adding torrent data directly to rTorrent for: %s with commands: %s with data size: %s",
name,
commands,
len(torrent_info.torrent_data),
)
self._rpc.load.raw_start("", torrent_info.torrent_data, ";".join(commands))
else:
logger.debug(f"Adding torrent URL to rTorrent for: {name} with commands: {commands} with URL: {url}")
logger.debug(
"Adding torrent URL to rTorrent for: %s with commands: %s with URL: %s",
name,
commands,
url,
)
add_url = torrent_info.magnet_url or url
self._rpc.load.start("", add_url, ";".join(commands))
torrent_hash = torrent_info.info_hash or expected_hash
if not torrent_hash:
raise Exception("Could not determine torrent hash from URL")
_raise_runtime_error("Could not determine torrent hash from URL")
logger.debug(f"Added torrent to rTorrent: {torrent_hash}")
logger.debug("Added torrent to rTorrent: %s", torrent_hash)
except Exception:
logger.exception("rTorrent add failed")
raise
else:
return torrent_hash
except Exception as e:
logger.error(f"rTorrent add failed: {e}")
raise
def get_status(self, download_id: str) -> DownloadStatus:
"""
Get torrent status by hash.
"""Get torrent status by hash.
Args:
download_id: Torrent info_hash
Returns:
Current download status.
"""
try:
# rtorrent is somehow case sensitive and requires uppercase hashes for look
@@ -169,25 +185,29 @@ class RTorrentClient(DownloadClient):
"d.complete=",
)
torrent_list = [t for t in all_torrents if t and t[0] == download_id]
logger.debug(f"Fetched torrent status from rTorrent for: {download_id} - {torrent_list}")
logger.debug(
"Fetched torrent status from rTorrent for: %s - %s",
download_id,
torrent_list,
)
if not torrent_list:
logger.warning(f"Torrent not found in rTorrent: {download_id}")
logger.warning("Torrent not found in rTorrent: %s", download_id)
return DownloadStatus.error("Torrent not found")
torrent = torrent_list[0]
if not torrent:
logger.warning(f"Torrent data is empty for: {download_id}")
logger.warning("Torrent data is empty for: %s", download_id)
return DownloadStatus.error("Torrent not found")
logger.debug(f"Torrent data for {download_id}: {torrent}")
logger.debug("Torrent data for %s: %s", download_id, torrent)
(
torrent_hash,
_torrent_hash,
state,
bytes_downloaded,
bytes_total,
down_rate,
up_rate,
custom_category,
_up_rate,
_custom_category,
complete,
) = torrent
@@ -198,10 +218,7 @@ class RTorrentClient(DownloadClient):
complete = bool(complete)
if bytes_total > 0:
progress = (bytes_downloaded / bytes_total) * 100
else:
progress = 0
progress = (bytes_downloaded / bytes_total) * 100 if bytes_total > 0 else 0
bytes_left = max(0, bytes_total - bytes_downloaded)
@@ -221,7 +238,7 @@ class RTorrentClient(DownloadClient):
eta = None
if down_rate > 0 and bytes_left > 0:
eta_seconds = bytes_left / down_rate
if eta_seconds < 604800:
if eta_seconds < _ETA_MAX_SECONDS:
eta = int(eta_seconds)
file_path = None
@@ -240,12 +257,11 @@ class RTorrentClient(DownloadClient):
except Exception as e:
error_type = type(e).__name__
logger.error(f"rTorrent get_status failed ({error_type}): {e}")
logger.exception("rTorrent get_status failed (%s)", error_type)
return DownloadStatus.error(f"{error_type}: {e}")
def remove(self, download_id: str, delete_files: bool = False) -> bool:
"""
Remove a torrent from rTorrent.
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
"""Remove a torrent from rTorrent.
Args:
download_id: Torrent info_hash
@@ -253,6 +269,7 @@ class RTorrentClient(DownloadClient):
Returns:
True if successful.
"""
try:
if delete_files:
@@ -263,35 +280,37 @@ class RTorrentClient(DownloadClient):
self._rpc.d.erase(download_id)
logger.info(
f"Removed torrent from rTorrent: {download_id}"
+ (" (with files)" if delete_files else "")
"Removed torrent from rTorrent: %s%s",
download_id,
" (with files)" if delete_files else "",
)
return True
except Exception as e:
error_type = type(e).__name__
logger.error(f"rTorrent remove failed ({error_type}): {e}")
logger.exception("rTorrent remove failed (%s)", error_type)
return False
else:
return True
def get_download_path(self, download_id: str) -> Optional[str]:
"""
Get the path where torrent files are located.
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where torrent files are located.
Args:
download_id: Torrent info_hash
Returns:
Content path (file or directory), or None.
"""
try:
return self._get_torrent_path(download_id)
except Exception as e:
error_type = type(e).__name__
logger.debug(f"rTorrent get_download_path failed ({error_type}): {e}")
logger.debug("rTorrent get_download_path failed (%s): %s", error_type, e)
return None
def find_existing(
self, url: str, category: Optional[str] = None
) -> Optional[Tuple[str, DownloadStatus]]:
self, url: str, category: str | None = None
) -> tuple[str, DownloadStatus] | None:
"""Check if a torrent for this URL already exists in rTorrent."""
try:
torrent_info = extract_torrent_info(url)
@@ -304,21 +323,20 @@ class RTorrentClient(DownloadClient):
return (torrent_info.info_hash, status)
except Exception:
pass
return None
except Exception as e:
logger.debug(f"Error checking for existing torrent: {e}")
logger.debug("Error checking for existing torrent: %s", e)
return None
else:
return None
def _get_download_dir(self) -> str:
"""Get the download directory from rTorrent config."""
try:
download_dir = self._rpc.directory.default()
return download_dir
return self._rpc.directory.default()
except Exception:
return "/downloads"
def _get_torrent_path(self, download_id: str) -> Optional[str]:
def _get_torrent_path(self, download_id: str) -> str | None:
"""Get the file path of a torrent by hash.
Uses `d.base_path` for the item output path. In the xmlrpc interface
@@ -337,6 +355,7 @@ class RTorrentClient(DownloadClient):
if not details:
return None
path = details[0][0]
return path if path else None
except Exception:
return None
else:
return path or None
shelfmark/download/clients/sabnzbd.py
+105 -82
View File
@@ -1,10 +1,8 @@
"""
SABnzbd download client for Prowlarr integration.
"""SABnzbd download client for Prowlarr integration.
Uses SABnzbd's REST API directly via requests (no external dependency).
"""
from typing import Any, Optional, Tuple
from urllib.parse import urlparse
import requests
@@ -12,38 +10,41 @@ import requests
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url
from shelfmark.download.network import get_ssl_verify
from shelfmark.download.clients import (
DownloadClient,
DownloadStatus,
register_client,
with_retry,
)
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_ETA_PART_COUNT = 3
_SPEED_PARTS_MIN = 2
def _parse_eta(eta_str: str) -> Optional[int]:
def _parse_eta(eta_str: str) -> int | None:
"""Parse SABnzbd ETA string (format: 'H:MM:SS') to seconds."""
if not eta_str or eta_str == "0:00:00":
return None
try:
parts = eta_str.split(":")
if len(parts) == 3:
if len(parts) == _ETA_PART_COUNT:
return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
except (ValueError, IndexError):
except ValueError, IndexError:
pass
return None
def _parse_speed(slot: dict) -> Optional[int]:
def _parse_speed(slot: dict) -> int | None:
"""Parse download speed from SABnzbd slot data, returning bytes/sec."""
# Prefer kbpersec field (more reliable numeric value)
kbpersec_str = slot.get("kbpersec", "")
if kbpersec_str:
try:
return int(float(kbpersec_str) * 1024)
except (ValueError, TypeError):
except ValueError, TypeError:
pass
# Fall back to human-readable speed field
@@ -53,7 +54,7 @@ def _parse_speed(slot: dict) -> Optional[int]:
try:
speed_parts = speed_str.split()
if len(speed_parts) < 2:
if len(speed_parts) < _SPEED_PARTS_MIN:
return None
speed_val = float(speed_parts[0])
unit = speed_parts[1].upper()
@@ -62,7 +63,7 @@ def _parse_speed(slot: dict) -> Optional[int]:
if prefix in unit:
return int(speed_val * mult)
return int(speed_val)
except (ValueError, IndexError):
except ValueError, IndexError:
return None
@@ -100,19 +101,22 @@ class SABnzbdClient(DownloadClient):
protocol = "usenet"
name = "sabnzbd"
def __init__(self):
def __init__(self) -> None:
"""Initialize SABnzbd client with settings from config."""
raw_url = config.get("SABNZBD_URL", "")
if not raw_url:
raise ValueError("SABNZBD_URL is required")
msg = "SABNZBD_URL is required"
raise ValueError(msg)
api_key = config.get("SABNZBD_API_KEY", "")
if not api_key:
raise ValueError("SABNZBD_API_KEY is required")
msg = "SABNZBD_API_KEY is required"
raise ValueError(msg)
self.url = normalize_http_url(raw_url)
if not self.url:
raise ValueError("SABNZBD_URL is invalid")
msg = "SABNZBD_URL is invalid"
raise ValueError(msg)
self.api_key = api_key
self._category = config.get("SABNZBD_CATEGORY", "books")
@@ -125,9 +129,8 @@ class SABnzbdClient(DownloadClient):
return client == "sabnzbd" and bool(url) and bool(api_key)
@with_retry()
def _api_call(self, mode: str, params: Optional[dict] = None) -> Any:
"""
Make an API call to SABnzbd.
def _api_call(self, mode: str, params: dict | None = None) -> object:
"""Make an API call to SABnzbd.
Args:
mode: API mode (e.g., "version", "addurl", "queue", "history")
@@ -138,6 +141,7 @@ class SABnzbdClient(DownloadClient):
Raises:
Exception: If API call fails after retries.
"""
api_url = f"{self.url}/api"
@@ -149,7 +153,9 @@ class SABnzbdClient(DownloadClient):
if params:
request_params.update(params)
response = requests.get(api_url, params=request_params, timeout=30, verify=get_ssl_verify(api_url))
response = requests.get(
api_url, params=request_params, timeout=30, verify=get_ssl_verify(api_url)
)
response.raise_for_status()
result = response.json()
@@ -157,16 +163,19 @@ class SABnzbdClient(DownloadClient):
# Check for error in response
if isinstance(result, dict) and result.get("status") is False:
error = result.get("error", "Unknown error")
raise Exception(f"SABnzbd error: {error}")
msg = f"SABnzbd error: {error}"
raise RuntimeError(msg)
return result
def _api_post_file(self, nzb_content: bytes, filename: str, nzb_name: str, category: str) -> Any:
"""
Upload an NZB file to SABnzbd using addfile.
def _api_post_file(
self, nzb_content: bytes, filename: str, nzb_name: str, category: str
) -> object:
"""Upload an NZB file to SABnzbd using addfile.
Returns:
JSON response from SABnzbd.
"""
api_url = f"{self.url}/api"
request_params = {
@@ -178,13 +187,20 @@ class SABnzbdClient(DownloadClient):
}
files = {"name": (filename, nzb_content, "application/x-nzb")}
response = requests.post(api_url, params=request_params, files=files, timeout=30, verify=get_ssl_verify(api_url))
response = requests.post(
api_url,
params=request_params,
files=files,
timeout=30,
verify=get_ssl_verify(api_url),
)
response.raise_for_status()
result = response.json()
if isinstance(result, dict) and result.get("status") is False:
error = result.get("error", "Unknown error")
raise Exception(f"SABnzbd error: {error}")
msg = f"SABnzbd error: {error}"
raise RuntimeError(msg)
return result
@@ -242,9 +258,10 @@ class SABnzbdClient(DownloadClient):
return f"{base_name}.nzb"
@staticmethod
def _extract_nzo_id(result: Any) -> str:
def _extract_nzo_id(result: object) -> str:
if not isinstance(result, dict):
raise Exception("SABnzbd returned invalid response")
msg = "SABnzbd returned invalid response"
raise TypeError(msg)
nzo_ids = result.get("nzo_ids") or result.get("nzo_id")
if isinstance(nzo_ids, list) and nzo_ids:
@@ -254,31 +271,32 @@ class SABnzbdClient(DownloadClient):
if isinstance(nzo_ids, int):
return str(nzo_ids)
raise Exception("SABnzbd returned no nzo_id")
msg = "SABnzbd returned no nzo_id"
raise RuntimeError(msg)
def test_connection(self) -> Tuple[bool, str]:
def test_connection(self) -> tuple[bool, str]:
"""Test connection to SABnzbd."""
try:
result = self._api_call("version")
version = result.get("version", "unknown")
return True, f"Connected to SABnzbd {version}"
except requests.exceptions.ConnectionError:
return False, "Could not connect to SABnzbd"
except requests.exceptions.Timeout:
return False, "Connection timed out"
except Exception as e:
return False, f"Connection failed: {str(e)}"
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to SABnzbd {version}"
def add_download(
self,
url: str,
name: str,
category: Optional[str] = None,
expected_hash: Optional[str] = None,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
) -> str:
"""
Add NZB by URL.
"""Add NZB by URL.
Args:
url: NZB URL (can be Prowlarr proxy URL)
@@ -291,20 +309,22 @@ class SABnzbdClient(DownloadClient):
Raises:
Exception: If adding fails.
"""
# Use configured category if not explicitly provided
category = category or self._category
try:
logger.debug(f"Adding NZB to SABnzbd: {name}")
logger.debug("Adding NZB to SABnzbd: %s", name)
nzb_filename = self._build_nzb_filename(name, url)
nzb_content = self._fetch_nzb_content(url)
result = self._api_post_file(nzb_content, nzb_filename, name, category)
nzo_id = self._extract_nzo_id(result)
logger.info(f"Added NZB to SABnzbd: {nzo_id}")
return nzo_id
logger.info("Added NZB to SABnzbd: %s", nzo_id)
except Exception as e:
logger.warning(f"SABnzbd addfile failed, falling back to addurl: {e}")
logger.warning("SABnzbd addfile failed, falling back to addurl: %s", e)
else:
return nzo_id
try:
result = self._api_call(
@@ -316,21 +336,22 @@ class SABnzbdClient(DownloadClient):
},
)
nzo_id = self._extract_nzo_id(result)
logger.info(f"Added NZB to SABnzbd via addurl: {nzo_id}")
return nzo_id
except Exception as e:
logger.error(f"SABnzbd add failed: {e}")
logger.info("Added NZB to SABnzbd via addurl: %s", nzo_id)
except Exception:
logger.exception("SABnzbd add failed")
raise
else:
return nzo_id
def get_status(self, download_id: str) -> DownloadStatus:
"""
Get NZB status by nzo_id.
"""Get NZB status by nzo_id.
Args:
download_id: SABnzbd nzo_id
Returns:
Current download status.
"""
try:
# Check active queue first
@@ -383,7 +404,12 @@ class SABnzbdClient(DownloadClient):
storage = slot.get("storage", "")
if storage is None:
storage = ""
logger.debug(f"SABnzbd history: {download_id} status={status_text} storage='{storage}'")
logger.debug(
"SABnzbd history: %s status=%s storage='%s'",
download_id,
status_text,
storage,
)
if status_text == "COMPLETED":
title = slot.get("name") or slot.get("nzb_name") or ""
@@ -396,7 +422,7 @@ class SABnzbdClient(DownloadClient):
complete=True,
file_path=resolved_storage,
)
elif status_text == "FAILED":
if status_text == "FAILED":
fail_message = slot.get("fail_message", "Download failed")
title = slot.get("name") or slot.get("nzb_name") or ""
resolved_storage = self._resolve_completed_storage_path(storage, title)
@@ -407,27 +433,25 @@ class SABnzbdClient(DownloadClient):
complete=True,
file_path=resolved_storage,
)
else:
# Post-processing states: Queued, QuickCheck, Verifying,
# Repairing, Fetching, Extracting, Moving, Running
# Keep polling - not yet complete
return DownloadStatus(
progress=100,
state="processing",
message=status_text.title(),
complete=False,
file_path=None,
)
# Post-processing states: Queued, QuickCheck, Verifying,
# Repairing, Fetching, Extracting, Moving, Running
# Keep polling - not yet complete
return DownloadStatus(
progress=100,
state="processing",
message=status_text.title(),
complete=False,
file_path=None,
)
# Not found
logger.warning(f"SABnzbd: download {download_id} not found in queue or history")
logger.warning("SABnzbd: download %s not found in queue or history", download_id)
return DownloadStatus.error("Download not found")
except Exception as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, delete_files: bool = False, archive: bool = True) -> bool:
"""
Remove a download from SABnzbd.
def remove(self, download_id: str, *, delete_files: bool = False, archive: bool = True) -> bool:
"""Remove a download from SABnzbd.
Args:
download_id: SABnzbd nzo_id
@@ -436,6 +460,7 @@ class SABnzbdClient(DownloadClient):
Returns:
True if successful.
"""
# First try to remove from queue. If it isn't there (common for completed jobs),
# fall back to history removal instead of failing fast on a SABnzbd error response.
@@ -450,10 +475,10 @@ class SABnzbdClient(DownloadClient):
)
if result.get("status"):
logger.info(f"Removed NZB from SABnzbd queue: {download_id}")
logger.info("Removed NZB from SABnzbd queue: %s", download_id)
return True
except Exception as e:
logger.debug(f"SABnzbd queue delete skipped for {download_id}: {e}")
logger.debug("SABnzbd queue delete skipped for %s: %s", download_id, e)
# If not in queue (or queue delete failed), try to remove from history.
try:
@@ -469,7 +494,7 @@ class SABnzbdClient(DownloadClient):
if result.get("status"):
action = "archived" if archive else "removed"
logger.info(f"NZB {action} from SABnzbd history: {download_id}")
logger.info("NZB %s from SABnzbd history: %s", action, download_id)
return True
except Exception as e:
self._log_error("remove", e)
@@ -477,24 +502,23 @@ class SABnzbdClient(DownloadClient):
return False
def get_download_path(self, download_id: str) -> Optional[str]:
"""
Get the path where NZB files are located.
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where NZB files are located.
Args:
download_id: SABnzbd nzo_id
Returns:
Storage directory, or None.
"""
status = self.get_status(download_id)
return status.file_path
def find_existing(
self, url: str, category: Optional[str] = None
) -> Optional[Tuple[str, DownloadStatus]]:
"""
Check if an NZB for this URL already exists in SABnzbd.
self, url: str, category: str | None = None
) -> tuple[str, DownloadStatus] | None:
"""Check if an NZB for this URL already exists in SABnzbd.
Note: Unlike torrents which have a unique info_hash, usenet NZBs don't have
a universal unique identifier. SABnzbd generates an nzo_id when adding,
@@ -507,23 +531,22 @@ class SABnzbdClient(DownloadClient):
Returns:
Tuple of (nzo_id, status) if found, None if not found.
"""
try:
# Extract NZB name from URL (last path component without extension)
from urllib.parse import unquote, urlparse
parsed = urlparse(url)
path = unquote(parsed.path)
# Get filename from path
if "/" in path:
filename = path.rsplit("/", 1)[-1]
else:
filename = path
filename = path.rsplit("/", 1)[-1] if "/" in path else path
# Remove common NZB extensions
for ext in [".nzb", ".nzb.gz"]:
if filename.lower().endswith(ext):
filename = filename[:-len(ext)]
filename = filename[: -len(ext)]
break
if not filename:
@@ -543,7 +566,7 @@ class SABnzbdClient(DownloadClient):
nzo_id = slot.get("nzo_id")
if nzo_id:
status = self.get_status(nzo_id)
logger.debug(f"Found existing NZB in SABnzbd queue: {nzo_id}")
logger.debug("Found existing NZB in SABnzbd queue: %s", nzo_id)
return (nzo_id, status)
# Search history (SABnzbd uses "category" field in history)
@@ -557,11 +580,11 @@ class SABnzbdClient(DownloadClient):
nzo_id = slot.get("nzo_id")
if nzo_id:
status = self.get_status(nzo_id)
logger.debug(f"Found existing NZB in SABnzbd history: {nzo_id}")
logger.debug("Found existing NZB in SABnzbd history: %s", nzo_id)
return (nzo_id, status)
return None
except Exception as e:
logger.debug(f"Error checking for existing NZB: {e}")
logger.debug("Error checking for existing NZB: %s", e)
return None
else:
return None
shelfmark/download/clients/settings.py
+112 -64
View File
@@ -1,25 +1,33 @@
"""Shared download client settings registration."""
from contextlib import contextmanager
from typing import Any, Dict, Optional
from contextlib import contextmanager, suppress
from typing import TYPE_CHECKING, Any, NoReturn
from shelfmark.core.settings_registry import (
register_settings,
HeadingField,
TextField,
PasswordField,
ActionButton,
HeadingField,
PasswordField,
SelectField,
SettingsField,
TagListField,
TextField,
register_settings,
)
from shelfmark.core.utils import normalize_http_url, get_hardened_xmlrpc_client
from shelfmark.core.utils import get_hardened_xmlrpc_client, normalize_http_url
from shelfmark.download.network import get_ssl_verify
if TYPE_CHECKING:
from collections.abc import Iterator
# ==================== Test Connection Callbacks ====================
def _raise_runtime_error(message: str) -> NoReturn:
raise RuntimeError(message)
@contextmanager
def _transmission_session_verify_override(url: str):
def _transmission_session_verify_override(url: str) -> Iterator[None]:
"""Ensure transmission-rpc constructor uses the configured TLS verify mode."""
verify = get_ssl_verify(url)
if verify:
@@ -28,7 +36,7 @@ def _transmission_session_verify_override(url: str):
try:
import transmission_rpc.client as transmission_rpc_client
except Exception:
except ImportError:
yield
return
@@ -46,7 +54,7 @@ def _transmission_session_verify_override(url: str):
transmission_rpc_client.requests.Session = original_session_factory
def _test_qbittorrent_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
def _test_qbittorrent_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the qBittorrent connection using current form values."""
from shelfmark.core.config import config
@@ -66,17 +74,23 @@ def _test_qbittorrent_connection(current_values: Optional[Dict[str, Any]] = None
if not url:
return {"success": False, "message": "qBittorrent URL is invalid"}
client = Client(host=url, username=username, password=password, VERIFY_WEBUI_CERTIFICATE=get_ssl_verify(url))
client = Client(
host=url,
username=username,
password=password,
VERIFY_WEBUI_CERTIFICATE=get_ssl_verify(url),
)
client.auth_log_in()
api_version = client.app.web_api_version
return {"success": True, "message": f"Connected to qBittorrent (API v{api_version})"}
except ImportError:
return {"success": False, "message": "qbittorrent-api package not installed"}
except Exception as e:
return {"success": False, "message": f"Connection failed: {str(e)}"}
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to qBittorrent (API v{api_version})"}
def _test_transmission_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
def _test_transmission_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the Transmission connection using current form values."""
from shelfmark.core.config import config
from shelfmark.download.clients.torrent_utils import (
@@ -86,8 +100,12 @@ def _test_transmission_connection(current_values: Optional[Dict[str, Any]] = Non
current_values = current_values or {}
raw_url = current_values.get("TRANSMISSION_URL") or config.get("TRANSMISSION_URL", "")
username = current_values.get("TRANSMISSION_USERNAME") or config.get("TRANSMISSION_USERNAME", "")
password = current_values.get("TRANSMISSION_PASSWORD") or config.get("TRANSMISSION_PASSWORD", "")
username = current_values.get("TRANSMISSION_USERNAME") or config.get(
"TRANSMISSION_USERNAME", ""
)
password = current_values.get("TRANSMISSION_PASSWORD") or config.get(
"TRANSMISSION_PASSWORD", ""
)
if not raw_url:
return {"success": False, "message": "Transmission URL is required"}
@@ -106,8 +124,8 @@ def _test_transmission_connection(current_values: Optional[Dict[str, Any]] = Non
"host": host,
"port": port,
"path": path,
"username": username if username else None,
"password": password if password else None,
"username": username or None,
"password": password or None,
"protocol": protocol,
}
try:
@@ -119,11 +137,9 @@ def _test_transmission_connection(current_values: Optional[Dict[str, Any]] = Non
client_kwargs.pop("protocol", None)
with _transmission_session_verify_override(url):
client = Client(**client_kwargs)
if protocol == "https" and hasattr(client, "protocol"):
try:
setattr(client, "protocol", protocol)
except Exception:
pass
if protocol == "https" and hasattr(client, "protocol"):
with suppress(Exception):
client.protocol = protocol
# Keep session verify aligned for subsequent calls beyond constructor bootstrap.
http_session = getattr(client, "_http_session", None)
@@ -132,18 +148,20 @@ def _test_transmission_connection(current_values: Optional[Dict[str, Any]] = Non
session = client.get_session()
version = session.version
return {"success": True, "message": f"Connected to Transmission {version}"}
except ImportError:
return {"success": False, "message": "transmission-rpc package not installed"}
except Exception as e:
return {"success": False, "message": f"Connection failed: {str(e)}"}
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to Transmission {version}"}
def _test_deluge_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
def _test_deluge_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test Deluge Web UI JSON-RPC connection using current form values."""
from urllib.parse import urlparse
import requests
from shelfmark.core.config import config
current_values = current_values or {}
@@ -177,13 +195,12 @@ def _test_deluge_connection(current_values: Optional[Dict[str, Any]] = None) ->
if parsed.port is not None:
port = parsed.port
base_path = (parsed.path or "").rstrip("/")
else:
# Allow "host:port" in DELUGE_HOST for convenience.
if ":" in raw_host and raw_host.count(":") == 1:
host_part, port_part = raw_host.split(":", 1)
if host_part and port_part.isdigit():
host = host_part
port = int(port_part)
# Allow "host:port" in DELUGE_HOST for convenience.
elif ":" in raw_host and raw_host.count(":") == 1:
host_part, port_part = raw_host.split(":", 1)
if host_part and port_part.isdigit():
host = host_part
port = int(port_part)
rpc_url = f"{scheme}://{host}:{port}{base_path}/json"
@@ -195,8 +212,8 @@ def _test_deluge_connection(current_values: Optional[Dict[str, Any]] = None) ->
if data.get("error"):
error = data["error"]
if isinstance(error, dict):
raise Exception(error.get("message") or str(error))
raise Exception(str(error))
raise RuntimeError(error.get("message") or str(error))
raise RuntimeError(str(error))
return data.get("result")
def get_daemon_version(session: requests.Session, rpc_id: int) -> Any:
@@ -204,7 +221,7 @@ def _test_deluge_connection(current_values: Optional[Dict[str, Any]] = None) ->
methods = rpc_call(session, rpc_id, "system.listMethods")
if isinstance(methods, list) and "daemon.get_version" in methods:
return rpc_call(session, rpc_id + 1, "daemon.get_version")
except Exception:
except requests.exceptions.RequestException, RuntimeError, ValueError, TypeError:
# Fall back to daemon.info to preserve existing behavior.
pass
@@ -226,7 +243,11 @@ def _test_deluge_connection(current_values: Optional[Dict[str, Any]] = None) ->
host_id = hosts[0][0]
for entry in hosts:
if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
if (
isinstance(entry, list)
and len(entry) >= 2
and entry[1] in {"127.0.0.1", "localhost"}
):
host_id = entry[0]
break
@@ -239,22 +260,31 @@ def _test_deluge_connection(current_values: Optional[Dict[str, Any]] = None) ->
}
version = get_daemon_version(session, 6)
return {"success": True, "message": f"Connected to Deluge {version}"}
except requests.exceptions.ConnectionError:
return {"success": False, "message": "Could not connect to Deluge Web UI"}
except requests.exceptions.Timeout:
return {"success": False, "message": "Connection timed out"}
except Exception as e:
return {"success": False, "message": f"Connection failed: {str(e)}"}
except (
requests.exceptions.RequestException,
RuntimeError,
ValueError,
TypeError,
KeyError,
IndexError,
AttributeError,
) as e:
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to Deluge {version}"}
def _test_rtorrent_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
def _test_rtorrent_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the rTorrent connection using current form values."""
from shelfmark.core.config import config
import ssl
from urllib.parse import urlparse
from shelfmark.core.config import config
current_values = current_values or {}
raw_url = current_values.get("RTORRENT_URL") or config.get("RTORRENT_URL", "")
@@ -290,14 +320,17 @@ def _test_rtorrent_connection(current_values: Optional[Dict[str, Any]] = None) -
rpc = xmlrpc_client.ServerProxy(rpc_url)
version = rpc.system.client_version()
except (xmlrpc_client.Error, RuntimeError, OSError, ValueError, TypeError) as e:
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to rTorrent {version}"}
except Exception as e:
return {"success": False, "message": f"Connection failed: {str(e)}"}
def _test_nzbget_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
def _test_nzbget_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the NZBGet connection using current form values."""
import requests
from shelfmark.core.config import config
current_values = current_values or {}
@@ -316,24 +349,38 @@ def _test_nzbget_connection(current_values: Optional[Dict[str, Any]] = None) ->
try:
rpc_url = f"{url.rstrip('/')}/jsonrpc"
payload = {"jsonrpc": "2.0", "method": "status", "params": [], "id": 1}
response = requests.post(rpc_url, json=payload, auth=(username, password), timeout=30, verify=get_ssl_verify(rpc_url))
response = requests.post(
rpc_url,
json=payload,
auth=(username, password),
timeout=30,
verify=get_ssl_verify(rpc_url),
)
response.raise_for_status()
result = response.json()
if "error" in result and result["error"]:
raise Exception(result["error"].get("message", "RPC error"))
if result.get("error"):
_raise_runtime_error(result["error"].get("message", "RPC error"))
version = result.get("result", {}).get("Version", "unknown")
return {"success": True, "message": f"Connected to NZBGet {version}"}
except requests.exceptions.ConnectionError:
return {"success": False, "message": "Could not connect to NZBGet"}
except requests.exceptions.Timeout:
return {"success": False, "message": "Connection timed out"}
except Exception as e:
return {"success": False, "message": f"Connection failed: {str(e)}"}
except (
requests.exceptions.RequestException,
RuntimeError,
ValueError,
AttributeError,
TypeError,
) as e:
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to NZBGet {version}"}
def _test_sabnzbd_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
def _test_sabnzbd_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the SABnzbd connection using current form values."""
import requests
from shelfmark.core.config import config
current_values = current_values or {}
@@ -357,24 +404,32 @@ def _test_sabnzbd_connection(current_values: Optional[Dict[str, Any]] = None) ->
response.raise_for_status()
result = response.json()
version = result.get("version", "unknown")
return {"success": True, "message": f"Connected to SABnzbd {version}"}
except requests.exceptions.ConnectionError:
return {"success": False, "message": "Could not connect to SABnzbd"}
except requests.exceptions.Timeout:
return {"success": False, "message": "Connection timed out"}
except Exception as e:
return {"success": False, "message": f"Connection failed: {str(e)}"}
except (
requests.exceptions.RequestException,
RuntimeError,
ValueError,
AttributeError,
TypeError,
) as e:
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to SABnzbd {version}"}
# ==================== Download Clients Tab ====================
@register_settings(
name="prowlarr_clients",
display_name="Download Clients",
icon="cog",
order=110,
)
def prowlarr_clients_settings():
def prowlarr_clients_settings() -> list[SettingsField]:
"""Download client settings shared by external release sources."""
return [
# --- Torrent Client Selection ---
@@ -396,7 +451,6 @@ def prowlarr_clients_settings():
],
default="",
),
# --- qBittorrent Settings ---
TextField(
key="QBITTORRENT_URL",
@@ -458,7 +512,6 @@ def prowlarr_clients_settings():
normalize_urls=False,
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "qbittorrent"},
),
# --- Transmission Settings ---
TextField(
key="TRANSMISSION_URL",
@@ -510,7 +563,6 @@ def prowlarr_clients_settings():
placeholder="/downloads",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "transmission"},
),
# --- Deluge Settings ---
TextField(
key="DELUGE_HOST",
@@ -565,7 +617,6 @@ def prowlarr_clients_settings():
placeholder="/downloads",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "deluge"},
),
# --- rTorrent Settings ---
TextField(
key="RTORRENT_URL",
@@ -621,7 +672,6 @@ def prowlarr_clients_settings():
default="keep",
show_when={"field": "PROWLARR_TORRENT_CLIENT", "notEmpty": True},
),
# --- Usenet Client Selection ---
HeadingField(
key="usenet_heading",
@@ -639,7 +689,6 @@ def prowlarr_clients_settings():
],
default="",
),
# --- NZBGet Settings ---
TextField(
key="NZBGET_URL",
@@ -686,7 +735,6 @@ def prowlarr_clients_settings():
default="",
show_when={"field": "PROWLARR_USENET_CLIENT", "value": "nzbget"},
),
# --- SABnzbd Settings ---
TextField(
key="SABNZBD_URL",
shelfmark/download/clients/torrent_utils.py
+98 -66
View File
@@ -4,7 +4,6 @@ import base64
import hashlib
import re
from dataclasses import dataclass
from typing import Optional, Tuple
from urllib.parse import parse_qs, urljoin, urlparse
import requests
@@ -15,24 +14,32 @@ from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_MAGNET_RESPONSE_MAX_BYTES = 2000
_BASE32_BTMH_TAG_BYTES = 34
_BTIH_INFO_BYTE_HEX = 0x20
_BTIH_PREFIX_BYTE = 0x12
_BTIH_DIGEST_LENGTH = 32
_BTIH_HASH_LENGTH_40 = 40
_BTIH_HASH_LENGTH_32 = 32
@dataclass
class TorrentInfo:
"""Parsed information from a torrent URL."""
info_hash: Optional[str]
info_hash: str | None
"""Lowercase hex info_hash (32 or 40 chars), or None if extraction failed."""
torrent_data: Optional[bytes]
torrent_data: bytes | None
"""Raw .torrent file content, only populated for .torrent URLs."""
is_magnet: bool
"""True if the URL was a magnet link."""
magnet_url: Optional[str] = None
magnet_url: str | None = None
"""The actual magnet URL, if available."""
def with_info_hash(self, info_hash: Optional[str]) -> "TorrentInfo":
def with_info_hash(self, info_hash: str | None) -> TorrentInfo:
"""Return a copy with the info_hash replaced when provided."""
if info_hash:
return TorrentInfo(
@@ -46,8 +53,9 @@ class TorrentInfo:
def extract_torrent_info(
url: str,
*,
fetch_torrent: bool = True,
expected_hash: Optional[str] = None,
expected_hash: str | None = None,
) -> TorrentInfo:
"""Extract info_hash from magnet link or .torrent URL.
@@ -58,6 +66,7 @@ def extract_torrent_info(
Redirects to magnet links are handled explicitly so we can extract a
hash from the magnet when available.
"""
is_magnet = url.startswith("magnet:")
@@ -85,11 +94,17 @@ def extract_torrent_info(
return urljoin(current, location)
try:
logger.debug(f"Fetching torrent file from: {url[:80]}...")
logger.debug("Fetching torrent file from: %s...", url[:80])
# Use allow_redirects=False to handle magnet link redirects manually
# Some indexers redirect download URLs to magnet links
resp = requests.get(url, timeout=30, allow_redirects=False, headers=headers, verify=get_ssl_verify(url))
resp = requests.get(
url,
timeout=30,
allow_redirects=False,
headers=headers,
verify=get_ssl_verify(url),
)
# Check if this is a redirect to a magnet link
if resp.status_code in (301, 302, 303, 307, 308):
@@ -100,18 +115,26 @@ def extract_torrent_info(
if not info_hash and expected_hash:
info_hash = expected_hash
return TorrentInfo(
info_hash=info_hash, torrent_data=None, is_magnet=True, magnet_url=redirect_url
info_hash=info_hash,
torrent_data=None,
is_magnet=True,
magnet_url=redirect_url,
)
# Not a magnet redirect, follow it manually
logger.debug(f"Following redirect to: {redirect_url[:80]}...")
resp = requests.get(redirect_url, timeout=30, headers=headers, verify=get_ssl_verify(redirect_url))
logger.debug("Following redirect to: %s...", redirect_url[:80])
resp = requests.get(
redirect_url,
timeout=30,
headers=headers,
verify=get_ssl_verify(redirect_url),
)
resp.raise_for_status()
torrent_data = resp.content
# Check if response is actually a magnet link (text response)
# Some indexers return magnet links as plain text instead of redirecting
if len(torrent_data) < 2000: # Magnet links are typically short
if len(torrent_data) < _MAGNET_RESPONSE_MAX_BYTES: # Magnet links are typically short
try:
text_content = torrent_data.decode("utf-8", errors="ignore").strip()
if text_content.startswith("magnet:"):
@@ -120,23 +143,26 @@ def extract_torrent_info(
if not info_hash and expected_hash:
info_hash = expected_hash
return TorrentInfo(
info_hash=info_hash, torrent_data=None, is_magnet=True, magnet_url=text_content
info_hash=info_hash,
torrent_data=None,
is_magnet=True,
magnet_url=text_content,
)
except Exception:
pass # Not text, continue with torrent parsing
info_hash = extract_info_hash_from_torrent(torrent_data) or expected_hash
if info_hash:
logger.debug(f"Extracted hash from torrent file: {info_hash}")
logger.debug("Extracted hash from torrent file: %s", info_hash)
else:
logger.warning("Could not extract hash from torrent file")
return TorrentInfo(info_hash=info_hash, torrent_data=torrent_data, is_magnet=False)
except Exception as e:
logger.debug(f"Could not fetch torrent file: {e}")
logger.debug("Could not fetch torrent file: %s", e)
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
def parse_transmission_url(url: str) -> Tuple[str, str, int, str]:
def parse_transmission_url(url: str) -> tuple[str, str, int, str]:
"""Parse Transmission URL into (protocol, host, port, path)."""
parsed = urlparse(url)
protocol = (parsed.scheme or "http").lower()
@@ -155,89 +181,89 @@ def parse_transmission_url(url: str) -> Tuple[str, str, int, str]:
def bencode_decode(data: bytes) -> tuple:
"""Decode bencoded data. Returns (value, remaining_bytes)."""
if data[0:1] == b'd':
if data[0:1] == b"d":
# Dictionary
result = {}
data = data[1:]
while data[0:1] != b'e':
while data[0:1] != b"e":
key, data = bencode_decode(data)
value, data = bencode_decode(data)
result[key] = value
return result, data[1:]
elif data[0:1] == b'l':
if data[0:1] == b"l":
# List
result = []
data = data[1:]
while data[0:1] != b'e':
while data[0:1] != b"e":
value, data = bencode_decode(data)
result.append(value)
return result, data[1:]
elif data[0:1] == b'i':
if data[0:1] == b"i":
# Integer
end = data.index(b'e')
return int(data[1:end]), data[end + 1:]
elif data[0:1].isdigit():
end = data.index(b"e")
return int(data[1:end]), data[end + 1 :]
if data[0:1].isdigit():
# Byte string
colon = data.index(b':')
colon = data.index(b":")
length = int(data[:colon])
start = colon + 1
return data[start:start + length], data[start + length:]
else:
first_byte = data[0:1]
raise ValueError(
f"Invalid bencode data: expected 'd', 'l', 'i', or digit, "
f"got {first_byte!r}. First 20 bytes: {data[:20]!r}"
)
return data[start : start + length], data[start + length :]
first_byte = data[0:1]
msg = (
f"Invalid bencode data: expected 'd', 'l', 'i', or digit, "
f"got {first_byte!r}. First 20 bytes: {data[:20]!r}"
)
raise ValueError(msg)
def bencode_encode(data) -> bytes:
def bencode_encode(data: dict[str | bytes, object] | list[object] | int | bytes | str) -> bytes:
"""Encode data to bencode format."""
if isinstance(data, dict):
# Keys must be sorted (bencode spec requirement)
result = b'd'
result = b"d"
for key in sorted(data.keys()):
result += bencode_encode(key)
result += bencode_encode(data[key])
result += b'e'
result += b"e"
return result
elif isinstance(data, list):
result = b'l'
if isinstance(data, list):
result = b"l"
for item in data:
result += bencode_encode(item)
result += b'e'
result += b"e"
return result
elif isinstance(data, int):
return f'i{data}e'.encode()
elif isinstance(data, bytes):
return f'{len(data)}:'.encode() + data
elif isinstance(data, str):
encoded = data.encode('utf-8')
return f'{len(encoded)}:'.encode() + encoded
else:
raise ValueError(
f"Cannot bencode type {type(data).__name__}: "
f"expected dict, list, int, bytes, or str. Value: {data!r}"
)
if isinstance(data, int):
return f"i{data}e".encode()
if isinstance(data, bytes):
return f"{len(data)}:".encode() + data
if isinstance(data, str):
encoded = data.encode("utf-8")
return f"{len(encoded)}:".encode() + encoded
msg = (
f"Cannot bencode type {type(data).__name__}: "
f"expected dict, list, int, bytes, or str. Value: {data!r}"
)
raise ValueError(msg)
def extract_info_hash_from_torrent(torrent_data: bytes) -> Optional[str]:
def extract_info_hash_from_torrent(torrent_data: bytes) -> str | None:
"""Extract info_hash from .torrent file data."""
try:
decoded, _ = bencode_decode(torrent_data)
if b'info' not in decoded:
if b"info" not in decoded:
return None
info_bencoded = bencode_encode(decoded[b'info'])
info_dict = decoded[b'info']
if isinstance(info_dict, dict) and b'pieces' in info_dict:
info_bencoded = bencode_encode(decoded[b"info"])
info_dict = decoded[b"info"]
if isinstance(info_dict, dict) and b"pieces" in info_dict:
return hashlib.sha1(info_bencoded).hexdigest().lower()
return hashlib.sha256(info_bencoded).hexdigest().lower()
except Exception as e:
logger.debug(f"Failed to parse torrent file: {e}")
logger.debug("Failed to parse torrent file: %s", e)
return None
def extract_hash_from_magnet(magnet_url: str) -> Optional[str]:
def extract_hash_from_magnet(magnet_url: str) -> str | None:
"""Extract info_hash from a magnet URL."""
if not magnet_url.startswith("magnet:"):
return None
@@ -245,12 +271,12 @@ def extract_hash_from_magnet(magnet_url: str) -> Optional[str]:
parsed = urlparse(magnet_url)
params = parse_qs(parsed.query)
def extract_btmh(value: str) -> Optional[str]:
def extract_btmh(value: str) -> str | None:
raw_value = value.strip()
if not raw_value:
return None
data: Optional[bytes] = None
data: bytes | None = None
if re.fullmatch(r"[a-fA-F0-9]+", raw_value):
if len(raw_value) % 2 != 0:
return None
@@ -268,12 +294,16 @@ def extract_hash_from_magnet(magnet_url: str) -> Optional[str]:
if not data:
return None
if len(data) >= 34 and data[0] == 0x12 and data[1] == 0x20:
digest = data[2:34]
if len(digest) == 32:
if (
len(data) >= _BASE32_BTMH_TAG_BYTES
and data[0] == _BTIH_PREFIX_BYTE
and data[1] == _BTIH_INFO_BYTE_HEX
):
digest = data[2:_BASE32_BTMH_TAG_BYTES]
if len(digest) == _BTIH_DIGEST_LENGTH:
return digest.hex().lower()
if len(data) == 32:
if len(data) == _BTIH_HASH_LENGTH_32:
return data.hex().lower()
return None
@@ -287,11 +317,13 @@ def extract_hash_from_magnet(magnet_url: str) -> Optional[str]:
hash_value = match.group(1)
# 40-char hex or 32-char hex (ED2K) - return as-is
if len(hash_value) == 40 or re.match(r'^[a-fA-F0-9]{32}$', hash_value):
if len(hash_value) == _BTIH_HASH_LENGTH_40 or re.match(
r"^[a-fA-F0-9]{32}$", hash_value
):
return hash_value.lower()
# 32-char base32 - decode to hex
if re.match(r'^[A-Z2-7]{32}$', hash_value.upper()):
if re.match(r"^[A-Z2-7]{32}$", hash_value.upper()):
try:
return base64.b32decode(hash_value.upper()).hex().lower()
except Exception:
@@ -302,7 +334,7 @@ def extract_hash_from_magnet(magnet_url: str) -> Optional[str]:
for xt in xt_values:
if xt.startswith("urn:btmh:"):
btmh_value = xt[len("urn:btmh:"):]
btmh_value = xt[len("urn:btmh:") :]
btmh_hash = extract_btmh(btmh_value)
if btmh_hash:
return btmh_hash
shelfmark/download/clients/transmission.py
+71 -61
View File
@@ -1,17 +1,14 @@
"""
Transmission download client for Prowlarr integration.
"""Transmission download client for Prowlarr integration.
Uses the transmission-rpc library to communicate with Transmission's RPC API.
"""
from contextlib import contextmanager
from typing import Any, Iterator, Optional, Tuple
from contextlib import contextmanager, suppress
from typing import TYPE_CHECKING
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url
from shelfmark.download.network import get_ssl_verify
from shelfmark.download.clients import (
DownloadClient,
DownloadStatus,
@@ -21,9 +18,16 @@ from shelfmark.download.clients.torrent_utils import (
extract_torrent_info,
parse_transmission_url,
)
from shelfmark.download.network import get_ssl_verify
if TYPE_CHECKING:
from collections.abc import Iterator
logger = setup_logger(__name__)
_SEEDING_PROGRESS_PERCENT = 100
_ETA_MAX_SECONDS = 604800
@contextmanager
def _transmission_session_verify_override(url: str) -> Iterator[None]:
@@ -46,7 +50,7 @@ def _transmission_session_verify_override(url: str) -> Iterator[None]:
original_session_factory = transmission_rpc_client.requests.Session
def _session_factory(*args: Any, **kwargs: Any) -> Any:
def _session_factory(*args: object, **kwargs: object) -> object:
session = original_session_factory(*args, **kwargs)
session.verify = False
return session
@@ -58,7 +62,7 @@ def _transmission_session_verify_override(url: str) -> Iterator[None]:
transmission_rpc_client.requests.Session = original_session_factory
def _apply_transmission_ssl_verify(client: Any, url: str) -> None:
def _apply_transmission_ssl_verify(client: object, url: str) -> None:
"""Apply global certificate validation policy to transmission-rpc client."""
session = getattr(client, "_http_session", None)
if session is None:
@@ -76,17 +80,19 @@ class TransmissionClient(DownloadClient):
protocol = "torrent"
name = "transmission"
def __init__(self):
def __init__(self) -> None:
"""Initialize Transmission client with settings from config."""
from transmission_rpc import Client
raw_url = config.get("TRANSMISSION_URL", "")
if not raw_url:
raise ValueError("TRANSMISSION_URL is required")
msg = "TRANSMISSION_URL is required"
raise ValueError(msg)
url = normalize_http_url(raw_url)
if not url:
raise ValueError("TRANSMISSION_URL is invalid")
msg = "TRANSMISSION_URL is invalid"
raise ValueError(msg)
username = config.get("TRANSMISSION_USERNAME", "")
password = config.get("TRANSMISSION_PASSWORD", "")
@@ -98,8 +104,8 @@ class TransmissionClient(DownloadClient):
"host": host,
"port": port,
"path": path,
"username": username if username else None,
"password": password if password else None,
"username": username or None,
"password": password or None,
"protocol": protocol,
}
try:
@@ -114,10 +120,8 @@ class TransmissionClient(DownloadClient):
self._client = Client(**client_kwargs)
# Some versions expose protocol as an attribute rather than kwarg.
if protocol == "https" and hasattr(self._client, "protocol"):
try:
setattr(self._client, "protocol", protocol)
except Exception:
pass
with suppress(Exception):
self._client.protocol = protocol
_apply_transmission_ssl_verify(self._client, url)
self._category = config.get("TRANSMISSION_CATEGORY", "books")
self._download_dir = config.get("TRANSMISSION_DOWNLOAD_DIR", "")
@@ -129,25 +133,25 @@ class TransmissionClient(DownloadClient):
url = normalize_http_url(config.get("TRANSMISSION_URL", ""))
return client == "transmission" and bool(url)
def test_connection(self) -> Tuple[bool, str]:
def test_connection(self) -> tuple[bool, str]:
"""Test connection to Transmission."""
try:
session = self._client.get_session()
version = session.version
return True, f"Connected to Transmission {version}"
except Exception as e:
return False, f"Connection failed: {str(e)}"
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to Transmission {version}"
def add_download(
self,
url: str,
name: str,
category: Optional[str] = None,
expected_hash: Optional[str] = None,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
) -> str:
"""
Add torrent by URL (magnet or .torrent).
"""Add torrent by URL (magnet or .torrent).
Args:
url: Magnet link or .torrent URL
@@ -160,6 +164,7 @@ class TransmissionClient(DownloadClient):
Raises:
Exception: If adding fails.
"""
try:
resolved_category = category or self._category or ""
@@ -186,7 +191,7 @@ class TransmissionClient(DownloadClient):
)
torrent_hash = torrent.hashString.lower()
logger.info(f"Added torrent to Transmission: {torrent_hash}")
logger.info("Added torrent to Transmission: %s", torrent_hash)
# Apply per-torrent seeding limits from indexer
seed_kwargs = {}
@@ -202,23 +207,23 @@ class TransmissionClient(DownloadClient):
try:
self._client.change_torrent(ids=torrent_hash, **seed_kwargs)
except Exception as e:
logger.warning(f"Failed to set seeding limits for {torrent_hash}: {e}")
logger.warning("Failed to set seeding limits for %s: %s", torrent_hash, e)
except Exception:
logger.exception("Transmission add failed")
raise
else:
return torrent_hash
except Exception as e:
logger.error(f"Transmission add failed: {e}")
raise
def get_status(self, download_id: str) -> DownloadStatus:
"""
Get torrent status by hash.
"""Get torrent status by hash.
Args:
download_id: Torrent info_hash
Returns:
Current download status.
"""
try:
torrent = self._client.get_torrent(download_id)
@@ -232,7 +237,9 @@ class TransmissionClient(DownloadClient):
# 5: seed pending
# 6: seeding
# torrent.status is an enum with .value as string
status_value = torrent.status.value if hasattr(torrent.status, 'value') else str(torrent.status)
status_value = (
torrent.status.value if hasattr(torrent.status, "value") else str(torrent.status)
)
status_map = {
"stopped": ("paused", "Paused"),
"check pending": ("checking", "Waiting to check"),
@@ -246,30 +253,30 @@ class TransmissionClient(DownloadClient):
state, message = status_map.get(status_value, ("downloading", "Downloading"))
progress = torrent.percent_done * 100
# Only mark complete when seeding - seed pending means files still being moved
complete = progress >= 100 and status_value == "seeding"
complete = progress >= _SEEDING_PROGRESS_PERCENT and status_value == "seeding"
if complete:
message = "Complete"
# Get ETA if available and reasonable (less than 1 week)
eta = None
if hasattr(torrent, 'eta') and torrent.eta:
if hasattr(torrent, "eta") and torrent.eta:
eta_seconds = torrent.eta.total_seconds()
if 0 < eta_seconds < 604800:
if 0 < eta_seconds < _ETA_MAX_SECONDS:
eta = int(eta_seconds)
# Get download speed
download_speed = torrent.rate_download if hasattr(torrent, 'rate_download') else None
download_speed = torrent.rate_download if hasattr(torrent, "rate_download") else None
# Get file path for completed downloads
file_path = None
if complete:
# Output path is downloadDir + torrent name (with ':' replaced)
torrent_name = getattr(torrent, 'name', '')
torrent_name = getattr(torrent, "name", "")
if isinstance(torrent_name, str):
torrent_name = torrent_name.replace(':', '_')
torrent_name = torrent_name.replace(":", "_")
file_path = self._build_path(
getattr(torrent, 'download_dir', ''),
getattr(torrent, "download_dir", ""),
torrent_name,
)
@@ -288,9 +295,8 @@ class TransmissionClient(DownloadClient):
except Exception as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, delete_files: bool = False) -> bool:
"""
Remove a torrent from Transmission.
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
"""Remove a torrent from Transmission.
Args:
download_id: Torrent info_hash
@@ -298,6 +304,7 @@ class TransmissionClient(DownloadClient):
Returns:
True if successful.
"""
try:
self._client.remove_torrent(
@@ -305,40 +312,42 @@ class TransmissionClient(DownloadClient):
delete_data=delete_files,
)
logger.info(
f"Removed torrent from Transmission: {download_id}"
+ (" (with files)" if delete_files else "")
"Removed torrent from Transmission: %s%s",
download_id,
" (with files)" if delete_files else "",
)
return True
except Exception as e:
self._log_error("remove", e)
return False
else:
return True
def get_download_path(self, download_id: str) -> Optional[str]:
"""
Get the path where torrent files are located.
def get_download_path(self, download_id: str) -> str | None:
"""Get the path where torrent files are located.
Args:
download_id: Torrent info_hash
Returns:
Content path (file or directory), or None.
"""
try:
torrent = self._client.get_torrent(download_id)
torrent_name = getattr(torrent, 'name', '')
if isinstance(torrent_name, str):
torrent_name = torrent_name.replace(':', '_')
return self._build_path(
getattr(torrent, 'download_dir', ''),
torrent_name,
)
torrent = self._client.get_torrent(download_id)
torrent_name = getattr(torrent, "name", "")
if isinstance(torrent_name, str):
torrent_name = torrent_name.replace(":", "_")
return self._build_path(
getattr(torrent, "download_dir", ""),
torrent_name,
)
except Exception as e:
self._log_error("get_download_path", e, level="debug")
return None
def find_existing(
self, url: str, category: Optional[str] = None
) -> Optional[Tuple[str, DownloadStatus]]:
self, url: str, category: str | None = None
) -> tuple[str, DownloadStatus] | None:
"""Check if a torrent for this URL already exists in Transmission."""
try:
torrent_info = extract_torrent_info(url)
@@ -348,9 +357,10 @@ class TransmissionClient(DownloadClient):
try:
self._client.get_torrent(torrent_info.info_hash)
status = self.get_status(torrent_info.info_hash)
return (torrent_info.info_hash, status)
except KeyError:
return None
else:
return (torrent_info.info_hash, status)
except Exception as e:
logger.debug(f"Error checking for existing torrent: {e}")
logger.debug("Error checking for existing torrent: %s", e)
return None
shelfmark/download/fs.py
+70 -51
View File
@@ -11,33 +11,36 @@ import subprocess
import tempfile
import time
from pathlib import Path
from typing import Any, Callable, Optional, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeVar, cast
from shelfmark.core.logger import setup_logger
from shelfmark.download.permissions_debug import log_transfer_permission_context
if TYPE_CHECKING:
from collections.abc import Callable
from gevent.threadpool import ThreadPool
logger = setup_logger(__name__)
try:
from gevent import monkey as _gevent_monkey
from gevent.threadpool import ThreadPool as _GeventThreadPool
except Exception:
except ImportError:
_gevent_monkey = None
_GeventThreadPool = None
T = TypeVar("T")
_IO_THREADPOOL: Optional["_GeventThreadPool"] = None
_IO_THREADPOOL: ThreadPool | None = None
def _use_gevent_threadpool() -> bool:
return bool(
_gevent_monkey
and _GeventThreadPool
and _gevent_monkey.is_module_patched("threading")
_gevent_monkey and _GeventThreadPool and _gevent_monkey.is_module_patched("threading")
)
def _get_io_threadpool() -> "_GeventThreadPool":
def _get_io_threadpool() -> ThreadPool:
global _IO_THREADPOOL
if _IO_THREADPOOL is None:
pool_size = max(2, min(8, os.cpu_count() or 2))
@@ -45,7 +48,9 @@ def _get_io_threadpool() -> "_GeventThreadPool":
return _IO_THREADPOOL
def _call_and_capture(func: Callable[..., T], args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[bool, T | Exception]:
def _call_and_capture[T](
func: Callable[..., T], args: tuple[Any, ...], kwargs: dict[str, Any]
) -> tuple[bool, T | Exception]:
try:
return True, func(*args, **kwargs)
except Exception as exc:
@@ -60,13 +65,10 @@ def _must_avoid_gevent_threadpool(func: Callable[..., Any]) -> bool:
# gevent.subprocess requires child watchers on the default event loop.
# Executing patched subprocess functions in a worker thread can raise:
# "TypeError: child watchers are only available on the default loop".
if _gevent_monkey.is_object_patched("subprocess", "run") and func is subprocess.run:
return True
return False
return _gevent_monkey.is_object_patched("subprocess", "run") and func is subprocess.run
def run_blocking_io(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
def run_blocking_io[T](func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
"""Run blocking I/O in a native thread when under gevent.
gevent's threadpool will eagerly log exceptions raised inside worker threads,
@@ -80,13 +82,12 @@ def run_blocking_io(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
if _use_gevent_threadpool():
ok, result = _get_io_threadpool().apply(_call_and_capture, (func, args, kwargs))
if ok:
return cast(T, result)
exc = cast(Exception, result)
return cast("T", result)
exc = cast("Exception", result)
raise exc
return func(*args, **kwargs)
_VERIFY_IO_WAIT_SECONDS = 3.0
_PUBLISH_VERIFY_RETRY_SECONDS = 0.25
@@ -107,14 +108,17 @@ def _verify_transfer_size(
return
logger.debug(
f"File {action} size mismatch, waiting for filesystem sync: {dest} "
f"({actual_size} != {expected_size})"
"File %s size mismatch, waiting for filesystem sync: %s (%s != %s)",
action,
dest,
actual_size,
expected_size,
)
time.sleep(_VERIFY_IO_WAIT_SECONDS)
actual_size = run_blocking_io(dest.stat).st_size
if actual_size != expected_size:
raise IOError(
raise OSError(
f"File {action} incomplete, data loss may have occurred. "
f"'{dest}' was {actual_size} bytes instead of expected {expected_size}."
)
@@ -138,10 +142,11 @@ def _verify_published_file(
"""
try:
_verify_transfer_size(dest, expected_size, action)
return
except OSError as error:
if not _is_stale_handle_error(error):
raise
else:
return
time.sleep(_PUBLISH_VERIFY_RETRY_SECONDS)
@@ -174,6 +179,7 @@ def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
@@ -194,10 +200,11 @@ def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
finally:
run_blocking_io(os.close, fd)
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
logger.info("File collision resolved: %s", try_path.name)
except FileExistsError:
continue
else:
return try_path
raise RuntimeError(f"Could not write file after {max_attempts} attempts: {dest_path}")
@@ -219,7 +226,7 @@ def _system_op(op: str, source: Path, dest: Path) -> None:
)
def _perform_nfs_fallback(source: Path, dest: Path, is_move: bool) -> None:
def _perform_nfs_fallback(source: Path, dest: Path, *, is_move: bool) -> None:
"""Handle NFS/SMB permission errors by falling back to copyfile -> system op."""
expected_size = run_blocking_io(source.stat).st_size
@@ -230,15 +237,16 @@ def _perform_nfs_fallback(source: Path, dest: Path, is_move: bool) -> None:
if is_move:
run_blocking_io(source.unlink)
return
except Exception as copy_error:
# Clean up failed copy attempt if it exists
run_blocking_io(dest.unlink, missing_ok=True)
if _is_permission_error(copy_error):
log_transfer_permission_context("nfs_fallback_copyfile", source=source, dest=dest, error=copy_error)
logger.error("Fallback copyfile failed (%s -> %s): %s", source, dest, copy_error)
log_transfer_permission_context(
"nfs_fallback_copyfile", source=source, dest=dest, error=copy_error
)
logger.exception("Fallback copyfile failed (%s -> %s)", source, dest)
# Fallback 2: system command
op = "mv" if is_move else "cp"
@@ -250,10 +258,14 @@ def _perform_nfs_fallback(source: Path, dest: Path, is_move: bool) -> None:
if is_move:
run_blocking_io(source.unlink, missing_ok=True)
except subprocess.CalledProcessError as sys_error:
log_transfer_permission_context("nfs_fallback_system", source=source, dest=dest, error=sys_error)
logger.error("System %s failed (%s -> %s): %s", op, source, dest, sys_error.stderr)
log_transfer_permission_context(
"nfs_fallback_system", source=source, dest=dest, error=sys_error
)
logger.exception("System %s failed (%s -> %s): %s", op, source, dest, sys_error.stderr)
run_blocking_io(dest.unlink, missing_ok=True)
raise
else:
return
def _is_enoent_error(error: Exception) -> bool:
@@ -263,7 +275,7 @@ def _is_enoent_error(error: Exception) -> bool:
def _can_use_partial_copy_after_enoent(
temp_path: Optional[Path],
temp_path: Path | None,
expected_size: int,
action: str,
) -> bool:
@@ -273,9 +285,10 @@ def _can_use_partial_copy_after_enoent(
try:
_verify_transfer_size(temp_path, expected_size, action)
return True
except Exception:
except OSError:
return False
else:
return True
def _claim_destination(path: Path) -> bool:
@@ -345,7 +358,6 @@ def _publish_temp_file(temp_path: Path, dest_path: Path) -> bool:
run_blocking_io(os.close, fd)
except OSError:
pass
return True
except Exception as e:
if _is_permission_error(e):
log_transfer_permission_context(
@@ -356,6 +368,8 @@ def _publish_temp_file(temp_path: Path, dest_path: Path) -> bool:
)
run_blocking_io(dest_path.unlink, missing_ok=True)
raise
else:
return True
def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
@@ -378,6 +392,7 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
@@ -402,8 +417,7 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
else:
run_blocking_io(os.rename, str(source_path), str(try_path))
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
logger.info("File collision resolved: %s", try_path.name)
except FileExistsError:
# Race condition: file created between exists() check and rename()
if claimed:
@@ -421,7 +435,7 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
run_blocking_io(try_path.unlink, missing_ok=True)
claimed = False
temp_path: Optional[Path] = None
temp_path: Path | None = None
try:
try:
temp_path = _create_temp_path(try_path)
@@ -464,9 +478,7 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
run_blocking_io(source_path.unlink)
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
logger.info("File collision resolved: %s", try_path.name)
except FileExistsError:
if temp_path:
run_blocking_io(temp_path.unlink, missing_ok=True)
@@ -475,6 +487,8 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
if temp_path:
run_blocking_io(temp_path.unlink, missing_ok=True)
raise
else:
return try_path
except (PermissionError, OSError) as e:
if _is_permission_error(e):
@@ -493,17 +507,19 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
try:
_perform_nfs_fallback(source_path, try_path, is_move=True)
if attempt > 0:
logger.info(f"File collision resolved (fallback): {try_path.name}")
return try_path
logger.info("File collision resolved (fallback): %s", try_path.name)
except Exception as fallback_error:
logger.error(
"NFS fallback also failed (%s -> %s): %s",
logger.exception(
"NFS fallback also failed (%s -> %s)",
source_path,
try_path,
fallback_error,
)
raise e from fallback_error
else:
return try_path
raise
else:
return try_path
raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}")
@@ -521,6 +537,7 @@ def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100)
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
@@ -531,8 +548,7 @@ def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100)
try:
run_blocking_io(os.link, str(source_path), str(try_path))
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
logger.info("File collision resolved: %s", try_path.name)
except FileExistsError:
continue
except OSError as e:
@@ -553,6 +569,8 @@ def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100)
)
return atomic_copy(source_path, dest_path, max_attempts=max_attempts)
raise
else:
return try_path
raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
@@ -573,6 +591,7 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
@@ -583,7 +602,7 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
if run_blocking_io(try_path.exists):
continue
temp_path: Optional[Path] = None
temp_path: Path | None = None
try:
temp_path = _create_temp_path(try_path)
try:
@@ -606,11 +625,10 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
try:
_perform_nfs_fallback(source_path, temp_path, is_move=False)
except Exception as fallback_error:
logger.error(
"NFS fallback also failed (%s -> %s): %s",
logger.exception(
"NFS fallback also failed (%s -> %s)",
source_path,
temp_path,
fallback_error,
)
raise e from fallback_error
elif _is_enoent_error(e) and _can_use_partial_copy_after_enoent(
@@ -639,11 +657,12 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
raise
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
logger.info("File collision resolved: %s", try_path.name)
except Exception:
if temp_path:
run_blocking_io(temp_path.unlink, missing_ok=True)
raise
else:
return try_path
raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}")
shelfmark/download/http.py
+214 -112
View File
@@ -2,54 +2,76 @@
import random
import time
from http import HTTPStatus
from io import BytesIO
from threading import Event, Thread
from typing import Callable, Optional
from urllib.parse import urlparse, urljoin
from typing import TYPE_CHECKING, NoReturn
from urllib.parse import urljoin, urlparse
import requests
from tqdm import tqdm
from shelfmark.download import network
from shelfmark.download.network import get_proxies, get_ssl_verify
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
from shelfmark.download import network
from shelfmark.download.network import get_proxies, get_ssl_verify
if TYPE_CHECKING:
from collections.abc import Callable
from types import ModuleType
logger = setup_logger(__name__)
_MAX_REDIRECTS = 5
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
_HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
_HTTP_STATUS_RATE_LIMITED = HTTPStatus.TOO_MANY_REQUESTS
_HTTP_STATUS_OK = HTTPStatus.OK
_HTTP_STATUS_RANGE_NOT_SATISFIABLE = HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE
_HTTP_STATUS_PARTIAL_CONTENT = HTTPStatus.PARTIAL_CONTENT
_HTTP_STATUS_NON_RETRYABLE = (_HTTP_STATUS_FORBIDDEN, _HTTP_STATUS_NOT_FOUND)
# Bypasser modules are imported lazily to support dynamic selection based on config
_internal_bypasser = None
_external_bypasser = None
def _get_internal_bypasser():
def _raise_too_many_redirects(message: str) -> NoReturn:
raise requests.exceptions.TooManyRedirects(message)
def _get_internal_bypasser() -> ModuleType:
"""Lazy import of internal bypasser module."""
global _internal_bypasser
if _internal_bypasser is None:
try:
from shelfmark.bypass import internal_bypasser
_internal_bypasser = internal_bypasser
except ImportError as e:
raise RuntimeError(
msg = (
f"Failed to import internal bypasser: {e}. "
"Check that all dependencies are installed. "
"You may need to disable CF bypass or use the external bypasser."
) from e
)
raise RuntimeError(msg) from e
return _internal_bypasser
def _get_external_bypasser():
def _get_external_bypasser() -> ModuleType:
"""Lazy import of external bypasser module."""
global _external_bypasser
if _external_bypasser is None:
try:
from shelfmark.bypass import external_bypasser
_external_bypasser = external_bypasser
except ImportError as e:
raise RuntimeError(
msg = (
f"Failed to import external bypasser: {e}. "
"Check that the external bypasser is properly configured."
) from e
)
raise RuntimeError(msg) from e
return _external_bypasser
@@ -63,25 +85,29 @@ def _is_cf_bypass_enabled() -> bool:
return app_config.get("USE_CF_BYPASS", True)
def get_bypassed_page(url, selector=None, cancel_flag=None):
def get_bypassed_page(
url: str,
selector: network.AAMirrorSelector | None = None,
cancel_flag: Event | None = None,
) -> str | None:
"""Wrapper that delegates to the appropriate bypasser based on config."""
if _is_using_external_bypasser():
return _get_external_bypasser().get_bypassed_page(url, selector, cancel_flag)
return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag)
def get_cf_cookies_for_domain(domain):
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
"""Get CF cookies - only available with internal bypasser."""
if _is_using_external_bypasser():
logger.debug(f"External bypasser in use, CF cookies not available for {domain}")
logger.debug("External bypasser in use, CF cookies not available for %s", domain)
return {}
return _get_internal_bypasser().get_cf_cookies_for_domain(domain)
def get_cf_user_agent_for_domain(domain):
def get_cf_user_agent_for_domain(domain: str) -> str | None:
"""Get CF user agent - only available with internal bypasser."""
if _is_using_external_bypasser():
logger.debug(f"External bypasser in use, CF user agent not available for {domain}")
logger.debug("External bypasser in use, CF user agent not available for %s", domain)
return None
return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
@@ -100,7 +126,7 @@ def _apply_cf_bypass(url: str, headers: dict) -> dict:
cookies = get_cf_cookies_for_domain(hostname)
stored_ua = get_cf_user_agent_for_domain(hostname)
if stored_ua:
headers['User-Agent'] = stored_ua
headers["User-Agent"] = stored_ua
return cookies
@@ -110,19 +136,22 @@ MAX_DOWNLOAD_RETRIES = 2
MAX_RESUME_ATTEMPTS = 3
RETRYABLE_CODES = (429, 500, 502, 503, 504)
CONNECTION_ERRORS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout,
requests.exceptions.SSLError, requests.exceptions.ChunkedEncodingError)
CONNECTION_ERRORS = (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.SSLError,
requests.exceptions.ChunkedEncodingError,
)
DOWNLOAD_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
}
def parse_size_string(size: str) -> Optional[float]:
def parse_size_string(size: str) -> float | None:
"""Parse a human-readable size string (e.g., '10.5 MB') into bytes."""
if not size:
return None
@@ -133,20 +162,22 @@ def parse_size_string(size: str) -> Optional[float]:
if normalized.endswith(suffix):
return float(normalized[:-2]) * mult
return float(normalized)
except (ValueError, IndexError):
except ValueError, IndexError:
return None
def _backoff_delay(attempt: int, base: float = 0.25, cap: float = 3.0) -> float:
"""Exponential backoff with jitter."""
return min(cap, base * (2 ** (attempt - 1))) + random.random() * base
def _get_status_code(e: Exception) -> Optional[int]:
def _get_status_code(e: Exception) -> int | None:
"""Extract HTTP status code from an exception, or None if not applicable."""
if isinstance(e, requests.exceptions.HTTPError) and e.response is not None:
return e.response.status_code
return None
def _is_retryable_error(e: Exception) -> bool:
"""Check if error is retryable (connection error or retryable HTTP status)."""
if isinstance(e, CONNECTION_ERRORS):
@@ -155,31 +186,34 @@ def _is_retryable_error(e: Exception) -> bool:
return status is not None and status in RETRYABLE_CODES
def _try_rotation(original_url: str, current_url: str, selector: network.AAMirrorSelector) -> Optional[str]:
def _try_rotation(
original_url: str, current_url: str, selector: network.AAMirrorSelector
) -> str | None:
"""Try mirror/DNS rotation. Returns new URL or None."""
if current_url.startswith(network.get_aa_base_url()):
new_base, action = selector.next_mirror_or_rotate_dns()
if action in ("mirror", "dns") and new_base:
new_url = selector.rewrite(original_url)
logger.info(f"[{action}] switching to: {new_url}")
logger.info("[%s] switching to: %s", action, new_url)
return new_url
elif network.should_rotate_dns_for_url(current_url) and network.rotate_dns_provider():
logger.info(f"[dns-rotate] retrying: {original_url}")
logger.info("[dns-rotate] retrying: %s", original_url)
return original_url
return None
def html_get_page(
url: str,
retry: Optional[int] = None,
retry: int | None = None,
selector: network.AAMirrorSelector | None = None,
cancel_flag: Event | None = None,
status_callback: Callable[[str, str | None], None] | None = None,
*,
use_bypasser: bool = False,
selector: Optional[network.AAMirrorSelector] = None,
cancel_flag: Optional[Event] = None,
status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
allow_bypasser_fallback: bool = True,
include_response_url: bool = False,
success_delay: float = 1.0,
session: Optional[requests.Session] = None,
session: requests.Session | None = None,
) -> str | tuple[str, str]:
"""Fetch HTML content from a URL with retry mechanism.
@@ -189,7 +223,9 @@ def html_get_page(
include_response_url: If True, return `(html, final_url)` to expose the
resolved response URL after redirects.
success_delay: Optional delay (seconds) after successful fetch.
"""
def _result(html: str, response_url: str) -> str | tuple[str, str]:
if include_response_url:
return html, response_url
@@ -204,41 +240,43 @@ def html_get_page(
for attempt in range(1, retry + 1):
# Check for cancellation before each attempt
if cancel_flag and cancel_flag.is_set():
logger.info(f"html_get_page cancelled before attempt {attempt}")
logger.info("html_get_page cancelled before attempt %s", attempt)
return _result("", current_url)
try:
if use_bypasser_now and _is_cf_bypass_enabled():
logger.debug(f"GET (bypasser): {current_url}")
if status_callback:
status_callback("resolving", "Bypassing protection...")
heartbeat_stop = Event()
heartbeat_thread: Optional[Thread] = None
heartbeat_thread: Thread | None = None
if status_callback:
def _heartbeat() -> None:
def _heartbeat(stop_event: Event = heartbeat_stop) -> None:
# Keep the download "alive" during long bypass operations so the orchestrator
# doesn't flag it as stalled.
while not heartbeat_stop.wait(timeout=30):
if cancel_flag and cancel_flag.is_set():
return
try:
status_callback("resolving", "Bypassing protection...")
except Exception:
return
heartbeat_thread = Thread(target=_heartbeat, daemon=True, name="BypassHeartbeat")
if cancel_flag and cancel_flag.is_set():
return
try:
status_callback("resolving", "Bypassing protection...")
except Exception:
return
heartbeat_thread = Thread(
target=_heartbeat, daemon=True, name="BypassHeartbeat"
)
heartbeat_thread.start()
try:
result = get_bypassed_page(current_url, selector, cancel_flag)
return _result(result or "", current_url)
except Exception as e:
logger.warning(f"Bypasser error: {type(e).__name__}: {e}")
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
return _result("", current_url)
finally:
heartbeat_stop.set()
if heartbeat_thread:
heartbeat_thread.join(timeout=1)
logger.debug(f"GET: {current_url}")
logger.debug("GET: %s", current_url)
# Use a browser-like UA by default (AA can behave differently for python-requests UA).
headers = {"User-Agent": DOWNLOAD_HEADERS["User-Agent"]}
@@ -267,7 +305,9 @@ def html_get_page(
if is_aa_url and response.is_redirect:
location = response.headers.get("Location", "")
if not location:
raise requests.exceptions.TooManyRedirects(f"Redirect with no Location header: {current_url}")
_raise_too_many_redirects(
f"Redirect with no Location header: {current_url}"
)
redirect_url = urljoin(current_url, location)
current_host = urlparse(current_url).hostname or ""
@@ -305,8 +345,8 @@ def html_get_page(
# Same-host redirect (relative or absolute) - follow manually.
redirects_followed += 1
if redirects_followed > 5:
raise requests.exceptions.TooManyRedirects(f"Too many redirects for {current_url}")
if redirects_followed > _MAX_REDIRECTS:
_raise_too_many_redirects(f"Too many redirects for {current_url}")
current_url = redirect_url
continue
@@ -319,14 +359,14 @@ def html_get_page(
status = _get_status_code(e)
# 403 = Cloudflare/DDoS-Guard protection
if status == 403:
if status == _HTTP_STATUS_FORBIDDEN:
# If bypasser fallback is disabled, try mirrors instead
if not allow_bypasser_fallback:
new_url = _try_rotation(original_url, current_url, selector)
if new_url:
current_url = new_url
continue
logger.warning(f"403 error, mirrors exhausted: {current_url}")
logger.warning("403 error, mirrors exhausted: %s", current_url)
return _result("", current_url)
if _is_cf_bypass_enabled() and not use_bypasser_now:
@@ -336,19 +376,22 @@ def html_get_page(
fresh_cookies = get_cf_cookies_for_domain(parsed.hostname or "")
if fresh_cookies and not cookies:
# Cookies are now available - retry with cookies before using bypasser
logger.debug(f"403 but cookies now available - retrying with cookies: {current_url}")
logger.debug(
"403 but cookies now available - retrying with cookies: %s",
current_url,
)
continue
logger.info(f"403 detected; switching to bypasser: {current_url}")
logger.info("403 detected; switching to bypasser: %s", current_url)
if status_callback:
status_callback("resolving", "Bypassing protection...")
use_bypasser_now = True
continue
logger.warning(f"403 error, giving up: {current_url}")
logger.warning("403 error, giving up: %s", current_url)
return _result("", current_url)
# 404 = Not found
if status == 404:
logger.warning(f"404 error: {current_url}")
if status == _HTTP_STATUS_NOT_FOUND:
logger.warning("404 error: %s", current_url)
return _result("", current_url)
# Try mirror/DNS rotation on retryable errors
@@ -360,10 +403,17 @@ def html_get_page(
# Retry with backoff
if attempt < retry:
logger.warning(f"Retry {attempt}/{retry} for {current_url}: {type(e).__name__}: {e}")
logger.warning(
"Retry %s/%s for %s: %s: %s",
attempt,
retry,
current_url,
type(e).__name__,
e,
)
time.sleep(_backoff_delay(attempt))
else:
logger.error(f"Giving up after {retry} attempts: {current_url}")
logger.exception("Giving up after %s attempts: %s", retry, current_url)
return _result("", current_url)
@@ -371,12 +421,12 @@ def html_get_page(
def download_url(
link: str,
size: str = "",
progress_callback: Optional[Callable[[float], None]] = None,
cancel_flag: Optional[Event] = None,
_selector: Optional[network.AAMirrorSelector] = None,
status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
referer: Optional[str] = None,
) -> Optional[BytesIO]:
progress_callback: Callable[[float], None] | None = None,
cancel_flag: Event | None = None,
_selector: network.AAMirrorSelector | None = None,
status_callback: Callable[[str, str | None], None] | None = None,
referer: str | None = None,
) -> BytesIO | None:
"""Download content from URL with automatic retry and resume support."""
selector = _selector or network.AAMirrorSelector()
current_url = selector.rewrite(link)
@@ -384,7 +434,7 @@ def download_url(
# Build headers with optional referer
headers = DOWNLOAD_HEADERS.copy()
if referer:
headers['Referer'] = referer
headers["Referer"] = referer
total_size = parse_size_string(size) or 0
attempt = 0
@@ -399,19 +449,35 @@ def download_url(
try:
if attempt > 0 and status_callback:
status_callback("resolving", f"Connecting (Attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
status_callback(
"resolving",
f"Connecting (Attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})",
)
logger.info(f"Downloading: {current_url} (attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
logger.info(
"Downloading: %s (attempt %s/%s)",
current_url,
attempt + 1,
MAX_DOWNLOAD_RETRIES,
)
# Try with CF cookies/UA if available
cookies = _apply_cf_bypass(current_url, headers)
response = requests.get(current_url, stream=True, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers, verify=get_ssl_verify(current_url))
response = requests.get(
current_url,
stream=True,
proxies=get_proxies(current_url),
timeout=REQUEST_TIMEOUT,
cookies=cookies,
headers=headers,
verify=get_ssl_verify(current_url),
)
response.raise_for_status()
if status_callback:
status_callback("downloading", "")
total_size = total_size or float(response.headers.get('content-length', 0))
pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading')
total_size = total_size or float(response.headers.get("content-length", 0))
pbar = tqdm(total=total_size, unit="B", unit_scale=True, desc="Downloading")
for chunk in response.iter_content(chunk_size=8192):
if chunk:
@@ -426,55 +492,69 @@ def download_url(
pbar.close()
# Validate - check we didn't get HTML instead of file
if total_size > 0 and bytes_downloaded < total_size * 0.9:
if response.headers.get('content-type', '').startswith('text/html'):
logger.warning(f"Received HTML instead of file: {current_url}")
return None
if (
total_size > 0
and bytes_downloaded < total_size * 0.9
and response.headers.get("content-type", "").startswith("text/html")
):
logger.warning("Received HTML instead of file: %s", current_url)
return None
logger.debug(f"Download completed: {bytes_downloaded} bytes")
return buffer
logger.debug("Download completed: %s bytes", bytes_downloaded)
except requests.exceptions.RequestException as e:
status = _get_status_code(e)
retryable = _is_retryable_error(e)
# Z-Library 403 - try refreshing cookies via bypasser once before giving up
if status == 403 and _is_cf_bypass_enabled() and not zlib_cookie_refresh_attempted:
if (
status == _HTTP_STATUS_FORBIDDEN
and _is_cf_bypass_enabled()
and not zlib_cookie_refresh_attempted
):
parsed = urlparse(current_url)
if parsed.hostname and 'z-lib' in parsed.hostname and referer:
if parsed.hostname and "z-lib" in parsed.hostname and referer:
zlib_cookie_refresh_attempted = True
logger.info(f"Z-Library 403 - refreshing cookies via referer: {referer}")
logger.info("Z-Library 403 - refreshing cookies via referer: %s", referer)
try:
get_bypassed_page(referer, selector, cancel_flag)
time.sleep(0.5)
# Retry with fresh cookies (don't increment attempt)
continue
except Exception as cookie_err:
logger.warning(f"Z-Library cookie refresh failed: {cookie_err}")
logger.warning("Z-Library cookie refresh failed: %s", cookie_err)
# Non-retryable errors
if status in (403, 404):
logger.warning(f"Download failed ({status}): {current_url}")
if status in _HTTP_STATUS_NON_RETRYABLE:
logger.warning("Download failed (%s): %s", status, current_url)
return None
# Rate limited - skip to next source immediately
# (waiting doesn't help with concurrent downloads hitting the same server)
if status == 429:
logger.info(f"Rate limited (429) - trying next source")
if status == _HTTP_STATUS_RATE_LIMITED:
logger.info("Rate limited (429) - trying next source")
if status_callback:
status_callback("resolving", "Server busy, trying next")
return None
# Timeout - don't retry, server likely overloaded
if isinstance(e, requests.exceptions.Timeout):
logger.warning(f"Timeout: {current_url} - skipping to next source")
logger.warning("Timeout: %s - skipping to next source", current_url)
if status_callback:
status_callback("resolving", "Server timed out, trying next")
return None
# Try to resume if we got some data
if bytes_downloaded > 0 and retryable:
resumed = _try_resume(current_url, buffer, bytes_downloaded, total_size, progress_callback, cancel_flag, headers)
resumed = _try_resume(
current_url,
buffer,
bytes_downloaded,
total_size,
progress_callback,
cancel_flag,
headers,
)
if resumed:
return resumed
@@ -486,12 +566,14 @@ def download_url(
attempt += 1
continue
logger.warning(f"Download error: {type(e).__name__}: {e}")
logger.warning("Download error: %s: %s", type(e).__name__, e)
if attempt < MAX_DOWNLOAD_RETRIES - 1:
time.sleep(_backoff_delay(attempt + 1))
attempt += 1
else:
return buffer
logger.error(f"Download failed after {MAX_DOWNLOAD_RETRIES} attempts: {link}")
logger.error("Download failed after %s attempts: %s", MAX_DOWNLOAD_RETRIES, link)
return None
@@ -500,35 +582,54 @@ def _try_resume(
buffer: BytesIO,
start_byte: int,
total_size: float,
progress_callback: Optional[Callable[[float], None]],
cancel_flag: Optional[Event],
base_headers: Optional[dict] = None,
) -> Optional[BytesIO]:
progress_callback: Callable[[float], None] | None,
cancel_flag: Event | None,
base_headers: dict | None = None,
) -> BytesIO | None:
"""Try to resume an interrupted download."""
for attempt in range(MAX_RESUME_ATTEMPTS):
logger.info(f"Resuming from {start_byte} bytes (attempt {attempt + 1}/{MAX_RESUME_ATTEMPTS})")
logger.info(
"Resuming from %s bytes (attempt %s/%s)",
start_byte,
attempt + 1,
MAX_RESUME_ATTEMPTS,
)
time.sleep(_backoff_delay(attempt + 1, base=0.5, cap=5.0))
try:
# Try with CF cookies/UA if available
resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'}
resume_headers = {
**(base_headers or DOWNLOAD_HEADERS),
"Range": f"bytes={start_byte}-",
}
cookies = _apply_cf_bypass(url, resume_headers)
response = requests.get(
url, stream=True, proxies=get_proxies(url), timeout=REQUEST_TIMEOUT,
headers=resume_headers, cookies=cookies, verify=get_ssl_verify(url)
url,
stream=True,
proxies=get_proxies(url),
timeout=REQUEST_TIMEOUT,
headers=resume_headers,
cookies=cookies,
verify=get_ssl_verify(url),
)
# Check resume support
if response.status_code == 200: # Server doesn't support resume
if response.status_code == _HTTP_STATUS_OK: # Server doesn't support resume
logger.info("Server doesn't support resume")
return None
if response.status_code == 416: # Range not satisfiable
if response.status_code == _HTTP_STATUS_RANGE_NOT_SATISFIABLE: # Range not satisfiable
logger.warning("Range not satisfiable")
return None
if response.status_code != 206:
if response.status_code != _HTTP_STATUS_PARTIAL_CONTENT:
response.raise_for_status()
pbar = tqdm(total=total_size, initial=start_byte, unit='B', unit_scale=True, desc='Resuming')
pbar = tqdm(
total=total_size,
initial=start_byte,
unit="B",
unit_scale=True,
desc="Resuming",
)
for chunk in response.iter_content(chunk_size=8192):
if chunk:
buffer.write(chunk)
@@ -540,14 +641,15 @@ def _try_resume(
pbar.close()
return None
pbar.close()
logger.info(f"Resume completed: {start_byte} bytes")
return buffer
logger.info("Resume completed: %s bytes", start_byte)
except requests.exceptions.RequestException as e:
logger.debug(f"Resume attempt {attempt + 1} failed: {e}")
logger.warning(f"Resume failed after {MAX_RESUME_ATTEMPTS} attempts")
logger.debug("Resume attempt %s failed: %s", attempt + 1, e)
else:
return buffer
logger.warning("Resume failed after %s attempts", MAX_RESUME_ATTEMPTS)
return None
shelfmark/download/network.py
+395 -300
View File
File diff suppressed because it is too large Load Diff
shelfmark/download/orchestrator.py
+160 -140
View File
@@ -12,15 +12,18 @@ from concurrent.futures import Future, ThreadPoolExecutor
from email.utils import parseaddr
from pathlib import Path
from threading import Event, Lock
from typing import Any, Dict, List, Optional, Tuple
from typing import Any
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask, QueueStatus, SearchMode
from shelfmark.core.queue import book_queue
from shelfmark.core.request_helpers import normalize_optional_text, normalize_positive_int
from shelfmark.core.utils import transform_cover_url, is_audiobook as check_audiobook
from shelfmark.config import env as env_config
from shelfmark.core.request_helpers import (
normalize_optional_text,
normalize_positive_int,
)
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.core.utils import transform_cover_url
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.postprocess.pipeline import is_torrent_source, safe_cleanup_path
from shelfmark.download.postprocess.router import post_process_download
@@ -46,21 +49,25 @@ WEBSOCKET_AVAILABLE = True
try:
from shelfmark.api.websocket import ws_manager
except ImportError:
logger.error("WebSocket unavailable - real-time updates disabled")
logger.exception("WebSocket unavailable - real-time updates disabled")
ws_manager = None
WEBSOCKET_AVAILABLE = False
# Progress update throttling - track last broadcast time per book
_progress_last_broadcast: Dict[str, float] = {}
_progress_last_broadcast: dict[str, float] = {}
_progress_lock = Lock()
# Stall detection - track last activity time per download
_last_activity: Dict[str, float] = {}
_last_progress_value: Dict[str, float] = {}
_last_activity: dict[str, float] = {}
_last_progress_value: dict[str, float] = {}
# De-duplicate status updates (keep-alive updates shouldn't spam clients)
_last_status_event: Dict[str, Tuple[str, Optional[str]]] = {}
_last_status_event: dict[str, tuple[str, str | None]] = {}
STALL_TIMEOUT = 300 # 5 minutes without progress/status update = stalled
COORDINATOR_LOOP_ERROR_RETRY_DELAY = 1.0
_PROGRESS_BROADCAST_START_PERCENT = 1
_PROGRESS_BROADCAST_COMPLETE_PERCENT = 99
_PROGRESS_BROADCAST_MIN_DELTA = 10
def _is_plain_email_address(value: str) -> bool:
parsed = parseaddr(value or "")[1]
@@ -68,12 +75,13 @@ def _is_plain_email_address(value: str) -> bool:
def _resolve_email_destination(
user_id: Optional[int] = None,
) -> Tuple[Optional[str], Optional[str]]:
user_id: int | None = None,
) -> tuple[str | None, str | None]:
"""Resolve the destination email address for email output mode.
Returns:
(email_to, error_message)
"""
configured_recipient = str(config.get("EMAIL_RECIPIENT", "", user_id=user_id) or "").strip()
if configured_recipient:
@@ -84,7 +92,7 @@ def _resolve_email_destination(
return None, None
def _parse_release_search_mode(value: Any) -> SearchMode:
def _parse_release_search_mode(value: object) -> SearchMode:
if isinstance(value, SearchMode):
return value
if value is None:
@@ -93,29 +101,32 @@ def _parse_release_search_mode(value: Any) -> SearchMode:
try:
return SearchMode(value.strip().lower())
except ValueError as exc:
raise ValueError(f"Invalid search_mode: {value}") from exc
raise ValueError(f"Invalid search_mode: {value}")
msg = f"Invalid search_mode: {value}"
raise ValueError(msg) from exc
msg = f"Invalid search_mode: {value}"
raise ValueError(msg)
def _optional_number(value: Any) -> Optional[float]:
def _optional_number(value: object) -> float | None:
if isinstance(value, (int, float)) and not isinstance(value, bool):
return float(value)
try:
return float(value)
except (TypeError, ValueError):
except TypeError, ValueError:
return None
def _optional_positive_int(value: Any) -> Optional[int]:
def _optional_positive_int(value: object) -> int | None:
if isinstance(value, bool):
return None
try:
parsed = int(value)
except (TypeError, ValueError):
except TypeError, ValueError:
return None
return parsed if parsed > 0 else None
def _seed_time_seconds_to_minutes(value: Any) -> Optional[int]:
def _seed_time_seconds_to_minutes(value: object) -> int | None:
seed_time_seconds = _optional_positive_int(value)
if seed_time_seconds is None:
return None
@@ -124,7 +135,7 @@ def _seed_time_seconds_to_minutes(value: Any) -> Optional[int]:
def _build_retry_resolution_fields(
release_data: dict[str, Any],
) -> Dict[str, Any]:
) -> dict[str, Any]:
"""Persist generic resolved-download data needed for restart-safe retries."""
extra = release_data.get("extra")
if not isinstance(extra, dict):
@@ -139,9 +150,7 @@ def _build_retry_resolution_fields(
release_data.get("seeding_time_limit_minutes")
)
if seeding_time_limit_minutes is None:
seeding_time_limit_minutes = _seed_time_seconds_to_minutes(
extra.get("minimum_seed_time")
)
seeding_time_limit_minutes = _seed_time_seconds_to_minutes(extra.get("minimum_seed_time"))
return {
"retry_download_url": normalize_optional_text(release_data.get("download_url")),
@@ -159,47 +168,49 @@ def _build_retry_resolution_fields(
def queue_release(
release_data: dict,
priority: int = 0,
user_id: Optional[int] = None,
username: Optional[str] = None,
) -> Tuple[bool, Optional[str]]:
user_id: int | None = None,
username: str | None = None,
) -> tuple[bool, str | None]:
"""Add a release to the download queue. Returns (success, error_message)."""
try:
source = release_data['source']
extra = release_data.get('extra', {})
raw_request_id = release_data.get('_request_id')
request_id: Optional[int] = None
source = release_data["source"]
extra = release_data.get("extra", {})
raw_request_id = release_data.get("_request_id")
request_id: int | None = None
if isinstance(raw_request_id, int) and raw_request_id > 0:
request_id = raw_request_id
search_mode = _parse_release_search_mode(release_data.get("search_mode"))
# Get author, year, preview, and content_type from top-level (preferred) or extra (fallback)
author = release_data.get('author') or extra.get('author')
year = release_data.get('year') or extra.get('year')
preview = release_data.get('preview') or extra.get('preview')
content_type = release_data.get('content_type') or extra.get('content_type')
author = release_data.get("author") or extra.get("author")
year = release_data.get("year") or extra.get("year")
preview = release_data.get("preview") or extra.get("preview")
content_type = release_data.get("content_type") or extra.get("content_type")
source_url_raw = (
release_data.get('download_url')
or release_data.get('source_url')
or release_data.get('info_url')
or extra.get('detail_url')
or extra.get('source_url')
release_data.get("download_url")
or release_data.get("source_url")
or release_data.get("info_url")
or extra.get("detail_url")
or extra.get("source_url")
)
source_url = source_url_raw.strip() if isinstance(source_url_raw, str) else None
if source_url == "":
source_url = None
# Get series info for library naming templates
series_name = release_data.get('series_name') or extra.get('series_name')
series_position = release_data.get('series_position') or extra.get('series_position')
subtitle = release_data.get('subtitle') or extra.get('subtitle')
series_name = release_data.get("series_name") or extra.get("series_name")
series_position = release_data.get("series_position") or extra.get("series_position")
subtitle = release_data.get("subtitle") or extra.get("subtitle")
books_output_mode = str(
config.get("BOOKS_OUTPUT_MODE", "folder", user_id=user_id) or "folder"
).strip().lower()
books_output_mode = (
str(config.get("BOOKS_OUTPUT_MODE", "folder", user_id=user_id) or "folder")
.strip()
.lower()
)
is_audiobook = check_audiobook(content_type)
output_mode = "folder" if is_audiobook else books_output_mode
output_args: Dict[str, Any] = {}
output_args: dict[str, Any] = {}
retry_resolution_fields = _build_retry_resolution_fields(release_data)
if output_mode == "email" and not is_audiobook:
@@ -211,13 +222,13 @@ def queue_release(
# Create a source-agnostic download task from release data
task = DownloadTask(
task_id=release_data['source_id'],
task_id=release_data["source_id"],
source=source,
title=release_data.get('title', 'Unknown'),
title=release_data.get("title", "Unknown"),
author=author,
year=year,
format=release_data.get('format'),
size=release_data.get('size'),
format=release_data.get("format"),
size=release_data.get("size"),
preview=preview,
content_type=content_type,
source_url=source_url,
@@ -235,17 +246,15 @@ def queue_release(
)
if not book_queue.add(task):
logger.info(f"Release already in queue: {task.title}")
logger.info("Release already in queue: %s", task.title)
return False, "Release is already in the download queue"
logger.info(f"Release queued with priority {priority}: {task.title}")
logger.info("Release queued with priority %s: %s", priority, task.title)
# Broadcast status update via WebSocket
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
return True, None
except ValueError as e:
error_msg = str(e)
logger.warning(error_msg)
@@ -258,12 +267,15 @@ def queue_release(
error_msg = f"Error queueing release: {e}"
logger.error_trace(error_msg)
return False, error_msg
else:
return True, None
def queue_status(user_id: Optional[int] = None) -> Dict[str, Dict[str, Any]]:
def queue_status(user_id: int | None = None) -> dict[str, dict[str, Any]]:
"""Get current status of the download queue."""
status = book_queue.get_status(user_id=user_id)
for _, tasks in status.items():
for _, task in tasks.items():
for tasks in status.values():
for task in tasks.values():
if task.download_path and not run_blocking_io(os.path.exists, task.download_path):
task.download_path = None
@@ -276,7 +288,8 @@ def queue_status(user_id: Optional[int] = None) -> Dict[str, Dict[str, Any]]:
for status_type, tasks in status.items()
}
def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]]:
def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]:
"""Get downloaded file data for a specific task."""
task = None
try:
@@ -288,7 +301,7 @@ def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]
if not path:
return None, task
with open(path, "rb") as f:
with Path(path).open("rb") as f:
return f.read(), task
except Exception as e:
logger.error_trace(f"Error getting book data: {e}")
@@ -296,6 +309,7 @@ def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]
task.download_path = None
return None, task
def _has_staged_retry_source(task: DownloadTask) -> bool:
"""Whether a failed task still has a staged file available for retry."""
staged_path = task.staged_path.strip() if isinstance(task.staged_path, str) else ""
@@ -313,8 +327,8 @@ def _has_fresh_retry_context(task: DownloadTask) -> bool:
def can_retry_download_task(
task: Optional[DownloadTask],
status: Optional[QueueStatus],
task: DownloadTask | None,
status: QueueStatus | None,
) -> bool:
"""Whether the task can be manually retried from the Activity UI."""
if task is None or status not in (QueueStatus.ERROR, QueueStatus.CANCELLED):
@@ -329,10 +343,10 @@ def can_retry_download_task(
return _has_staged_retry_source(task)
def serialize_task_for_retry(task: DownloadTask) -> Dict[str, Any]:
def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
"""Serialize the task state needed for restart-safe retries."""
raw_search_mode = getattr(task, "search_mode", None)
search_mode: Optional[str] = None
search_mode: str | None = None
if isinstance(raw_search_mode, SearchMode):
search_mode = raw_search_mode.value
elif isinstance(raw_search_mode, str):
@@ -374,7 +388,7 @@ def serialize_task_for_retry(task: DownloadTask) -> Dict[str, Any]:
}
def _restore_task_from_retry_payload(payload: Any) -> Optional[DownloadTask]:
def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None:
if not isinstance(payload, dict):
return None
@@ -423,18 +437,16 @@ def _restore_task_from_retry_payload(payload: Any) -> Optional[DownloadTask]:
retry_seeding_time_limit_minutes=_optional_positive_int(
payload.get("retry_seeding_time_limit_minutes")
),
can_retry_without_staged_source=bool(
payload.get("can_retry_without_staged_source", True)
),
can_retry_without_staged_source=bool(payload.get("can_retry_without_staged_source", True)),
)
def retry_persisted_download(
payload: Any,
payload: object,
*,
final_status: Any,
final_status: object,
priority: int = -10,
) -> Tuple[bool, Optional[str]]:
) -> tuple[bool, str | None]:
"""Retry a persisted download row after the in-memory task has been lost."""
task = _restore_task_from_retry_payload(payload)
if task is None:
@@ -453,13 +465,8 @@ def retry_persisted_download(
if normalized_status in {"active", "cancelled"} and not has_fresh_retry_context:
return False, "Download cannot be retried"
if (
task.request_id is not None
and normalized_status == "error"
and not has_staged_retry_source
):
if task.request_id is not None:
return False, "Request-linked downloads must be retried from requests"
if task.request_id is not None and normalized_status == "error" and not has_staged_retry_source:
return False, "Request-linked downloads must be retried from requests"
if (
task.request_id is None
@@ -486,33 +493,33 @@ def retry_persisted_download(
def _task_to_dict(
task: DownloadTask,
current_status: Optional[QueueStatus] = None,
) -> Dict[str, Any]:
current_status: QueueStatus | None = None,
) -> dict[str, Any]:
"""Convert DownloadTask to dict for frontend, transforming cover URLs."""
# Transform external preview URLs to local proxy URLs
preview = transform_cover_url(task.preview, task.task_id)
retry_status = current_status or book_queue.get_task_status(task.task_id)
return {
'id': task.task_id,
'title': task.title,
'author': task.author,
'format': task.format,
'size': task.size,
'preview': preview,
'content_type': task.content_type,
'source': task.source,
'source_display_name': get_source_display_name(task.source),
'priority': task.priority,
'added_time': task.added_time,
'progress': task.progress,
'status': task.status,
'status_message': task.status_message,
'download_path': task.download_path,
'user_id': task.user_id,
'username': task.username,
'request_id': task.request_id,
'retry_available': can_retry_download_task(task, retry_status),
"id": task.task_id,
"title": task.title,
"author": task.author,
"format": task.format,
"size": task.size,
"preview": preview,
"content_type": task.content_type,
"source": task.source,
"source_display_name": get_source_display_name(task.source),
"priority": task.priority,
"added_time": task.added_time,
"progress": task.progress,
"status": task.status,
"status_message": task.status_message,
"download_path": task.download_path,
"user_id": task.user_id,
"username": task.username,
"request_id": task.request_id,
"retry_available": can_retry_download_task(task, retry_status),
}
@@ -524,8 +531,8 @@ def _clear_task_error_state(task: DownloadTask) -> None:
def _capture_task_error(
task: DownloadTask,
*,
message: Optional[str] = None,
exc_type: Optional[str] = None,
message: str | None = None,
exc_type: str | None = None,
) -> None:
if isinstance(message, str):
normalized = message.strip()
@@ -546,7 +553,7 @@ def _format_download_exception_message(exc: Exception) -> str:
return f"Download failed: {type(exc).__name__}"
def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
def _download_task(task_id: str, cancel_flag: Event) -> str | None:
"""Download a task via appropriate handler, then post-process to ingest."""
try:
# Check for cancellation before starting
@@ -570,7 +577,7 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
def progress_callback(progress: float) -> None:
update_download_progress(task_id, progress)
def status_callback(status: str, message: Optional[str] = None) -> None:
def status_callback(status: str, message: str | None = None) -> None:
status_key = status.lower()
if status_key == "error":
_capture_task_error(
@@ -591,7 +598,7 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
# Get the download handler based on the task's source
handler = get_handler(task.source)
temp_file: Optional[Path] = None
temp_file: Path | None = None
if task.staged_path:
staged_file = Path(task.staged_path)
@@ -615,7 +622,7 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
temp_file = Path(temp_path)
if not run_blocking_io(temp_file.exists):
logger.error(f"Handler returned non-existent path: {temp_path}")
logger.error("Handler returned non-existent path: %s", temp_path)
_capture_task_error(
task,
message=f"Download file missing: {temp_path}",
@@ -667,8 +674,6 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
task.staged_path = None
_clear_task_error_state(task)
return result
except Exception as e:
if cancel_flag.is_set():
logger.info("Task %s: cancelled during error handling", task_id)
@@ -683,6 +688,8 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
)
return None
else:
return result
def update_download_progress(book_id: str, progress: float) -> None:
@@ -696,37 +703,38 @@ def update_download_progress(book_id: str, progress: float) -> None:
if last_progress is None or progress != last_progress:
_last_activity[book_id] = time.time()
_last_progress_value[book_id] = progress
# Broadcast progress via WebSocket with throttling
if ws_manager:
current_time = time.time()
should_broadcast = False
with _progress_lock:
last_broadcast = _progress_last_broadcast.get(book_id, 0)
last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0)
time_elapsed = current_time - last_broadcast
# Always broadcast at start (0%) or completion (>=99%)
if progress <= 1 or progress >= 99:
should_broadcast = True
# Broadcast if enough time has passed (convert interval from seconds)
elif time_elapsed >= config.DOWNLOAD_PROGRESS_UPDATE_INTERVAL:
should_broadcast = True
# Broadcast on significant progress jumps (>10%)
elif progress - last_progress >= 10:
should_broadcast = True
should_broadcast = (
progress <= _PROGRESS_BROADCAST_START_PERCENT
or progress >= _PROGRESS_BROADCAST_COMPLETE_PERCENT
or time_elapsed >= config.DOWNLOAD_PROGRESS_UPDATE_INTERVAL
or progress - last_progress >= _PROGRESS_BROADCAST_MIN_DELTA
)
if should_broadcast:
_progress_last_broadcast[book_id] = current_time
_progress_last_broadcast[f"{book_id}_progress"] = progress
if should_broadcast:
task = book_queue.get_task(book_id)
task_user_id = task.user_id if task else None
ws_manager.broadcast_download_progress(book_id, progress, 'downloading', user_id=task_user_id)
ws_manager.broadcast_download_progress(
book_id, progress, "downloading", user_id=task_user_id
)
def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None:
def update_download_status(book_id: str, status: str, message: str | None = None) -> None:
"""Update download status with optional message for UI display."""
status_key = status.lower()
try:
@@ -752,18 +760,19 @@ def update_download_status(book_id: str, status: str, message: Optional[str] = N
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
def cancel_download(book_id: str) -> bool:
"""Cancel a download."""
result = book_queue.cancel_download(book_id)
# Broadcast status update via WebSocket
if result and ws_manager and ws_manager.is_enabled():
ws_manager.broadcast_status_update(queue_status())
return result
def retry_download(book_id: str) -> Tuple[bool, Optional[str]]:
def retry_download(book_id: str) -> tuple[bool, str | None]:
"""Retry a failed or cancelled download.
Request-linked downloads can only be manually retried when cancelled or
@@ -794,22 +803,27 @@ def retry_download(book_id: str) -> Tuple[bool, Optional[str]]:
return True, None
def set_book_priority(book_id: str, priority: int) -> bool:
"""Set priority for a queued book (lower = higher priority)."""
return book_queue.set_priority(book_id, priority)
def reorder_queue(book_priorities: Dict[str, int]) -> bool:
def reorder_queue(book_priorities: dict[str, int]) -> bool:
"""Bulk reorder queue by mapping book_id to new priority."""
return book_queue.reorder_queue(book_priorities)
def get_queue_order() -> List[Dict[str, Any]]:
def get_queue_order() -> list[dict[str, Any]]:
"""Get current queue order for display."""
return book_queue.get_queue_order()
def get_active_downloads() -> List[str]:
def get_active_downloads() -> list[str]:
"""Get list of currently active downloads."""
return book_queue.get_active_downloads()
def _cleanup_progress_tracking(task_id: str) -> None:
"""Clean up progress tracking data for a completed/cancelled download."""
with _progress_lock:
@@ -875,25 +889,26 @@ def _process_single_download(task_id: str, cancel_flag: Event) -> None:
if task:
_capture_task_error(
task,
message=f"Download failed: {type(e).__name__}: {str(e)}",
message=f"Download failed: {type(e).__name__}: {e!s}",
exc_type=type(e).__name__,
)
_finalize_download_failure(task_id)
else:
logger.info(f"Download cancelled: {task_id}")
logger.info("Download cancelled: %s", task_id)
book_queue.update_status(task_id, QueueStatus.CANCELLED)
# Broadcast error/cancelled status
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
def concurrent_download_loop() -> None:
"""Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
max_workers = config.MAX_CONCURRENT_DOWNLOADS
logger.info(f"Starting concurrent download loop with {max_workers} workers")
logger.info("Starting concurrent download loop with %s workers", max_workers)
with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="Download") as executor:
active_futures: Dict[Future, str] = {} # Track active download futures
active_futures: dict[Future, str] = {} # Track active download futures
stalled_tasks: set[str] = set() # Track tasks already cancelled due to stall
while True:
@@ -911,14 +926,17 @@ def concurrent_download_loop() -> None:
# Check for stalled downloads (no activity in STALL_TIMEOUT seconds)
current_time = time.time()
with _progress_lock:
for future, task_id in list(active_futures.items()):
for _future, task_id in list(active_futures.items()):
if task_id in stalled_tasks:
continue
last_active = _last_activity.get(task_id, current_time)
if current_time - last_active > STALL_TIMEOUT:
logger.warning(f"Download stalled for {task_id}, cancelling")
logger.warning("Download stalled for %s, cancelling", task_id)
book_queue.cancel_download(task_id)
book_queue.update_status_message(task_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)")
book_queue.update_status_message(
task_id,
f"Download stalled (no activity for {STALL_TIMEOUT}s)",
)
stalled_tasks.add(task_id)
# Start new downloads if we have capacity
@@ -931,7 +949,7 @@ def concurrent_download_loop() -> None:
# Only delay if other downloads are already active
if active_futures:
stagger_delay = random.uniform(2, 5)
logger.debug(f"Staggering download start by {stagger_delay:.1f}s")
logger.debug("Staggering download start by %.1fs", stagger_delay)
time.sleep(stagger_delay)
task_id, cancel_flag = next_download
@@ -946,8 +964,9 @@ def concurrent_download_loop() -> None:
logger.error_trace("Download coordinator loop error: %s", e)
time.sleep(COORDINATOR_LOOP_ERROR_RETRY_DELAY)
# Download coordinator thread (started explicitly via start())
_coordinator_thread: Optional[threading.Thread] = None
_coordinator_thread: threading.Thread | None = None
_coordinator_lock = Lock()
@@ -964,10 +983,11 @@ def start() -> None:
logger.warning("Download coordinator thread is not alive; starting a new one")
_coordinator_thread = threading.Thread(
target=concurrent_download_loop,
daemon=True,
name="DownloadCoordinator"
target=concurrent_download_loop, daemon=True, name="DownloadCoordinator"
)
_coordinator_thread.start()
logger.info(f"Download coordinator started with {config.MAX_CONCURRENT_DOWNLOADS} concurrent workers")
logger.info(
"Download coordinator started with %s concurrent workers",
config.MAX_CONCURRENT_DOWNLOADS,
)
shelfmark/download/outputs/__init__.py
+7 -8
View File
@@ -1,14 +1,14 @@
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from threading import Event
from typing import Callable, Optional
from shelfmark.core.models import DownloadTask
StatusCallback = Callable[[str, Optional[str]], None]
OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback, bool], Optional[str]]
StatusCallback = Callable[[str, str | None], None]
OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback, bool], str | None]
@dataclass(frozen=True)
@@ -48,9 +48,9 @@ def load_output_handlers() -> None:
if _OUTPUTS_LOADED:
return
from . import booklore # noqa: F401
from . import email # noqa: F401
from . import folder # noqa: F401
from . import booklore as booklore
from . import email as email
from . import folder as folder
_OUTPUTS_LOADED = True
@@ -65,7 +65,6 @@ def _derive_output_mode(task: DownloadTask) -> str:
Prefer the mode captured at queue time. Fall back to current config for
legacy tasks that do not have `output_mode` populated.
"""
mode = _normalize_output_mode(getattr(task, "output_mode", None))
if mode:
return mode
@@ -80,7 +79,7 @@ def _derive_output_mode(task: DownloadTask) -> str:
return _normalize_output_mode(config.get("BOOKS_OUTPUT_MODE", "folder")) or "folder"
def resolve_output_handler(task: DownloadTask) -> Optional[OutputRegistration]:
def resolve_output_handler(task: DownloadTask) -> OutputRegistration | None:
load_output_handlers()
desired_mode = _derive_output_mode(task)
shelfmark/download/outputs/booklore.py
+116 -48
View File
@@ -3,24 +3,44 @@ from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
from threading import Event
from typing import Any, Dict, List, Mapping, Optional
from typing import TYPE_CHECKING, Any
import requests
import shelfmark.core.config as core_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.download.outputs import register_output
from shelfmark.download.staging import STAGE_COPY, STAGE_MOVE, STAGE_NONE, build_staging_dir, get_staging_dir
from shelfmark.download.outputs import StatusCallback, register_output
from shelfmark.download.staging import (
STAGE_COPY,
STAGE_MOVE,
STAGE_NONE,
build_staging_dir,
get_staging_dir,
)
if TYPE_CHECKING:
from collections.abc import Mapping
from threading import Event
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
BOOKLORE_OUTPUT_MODE = "booklore"
BOOKLORE_DESTINATION_LIBRARY = "library"
BOOKLORE_DESTINATION_BOOKDROP = "bookdrop"
BOOKLORE_SUPPORTED_EXTENSIONS = {".azw", ".azw3", ".cb7", ".cbr", ".cbz", ".epub", ".fb2", ".mobi", ".pdf"}
BOOKLORE_SUPPORTED_EXTENSIONS = {
".azw",
".azw3",
".cb7",
".cbr",
".cbz",
".epub",
".fb2",
".mobi",
".pdf",
}
BOOKLORE_SUPPORTED_FORMATS_LABEL = ", ".join(
ext.lstrip(".").upper() for ext in sorted(BOOKLORE_SUPPORTED_EXTENSIONS)
)
@@ -43,16 +63,18 @@ class BookloreConfig:
refresh_after_upload: bool = False
def _parse_int(value: Any, label: str) -> int:
def _parse_int(value: object, label: str) -> int:
if value is None or value == "":
raise BookloreError(f"{label} is required")
msg = f"{label} is required"
raise BookloreError(msg)
try:
return int(value)
except (TypeError, ValueError) as exc:
raise BookloreError(f"{label} must be a number") from exc
msg = f"{label} must be a number"
raise BookloreError(msg) from exc
def _parse_destination(value: Any) -> str:
def _parse_destination(value: object) -> str:
normalized = str(value or "").strip().lower()
if normalized == BOOKLORE_DESTINATION_BOOKDROP:
return BOOKLORE_DESTINATION_BOOKDROP
@@ -61,18 +83,21 @@ def _parse_destination(value: Any) -> str:
def build_booklore_config(
values: Mapping[str, Any],
user_id: Optional[int] = None,
user_id: int | None = None,
) -> BookloreConfig:
base_url = str(values.get("BOOKLORE_HOST", "")).strip()
username = str(values.get("BOOKLORE_USERNAME", "")).strip()
password = values.get("BOOKLORE_PASSWORD", "") or ""
if not base_url:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} URL is required")
msg = f"{BOOKLORE_DISPLAY_NAME} URL is required"
raise BookloreError(msg)
if not username:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} username is required")
msg = f"{BOOKLORE_DISPLAY_NAME} username is required"
raise BookloreError(msg)
if not password:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} password is required")
msg = f"{BOOKLORE_DISPLAY_NAME} password is required"
raise BookloreError(msg)
destination = _parse_destination(
values.get("BOOKLORE_DESTINATION", BOOKLORE_DESTINATION_LIBRARY)
@@ -115,33 +140,43 @@ def build_booklore_config(
def booklore_login(booklore_config: BookloreConfig) -> str:
url = f"{booklore_config.base_url}/api/v1/auth/login"
payload = {"username": booklore_config.username, "password": booklore_config.password}
payload = {
"username": booklore_config.username,
"password": booklore_config.password,
}
try:
response = requests.post(url, json=payload, timeout=30, verify=booklore_config.verify_tls)
except requests.exceptions.ConnectionError as exc:
raise BookloreError(f"Could not connect to {BOOKLORE_DISPLAY_NAME}") from exc
msg = f"Could not connect to {BOOKLORE_DISPLAY_NAME}"
raise BookloreError(msg) from exc
except requests.exceptions.Timeout as exc:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} connection timed out") from exc
msg = f"{BOOKLORE_DISPLAY_NAME} connection timed out"
raise BookloreError(msg) from exc
except requests.exceptions.RequestException as exc:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} login failed: {exc}") from exc
msg = f"{BOOKLORE_DISPLAY_NAME} login failed: {exc}"
raise BookloreError(msg) from exc
if response.status_code in {401, 403}:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} authentication failed")
msg = f"{BOOKLORE_DISPLAY_NAME} authentication failed"
raise BookloreError(msg)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as exc:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} login failed ({response.status_code})") from exc
msg = f"{BOOKLORE_DISPLAY_NAME} login failed ({response.status_code})"
raise BookloreError(msg) from exc
try:
data = response.json()
except ValueError as exc:
raise BookloreError(f"Invalid {BOOKLORE_DISPLAY_NAME} login response") from exc
msg = f"Invalid {BOOKLORE_DISPLAY_NAME} login response"
raise BookloreError(msg) from exc
token = data.get("accessToken")
if not token:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} did not return an access token")
msg = f"{BOOKLORE_DISPLAY_NAME} did not return an access token"
raise BookloreError(msg)
return token
@@ -154,12 +189,14 @@ def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list
response = requests.get(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
response.raise_for_status()
except requests.exceptions.RequestException as exc:
raise BookloreError(f"Failed to fetch {BOOKLORE_DISPLAY_NAME} libraries: {exc}") from exc
msg = f"Failed to fetch {BOOKLORE_DISPLAY_NAME} libraries: {exc}"
raise BookloreError(msg) from exc
try:
return response.json()
except ValueError as exc:
raise BookloreError(f"Invalid {BOOKLORE_DISPLAY_NAME} libraries response") from exc
msg = f"Invalid {BOOKLORE_DISPLAY_NAME} libraries response"
raise BookloreError(msg) from exc
def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: Path) -> None:
@@ -168,7 +205,10 @@ def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path:
params = None
else:
url = f"{booklore_config.base_url}/api/v1/files/upload"
params = {"libraryId": booklore_config.library_id, "pathId": booklore_config.path_id}
params = {
"libraryId": booklore_config.library_id,
"pathId": booklore_config.path_id,
}
headers = {"Authorization": f"Bearer {token}"}
@@ -190,13 +230,17 @@ def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path:
if message:
message = f": {message[:200]}"
status_code = response.status_code if response is not None else "unknown"
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} upload failed ({status_code}){message}") from exc
msg = f"{BOOKLORE_DISPLAY_NAME} upload failed ({status_code}){message}"
raise BookloreError(msg) from exc
except requests.exceptions.ConnectionError as exc:
raise BookloreError(f"Could not connect to {BOOKLORE_DISPLAY_NAME}") from exc
msg = f"Could not connect to {BOOKLORE_DISPLAY_NAME}"
raise BookloreError(msg) from exc
except requests.exceptions.Timeout as exc:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} upload timed out") from exc
msg = f"{BOOKLORE_DISPLAY_NAME} upload timed out"
raise BookloreError(msg) from exc
except requests.exceptions.RequestException as exc:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} upload failed: {exc}") from exc
msg = f"{BOOKLORE_DISPLAY_NAME} upload failed: {exc}"
raise BookloreError(msg) from exc
def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> None:
@@ -207,14 +251,15 @@ def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> Non
response = requests.put(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
response.raise_for_status()
except requests.exceptions.RequestException as exc:
raise BookloreError(f"{BOOKLORE_DISPLAY_NAME} refresh failed: {exc}") from exc
msg = f"{BOOKLORE_DISPLAY_NAME} refresh failed: {exc}"
raise BookloreError(msg) from exc
def _supports_booklore(task: DownloadTask) -> bool:
return not check_audiobook(task.content_type)
def _get_booklore_settings() -> Dict[str, Any]:
def _get_booklore_settings() -> dict[str, Any]:
return {
"BOOKLORE_HOST": core_config.config.get("BOOKLORE_HOST", ""),
"BOOKLORE_USERNAME": core_config.config.get("BOOKLORE_USERNAME", ""),
@@ -228,8 +273,8 @@ def _get_booklore_settings() -> Dict[str, Any]:
}
def _booklore_format_error(rejected_files: List[Path]) -> str:
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
def _booklore_format_error(rejected_files: list[Path]) -> str:
rejected_exts = sorted({f.suffix.lower() for f in rejected_files})
rejected_list = ", ".join(rejected_exts)
return (
f"{BOOKLORE_DISPLAY_NAME} does not support {rejected_list}. "
@@ -241,9 +286,10 @@ def _post_process_booklore(
temp_file: Path,
task: DownloadTask,
cancel_flag: Event,
status_callback,
status_callback: StatusCallback,
*,
preserve_source_on_failure: bool = False,
) -> Optional[str]:
) -> str | None:
from shelfmark.download.postprocess.pipeline import (
CustomScriptContext,
OutputPlan,
@@ -273,7 +319,11 @@ def _post_process_booklore(
stage_action = STAGE_NONE
if is_managed_workspace_path(temp_file):
stage_action = STAGE_COPY if preserve_source_on_failure else STAGE_MOVE
staging_dir = build_staging_dir("booklore", task.task_id) if stage_action != STAGE_NONE else get_staging_dir()
staging_dir = (
build_staging_dir("booklore", task.task_id)
if stage_action != STAGE_NONE
else get_staging_dir()
)
output_plan = OutputPlan(
mode=BOOKLORE_OUTPUT_MODE,
@@ -293,7 +343,11 @@ def _post_process_booklore(
if not prepared:
return None
logger.debug("Task %s: prepared %d file(s) for Booklore upload", task.task_id, len(prepared.files))
logger.debug(
"Task %s: prepared %d file(s) for Booklore upload",
task.task_id,
len(prepared.files),
)
success = False
try:
@@ -309,13 +363,20 @@ def _post_process_booklore(
return None
token = booklore_login(booklore_config)
logger.info("Task %s: uploading %d file(s) to Booklore", task.task_id, len(prepared.files))
logger.info(
"Task %s: uploading %d file(s) to Booklore",
task.task_id,
len(prepared.files),
)
for index, file_path in enumerate(prepared.files, start=1):
if cancel_flag.is_set():
logger.info("Task %s: cancelled during Booklore upload", task.task_id)
return None
status_callback("resolving", f"Uploading to {BOOKLORE_DISPLAY_NAME} ({index}/{len(prepared.files)})")
status_callback(
"resolving",
f"Uploading to {BOOKLORE_DISPLAY_NAME} ({index}/{len(prepared.files)})",
)
booklore_upload_file(booklore_config, token, file_path)
if booklore_config.refresh_after_upload:
@@ -324,9 +385,13 @@ def _post_process_booklore(
except BookloreError as e:
logger.warning("Task %s: Booklore refresh failed: %s", task.task_id, e)
logger.info("Task %s: uploaded %d file(s) to Booklore", task.task_id, len(prepared.files))
logger.info(
"Task %s: uploaded %d file(s) to Booklore",
task.task_id,
len(prepared.files),
)
destination: Optional[Path]
destination: Path | None
if len(prepared.files) == 1:
destination = prepared.files[0].parent
else:
@@ -350,11 +415,11 @@ def _post_process_booklore(
else BOOKLORE_DESTINATION_LIBRARY
),
"library_id": (
None
if booklore_config.upload_to_bookdrop
else booklore_config.library_id
None if booklore_config.upload_to_bookdrop else booklore_config.library_id
),
"path_id": None if booklore_config.upload_to_bookdrop else booklore_config.path_id,
"path_id": None
if booklore_config.upload_to_bookdrop
else booklore_config.path_id,
"refresh_after_upload": bool(booklore_config.refresh_after_upload),
}
},
@@ -367,7 +432,7 @@ def _post_process_booklore(
message = f"Uploaded to {BOOKLORE_DISPLAY_NAME} ({len(prepared.files)} files)"
status_callback("complete", message)
success = True
return f"booklore://{task.task_id}"
output_path = f"booklore://{task.task_id}"
except BookloreError as e:
logger.warning("Task %s: Booklore upload failed: %s", task.task_id, e)
@@ -377,6 +442,8 @@ def _post_process_booklore(
logger.error_trace("Task %s: unexpected error uploading to Booklore: %s", task.task_id, e)
status_callback("error", f"{BOOKLORE_DISPLAY_NAME} upload failed: {e}")
return None
else:
return output_path
finally:
cleanup_output_staging(
prepared.output_plan,
@@ -393,9 +460,10 @@ def process_booklore_output(
temp_file: Path,
task: DownloadTask,
cancel_flag: Event,
status_callback,
status_callback: StatusCallback,
*,
preserve_source_on_failure: bool = False,
) -> Optional[str]:
) -> str | None:
return _post_process_booklore(
temp_file,
task,
shelfmark/download/outputs/email.py
+63 -44
View File
@@ -3,19 +3,30 @@ from __future__ import annotations
import mimetypes
import smtplib
import ssl
from contextlib import suppress
from dataclasses import dataclass
from email.message import EmailMessage
from email.utils import formatdate, make_msgid, parseaddr
from pathlib import Path
from threading import Event
from typing import Any, Dict, Mapping, Optional
from typing import TYPE_CHECKING, Any
import shelfmark.core.config as core_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.download.outputs import register_output
from shelfmark.download.staging import STAGE_COPY, STAGE_MOVE, STAGE_NONE, build_staging_dir, get_staging_dir
from shelfmark.download.staging import (
STAGE_COPY,
STAGE_MOVE,
STAGE_NONE,
build_staging_dir,
get_staging_dir,
)
if TYPE_CHECKING:
from collections.abc import Callable, Mapping
from pathlib import Path
from threading import Event
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
@@ -62,14 +73,18 @@ def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig:
security = str(values.get("EMAIL_SMTP_SECURITY", SECURITY_STARTTLS) or "").strip().lower()
if security not in ALLOWED_SECURITY:
raise EmailOutputError(f"SMTP security must be one of: {', '.join(sorted(ALLOWED_SECURITY))}")
raise EmailOutputError(
f"SMTP security must be one of: {', '.join(sorted(ALLOWED_SECURITY))}"
)
username = str(values.get("EMAIL_SMTP_USERNAME", "") or "").strip()
password = values.get("EMAIL_SMTP_PASSWORD", "") or ""
from_addr = str(values.get("EMAIL_FROM", "") or "").strip()
subject_template = str(values.get("EMAIL_SUBJECT_TEMPLATE", "{Title}") or "").strip()
timeout_seconds = _parse_int(values.get("EMAIL_SMTP_TIMEOUT_SECONDS", 60), "SMTP timeout (seconds)", minimum=1)
timeout_seconds = _parse_int(
values.get("EMAIL_SMTP_TIMEOUT_SECONDS", 60), "SMTP timeout (seconds)", minimum=1
)
allow_unverified_tls = bool(values.get("EMAIL_ALLOW_UNVERIFIED_TLS", False))
if not host:
@@ -83,7 +98,9 @@ def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig:
if username_email and "@" in username_email:
from_addr = f"Shelfmark <{username_email}>"
else:
raise EmailOutputError("From address is required (or set SMTP username to an email address).")
raise EmailOutputError(
"From address is required (or set SMTP username to an email address)."
)
return EmailSmtpConfig(
host=host,
@@ -98,7 +115,7 @@ def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig:
)
def _get_email_settings() -> Dict[str, Any]:
def _get_email_settings() -> dict[str, Any]:
return {
"EMAIL_SMTP_HOST": core_config.config.get("EMAIL_SMTP_HOST", ""),
"EMAIL_SMTP_PORT": core_config.config.get("EMAIL_SMTP_PORT", 587),
@@ -124,7 +141,7 @@ def _render_subject(template: str, task: DownloadTask) -> str:
}
try:
rendered = template.format(**mapping)
except Exception:
except IndexError, KeyError, ValueError:
rendered = template
rendered = " ".join(str(rendered).split()).strip()
@@ -132,11 +149,8 @@ def _render_subject(template: str, task: DownloadTask) -> str:
def _msgid_domain(from_addr: str) -> str:
try:
from_email = parseaddr(from_addr)[1]
domain = (from_email.partition("@")[2] or "").strip().rstrip(">")
except Exception:
domain = ""
from_email = parseaddr(from_addr)[1]
domain = (from_email.partition("@")[2] or "").strip().rstrip(">")
return domain or "shelfmark.local"
@@ -171,7 +185,7 @@ def compose_email_message(
return message
def _create_tls_context(allow_unverified: bool) -> ssl.SSLContext:
def _create_tls_context(*, allow_unverified: bool) -> ssl.SSLContext:
context = ssl.create_default_context()
if allow_unverified:
context.check_hostname = False
@@ -181,11 +195,10 @@ def _create_tls_context(allow_unverified: bool) -> ssl.SSLContext:
def test_smtp_connection(smtp_config: EmailSmtpConfig) -> None:
"""Connect and (optionally) authenticate to the SMTP server. Does not send mail."""
smtp: Optional[smtplib.SMTP] = None
smtp: smtplib.SMTP | None = None
try:
if smtp_config.security == SECURITY_SSL:
context = _create_tls_context(smtp_config.allow_unverified_tls)
context = _create_tls_context(allow_unverified=smtp_config.allow_unverified_tls)
smtp = smtplib.SMTP_SSL(
smtp_config.host,
smtp_config.port,
@@ -193,12 +206,14 @@ def test_smtp_connection(smtp_config: EmailSmtpConfig) -> None:
context=context,
)
else:
smtp = smtplib.SMTP(smtp_config.host, smtp_config.port, timeout=smtp_config.timeout_seconds)
smtp = smtplib.SMTP(
smtp_config.host, smtp_config.port, timeout=smtp_config.timeout_seconds
)
smtp.ehlo()
if smtp_config.security == SECURITY_STARTTLS:
context = _create_tls_context(smtp_config.allow_unverified_tls)
context = _create_tls_context(allow_unverified=smtp_config.allow_unverified_tls)
smtp.starttls(context=context)
smtp.ehlo()
@@ -210,20 +225,17 @@ def test_smtp_connection(smtp_config: EmailSmtpConfig) -> None:
raise EmailOutputError(f"Could not connect to SMTP server: {exc}") from exc
finally:
if smtp is not None:
try:
with suppress(Exception):
smtp.quit()
except Exception:
try:
smtp.close()
except Exception:
pass
with suppress(Exception):
smtp.close()
def send_email_message(smtp_config: EmailSmtpConfig, message: EmailMessage) -> None:
smtp: Optional[smtplib.SMTP] = None
smtp: smtplib.SMTP | None = None
try:
if smtp_config.security == SECURITY_SSL:
context = _create_tls_context(smtp_config.allow_unverified_tls)
context = _create_tls_context(allow_unverified=smtp_config.allow_unverified_tls)
smtp = smtplib.SMTP_SSL(
smtp_config.host,
smtp_config.port,
@@ -231,12 +243,14 @@ def send_email_message(smtp_config: EmailSmtpConfig, message: EmailMessage) -> N
context=context,
)
else:
smtp = smtplib.SMTP(smtp_config.host, smtp_config.port, timeout=smtp_config.timeout_seconds)
smtp = smtplib.SMTP(
smtp_config.host, smtp_config.port, timeout=smtp_config.timeout_seconds
)
smtp.ehlo()
if smtp_config.security == SECURITY_STARTTLS:
context = _create_tls_context(smtp_config.allow_unverified_tls)
context = _create_tls_context(allow_unverified=smtp_config.allow_unverified_tls)
smtp.starttls(context=context)
smtp.ehlo()
@@ -250,13 +264,10 @@ def send_email_message(smtp_config: EmailSmtpConfig, message: EmailMessage) -> N
raise EmailOutputError(f"Failed to send email: {exc}") from exc
finally:
if smtp is not None:
try:
with suppress(Exception):
smtp.quit()
except Exception:
try:
smtp.close()
except Exception:
pass
with suppress(Exception):
smtp.close()
def _supports_email(task: DownloadTask) -> bool:
@@ -267,9 +278,10 @@ def _post_process_email(
temp_file: Path,
task: DownloadTask,
cancel_flag: Event,
status_callback,
status_callback: Callable[[str, str | None], None],
*,
preserve_source_on_failure: bool = False,
) -> Optional[str]:
) -> str | None:
from shelfmark.download.postprocess.pipeline import (
CustomScriptContext,
OutputPlan,
@@ -309,7 +321,11 @@ def _post_process_email(
stage_action = STAGE_NONE
if is_managed_workspace_path(temp_file):
stage_action = STAGE_COPY if preserve_source_on_failure else STAGE_MOVE
staging_dir = build_staging_dir("email", task.task_id) if stage_action != STAGE_NONE else get_staging_dir()
staging_dir = (
build_staging_dir("email", task.task_id)
if stage_action != STAGE_NONE
else get_staging_dir()
)
output_plan = OutputPlan(
mode=EMAIL_OUTPUT_MODE,
@@ -334,7 +350,7 @@ def _post_process_email(
limit_mb_raw = core_config.config.get("EMAIL_ATTACHMENT_SIZE_LIMIT_MB", 25)
try:
attachment_limit_mb = int(limit_mb_raw)
except (TypeError, ValueError):
except TypeError, ValueError:
attachment_limit_mb = 25
if attachment_limit_mb > 0:
@@ -406,7 +422,7 @@ def _post_process_email(
status_callback("complete", f"Sent to {label}")
success = True
return f"email://{task.task_id}"
output_path = f"email://{task.task_id}"
except EmailOutputError as exc:
logger.warning("Task %s: email send failed: %s", task.task_id, exc)
@@ -416,6 +432,8 @@ def _post_process_email(
logger.error_trace("Task %s: unexpected error sending email: %s", task.task_id, exc)
status_callback("error", f"Email send failed: {exc}")
return None
else:
return output_path
finally:
cleanup_output_staging(
prepared.output_plan,
@@ -432,9 +450,10 @@ def process_email_output(
temp_file: Path,
task: DownloadTask,
cancel_flag: Event,
status_callback,
status_callback: Callable[[str, str | None], None],
*,
preserve_source_on_failure: bool = False,
) -> Optional[str]:
) -> str | None:
return _post_process_email(
temp_file,
task,
shelfmark/download/outputs/folder.py
+30 -18
View File
@@ -1,17 +1,19 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
from threading import Event
from typing import Any, Optional, List
from typing import TYPE_CHECKING, Any
import shelfmark.core.config as core_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.download.outputs import register_output
from shelfmark.download.staging import StageAction, STAGE_NONE
from shelfmark.download.outputs import StatusCallback, register_output
from shelfmark.download.staging import STAGE_NONE, StageAction
if TYPE_CHECKING:
from pathlib import Path
from threading import Event
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
@@ -31,7 +33,7 @@ class _ProcessingPlan:
allow_archive_extraction: bool
stage_action: StageAction
staging_dir: Path
hardlink_source: Optional[Path]
hardlink_source: Path | None
output_mode: str = FOLDER_OUTPUT_MODE
@@ -42,8 +44,8 @@ def _supports_folder_output(task: DownloadTask) -> bool:
def _build_processing_plan(
temp_file: Path,
task: DownloadTask,
status_callback,
) -> Optional[_ProcessingPlan]:
status_callback: StatusCallback,
) -> _ProcessingPlan | None:
from shelfmark.download.postprocess.pipeline import (
build_output_plan,
get_final_destination,
@@ -52,7 +54,7 @@ def _build_processing_plan(
from shelfmark.download.postprocess.policy import get_file_organization
is_audiobook = check_audiobook(task.content_type)
organization_mode = get_file_organization(is_audiobook)
organization_mode = get_file_organization(is_audiobook=is_audiobook)
destination = get_final_destination(task)
if not validate_destination(destination, status_callback):
@@ -87,9 +89,10 @@ def process_folder_output(
temp_file: Path,
task: DownloadTask,
cancel_flag: Event,
status_callback,
status_callback: StatusCallback,
*,
preserve_source_on_failure: bool = False,
) -> Optional[str]:
) -> str | None:
"""Post-process download to the configured folder destination."""
from shelfmark.download.postprocess.pipeline import (
CustomScriptContext,
@@ -97,8 +100,8 @@ def process_folder_output(
cleanup_output_staging,
is_torrent_source,
log_plan_steps,
prepare_output_files,
maybe_run_custom_script,
prepare_output_files,
record_step,
transfer_book_files,
)
@@ -128,16 +131,23 @@ def process_folder_output(
if not prepared:
return None
steps: List[Any] = []
steps: list[Any] = []
if prepared.output_plan.stage_action != STAGE_NONE:
step_name = f"stage_{prepared.output_plan.stage_action}"
record_step(steps, step_name, source=str(temp_file), dest=str(prepared.output_plan.staging_dir))
record_step(
steps,
step_name,
source=str(temp_file),
dest=str(prepared.output_plan.staging_dir),
)
# Custom script is run post-transfer (see below).
# If we staged into TMP_DIR, transfer from the staged path and disable hardlinking.
use_hardlink = plan.use_hardlink and prepared.output_plan.stage_action == STAGE_NONE
source_path = plan.hardlink_source if use_hardlink and plan.hardlink_source else prepared.working_path
source_path = (
plan.hardlink_source if use_hardlink and plan.hardlink_source else prepared.working_path
)
is_torrent = is_torrent_source(source_path, task)
usenet_action = core_config.config.get("PROWLARR_USENET_ACTION", "move")
@@ -147,7 +157,9 @@ def process_folder_output(
# "Move" is implemented as a client-side cleanup after import.
preserve_source = is_usenet or preserve_source_on_failure
copy_for_label = is_torrent or preserve_source or prepared.output_plan.stage_action != STAGE_NONE
copy_for_label = (
is_torrent or preserve_source or prepared.output_plan.stage_action != STAGE_NONE
)
if cancel_flag.is_set():
logger.info("Task %s: cancelled before final transfer", task.task_id)
shelfmark/download/permissions_debug.py
+36 -21
View File
@@ -10,13 +10,40 @@ original error.
from __future__ import annotations
import os
from pathlib import Path
from typing import TYPE_CHECKING, Any, TypeVar
from shelfmark.core.logger import setup_logger
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
logger = setup_logger(__name__)
def _run_io(func, *args, **kwargs):
_T = TypeVar("_T")
def _log_path_permissions(probe: Path, label: str) -> None:
"""Best-effort logging for one path probe."""
try:
st = _run_io(probe.stat)
logger.debug(
"Path permissions (%s): path=%s mode=%s owner=%s(%d) group=%s(%d) exists=%s dir=%s",
label,
probe,
oct(st.st_mode & 0o777),
_format_uid(st.st_uid),
st.st_uid,
_format_gid(st.st_gid),
st.st_gid,
_run_io(probe.exists),
_run_io(probe.is_dir),
)
except Exception as stat_error:
logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
def _run_io[T](func: Callable[..., _T], *args: Any, **kwargs: Any) -> _T:
"""Best-effort offload for potentially blocking filesystem calls.
Keep this module import-cycle safe: `shelfmark.download.fs` imports this module,
@@ -57,7 +84,6 @@ def log_path_permission_context(label: str, path: Path) -> None:
Only call this from failure paths.
"""
try:
euid = os.geteuid() if hasattr(os, "geteuid") else None
egid = os.getegid() if hasattr(os, "getegid") else None
@@ -96,14 +122,18 @@ def log_path_permission_context(label: str, path: Path) -> None:
_run_io(probe.is_symlink),
)
except Exception as stat_error:
logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
logger.debug(
"Path permissions (%s): stat failed for %s: %s",
label,
probe,
stat_error,
)
except Exception as context_error:
logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
def log_transfer_permission_context(label: str, source: Path, dest: Path, error: Exception) -> None:
"""Log useful permission/ownership context when a file transfer fails."""
try:
euid = os.geteuid() if hasattr(os, "geteuid") else None
egid = os.getegid() if hasattr(os, "getegid") else None
@@ -122,21 +152,6 @@ def log_transfer_permission_context(label: str, source: Path, dest: Path, error:
)
for probe in [source, dest, dest.parent]:
try:
st = _run_io(probe.stat)
logger.debug(
"Path permissions (%s): path=%s mode=%s owner=%s(%d) group=%s(%d) exists=%s dir=%s",
label,
probe,
oct(st.st_mode & 0o777),
_format_uid(st.st_uid),
st.st_uid,
_format_gid(st.st_gid),
st.st_gid,
_run_io(probe.exists),
_run_io(probe.is_dir),
)
except Exception as stat_error:
logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
_log_path_permissions(probe, label)
except Exception as context_error:
logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
shelfmark/download/postprocess/__init__.py
+2
View File
@@ -9,3 +9,5 @@ Output handlers live in `shelfmark.download.outputs` and should depend on
"""
from .router import post_process_download
__all__ = ["post_process_download"]
shelfmark/download/postprocess/custom_script.py
+36 -26
View File
@@ -5,15 +5,20 @@ import os
import subprocess
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional
from typing import TYPE_CHECKING, Any
import shelfmark.core.config as core_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.download.fs import run_blocking_io
from .steps import log_plan_steps, record_step
from .types import PlanStep
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.models import DownloadTask
from .types import PlanStep
logger = setup_logger(__name__)
@@ -29,7 +34,6 @@ def resolve_custom_script_target(target_path: Path, destination: Path, path_mode
target is not within the destination, fall back to just the filename to
avoid leaking unrelated absolute paths.
"""
mode = (path_mode or "absolute").strip().lower()
if mode != "relative":
return target_path
@@ -50,7 +54,7 @@ class CustomScriptExecution:
destination: Path
mode: str
phase: str
payload_json: Optional[str] = None
payload_json: str | None = None
@dataclass(frozen=True)
@@ -66,11 +70,11 @@ class CustomScriptContext:
task: DownloadTask
phase: str
output_mode: str
destination: Optional[Path] = None
destination: Path | None = None
final_paths: list[Path] = field(default_factory=list)
target_path: Optional[Path] = None
organization_mode: Optional[str] = None
transfer: Optional[CustomScriptTransferSummary] = None
target_path: Path | None = None
organization_mode: str | None = None
transfer: CustomScriptTransferSummary | None = None
output_details: dict[str, Any] = field(default_factory=dict)
@@ -81,7 +85,7 @@ def prepare_custom_script_execution(
destination: Path,
path_mode: str,
phase: str,
payload: Optional[dict[str, Any]] = None,
payload: dict[str, Any] | None = None,
) -> CustomScriptExecution:
mode = (path_mode or "absolute").strip().lower()
if mode != "relative":
@@ -103,10 +107,10 @@ def run_custom_script(
execution: CustomScriptExecution,
*,
task_id: str,
status_callback,
status_callback: Callable[[str, str | None], None],
timeout_seconds: int = DEFAULT_CUSTOM_SCRIPT_TIMEOUT_SECONDS,
) -> bool:
cwd: Optional[str] = None
cwd: str | None = None
if execution.mode == "relative":
# Make relative paths unambiguous by running the script from the destination folder.
cwd = str(execution.destination)
@@ -136,17 +140,18 @@ def run_custom_script(
)
if result.stdout:
logger.debug("Task %s: custom script stdout: %s", task_id, result.stdout.strip())
return True
except FileNotFoundError:
logger.error("Task %s: custom script not found: %s", task_id, execution.script_path)
logger.exception("Task %s: custom script not found: %s", task_id, execution.script_path)
status_callback("error", f"Custom script not found: {execution.script_path}")
return False
except PermissionError:
logger.error("Task %s: custom script not executable: %s", task_id, execution.script_path)
logger.exception(
"Task %s: custom script not executable: %s", task_id, execution.script_path
)
status_callback("error", f"Custom script not executable: {execution.script_path}")
return False
except subprocess.TimeoutExpired:
logger.error(
logger.exception(
"Task %s: custom script timed out after %ss: %s",
task_id,
timeout_seconds,
@@ -156,7 +161,7 @@ def run_custom_script(
return False
except subprocess.CalledProcessError as exc:
stderr = exc.stderr.strip() if exc.stderr else "No error output"
logger.error(
logger.exception(
"Task %s: custom script failed (exit code %s): %s",
task_id,
exc.returncode,
@@ -164,14 +169,16 @@ def run_custom_script(
)
status_callback("error", f"Custom script failed: {stderr[:100]}")
return False
else:
return True
def _choose_custom_script_target(
*,
explicit_target: Optional[Path],
destination: Optional[Path],
explicit_target: Path | None,
destination: Path | None,
final_paths: list[Path],
) -> Optional[Path]:
) -> Path | None:
if explicit_target is not None:
return explicit_target
@@ -187,7 +194,9 @@ def _choose_custom_script_target(
return destination
def _build_custom_script_payload(context: CustomScriptContext, *, target_path: Path) -> dict[str, Any]:
def _build_custom_script_payload(
context: CustomScriptContext, *, target_path: Path
) -> dict[str, Any]:
payload: dict[str, Any] = {
"version": 1,
"phase": context.phase,
@@ -233,8 +242,8 @@ def _build_custom_script_payload(context: CustomScriptContext, *, target_path: P
def maybe_run_custom_script(
context: CustomScriptContext,
*,
status_callback,
steps: Optional[list[PlanStep]] = None,
status_callback: Callable[[str, str | None], None],
steps: list[PlanStep] | None = None,
) -> bool:
"""Run the custom script hook (if configured).
@@ -242,7 +251,6 @@ def maybe_run_custom_script(
This function is responsible for choosing the script target, building the
optional JSON payload, and executing the script.
"""
script_path = getattr(core_config.config, "CUSTOM_SCRIPT", None)
if not isinstance(script_path, str) or not script_path.strip():
return True
@@ -261,7 +269,7 @@ def maybe_run_custom_script(
path_mode = core_config.config.get("CUSTOM_SCRIPT_PATH_MODE", "absolute")
payload: Optional[dict[str, Any]] = None
payload: dict[str, Any] | None = None
if core_config.config.get("CUSTOM_SCRIPT_JSON_PAYLOAD", False):
payload = _build_custom_script_payload(context, target_path=target_path)
@@ -293,4 +301,6 @@ def maybe_run_custom_script(
)
log_plan_steps(context.task.task_id, steps)
return run_custom_script(execution, task_id=context.task.task_id, status_callback=status_callback)
return run_custom_script(
execution, task_id=context.task.task_id, status_callback=status_callback
)
shelfmark/download/postprocess/destination.py
+21 -10
View File
@@ -1,32 +1,40 @@
from __future__ import annotations
import uuid
from pathlib import Path
from typing import TYPE_CHECKING
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.core.utils import (
get_destination,
)
from shelfmark.core.utils import (
is_audiobook as check_audiobook,
)
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.permissions_debug import log_path_permission_context
from shelfmark.release_sources import get_source
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from shelfmark.core.models import DownloadTask
logger = setup_logger("shelfmark.download.postprocess.pipeline")
def validate_destination(destination: Path, status_callback) -> bool:
def validate_destination(
destination: Path, status_callback: Callable[[str, str | None], None]
) -> bool:
"""Validate destination path is absolute, exists, and writable."""
if not destination.is_absolute():
logger.warning(f"Destination must be absolute: {destination}")
logger.warning("Destination must be absolute: %s", destination)
status_callback("error", f"Destination must be absolute: {destination}")
return False
destination_exists = run_blocking_io(destination.exists)
if destination_exists and not run_blocking_io(destination.is_dir):
logger.warning(f"Destination is not a directory: {destination}")
logger.warning("Destination is not a directory: %s", destination)
status_callback("error", f"Destination is not a directory: {destination}")
return False
@@ -35,7 +43,7 @@ def validate_destination(destination: Path, status_callback) -> bool:
run_blocking_io(destination.mkdir, parents=True, exist_ok=True)
except (OSError, PermissionError) as exc:
log_path_permission_context("destination_create", destination)
logger.warning(f"Cannot create destination: {destination} ({exc})")
logger.warning("Cannot create destination: %s (%s)", destination, exc)
status_callback("error", f"Cannot create destination: {destination} ({exc})")
return False
@@ -51,7 +59,7 @@ def validate_destination(destination: Path, status_callback) -> bool:
except Exception as exc:
logger.debug("Destination write probe path: %s", test_path)
log_path_permission_context("destination_write_probe", destination)
logger.warning(f"Destination not writable: {destination} ({exc})")
logger.warning("Destination not writable: %s (%s)", destination, exc)
status_callback("error", f"Destination not writable: {destination} ({exc})")
return False
@@ -60,7 +68,6 @@ def validate_destination(destination: Path, status_callback) -> bool:
def get_final_destination(task: DownloadTask) -> Path:
"""Get final destination directory, with content-type routing support."""
is_audiobook = check_audiobook(task.content_type)
try:
@@ -71,4 +78,8 @@ def get_final_destination(task: DownloadTask) -> Path:
if override:
return override
return get_destination(is_audiobook, user_id=task.user_id, username=task.username)
return get_destination(
is_audiobook=is_audiobook,
user_id=task.user_id,
username=task.username,
)
shelfmark/download/postprocess/pipeline.py
+7 -7
View File
@@ -18,8 +18,8 @@ implementation stay modular.
from __future__ import annotations
from .custom_script import (
CustomScriptExecution,
CustomScriptContext,
CustomScriptExecution,
CustomScriptTransferSummary,
maybe_run_custom_script,
prepare_custom_script_execution,
@@ -55,13 +55,13 @@ from .workspace import (
)
__all__ = [
"CustomScriptContext",
"CustomScriptExecution",
"CustomScriptTransferSummary",
"OutputPlan",
"PlanStep",
"PreparedFiles",
"TransferPlan",
"CustomScriptExecution",
"CustomScriptContext",
"CustomScriptTransferSummary",
"build_metadata_dict",
"build_output_plan",
"cleanup_output_staging",
@@ -75,12 +75,13 @@ __all__ = [
"is_within_tmp_dir",
"log_plan_steps",
"maybe_run_custom_script",
"prepare_output_files",
"prepare_custom_script_execution",
"prepare_output_files",
"process_directory",
"record_step",
"resolve_hardlink_source",
"resolve_custom_script_target",
"resolve_hardlink_source",
"run_custom_script",
"safe_cleanup_path",
"scan_directory_tree",
"should_hardlink",
@@ -88,5 +89,4 @@ __all__ = [
"transfer_directory_to_library",
"transfer_file_to_library",
"validate_destination",
"run_custom_script",
]
shelfmark/download/postprocess/policy.py
+4 -12
View File
@@ -11,18 +11,16 @@ Examples:
Implementation note:
Keep this module free of dependencies on archive extraction mechanics to avoid
circular imports (`archive` is used by the pipeline).
"""
from __future__ import annotations
from typing import List
import shelfmark.core.config as core_config
def get_supported_formats() -> List[str]:
def get_supported_formats() -> list[str]:
"""Get current supported formats from config singleton."""
formats = core_config.config.get(
"SUPPORTED_FORMATS",
["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
@@ -35,9 +33,8 @@ def get_supported_formats() -> List[str]:
return [fmt.lower() for fmt in formats]
def get_supported_audiobook_formats() -> List[str]:
def get_supported_audiobook_formats() -> list[str]:
"""Get current supported audiobook formats from config singleton."""
formats = core_config.config.get("SUPPORTED_AUDIOBOOK_FORMATS", ["m4b", "mp3"])
# Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
@@ -49,7 +46,6 @@ def get_supported_audiobook_formats() -> List[str]:
def get_file_organization(is_audiobook: bool) -> str:
"""Get the file organization mode for the content type."""
key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
mode = core_config.config.get(key, "rename")
@@ -68,7 +64,6 @@ def get_file_organization(is_audiobook: bool) -> str:
def get_template(is_audiobook: bool, organization_mode: str) -> str:
"""Get the template for the content type and organization mode."""
# Determine the correct key based on content type and organization mode
if is_audiobook:
if organization_mode == "organize":
@@ -76,10 +71,7 @@ def get_template(is_audiobook: bool, organization_mode: str) -> str:
else:
key = "TEMPLATE_AUDIOBOOK_RENAME"
else:
if organization_mode == "organize":
key = "TEMPLATE_ORGANIZE"
else:
key = "TEMPLATE_RENAME"
key = "TEMPLATE_ORGANIZE" if organization_mode == "organize" else "TEMPLATE_RENAME"
template = core_config.config.get(key, "")
shelfmark/download/postprocess/prepare.py
+25 -14
View File
@@ -1,17 +1,26 @@
from __future__ import annotations
from pathlib import Path
from typing import Optional
from typing import TYPE_CHECKING
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.download.staging import STAGE_COPY, STAGE_NONE, get_staging_dir, stage_path
from shelfmark.download.staging import (
STAGE_COPY,
STAGE_NONE,
get_staging_dir,
stage_path,
)
from .scan import collect_staged_files
from .transfer import resolve_hardlink_source
from .types import OutputPlan, PreparedFiles
from .workspace import cleanup_output_staging, is_managed_workspace_path
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from shelfmark.core.models import DownloadTask
logger = setup_logger("shelfmark.download.postprocess.pipeline")
@@ -19,11 +28,10 @@ def build_output_plan(
temp_file: Path,
task: DownloadTask,
output_mode: str,
destination: Optional[Path] = None,
status_callback=None,
destination: Path | None = None,
status_callback: Callable[[str, str | None], None] | None = None,
) -> OutputPlan:
"""Build an output plan that describes staging behavior for file-based outputs."""
transfer_plan = resolve_hardlink_source(temp_file, task, destination, status_callback)
staging_dir = get_staging_dir()
@@ -40,11 +48,12 @@ def prepare_output_files(
temp_file: Path,
task: DownloadTask,
output_mode: str,
status_callback,
destination: Optional[Path] = None,
output_plan: Optional[OutputPlan] = None,
status_callback: Callable[[str, str | None], None] | None,
destination: Path | None = None,
output_plan: OutputPlan | None = None,
*,
preserve_source_on_failure: bool = False,
) -> Optional[PreparedFiles]:
) -> PreparedFiles | None:
if output_plan is None:
output_plan = build_output_plan(
temp_file,
@@ -56,12 +65,14 @@ def prepare_output_files(
working_path = temp_file
if output_plan.stage_action != STAGE_NONE:
step_label = "Staging torrent files" if output_plan.stage_action == STAGE_COPY else "Staging files"
step_label = (
"Staging torrent files" if output_plan.stage_action == STAGE_COPY else "Staging files"
)
status_callback("resolving", step_label)
working_path = stage_path(working_path, output_plan.staging_dir, output_plan.stage_action)
can_delete_source_archives = output_plan.stage_action != STAGE_NONE or is_managed_workspace_path(
working_path
can_delete_source_archives = (
output_plan.stage_action != STAGE_NONE or is_managed_workspace_path(working_path)
)
cleanup_archives = can_delete_source_archives and not preserve_source_on_failure
shelfmark/download/postprocess/router.py
+11 -8
View File
@@ -10,14 +10,17 @@ Keeping this separate from `pipeline.py` avoids circular imports:
from __future__ import annotations
from pathlib import Path
from threading import Event
from typing import Optional
from typing import TYPE_CHECKING
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask, SearchMode
from shelfmark.download.outputs import resolve_output_handler
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from threading import Event
logger = setup_logger(__name__)
@@ -25,11 +28,11 @@ def post_process_download(
temp_file: Path,
task: DownloadTask,
cancel_flag: Event,
status_callback,
status_callback: Callable[[str, str | None], None],
*,
preserve_source_on_failure: bool = False,
) -> Optional[str]:
) -> str | None:
"""Post-process download using the selected output handler."""
if task.search_mode is None:
logger.warning(
"Task %s: missing search_mode; defaulting to Direct mode behavior",
@@ -50,7 +53,7 @@ def post_process_download(
task,
cancel_flag,
status_callback,
preserve_source_on_failure,
preserve_source_on_failure=preserve_source_on_failure,
)
from shelfmark.download.outputs.folder import process_folder_output
@@ -61,5 +64,5 @@ def post_process_download(
task,
cancel_flag,
status_callback,
preserve_source_on_failure,
preserve_source_on_failure=preserve_source_on_failure,
)
shelfmark/download/postprocess/scan.py
+80 -40
View File
@@ -2,33 +2,39 @@ from __future__ import annotations
import os
from pathlib import Path
from typing import List, Optional, Tuple
from typing import TYPE_CHECKING
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.permissions_debug import log_path_permission_context
from shelfmark.download.postprocess.policy import (
get_supported_audiobook_formats,
)
from shelfmark.download.postprocess.policy import (
get_supported_formats as get_book_formats,
)
from shelfmark.download.staging import build_staging_dir
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.models import DownloadTask
logger = setup_logger("shelfmark.download.postprocess.pipeline")
def get_supported_formats(content_type: Optional[str] = None) -> List[str]:
def get_supported_formats(content_type: str | None = None) -> list[str]:
if check_audiobook(content_type):
return get_supported_audiobook_formats()
return get_book_formats()
def _format_not_supported_error(rejected_files: List[Path], task: DownloadTask) -> str:
def _format_not_supported_error(rejected_files: list[Path], task: DownloadTask) -> str:
content_type = task.content_type
file_type_label = "audiobook" if check_audiobook(content_type) else "book"
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
rejected_exts = sorted({f.suffix.lower() for f in rejected_files})
rejected_list = ", ".join(rejected_exts)
supported_formats = get_supported_formats(content_type)
@@ -51,8 +57,9 @@ def extract_archive_files(
archive_path: Path,
output_dir: Path,
task: DownloadTask,
*,
cleanup_archive: bool,
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
) -> tuple[list[Path], list[Path], list[Path], str | None]:
content_type = task.content_type
try:
@@ -86,7 +93,12 @@ def extract_archive_files(
if not extracted_files:
if rejected_files:
return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
return (
[],
rejected_files,
cleanup_paths,
_format_not_supported_error(rejected_files, task),
)
file_type_label = "audiobook" if check_audiobook(content_type) else "book"
return [], rejected_files, cleanup_paths, f"No {file_type_label} files found in archive"
@@ -102,11 +114,11 @@ def extract_archive_files(
def scan_directory_tree(
directory: Path,
content_type: Optional[str],
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
content_type: str | None,
) -> tuple[list[Path], list[Path], list[Path], str | None]:
"""Scan a directory tree for book files, trackable-but-unsupported files, and archives."""
try:
def _probe_dir() -> None:
# Force a fast error if the dir is missing/inaccessible.
with os.scandir(directory) as it:
@@ -115,10 +127,10 @@ def scan_directory_tree(
run_blocking_io(_probe_dir)
except PermissionError as exc:
log_path_permission_context("scan_directory", directory)
logger.warning(f"Permission denied scanning directory: {directory} ({exc})")
logger.warning("Permission denied scanning directory: %s (%s)", directory, exc)
return [], [], [], f"Permission denied accessing download folder: {directory}"
except (FileNotFoundError, NotADirectoryError, OSError) as exc:
logger.warning(f"Cannot access download folder: {directory} ({exc})")
logger.warning("Cannot access download folder: %s (%s)", directory, exc)
return [], [], [], f"Cannot access download folder: {directory} ({exc})"
supported_formats = get_supported_formats(content_type)
@@ -126,11 +138,22 @@ def scan_directory_tree(
is_audiobook = check_audiobook(content_type)
if is_audiobook:
trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"}
else:
trackable_exts = {
'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
'.doc', '.docx', '.rtf', '.txt',
".pdf",
".epub",
".mobi",
".azw",
".azw3",
".fb2",
".djvu",
".cbz",
".cbr",
".doc",
".docx",
".rtf",
".txt",
}
logged_walk_permission_context = False
@@ -140,22 +163,19 @@ def scan_directory_tree(
if isinstance(error, PermissionError):
if not logged_walk_permission_context:
try:
error_path = Path(getattr(error, "filename", "") or str(directory))
except Exception:
error_path = directory
error_path = Path(getattr(error, "filename", "") or str(directory))
log_path_permission_context("scan_directory_walk", error_path)
logged_walk_permission_context = True
logger.debug(f"Skipping inaccessible path during scan: {error}")
logger.debug("Skipping inaccessible path during scan: %s", error)
else:
logger.debug(f"Error scanning directory tree: {error}")
logger.debug("Error scanning directory tree: %s", error)
def _walk_tree() -> Tuple[List[Path], List[Path], List[Path]]:
book_files: List[Path] = []
rejected_files: List[Path] = []
archive_files: List[Path] = []
def _walk_tree() -> tuple[list[Path], list[Path], list[Path]]:
book_files: list[Path] = []
rejected_files: list[Path] = []
archive_files: list[Path] = []
for root, _, files in os.walk(directory, onerror=onerror):
for filename in files:
@@ -176,10 +196,10 @@ def scan_directory_tree(
book_files, rejected_files, archive_files = run_blocking_io(_walk_tree)
except PermissionError as exc:
log_path_permission_context("scan_directory_walk", directory)
logger.warning(f"Permission denied scanning directory: {directory} ({exc})")
logger.warning("Permission denied scanning directory: %s (%s)", directory, exc)
return [], [], [], f"Permission denied accessing download folder: {directory}"
except (FileNotFoundError, NotADirectoryError, OSError) as exc:
logger.warning(f"Cannot access download folder: {directory} ({exc})")
logger.warning("Cannot access download folder: %s (%s)", directory, exc)
return [], [], [], f"Cannot access download folder: {directory} ({exc})"
return book_files, rejected_files, archive_files, None
@@ -188,12 +208,15 @@ def scan_directory_tree(
def collect_directory_files(
directory: Path,
task: DownloadTask,
*,
allow_archive_extraction: bool,
status_callback=None,
status_callback: Callable[[str, str | None], None] | None = None,
cleanup_archives: bool = False,
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
) -> tuple[list[Path], list[Path], list[Path], str | None]:
content_type = task.content_type
book_files, rejected_files, archive_files, scan_error = scan_directory_tree(directory, content_type)
book_files, rejected_files, archive_files, scan_error = scan_directory_tree(
directory, content_type
)
if scan_error:
return [], [], [], scan_error
@@ -206,7 +229,7 @@ def collect_directory_files(
len(book_files),
)
if rejected_files:
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
rejected_exts = sorted({f.suffix.lower() for f in rejected_files})
logger.debug(
"Task %s: also found %d file(s) with unsupported formats: %s",
task.task_id,
@@ -232,9 +255,9 @@ def collect_directory_files(
logger.info("Task %s: extracting %d archive(s)", task.task_id, len(archive_files))
all_files: List[Path] = []
all_errors: List[str] = []
cleanup_paths: List[Path] = []
all_files: list[Path] = []
all_errors: list[str] = []
cleanup_paths: list[Path] = []
for archive in archive_files:
extract_dir = build_staging_dir("extract", task.task_id)
@@ -267,7 +290,12 @@ def collect_directory_files(
return [], rejected_files, cleanup_paths, "; ".join(all_errors)
if rejected_files:
return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
return (
[],
rejected_files,
cleanup_paths,
_format_not_supported_error(rejected_files, task),
)
return [], rejected_files, cleanup_paths, "No book files found in archives"
@@ -280,10 +308,11 @@ def collect_directory_files(
def collect_staged_files(
working_path: Path,
task: DownloadTask,
*,
allow_archive_extraction: bool,
status_callback,
status_callback: Callable[[str, str | None], None] | None,
cleanup_archives: bool,
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
) -> tuple[list[Path], list[Path], list[Path], str | None]:
if run_blocking_io(working_path.is_dir):
if status_callback:
status_callback("resolving", "Processing download folder")
@@ -332,11 +361,22 @@ def collect_staged_files(
is_audiobook = check_audiobook(task.content_type)
if is_audiobook:
trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"}
else:
trackable_exts = {
'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
'.doc', '.docx', '.rtf', '.txt',
".pdf",
".epub",
".mobi",
".azw",
".azw3",
".fb2",
".djvu",
".cbz",
".cbr",
".doc",
".docx",
".rtf",
".txt",
}
if suffix in supported_exts:
shelfmark/download/postprocess/steps.py
+2 -4
View File
@@ -1,7 +1,5 @@
from __future__ import annotations
from typing import Any, List
from shelfmark.core.logger import setup_logger
from .types import PlanStep
@@ -9,11 +7,11 @@ from .types import PlanStep
logger = setup_logger("shelfmark.download.postprocess.pipeline")
def record_step(steps: List[PlanStep], name: str, **details: Any) -> None:
def record_step(steps: list[PlanStep], name: str, **details: object) -> None:
steps.append(PlanStep(name=name, details=details))
def log_plan_steps(task_id: str, steps: List[PlanStep]) -> None:
def log_plan_steps(task_id: str, steps: list[PlanStep]) -> None:
if not steps:
return
summary = " -> ".join(step.name for step in steps)
shelfmark/download/postprocess/transfer.py
+83 -59
View File
@@ -2,11 +2,10 @@ from __future__ import annotations
import os
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from typing import TYPE_CHECKING
import shelfmark.core.config as core_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.core.naming import (
assign_part_numbers,
build_library_path,
@@ -15,19 +14,28 @@ from shelfmark.core.naming import (
sanitize_filename,
)
from shelfmark.core.utils import is_audiobook as check_audiobook
from shelfmark.download.fs import atomic_copy, atomic_hardlink, atomic_move, run_blocking_io
from shelfmark.download.fs import (
atomic_copy,
atomic_hardlink,
atomic_move,
run_blocking_io,
)
from shelfmark.download.postprocess.policy import get_file_organization, get_template
from .scan import collect_directory_files, scan_directory_tree
from .types import TransferPlan
from .workspace import safe_cleanup_path
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.models import DownloadTask
logger = setup_logger("shelfmark.download.postprocess.pipeline")
def should_hardlink(task: DownloadTask) -> bool:
"""Check if hardlinking is enabled for this task (Prowlarr torrents only)."""
if task.source != "prowlarr":
return False
@@ -44,7 +52,6 @@ def should_hardlink(task: DownloadTask) -> bool:
return bool(hardlink_enabled)
def build_metadata_dict(task: DownloadTask) -> dict:
return {
"Author": task.author,
@@ -57,7 +64,9 @@ def build_metadata_dict(task: DownloadTask) -> dict:
}
def build_file_metadata(task: DownloadTask, source_file: Path, part_number: Optional[str] = None) -> dict:
def build_file_metadata(
task: DownloadTask, source_file: Path, part_number: str | None = None
) -> dict:
metadata = build_metadata_dict(task)
metadata["OriginalName"] = source_file.stem
if part_number is not None:
@@ -68,11 +77,10 @@ def build_file_metadata(task: DownloadTask, source_file: Path, part_number: Opti
def resolve_hardlink_source(
temp_file: Path,
task: DownloadTask,
destination: Optional[Path],
status_callback=None,
destination: Path | None,
status_callback: Callable[[str, str | None], None] | None = None,
) -> TransferPlan:
"""Resolve hardlink eligibility and source path for transfers."""
use_hardlink = False
source_path = temp_file
hardlink_enabled = should_hardlink(task)
@@ -80,13 +88,18 @@ def resolve_hardlink_source(
if hardlink_enabled and task.original_download_path:
hardlink_source = Path(task.original_download_path)
hardlink_source_exists = run_blocking_io(hardlink_source.exists)
if destination and hardlink_source_exists and run_blocking_io(same_filesystem, hardlink_source, destination):
if (
destination
and hardlink_source_exists
and run_blocking_io(same_filesystem, hardlink_source, destination)
):
use_hardlink = True
source_path = hardlink_source
elif hardlink_source_exists:
logger.warning(
f"Cannot hardlink: {hardlink_source} and {destination} are on different filesystems. "
"Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination."
"Cannot hardlink: %s and %s are on different filesystems. Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination.",
hardlink_source,
destination,
)
if status_callback:
status_callback("resolving", "Cannot hardlink (different filesystems), using copy")
@@ -101,14 +114,13 @@ def resolve_hardlink_source(
def is_torrent_source(source_path: Path, task: DownloadTask) -> bool:
"""Check if source is the torrent client path (needs copy to preserve seeding)."""
if not task.original_download_path:
return False
original_path = Path(task.original_download_path)
try:
return run_blocking_io(source_path.resolve) == run_blocking_io(original_path.resolve)
except (OSError, ValueError):
except OSError, ValueError:
try:
return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path))
except Exception:
@@ -124,11 +136,12 @@ def _max_attempts_for_batch(file_count: int, default: int = 100) -> int:
def _transfer_single_file(
source_path: Path,
dest_path: Path,
*,
use_hardlink: bool,
is_torrent: bool,
preserve_source: bool = False,
max_attempts: int = 100,
) -> Tuple[Path, str]:
) -> tuple[Path, str]:
if use_hardlink:
final_path = atomic_hardlink(source_path, dest_path, max_attempts=max_attempts)
try:
@@ -145,23 +158,24 @@ def _transfer_single_file(
def transfer_book_files(
book_files: List[Path],
book_files: list[Path],
destination: Path,
task: DownloadTask,
*,
use_hardlink: bool,
is_torrent: bool,
preserve_source: bool = False,
organization_mode: Optional[str] = None,
) -> Tuple[List[Path], Optional[str], Dict[str, int]]:
organization_mode: str | None = None,
) -> tuple[list[Path], str | None, dict[str, int]]:
if not book_files:
return [], "No book files found", {"hardlink": 0, "copy": 0, "move": 0}
is_audiobook = check_audiobook(task.content_type)
organization_mode = organization_mode or get_file_organization(is_audiobook)
organization_mode = organization_mode or get_file_organization(is_audiobook=is_audiobook)
max_attempts = _max_attempts_for_batch(len(book_files))
final_paths: List[Path] = []
op_counts: Dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0}
final_paths: list[Path] = []
op_counts: dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0}
if organization_mode == "organize":
template = get_template(is_audiobook, "organize")
@@ -182,14 +196,14 @@ def transfer_book_files(
final_path, op = _transfer_single_file(
source_file,
dest_path,
use_hardlink,
is_torrent,
use_hardlink=use_hardlink,
is_torrent=is_torrent,
preserve_source=preserve_source,
max_attempts=max_attempts,
)
final_paths.append(final_path)
op_counts[op] = op_counts.get(op, 0) + 1
logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
logger.debug("%s to destination: %s", op.capitalize(), final_path.name)
else:
zero_pad_width = max(len(str(len(book_files))), 2)
files_with_parts = assign_part_numbers(book_files, zero_pad_width)
@@ -209,14 +223,14 @@ def transfer_book_files(
final_path, op = _transfer_single_file(
source_file,
dest_path,
use_hardlink,
is_torrent,
use_hardlink=use_hardlink,
is_torrent=is_torrent,
preserve_source=preserve_source,
max_attempts=max_attempts,
)
final_paths.append(final_path)
op_counts[op] = op_counts.get(op, 0) + 1
logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
logger.debug("%s to destination: %s", op.capitalize(), final_path.name)
return final_paths, None, op_counts
@@ -242,14 +256,14 @@ def transfer_book_files(
final_path, op = _transfer_single_file(
book_file,
dest_path,
use_hardlink,
is_torrent,
use_hardlink=use_hardlink,
is_torrent=is_torrent,
preserve_source=preserve_source,
max_attempts=max_attempts,
)
final_paths.append(final_path)
op_counts[op] = op_counts.get(op, 0) + 1
logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
logger.debug("%s to destination: %s", op.capitalize(), final_path.name)
return final_paths, None, op_counts
@@ -258,11 +272,11 @@ def process_directory(
directory: Path,
ingest_dir: Path,
task: DownloadTask,
*,
allow_archive_extraction: bool = True,
use_hardlink: Optional[bool] = None,
) -> Tuple[List[Path], Optional[str]]:
use_hardlink: bool | None = None,
) -> tuple[list[Path], str | None]:
"""Process staged directory: find book files, extract archives, move to ingest."""
try:
is_torrent = is_torrent_source(directory, task)
book_files, _, cleanup_paths, error = collect_directory_files(
@@ -299,13 +313,17 @@ def process_directory(
for cleanup_path in cleanup_paths:
safe_cleanup_path(cleanup_path, task)
return final_paths, None
processed_paths = final_paths
except Exception as exc:
logger.error_trace("Task %s: error processing directory %s: %s", task.task_id, directory, exc)
logger.error_trace(
"Task %s: error processing directory %s: %s", task.task_id, directory, exc
)
if not is_torrent_source(directory, task):
safe_cleanup_path(directory, task)
return [], str(exc)
else:
return processed_paths, None
def transfer_file_to_library(
@@ -314,25 +332,28 @@ def transfer_file_to_library(
template: str,
metadata: dict,
task: DownloadTask,
temp_file: Optional[Path],
status_callback,
temp_file: Path | None,
status_callback: Callable[[str, str | None], None],
*,
use_hardlink: bool,
) -> Optional[str]:
) -> str | None:
extension = source_path.suffix.lstrip(".") or task.format
template_metadata = dict(metadata)
template_metadata.setdefault("OriginalName", source_path.stem)
dest_path = run_blocking_io(build_library_path, library_base, template, template_metadata, extension)
dest_path = run_blocking_io(
build_library_path, library_base, template, template_metadata, extension
)
run_blocking_io(dest_path.parent.mkdir, parents=True, exist_ok=True)
is_torrent = is_torrent_source(source_path, task)
final_path, op = _transfer_single_file(
source_path,
dest_path,
use_hardlink,
is_torrent,
use_hardlink=use_hardlink,
is_torrent=is_torrent,
max_attempts=_max_attempts_for_batch(1),
)
logger.info(f"Library {op}: {final_path}")
logger.info("Library %s: %s", op, final_path)
if use_hardlink and op != "hardlink":
logger.warning(
"Library hardlink requested but %s used instead for %s",
@@ -353,10 +374,11 @@ def transfer_directory_to_library(
template: str,
metadata: dict,
task: DownloadTask,
temp_file: Optional[Path],
status_callback,
temp_file: Path | None,
status_callback: Callable[[str, str | None], None],
*,
use_hardlink: bool,
) -> Optional[str]:
) -> str | None:
content_type = task.content_type.lower() if task.content_type else None
source_files, _, _, scan_error = scan_directory_tree(source_dir, content_type)
if scan_error:
@@ -367,7 +389,7 @@ def transfer_directory_to_library(
return None
if not source_files:
logger.warning(f"No supported files in {source_dir.name}")
logger.warning("No supported files in %s", source_dir.name)
status_callback("error", "No supported file formats found")
if temp_file:
safe_cleanup_path(temp_file, task)
@@ -383,8 +405,8 @@ def transfer_directory_to_library(
run_blocking_io(base_library_path.parent.mkdir, parents=True, exist_ok=True)
is_torrent = is_torrent_source(source_dir, task)
transferred_paths: List[Path] = []
op_counts: Dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0}
transferred_paths: list[Path] = []
op_counts: dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0}
max_attempts = _max_attempts_for_batch(len(source_files))
if len(source_files) == 1:
@@ -394,11 +416,11 @@ def transfer_directory_to_library(
final_path, op = _transfer_single_file(
source_file,
dest_path,
use_hardlink,
is_torrent,
use_hardlink=use_hardlink,
is_torrent=is_torrent,
max_attempts=max_attempts,
)
logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
logger.debug("Library %s: %s -> %s", op, source_file.name, final_path)
transferred_paths.append(final_path)
op_counts[op] = op_counts.get(op, 0) + 1
else:
@@ -408,23 +430,23 @@ def transfer_directory_to_library(
for source_file, part_number in files_with_parts:
ext = source_file.suffix.lstrip(".")
file_metadata = {**metadata, "PartNumber": part_number}
file_path = run_blocking_io(build_library_path, library_base, template, file_metadata, extension=ext)
file_path = run_blocking_io(
build_library_path, library_base, template, file_metadata, extension=ext
)
run_blocking_io(file_path.parent.mkdir, parents=True, exist_ok=True)
final_path, op = _transfer_single_file(
source_file,
file_path,
use_hardlink,
is_torrent,
use_hardlink=use_hardlink,
is_torrent=is_torrent,
max_attempts=max_attempts,
)
logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
logger.debug("Library %s: %s -> %s", op, source_file.name, final_path)
transferred_paths.append(final_path)
op_counts[op] = op_counts.get(op, 0) + 1
op_summary = ", ".join(
f"{op}={count}" for op, count in op_counts.items() if count
) or "none"
op_summary = ", ".join(f"{op}={count}" for op, count in op_counts.items() if count) or "none"
logger.info(
"Created %d library file(s) in %s (ops: %s)",
len(transferred_paths),
@@ -444,7 +466,9 @@ def transfer_directory_to_library(
safe_cleanup_path(temp_file, task)
safe_cleanup_path(source_dir, task)
message = f"Complete ({len(transferred_paths)} files)" if len(transferred_paths) > 1 else "Complete"
message = (
f"Complete ({len(transferred_paths)} files)" if len(transferred_paths) > 1 else "Complete"
)
status_callback("complete", message)
return str(transferred_paths[0])
shelfmark/download/postprocess/types.py
+10 -8
View File
@@ -1,10 +1,12 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import TYPE_CHECKING, Any
from shelfmark.download.staging import StageAction
if TYPE_CHECKING:
from pathlib import Path
from shelfmark.download.staging import StageAction
@dataclass(frozen=True)
@@ -21,19 +23,19 @@ class OutputPlan:
stage_action: StageAction
staging_dir: Path
allow_archive_extraction: bool
transfer_plan: Optional[TransferPlan] = None
transfer_plan: TransferPlan | None = None
@dataclass(frozen=True)
class PreparedFiles:
output_plan: OutputPlan
working_path: Path
files: List[Path]
rejected_files: List[Path]
cleanup_paths: List[Path]
files: list[Path]
rejected_files: list[Path]
cleanup_paths: list[Path]
@dataclass(frozen=True)
class PlanStep:
name: str
details: Dict[str, Any]
details: dict[str, Any]
shelfmark/download/postprocess/workspace.py
+19 -15
View File
@@ -2,15 +2,17 @@ from __future__ import annotations
import shutil
from pathlib import Path
from typing import List, Optional
from typing import TYPE_CHECKING
from shelfmark.config import env as env_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.staging import STAGE_NONE
from .types import OutputPlan
if TYPE_CHECKING:
from shelfmark.core.models import DownloadTask
from .types import OutputPlan
logger = setup_logger("shelfmark.download.postprocess.pipeline")
@@ -21,24 +23,28 @@ def _tmp_dir() -> Path:
def is_within_tmp_dir(path: Path) -> bool:
"""Legacy helper: True if path is inside TMP_DIR."""
# Fast path: avoid `resolve()` (can block on NFS) for obviously-non-TMP paths.
# This is a *negative* check only; for potential TMP paths we still resolve to
# prevent symlink escapes from being treated as managed.
tmp_dir = _tmp_dir()
try:
if path.is_absolute() and tmp_dir.is_absolute():
if path != tmp_dir and tmp_dir not in path.parents:
return False
if (
path.is_absolute()
and tmp_dir.is_absolute()
and path != tmp_dir
and tmp_dir not in path.parents
):
return False
except Exception:
# Fall back to the slower resolve-based check below.
pass
try:
run_blocking_io(path.resolve).relative_to(run_blocking_io(tmp_dir.resolve))
return True
except (OSError, ValueError):
except OSError, ValueError:
return False
else:
return True
def is_managed_workspace_path(path: Path) -> bool:
@@ -47,23 +53,21 @@ def is_managed_workspace_path(path: Path) -> bool:
The managed workspace is `TMP_DIR`. Anything outside it should be treated as
read-only for safety (e.g. torrent seeding directories).
"""
return is_within_tmp_dir(path)
def _is_original_download(path: Optional[Path], task: DownloadTask) -> bool:
def _is_original_download(path: Path | None, task: DownloadTask) -> bool:
if not path or not task.original_download_path:
return False
try:
original = Path(task.original_download_path)
return run_blocking_io(path.resolve) == run_blocking_io(original.resolve)
except (OSError, ValueError):
except OSError, ValueError:
return False
def safe_cleanup_path(path: Optional[Path], task: DownloadTask) -> None:
def safe_cleanup_path(path: Path | None, task: DownloadTask) -> None:
"""Remove a temp path only if it is safe and in our managed workspace."""
if not path or _is_original_download(path, task):
return
@@ -84,7 +88,7 @@ def cleanup_output_staging(
output_plan: OutputPlan,
working_path: Path,
task: DownloadTask,
cleanup_paths: Optional[List[Path]] = None,
cleanup_paths: list[Path] | None = None,
) -> None:
if output_plan.stage_action != STAGE_NONE:
cleanup_target = output_plan.staging_dir
shelfmark/download/staging.py
+5 -3
View File
@@ -2,13 +2,15 @@ from __future__ import annotations
import hashlib
import shutil
from pathlib import Path
from typing import Literal
from typing import TYPE_CHECKING, Literal
from shelfmark.config import env as env_config
from shelfmark.core.logger import setup_logger
from shelfmark.download.fs import run_blocking_io
if TYPE_CHECKING:
from pathlib import Path
logger = setup_logger(__name__)
StageAction = Literal["none", "copy", "move"]
@@ -49,7 +51,7 @@ def build_staging_dir(prefix: str | None, task_id: str) -> Path:
return staging_dir
def stage_file(source_path: Path, task_id: str, copy: bool = False) -> Path:
def stage_file(source_path: Path, task_id: str, *, copy: bool = False) -> Path:
"""Stage a file for ingest processing. Use copy=True for torrents to preserve seeding."""
staging_dir = get_staging_dir()
return stage_path(source_path, staging_dir, STAGE_COPY if copy else STAGE_MOVE)
shelfmark/main.py
+855 -652
View File
File diff suppressed because it is too large Load Diff
shelfmark/metadata_providers/__init__.py
+164 -141
View File
@@ -1,31 +1,37 @@
"""Metadata provider plugin system - base classes and registry."""
from abc import ABC, abstractmethod
from contextlib import suppress
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional, Type, Union
from enum import StrEnum
from typing import TYPE_CHECKING, Any, ClassVar
if TYPE_CHECKING:
from collections.abc import Callable
class SearchType(str, Enum):
class SearchType(StrEnum):
"""Type of search to perform."""
GENERAL = "general" # Search all fields (title, author, ISBN, etc.)
TITLE = "title" # Search by title only
AUTHOR = "author" # Search by author only
ISBN = "isbn" # Search by ISBN
TITLE = "title" # Search by title only
AUTHOR = "author" # Search by author only
ISBN = "isbn" # Search by ISBN
class SortOrder(str, Enum):
class SortOrder(StrEnum):
"""Sort order for search results."""
RELEVANCE = "relevance" # Best match first (default)
RELEVANCE = "relevance" # Best match first (default)
POPULARITY = "popularity" # Most popular first
RATING = "rating" # Highest rated first
NEWEST = "newest" # Most recently published first
OLDEST = "oldest" # Oldest published first
RATING = "rating" # Highest rated first
NEWEST = "newest" # Most recently published first
OLDEST = "oldest" # Oldest published first
SERIES_ORDER = "series_order" # By series position (requires series field)
# Display labels for sort options
SORT_LABELS: Dict[SortOrder, str] = {
SORT_LABELS: dict[SortOrder, str] = {
SortOrder.RELEVANCE: "Most relevant",
SortOrder.POPULARITY: "Most popular",
SortOrder.RATING: "Highest rated",
@@ -38,40 +44,44 @@ SORT_LABELS: Dict[SortOrder, str] = {
@dataclass
class MetadataCapability:
"""Declarative provider capability consumed by shared UI code."""
key: str
field_key: Optional[str] = None
sort: Optional[SortOrder] = None
field_key: str | None = None
sort: SortOrder | None = None
@dataclass
class TextSearchField:
"""Text input search field."""
key: str # Field identifier (e.g., "author", "publisher")
label: str # Display label in UI
placeholder: str = "" # Placeholder text
description: str = "" # Help text
suggestions_endpoint: Optional[str] = None # Remote suggestions endpoint for typeahead
suggestions_min_query_length: int = 2 # Minimum chars before requesting suggestions
key: str # Field identifier (e.g., "author", "publisher")
label: str # Display label in UI
placeholder: str = "" # Placeholder text
description: str = "" # Help text
suggestions_endpoint: str | None = None # Remote suggestions endpoint for typeahead
suggestions_min_query_length: int = 2 # Minimum chars before requesting suggestions
@dataclass
class NumberSearchField:
"""Numeric input search field."""
key: str
label: str
placeholder: str = ""
description: str = ""
min_value: Optional[int] = None
max_value: Optional[int] = None
min_value: int | None = None
max_value: int | None = None
step: int = 1
@dataclass
class SelectSearchField:
"""Single-choice dropdown search field."""
key: str
label: str
options: List[Dict[str, str]] = field(default_factory=list) # [{value: "", label: ""}]
options: list[dict[str, str]] = field(default_factory=list) # [{value: "", label: ""}]
placeholder: str = ""
description: str = ""
@@ -79,6 +89,7 @@ class SelectSearchField:
@dataclass
class CheckboxSearchField:
"""Boolean checkbox search field."""
key: str
label: str
description: str = ""
@@ -88,6 +99,7 @@ class CheckboxSearchField:
@dataclass
class DynamicSelectSearchField:
"""Single-choice dropdown field with options loaded from an API endpoint."""
key: str
label: str
options_endpoint: str
@@ -96,18 +108,18 @@ class DynamicSelectSearchField:
# Type alias for all search field types
SearchField = Union[
TextSearchField,
NumberSearchField,
SelectSearchField,
CheckboxSearchField,
DynamicSelectSearchField,
]
SearchField = (
TextSearchField
| NumberSearchField
| SelectSearchField
| CheckboxSearchField
| DynamicSelectSearchField
)
def serialize_metadata_capability(capability: MetadataCapability) -> Dict[str, Any]:
def serialize_metadata_capability(capability: MetadataCapability) -> dict[str, Any]:
"""Serialize a provider capability for API responses."""
result: Dict[str, Any] = {
result: dict[str, Any] = {
"key": capability.key,
}
@@ -120,14 +132,14 @@ def serialize_metadata_capability(capability: MetadataCapability) -> Dict[str, A
return result
def serialize_search_field(search_field: SearchField) -> Dict[str, Any]:
def serialize_search_field(search_field: SearchField) -> dict[str, Any]:
"""Serialize a search field to dict for API response."""
result: Dict[str, Any] = {
result: dict[str, Any] = {
"key": search_field.key,
"label": search_field.label,
"type": search_field.__class__.__name__,
"placeholder": getattr(search_field, 'placeholder', ''),
"description": getattr(search_field, 'description', ''),
"placeholder": getattr(search_field, "placeholder", ""),
"description": getattr(search_field, "description", ""),
}
# Add type-specific properties
@@ -152,70 +164,73 @@ def serialize_search_field(search_field: SearchField) -> Dict[str, Any]:
@dataclass
class MetadataSearchOptions:
"""Options for metadata search queries across all providers."""
query: str
search_type: SearchType = SearchType.GENERAL
language: Optional[str] = None # ISO 639-1 code (e.g., "en", "fr")
language: str | None = None # ISO 639-1 code (e.g., "en", "fr")
sort: SortOrder = SortOrder.RELEVANCE
limit: int = 40
page: int = 1
fields: Dict[str, Any] = field(default_factory=dict) # Custom search field values
fields: dict[str, Any] = field(default_factory=dict) # Custom search field values
@dataclass
class DisplayField:
"""A display field for metadata cards (ratings, page counts, etc.)."""
label: str # e.g., "Rating", "Pages", "Readers"
value: str # e.g., "4.5", "496", "8,041"
icon: Optional[str] = None # Icon name: "star", "book", "users", "editions"
label: str # e.g., "Rating", "Pages", "Readers"
value: str # e.g., "4.5", "496", "8,041"
icon: str | None = None # Icon name: "star", "book", "users", "editions"
@dataclass
class BookMetadata:
"""Book from metadata provider (not a specific release)."""
provider: str # Which provider this came from (internal name)
provider_id: str # ID in that provider's system
provider: str # Which provider this came from (internal name)
provider_id: str # ID in that provider's system
title: str
# Provider display name for UI (e.g., "Open Library" instead of "openlibrary")
provider_display_name: Optional[str] = None
provider_display_name: str | None = None
# Optional - not all providers have all fields
authors: List[str] = field(default_factory=list)
isbn_10: Optional[str] = None
isbn_13: Optional[str] = None
cover_url: Optional[str] = None
description: Optional[str] = None
publisher: Optional[str] = None
publish_year: Optional[int] = None
language: Optional[str] = None
genres: List[str] = field(default_factory=list)
source_url: Optional[str] = None # Link to book on provider's site
subtitle: Optional[str] = None # Book subtitle, if any
search_title: Optional[str] = None # Cleaner title for search queries (provider-specific)
search_author: Optional[str] = None # Cleaner author for search queries (provider-specific)
authors: list[str] = field(default_factory=list)
isbn_10: str | None = None
isbn_13: str | None = None
cover_url: str | None = None
description: str | None = None
publisher: str | None = None
publish_year: int | None = None
language: str | None = None
genres: list[str] = field(default_factory=list)
source_url: str | None = None # Link to book on provider's site
subtitle: str | None = None # Book subtitle, if any
search_title: str | None = None # Cleaner title for search queries (provider-specific)
search_author: str | None = None # Cleaner author for search queries (provider-specific)
# Cover aspect ratio hint for the frontend ("portrait" or "square")
cover_aspect: Optional[str] = None
cover_aspect: str | None = None
# Provider-specific display fields for cards/lists
display_fields: List[DisplayField] = field(default_factory=list)
display_fields: list[DisplayField] = field(default_factory=list)
# Series info (if book is part of a series)
series_id: Optional[str] = None # Provider-specific series ID
series_name: Optional[str] = None # Name of the series
series_position: Optional[float] = None # This book's position (e.g., 3, 1.5 for novellas)
series_count: Optional[int] = None # Total books in the series
series_id: str | None = None # Provider-specific series ID
series_name: str | None = None # Name of the series
series_position: float | None = None # This book's position (e.g., 3, 1.5 for novellas)
series_count: int | None = None # Total books in the series
# Alternative titles by language (for localized searches)
# Maps language code (e.g., "de", "German") to localized title
titles_by_language: Dict[str, str] = field(default_factory=dict)
titles_by_language: dict[str, str] = field(default_factory=dict)
def group_languages_by_localized_title(
base_title: str,
languages: Optional[List[str]],
titles_by_language: Optional[Dict[str, str]] = None,
) -> List[tuple[str, Optional[List[str]]]]:
languages: list[str] | None,
titles_by_language: dict[str, str] | None = None,
) -> list[tuple[str, list[str] | None]]:
"""Group language codes by localized title.
Release sources that support language filtering (e.g., Anna's Archive)
@@ -230,6 +245,7 @@ def group_languages_by_localized_title(
Returns:
List of (title, languages) tuples. If languages is None/empty, returns
[(base_title, None)].
"""
if not base_title:
return []
@@ -244,7 +260,7 @@ def group_languages_by_localized_title(
if not titles_by_language:
return [(base_title, normalized_langs)]
title_to_langs: Dict[str, List[str]] = {}
title_to_langs: dict[str, list[str]] = {}
for lang in normalized_langs:
localized_title = titles_by_language.get(lang) or base_title
title_to_langs.setdefault(localized_title, []).append(lang)
@@ -254,10 +270,10 @@ def group_languages_by_localized_title(
def build_localized_search_titles(
base_title: str,
languages: Optional[List[str]],
titles_by_language: Optional[Dict[str, str]] = None,
excluded_languages: Optional[set[str]] = None,
) -> List[str]:
languages: list[str] | None,
titles_by_language: dict[str, str] | None = None,
excluded_languages: set[str] | None = None,
) -> list[str]:
"""Build a list of titles to search for, including localized editions.
This is useful for release sources that *can't* pass language filters to
@@ -274,11 +290,12 @@ def build_localized_search_titles(
Returns:
List of unique titles to search for, in priority order.
"""
if not base_title:
return []
titles: List[str] = [base_title]
titles: list[str] = [base_title]
seen = {base_title}
if not languages or not titles_by_language:
@@ -309,12 +326,13 @@ def build_localized_search_titles(
@dataclass
class SearchResult:
"""Result from a metadata search with pagination info."""
books: List[BookMetadata]
books: list[BookMetadata]
page: int = 1
total_found: int = 0 # Total matching results (if known)
has_more: bool = False # True if more results available
source_url: Optional[str] = None # External URL for the result set (e.g. Hardcover list page)
source_title: Optional[str] = None # Display title for the result set (e.g. list name)
source_url: str | None = None # External URL for the result set (e.g. Hardcover list page)
source_title: str | None = None # Display title for the result set (e.g. list name)
class MetadataProvider(ABC):
@@ -330,33 +348,31 @@ class MetadataProvider(ABC):
supported_sorts: List of SortOrder values this provider supports
search_fields: List of provider-specific search fields
capabilities: Declarative capabilities exposed to shared UI code
"""
name: str
display_name: str
requires_auth: bool
supported_sorts: List[SortOrder] = [SortOrder.RELEVANCE]
search_fields: List[SearchField] = []
capabilities: List[MetadataCapability] = []
supported_sorts: ClassVar[tuple[SortOrder, ...]] = (SortOrder.RELEVANCE,)
search_fields: ClassVar[tuple[SearchField, ...]] = ()
capabilities: ClassVar[tuple[MetadataCapability, ...]] = ()
@abstractmethod
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
def search(self, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Search for books using the provided options."""
pass
@abstractmethod
def get_book(self, book_id: str) -> Optional[BookMetadata]:
def get_book(self, book_id: str) -> BookMetadata | None:
"""Get a specific book by provider ID."""
pass
@abstractmethod
def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
def search_by_isbn(self, isbn: str) -> BookMetadata | None:
"""Search for a book by ISBN."""
pass
@abstractmethod
def is_available(self) -> bool:
"""Check if this provider is configured and available."""
pass
def search_paginated(self, options: MetadataSearchOptions) -> SearchResult:
"""Search with pagination info. Override for accurate pagination."""
@@ -367,62 +383,71 @@ class MetadataProvider(ABC):
books=books,
page=options.page,
total_found=0, # Unknown without provider-specific implementation
has_more=has_more
has_more=has_more,
)
def get_search_field_options(
self,
field_key: str,
query: Optional[str] = None,
) -> List[Dict[str, str]]:
query: str | None = None,
) -> list[dict[str, str]]:
"""Get dynamic options for a provider-specific search field."""
return []
def get_book_targets(self, book_id: str) -> List[Dict[str, Any]]:
def get_book_targets(self, book_id: str) -> list[dict[str, Any]]:
"""Get provider-managed list or status targets for a specific book."""
raise NotImplementedError(f"{self.display_name} does not support book targets")
msg = f"{self.display_name} does not support book targets"
raise NotImplementedError(msg)
def get_book_targets_batch(self, book_ids: List[str]) -> Dict[str, List[Dict[str, Any]]]:
def get_book_targets_batch(self, book_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
"""Get provider-managed targets for multiple books.
Returns a dict mapping each book_id to its list of target options.
Default implementation calls get_book_targets per book.
"""
results: Dict[str, List[Dict[str, Any]]] = {}
for book_id in book_ids:
try:
results[book_id] = self.get_book_targets(book_id)
except (NotImplementedError, ValueError):
results[book_id] = []
return results
return {book_id: self._get_book_targets_for_batch(book_id) for book_id in book_ids}
def _get_book_targets_for_batch(self, book_id: str) -> list[dict[str, Any]]:
"""Safely fetch targets for one book, falling back to an empty list."""
try:
return self.get_book_targets(book_id)
except NotImplementedError, ValueError:
return []
def set_book_target_state(
self,
book_id: str,
target: str,
selected: bool,
) -> Dict[str, Any]:
) -> dict[str, Any]:
"""Set whether a book belongs to a provider-managed list or shelf.
Returns a dict with at least ``{"changed": bool}``.
"""
raise NotImplementedError(f"{self.display_name} does not support book targets")
msg = f"{self.display_name} does not support book targets"
raise NotImplementedError(msg)
# Provider registry
_PROVIDERS: Dict[str, Type[MetadataProvider]] = {}
_PROVIDER_KWARGS_FACTORIES: Dict[str, Any] = {} # Callable[[], Dict]
_PROVIDERS: dict[str, type[MetadataProvider]] = {}
_PROVIDER_KWARGS_FACTORIES: dict[str, Any] = {} # Callable[[], Dict]
def register_provider(name: str):
def register_provider(
name: str,
) -> Callable[[type[MetadataProvider]], type[MetadataProvider]]:
"""Decorator to register a metadata provider."""
def decorator(cls):
def decorator(cls: type[MetadataProvider]) -> type[MetadataProvider]:
_PROVIDERS[name] = cls
return cls
return decorator
def register_provider_kwargs(name: str):
def register_provider_kwargs(
name: str,
) -> Callable[[Callable[[], dict[str, Any]]], Callable[[], dict[str, Any]]]:
"""Decorator to register a provider's kwargs factory.
The decorated function should return a Dict of kwargs to pass to the
@@ -434,21 +459,25 @@ def register_provider_kwargs(name: str):
def _hardcover_kwargs() -> Dict:
from shelfmark.core.config import config
return {"api_key": config.get("HARDCOVER_API_KEY", "")}
"""
def decorator(fn):
def decorator(fn: Callable[[], dict[str, Any]]) -> Callable[[], dict[str, Any]]:
_PROVIDER_KWARGS_FACTORIES[name] = fn
return fn
return decorator
def get_provider(name: str, **kwargs) -> MetadataProvider:
"""Factory - instantiate any registered provider."""
if name not in _PROVIDERS:
raise ValueError(f"Unknown metadata provider: {name}")
msg = f"Unknown metadata provider: {name}"
raise ValueError(msg)
return _PROVIDERS[name](**kwargs)
def list_providers() -> List[dict]:
def list_providers() -> list[dict]:
"""For settings UI - list available providers with their requirements."""
return [
{"name": n, "display_name": c.display_name, "requires_auth": c.requires_auth}
@@ -456,7 +485,7 @@ def list_providers() -> List[dict]:
]
def get_provider_kwargs(provider_name: str) -> Dict:
def get_provider_kwargs(provider_name: str) -> dict:
"""Get provider-specific initialization kwargs from registered factory."""
factory = _PROVIDER_KWARGS_FACTORIES.get(provider_name)
if factory:
@@ -481,15 +510,15 @@ def is_provider_enabled(provider_name: str) -> bool:
return app_config.get(enabled_key, False) is True
def get_enabled_providers() -> List[str]:
def get_enabled_providers() -> list[str]:
"""Get list of all enabled provider names."""
return [name for name in _PROVIDERS if is_provider_enabled(name)]
def get_configured_provider(
content_type: str = "ebook",
user_id: Optional[int] = None,
) -> Optional[MetadataProvider]:
user_id: int | None = None,
) -> MetadataProvider | None:
"""Get the currently configured metadata provider for the content type."""
from shelfmark.core.config import config as app_config
@@ -520,7 +549,8 @@ def get_configured_provider(
def get_configured_provider_name(
content_type: str = "ebook",
user_id: Optional[int] = None,
user_id: int | None = None,
*,
fallback_to_main: bool = True,
) -> str:
"""Get the configured metadata provider name for a content type."""
@@ -550,16 +580,16 @@ def get_configured_provider_name(
def get_provider_sort_options(
provider_name: Optional[str] = None,
user_id: Optional[int] = None,
) -> List[Dict[str, str]]:
provider_name: str | None = None,
user_id: int | None = None,
) -> list[dict[str, str]]:
"""Get sort options for a metadata provider as {value, label} dicts."""
if provider_name is None:
provider_name = get_configured_provider_name(user_id=user_id)
if provider_name and provider_name in _PROVIDERS:
provider_class = _PROVIDERS[provider_name]
supported = getattr(provider_class, 'supported_sorts', [SortOrder.RELEVANCE])
supported = getattr(provider_class, "supported_sorts", [SortOrder.RELEVANCE])
else:
supported = [SortOrder.RELEVANCE]
@@ -570,16 +600,16 @@ def get_provider_sort_options(
def get_provider_search_fields(
provider_name: Optional[str] = None,
user_id: Optional[int] = None,
) -> List[Dict[str, Any]]:
provider_name: str | None = None,
user_id: int | None = None,
) -> list[dict[str, Any]]:
"""Get search fields for a metadata provider as serialized dicts."""
if provider_name is None:
provider_name = get_configured_provider_name(user_id=user_id)
if provider_name and provider_name in _PROVIDERS:
provider_class = _PROVIDERS[provider_name]
fields = getattr(provider_class, 'search_fields', [])
fields = getattr(provider_class, "search_fields", [])
else:
fields = []
@@ -587,9 +617,9 @@ def get_provider_search_fields(
def get_provider_capabilities(
provider_name: Optional[str] = None,
user_id: Optional[int] = None,
) -> List[Dict[str, Any]]:
provider_name: str | None = None,
user_id: int | None = None,
) -> list[dict[str, Any]]:
"""Get declarative capabilities for a metadata provider."""
if provider_name is None:
provider_name = get_configured_provider_name(user_id=user_id)
@@ -604,8 +634,8 @@ def get_provider_capabilities(
def get_provider_default_sort(
provider_name: Optional[str] = None,
user_id: Optional[int] = None,
provider_name: str | None = None,
user_id: int | None = None,
) -> str:
"""Get the default sort order for a metadata provider."""
from shelfmark.core.config import config as app_config
@@ -629,7 +659,7 @@ def sync_metadata_provider_selection() -> None:
enabling/disabling a provider.
"""
from shelfmark.core.config import config as app_config
from shelfmark.core.settings_registry import save_config_file, load_config_file
from shelfmark.core.settings_registry import load_config_file, save_config_file
app_config.refresh()
@@ -653,18 +683,11 @@ def sync_metadata_provider_selection() -> None:
# Import provider implementations to trigger registration
# These must be imported AFTER the base classes and registry are defined
try:
from shelfmark.metadata_providers import hardcover # noqa: F401, E402
except ImportError:
pass # Hardcover provider is optional
with suppress(ImportError):
from shelfmark.metadata_providers import hardcover as hardcover
try:
from shelfmark.metadata_providers import openlibrary # noqa: F401, E402
except ImportError:
pass # Open Library provider is optional
try:
from shelfmark.metadata_providers import googlebooks # noqa: F401, E402
except ImportError:
pass # Google Books provider is optional
with suppress(ImportError):
from shelfmark.metadata_providers import openlibrary as openlibrary
with suppress(ImportError):
from shelfmark.metadata_providers import googlebooks as googlebooks
shelfmark/metadata_providers/googlebooks.py
+75 -78
View File
@@ -6,20 +6,24 @@ Requires a free API key from Google Cloud Console (~1000 requests/day quota).
API Documentation: https://developers.google.com/books/docs/v1/using
"""
from contextlib import suppress
from http import HTTPStatus
from typing import Any, ClassVar
import requests
from typing import Any, Dict, List, Optional
from shelfmark.core.cache import cacheable
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import (
register_settings,
ActionButton,
CheckboxField,
HeadingField,
PasswordField,
SelectField,
ActionButton,
HeadingField,
SettingsField,
register_settings,
)
from shelfmark.core.config import config as app_config
from shelfmark.download.network import get_ssl_verify
from shelfmark.metadata_providers import (
BookMetadata,
@@ -28,18 +32,21 @@ from shelfmark.metadata_providers import (
MetadataSearchOptions,
SearchType,
SortOrder,
TextSearchField,
register_provider,
register_provider_kwargs,
TextSearchField,
)
logger = setup_logger(__name__)
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
_HTTP_STATUS_BAD_REQUEST = HTTPStatus.BAD_REQUEST
_HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
GOOGLE_BOOKS_BASE_URL = "https://www.googleapis.com/books/v1"
# Sort mapping - Google only supports "relevance" and "newest"
SORT_MAPPING: Dict[SortOrder, Optional[str]] = {
SORT_MAPPING: dict[SortOrder, str | None] = {
SortOrder.RELEVANCE: None, # Default, no param needed
SortOrder.NEWEST: "newest",
# POPULARITY, RATING, OLDEST not supported - fall back to relevance
@@ -47,7 +54,7 @@ SORT_MAPPING: Dict[SortOrder, Optional[str]] = {
@register_provider_kwargs("googlebooks")
def _googlebooks_kwargs() -> Dict[str, Any]:
def _googlebooks_kwargs() -> dict[str, Any]:
"""Provide Google Books-specific constructor kwargs."""
return {"api_key": app_config.get("GOOGLEBOOKS_API_KEY", "")}
@@ -59,8 +66,11 @@ class GoogleBooksProvider(MetadataProvider):
name = "googlebooks"
display_name = "Google Books"
requires_auth = True
supported_sorts = [SortOrder.RELEVANCE, SortOrder.NEWEST]
search_fields = [
supported_sorts: ClassVar[tuple[SortOrder, ...]] = (
SortOrder.RELEVANCE,
SortOrder.NEWEST,
)
search_fields: ClassVar[tuple[TextSearchField, ...]] = (
TextSearchField(
key="author",
label="Author",
@@ -71,9 +81,9 @@ class GoogleBooksProvider(MetadataProvider):
label="Title",
description="Search by book title",
),
]
)
def __init__(self, api_key: Optional[str] = None):
def __init__(self, api_key: str | None = None) -> None:
"""Initialize provider with optional API key (falls back to config)."""
self.api_key = api_key or app_config.get("GOOGLEBOOKS_API_KEY", "")
self.session = requests.Session()
@@ -82,7 +92,7 @@ class GoogleBooksProvider(MetadataProvider):
"""Check if provider is configured with an API key."""
return bool(self.api_key)
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
def search(self, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Search for books using Google Books API."""
if not self.api_key:
logger.warning("Google Books API key not configured")
@@ -106,9 +116,7 @@ class GoogleBooksProvider(MetadataProvider):
ttl_default=300,
key_prefix="googlebooks:search",
)
def _search_cached(
self, cache_key: str, options: MetadataSearchOptions
) -> List[BookMetadata]:
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Cached search implementation."""
# Build query string with Google Books operators
author_value = options.fields.get("author", "").strip()
@@ -134,7 +142,7 @@ class GoogleBooksProvider(MetadataProvider):
query = "+".join(query_parts)
# Build request params
params: Dict[str, Any] = {
params: dict[str, Any] = {
"q": query,
"maxResults": min(options.limit, 40), # Google max is 40
"startIndex": (options.page - 1) * options.limit,
@@ -150,32 +158,30 @@ class GoogleBooksProvider(MetadataProvider):
if options.language:
params["langRestrict"] = options.language
books: list[BookMetadata] = []
try:
result = self._make_request("/volumes", params)
if not result:
return []
if result:
items = result.get("items", [])
items = result.get("items", [])
books = []
for item in items:
book = self._parse_volume(item)
if book:
books.append(book)
for item in items:
book = self._parse_volume(item)
if book:
books.append(book)
logger.info("Google Books search '%s' returned %s results", query, len(books))
logger.info(f"Google Books search '{query}' returned {len(books)} results")
return books
except Exception as e:
logger.error(f"Google Books search error: {e}")
except Exception:
logger.exception("Google Books search error")
return []
return books
@cacheable(
ttl_key="METADATA_CACHE_BOOK_TTL",
ttl_default=600,
key_prefix="googlebooks:book",
)
def get_book(self, book_id: str) -> Optional[BookMetadata]:
def get_book(self, book_id: str) -> BookMetadata | None:
"""Get book details by Google Books volume ID."""
try:
result = self._make_request(f"/volumes/{book_id}", {})
@@ -184,8 +190,8 @@ class GoogleBooksProvider(MetadataProvider):
return self._parse_volume(result)
except Exception as e:
logger.error(f"Google Books get_book error: {e}")
except Exception:
logger.exception("Google Books get_book error")
return None
@cacheable(
@@ -193,13 +199,13 @@ class GoogleBooksProvider(MetadataProvider):
ttl_default=600,
key_prefix="googlebooks:isbn",
)
def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
def search_by_isbn(self, isbn: str) -> BookMetadata | None:
"""Search for a book by ISBN-10 or ISBN-13."""
# Clean ISBN (remove hyphens and spaces)
clean_isbn = isbn.replace("-", "").replace(" ", "").strip()
# Use ISBN operator for precise lookup
params: Dict[str, Any] = {
params: dict[str, Any] = {
"q": f"isbn:{clean_isbn}",
"maxResults": 1,
}
@@ -211,18 +217,16 @@ class GoogleBooksProvider(MetadataProvider):
items = result.get("items", [])
if not items:
logger.debug(f"No Google Books result for ISBN: {isbn}")
logger.debug("No Google Books result for ISBN: %s", isbn)
return None
return self._parse_volume(items[0])
except Exception as e:
logger.error(f"Google Books ISBN search error: {e}")
except Exception:
logger.exception("Google Books ISBN search error")
return None
def _make_request(
self, endpoint: str, params: Dict[str, Any]
) -> Optional[Dict[str, Any]]:
def _make_request(self, endpoint: str, params: dict[str, Any]) -> dict[str, Any] | None:
"""Make authenticated API request to endpoint."""
if not self.api_key:
logger.warning("Google Books API key not configured")
@@ -243,25 +247,25 @@ class GoogleBooksProvider(MetadataProvider):
return None
except requests.HTTPError as e:
if e.response is not None:
if e.response.status_code == 403:
if e.response.status_code == _HTTP_STATUS_FORBIDDEN:
# Quota exceeded or invalid API key
logger.error(
logger.exception(
"Google Books API: quota exceeded or invalid API key (HTTP 403)"
)
elif e.response.status_code == 400:
logger.warning(f"Google Books API: bad request - {e}")
elif e.response.status_code == 404:
elif e.response.status_code == _HTTP_STATUS_BAD_REQUEST:
logger.warning("Google Books API: bad request - %s", e)
elif e.response.status_code == _HTTP_STATUS_NOT_FOUND:
logger.debug("Google Books: volume not found")
else:
logger.error(f"Google Books API HTTP error: {e}")
logger.exception("Google Books API HTTP error")
else:
logger.error(f"Google Books API HTTP error: {e}")
logger.exception("Google Books API HTTP error")
return None
except Exception as e:
logger.error(f"Google Books API request failed: {e}")
except Exception:
logger.exception("Google Books API request failed")
return None
def _parse_volume(self, volume: Dict[str, Any]) -> Optional[BookMetadata]:
def _parse_volume(self, volume: dict[str, Any]) -> BookMetadata | None:
"""Parse a volume object into BookMetadata."""
try:
volume_id = volume.get("id")
@@ -296,9 +300,7 @@ class GoogleBooksProvider(MetadataProvider):
)
# Remove edge=curl parameter and upgrade to https
if cover_url:
cover_url = cover_url.replace("&edge=curl", "").replace(
"http://", "https://"
)
cover_url = cover_url.replace("&edge=curl", "").replace("http://", "https://")
# Publisher
publisher = volume_info.get("publisher")
@@ -307,10 +309,8 @@ class GoogleBooksProvider(MetadataProvider):
publish_year = None
published_date = volume_info.get("publishedDate", "")
if published_date:
try:
with suppress(ValueError, TypeError):
publish_year = int(published_date[:4])
except (ValueError, TypeError):
pass
# Language
language = volume_info.get("language")
@@ -325,7 +325,7 @@ class GoogleBooksProvider(MetadataProvider):
source_url = volume_info.get("infoLink")
# Build display fields - rating only
display_fields: List[DisplayField] = []
display_fields: list[DisplayField] = []
average_rating = volume_info.get("averageRating")
ratings_count = volume_info.get("ratingsCount")
@@ -333,9 +333,7 @@ class GoogleBooksProvider(MetadataProvider):
rating_str = f"{average_rating:.1f}"
if ratings_count:
rating_str += f" ({ratings_count:,})"
display_fields.append(
DisplayField(label="Rating", value=rating_str, icon="star")
)
display_fields.append(DisplayField(label="Rating", value=rating_str, icon="star"))
return BookMetadata(
provider="googlebooks",
@@ -356,11 +354,13 @@ class GoogleBooksProvider(MetadataProvider):
)
except Exception as e:
logger.debug(f"Failed to parse Google Books volume: {e}")
logger.debug("Failed to parse Google Books volume: %s", e)
return None
def _test_googlebooks_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
def _test_googlebooks_connection(
current_values: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Test the Google Books API connection using current form values."""
current_values = current_values or {}
@@ -377,25 +377,25 @@ def _test_googlebooks_connection(current_values: Dict[str, Any] = None) -> Dict[
provider = GoogleBooksProvider(api_key=api_key)
# Simple test search
result = provider._make_request("/volumes", {"q": "test", "maxResults": 1})
test_result = {
"success": False,
"message": "API request failed - check your API key",
}
if result is not None and "items" in result:
return {
test_result = {
"success": True,
"message": "Successfully connected to Google Books API",
}
elif result is not None:
return {
test_result = {
"success": True,
"message": "API connected but returned no results for test query",
}
else:
return {
"success": False,
"message": "API request failed - check your API key",
}
except Exception as e:
logger.exception("Google Books connection test failed")
return {"success": False, "message": f"Connection failed: {str(e)}"}
return {"success": False, "message": f"Connection failed: {e!s}"}
return test_result
# Sort options for settings UI
@@ -405,10 +405,8 @@ _GOOGLEBOOKS_SORT_OPTIONS = [
]
@register_settings(
"googlebooks", "Google Books", icon="book", order=53, group="metadata_providers"
)
def googlebooks_settings():
@register_settings("googlebooks", "Google Books", icon="book", order=53, group="metadata_providers")
def googlebooks_settings() -> list[SettingsField]:
"""Google Books metadata provider settings."""
return [
HeadingField(
@@ -431,8 +429,7 @@ def googlebooks_settings():
key="GOOGLEBOOKS_API_KEY",
label="API Key",
description=(
"Get your API key from Google Cloud Console "
"(APIs & Services > Credentials)"
"Get your API key from Google Cloud Console (APIs & Services > Credentials)"
),
required=True,
),
shelfmark/metadata_providers/hardcover.py
+361 -257
View File
File diff suppressed because it is too large Load Diff
shelfmark/metadata_providers/openlibrary.py
+75 -57
View File
@@ -1,23 +1,24 @@
"""Open Library metadata provider. No API key required, rate limited."""
import re
import time
import threading
import time
from collections import deque
from typing import Any, Deque, Dict, List, Optional
from typing import Any, ClassVar
import requests
from shelfmark.core.cache import cacheable
from shelfmark.core.logger import setup_logger
from shelfmark.download.network import get_ssl_verify
from shelfmark.core.settings_registry import (
register_settings,
CheckboxField,
SelectField,
ActionButton,
CheckboxField,
HeadingField,
SelectField,
SettingsField,
register_settings,
)
from shelfmark.download.network import get_ssl_verify
from shelfmark.metadata_providers import (
BookMetadata,
DisplayField,
@@ -25,8 +26,8 @@ from shelfmark.metadata_providers import (
MetadataSearchOptions,
SearchType,
SortOrder,
register_provider,
TextSearchField,
register_provider,
)
logger = setup_logger(__name__)
@@ -43,11 +44,11 @@ RATE_LIMIT_WINDOW_SECONDS = 60
class RateLimiter:
"""Simple sliding window rate limiter."""
def __init__(self, max_requests: int, window_seconds: int):
def __init__(self, max_requests: int, window_seconds: int) -> None:
"""Initialize rate limiter with max requests per time window."""
self.max_requests = max_requests
self.window_seconds = window_seconds
self.timestamps: Deque[float] = deque()
self.timestamps: deque[float] = deque()
self.lock = threading.Lock()
def wait_if_needed(self) -> None:
@@ -69,7 +70,7 @@ class RateLimiter:
# Sleep outside the lock to avoid blocking other threads
if wait_time > 0:
logger.debug(f"Rate limited, waiting {wait_time:.2f}s")
logger.debug("Rate limited, waiting %0.2fs", wait_time)
time.sleep(wait_time)
# Re-acquire lock and record request
@@ -90,7 +91,7 @@ _rate_limiter = RateLimiter(RATE_LIMIT_REQUESTS, RATE_LIMIT_WINDOW_SECONDS)
# Mapping from abstract sort order to Open Library sort parameter
# Note: Open Library only supports relevance (default), new, old, random
SORT_MAPPING: Dict[str, Optional[str]] = {
SORT_MAPPING: dict[str, str | None] = {
SortOrder.RELEVANCE: None, # Default (no sort param)
SortOrder.NEWEST: "new",
SortOrder.OLDEST: "old",
@@ -105,12 +106,12 @@ class OpenLibraryProvider(MetadataProvider):
name = "openlibrary"
display_name = "Open Library"
requires_auth = False
supported_sorts = [
supported_sorts: ClassVar[tuple[SortOrder, ...]] = (
SortOrder.RELEVANCE,
SortOrder.NEWEST,
SortOrder.OLDEST,
]
search_fields = [
)
search_fields: ClassVar[tuple[TextSearchField, ...]] = (
TextSearchField(
key="author",
label="Author",
@@ -121,9 +122,9 @@ class OpenLibraryProvider(MetadataProvider):
label="Title",
description="Search by book title",
),
]
)
def __init__(self):
def __init__(self) -> None:
"""Initialize provider."""
self.session = requests.Session()
@@ -131,7 +132,7 @@ class OpenLibraryProvider(MetadataProvider):
"""Open Library is always available (no auth required)."""
return True
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
def search(self, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Search for books using Open Library's search API."""
# Handle ISBN search separately
if options.search_type == SearchType.ISBN:
@@ -143,13 +144,15 @@ class OpenLibraryProvider(MetadataProvider):
cache_key = f"{options.query}:{options.search_type.value}:{options.sort.value}:{options.language}:{options.limit}:{options.page}:{fields_key}"
return self._search_cached(cache_key, options)
@cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="openlibrary:search")
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> List[BookMetadata]:
@cacheable(
ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="openlibrary:search"
)
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Cached search implementation."""
_rate_limiter.wait_if_needed()
# Build query params
params: Dict[str, Any] = {
params: dict[str, Any] = {
"limit": options.limit,
"page": options.page,
"fields": "key,title,author_name,first_publish_year,cover_i,isbn,publisher,language,subject,ratings_average,ratings_count",
@@ -185,6 +188,7 @@ class OpenLibraryProvider(MetadataProvider):
if options.language:
params["lang"] = options.language
books: list[BookMetadata] = []
try:
response = self.session.get(
f"{OPENLIBRARY_BASE_URL}/search.json",
@@ -195,14 +199,12 @@ class OpenLibraryProvider(MetadataProvider):
response.raise_for_status()
data = response.json()
books = []
for doc in data.get("docs", []):
book = self._parse_search_doc(doc)
if book:
books.append(book)
logger.info(f"Open Library search '{options.query}' returned {len(books)} results")
return books
logger.info("Open Library search '%s' returned %s results", options.query, len(books))
except requests.Timeout:
logger.warning("Open Library search timed out")
@@ -211,14 +213,18 @@ class OpenLibraryProvider(MetadataProvider):
if e.response.status_code == 503:
logger.warning("Open Library service unavailable (503)")
else:
logger.error(f"Open Library HTTP error: {e}")
logger.exception("Open Library HTTP error")
return []
except Exception as e:
logger.error(f"Open Library search error: {e}")
except requests.RequestException:
logger.exception("Open Library search request failed")
return []
except TypeError, ValueError:
logger.exception("Open Library search parsing error")
return []
return books
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="openlibrary:book")
def get_book(self, book_id: str) -> Optional[BookMetadata]:
def get_book(self, book_id: str) -> BookMetadata | None:
"""Get book details by Open Library work ID (e.g., 'OL12345W')."""
_rate_limiter.wait_if_needed()
@@ -244,16 +250,19 @@ class OpenLibraryProvider(MetadataProvider):
return None
except requests.HTTPError as e:
if e.response.status_code == 404:
logger.debug(f"Open Library work not found: {book_id}")
logger.debug("Open Library work not found: %s", book_id)
else:
logger.error(f"Open Library HTTP error: {e}")
logger.exception("Open Library HTTP error")
return None
except Exception as e:
logger.error(f"Open Library get_book error: {e}")
except requests.RequestException:
logger.exception("Open Library get_book request failed")
return None
except TypeError, ValueError:
logger.exception("Open Library get_book parsing error")
return None
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="openlibrary:isbn")
def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
def search_by_isbn(self, isbn: str) -> BookMetadata | None:
"""Search for a book by ISBN-10 or ISBN-13."""
# Clean ISBN
clean_isbn = isbn.replace("-", "").strip()
@@ -283,6 +292,7 @@ class OpenLibraryProvider(MetadataProvider):
# Update with ISBN from edition if not present
# Use dataclasses.replace() to avoid mutating cached object
from dataclasses import replace
updates = {}
if not book.isbn_10:
isbn_10_list = edition.get("isbn_10", [])
@@ -301,15 +311,18 @@ class OpenLibraryProvider(MetadataProvider):
except requests.HTTPError as e:
if e.response.status_code == 404:
logger.debug(f"Open Library ISBN not found: {isbn}")
logger.debug("Open Library ISBN not found: %s", isbn)
else:
logger.error(f"Open Library ISBN search HTTP error: {e}")
logger.exception("Open Library ISBN search HTTP error")
return None
except Exception as e:
logger.error(f"Open Library ISBN search error: {e}")
except requests.RequestException:
logger.exception("Open Library ISBN search request failed")
return None
except TypeError, ValueError:
logger.exception("Open Library ISBN search parsing error")
return None
def _parse_search_doc(self, doc: dict) -> Optional[BookMetadata]:
def _parse_search_doc(self, doc: dict) -> BookMetadata | None:
"""Parse a search document into BookMetadata."""
try:
# Extract work ID from key
@@ -374,11 +387,11 @@ class OpenLibraryProvider(MetadataProvider):
display_fields=display_fields,
)
except Exception as e:
logger.debug(f"Failed to parse Open Library search doc: {e}")
except (TypeError, ValueError, AttributeError, KeyError) as e:
logger.debug("Failed to parse Open Library search doc: %s", e)
return None
def _parse_work(self, work: dict, work_id: str) -> Optional[BookMetadata]:
def _parse_work(self, work: dict, work_id: str) -> BookMetadata | None:
"""Parse a work object into BookMetadata."""
try:
title = work.get("title")
@@ -424,11 +437,11 @@ class OpenLibraryProvider(MetadataProvider):
source_url=f"{OPENLIBRARY_BASE_URL}/works/{work_id}",
)
except Exception as e:
logger.debug(f"Failed to parse Open Library work: {e}")
except (TypeError, ValueError, AttributeError, KeyError) as e:
logger.debug("Failed to parse Open Library work: %s", e)
return None
def _parse_edition(self, edition: dict, isbn: str) -> Optional[BookMetadata]:
def _parse_edition(self, edition: dict, isbn: str) -> BookMetadata | None:
"""Parse an edition object into BookMetadata (fallback for ISBN lookup)."""
try:
title = edition.get("title")
@@ -461,7 +474,7 @@ class OpenLibraryProvider(MetadataProvider):
publish_date = edition.get("publish_date", "")
if publish_date:
# Try to extract year from various formats
year_match = re.search(r'\b(19|20)\d{2}\b', publish_date)
year_match = re.search(r"\b(19|20)\d{2}\b", publish_date)
if year_match:
publish_year = int(year_match.group())
@@ -478,11 +491,11 @@ class OpenLibraryProvider(MetadataProvider):
source_url=f"{OPENLIBRARY_BASE_URL}{key}" if key else None,
)
except Exception as e:
logger.debug(f"Failed to parse Open Library edition: {e}")
except (TypeError, ValueError, AttributeError, KeyError) as e:
logger.debug("Failed to parse Open Library edition: %s", e)
return None
def _get_author_name(self, author_key: str) -> Optional[str]:
def _get_author_name(self, author_key: str) -> str | None:
"""Get author name from author key (e.g., '/authors/OL123A')."""
_rate_limiter.wait_if_needed()
@@ -496,13 +509,14 @@ class OpenLibraryProvider(MetadataProvider):
author = response.json()
return author.get("name")
except Exception:
except requests.RequestException, ValueError:
# Don't log errors for author lookups - they're supplementary
return None
def _test_openlibrary_connection() -> Dict[str, Any]:
def _test_openlibrary_connection() -> dict[str, Any]:
"""Test the Open Library API connection."""
connection_result = {"success": False, "message": "Unexpected response from API"}
try:
provider = OpenLibraryProvider()
# Simple API call to test connectivity
@@ -515,15 +529,17 @@ def _test_openlibrary_connection() -> Dict[str, Any]:
response.raise_for_status()
data = response.json()
if "docs" in data:
return {"success": True, "message": "Successfully connected to Open Library API"}
else:
return {"success": False, "message": "Unexpected response from API"}
connection_result = {
"success": True,
"message": "Successfully connected to Open Library API",
}
except requests.Timeout:
return {"success": False, "message": "Connection timed out"}
except requests.RequestException as e:
return {"success": False, "message": f"Connection failed: {str(e)}"}
except Exception as e:
return {"success": False, "message": f"Error: {str(e)}"}
return {"success": False, "message": f"Connection failed: {e!s}"}
except (TypeError, ValueError, AttributeError) as e:
return {"success": False, "message": f"Error: {e!s}"}
return connection_result
# Open Library sort options for settings UI
@@ -534,8 +550,10 @@ _OPENLIBRARY_SORT_OPTIONS = [
]
@register_settings("openlibrary", "Open Library", icon="library", order=52, group="metadata_providers")
def openlibrary_settings():
@register_settings(
"openlibrary", "Open Library", icon="library", order=52, group="metadata_providers"
)
def openlibrary_settings() -> list[SettingsField]:
"""Open Library metadata provider settings."""
return [
HeadingField(
shelfmark/release_sources/__init__.py
+178 -135
View File
@@ -1,25 +1,28 @@
"""Release source plugin system - base classes and registry."""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field, asdict
from enum import Enum
from pathlib import Path
from threading import Event
from typing import List, Optional, Dict, Type, Callable, Literal, Any, TYPE_CHECKING
from dataclasses import dataclass, field
from enum import StrEnum
from typing import TYPE_CHECKING, Any, ClassVar, Literal
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from threading import Event
from shelfmark.core.models import DownloadTask
from shelfmark.core.search_plan import ReleaseSearchPlan
from shelfmark.core.models import DownloadTask
from shelfmark.metadata_providers import BookMetadata
class ReleaseProtocol(str, Enum):
class ReleaseProtocol(StrEnum):
"""Protocol for downloading a release."""
HTTP = "http" # Direct HTTP download
TORRENT = "torrent" # BitTorrent
NZB = "nzb" # Usenet NZB
DCC = "dcc" # IRC DCC
HTTP = "http" # Direct HTTP download
TORRENT = "torrent" # BitTorrent
NZB = "nzb" # Usenet NZB
DCC = "dcc" # IRC DCC
class SourceUnavailableError(Exception):
@@ -29,76 +32,82 @@ class SourceUnavailableError(Exception):
@dataclass
class BrowseRecord:
"""Source-native browse/search record used before normalization to Release."""
id: str
title: str
source: str
preview: Optional[str] = None
author: Optional[str] = None
publisher: Optional[str] = None
year: Optional[str] = None
language: Optional[str] = None
content: Optional[str] = None
format: Optional[str] = None
size: Optional[str] = None
info: Optional[Dict[str, List[str]]] = None
description: Optional[str] = None
download_urls: List[str] = field(default_factory=list)
download_path: Optional[str] = None
preview: str | None = None
author: str | None = None
publisher: str | None = None
year: str | None = None
language: str | None = None
content: str | None = None
format: str | None = None
size: str | None = None
info: dict[str, list[str]] | None = None
description: str | None = None
download_urls: list[str] = field(default_factory=list)
download_path: str | None = None
priority: int = 0
progress: Optional[float] = None
status_message: Optional[str] = None
added_time: Optional[float] = None
source_url: Optional[str] = None
progress: float | None = None
status_message: str | None = None
added_time: float | None = None
source_url: str | None = None
@dataclass
class Release:
"""A downloadable release - all sources return this same structure."""
source: str # "direct", "prowlarr", "irc", etc.
source_id: str # ID within that source
source: str # "direct", "prowlarr", "irc", etc.
source_id: str # ID within that source
title: str
format: Optional[str] = None
language: Optional[str] = None # ISO 639-1 code (e.g., "en", "de", "fr")
size: Optional[str] = None
size_bytes: Optional[int] = None
download_url: Optional[str] = None
info_url: Optional[str] = None # Link to release info page (e.g., tracker) - makes title clickable
protocol: Optional[ReleaseProtocol] = None
indexer: Optional[str] = None # Source name for display
seeders: Optional[int] = None # For torrents
peers: Optional[str] = None # For torrents: "seeders/leechers" display string
content_type: Optional[str] = None # "ebook" or "audiobook" - preserved from search
extra: Dict = field(default_factory=dict) # Source-specific metadata
format: str | None = None
language: str | None = None # ISO 639-1 code (e.g., "en", "de", "fr")
size: str | None = None
size_bytes: int | None = None
download_url: str | None = None
info_url: str | None = None # Link to release info page (e.g., tracker) - makes title clickable
protocol: ReleaseProtocol | None = None
indexer: str | None = None # Source name for display
seeders: int | None = None # For torrents
peers: str | None = None # For torrents: "seeders/leechers" display string
content_type: str | None = None # "ebook" or "audiobook" - preserved from search
extra: dict = field(default_factory=dict) # Source-specific metadata
@dataclass
class DownloadProgress:
"""DEPRECATED: Use progress_callback and status_callback instead."""
status: str # "queued", "resolving", "downloading", "complete", "failed"
progress: float # 0-100
status_message: Optional[str] = None
download_speed: Optional[int] = None
eta: Optional[int] = None
save_path: Optional[str] = None
status: str # "queued", "resolving", "downloading", "complete", "failed"
progress: float # 0-100
status_message: str | None = None
download_speed: int | None = None
eta: int | None = None
save_path: str | None = None
# --- Column Schema for Plugin-Driven UI ---
class ColumnRenderType(str, Enum):
class ColumnRenderType(StrEnum):
"""How the frontend should render the column value."""
TEXT = "text" # Plain text
BADGE = "badge" # Colored badge (format, language)
TAGS = "tags" # List of colored badges
SIZE = "size" # File size formatting
NUMBER = "number" # Numeric value
PEERS = "peers" # Peers display: "S/L" with color based on seeder count
TEXT = "text" # Plain text
BADGE = "badge" # Colored badge (format, language)
TAGS = "tags" # List of colored badges
SIZE = "size" # File size formatting
NUMBER = "number" # Numeric value
PEERS = "peers" # Peers display: "S/L" with color based on seeder count
INDEXER_PROTOCOL = "indexer_protocol" # Text + colored dot for torrent/usenet
FLAG_ICON = "flag_icon" # Icon with tooltip (VIP, freeleech, etc.)
FLAG_ICON = "flag_icon" # Icon with tooltip (VIP, freeleech, etc.)
FORMAT_CONTENT_TYPE = "format_content_type" # Content type icon + format badge
class ColumnAlign(str, Enum):
class ColumnAlign(StrEnum):
"""Column alignment options."""
LEFT = "left"
CENTER = "center"
RIGHT = "right"
@@ -107,74 +116,89 @@ class ColumnAlign(str, Enum):
@dataclass
class ColumnColorHint:
"""Color hint for badge-type columns."""
type: Literal["map", "static"] # "map" uses frontend colorMaps, "static" is fixed class
value: str # Map name ("format", "language") or Tailwind class
type: Literal["map", "static"] # "map" uses frontend colorMaps, "static" is fixed class
value: str # Map name ("format", "language") or Tailwind class
@dataclass
class ColumnSchema:
"""Definition for a single column in the release list."""
key: str # Data path (e.g., "format", "extra.language")
label: str # Accessibility label
key: str # Data path (e.g., "format", "extra.language")
label: str # Accessibility label
render_type: ColumnRenderType = ColumnRenderType.TEXT
align: ColumnAlign = ColumnAlign.LEFT
width: str = "auto" # CSS width (e.g., "80px", "minmax(0,2fr)")
hide_mobile: bool = False # Hide on small screens
color_hint: Optional[ColumnColorHint] = None # For BADGE render type
fallback: str = "-" # Value to show when data is missing
uppercase: bool = False # Force uppercase display
sortable: bool = False # Show in sort dropdown (opt-in)
sort_key: Optional[str] = None # Field to sort by (defaults to `key` if None)
width: str = "auto" # CSS width (e.g., "80px", "minmax(0,2fr)")
hide_mobile: bool = False # Hide on small screens
color_hint: ColumnColorHint | None = None # For BADGE render type
fallback: str = "-" # Value to show when data is missing
uppercase: bool = False # Force uppercase display
sortable: bool = False # Show in sort dropdown (opt-in)
sort_key: str | None = None # Field to sort by (defaults to `key` if None)
class LeadingCellType(str, Enum):
class LeadingCellType(StrEnum):
"""Type of leading cell to display in release rows."""
THUMBNAIL = "thumbnail" # Show book cover image
BADGE = "badge" # Show colored badge (e.g., "Torrent", "Usenet")
NONE = "none" # No leading cell
BADGE = "badge" # Show colored badge (e.g., "Torrent", "Usenet")
NONE = "none" # No leading cell
@dataclass
class LeadingCellConfig:
"""Configuration for the leading cell in release rows."""
type: LeadingCellType = LeadingCellType.THUMBNAIL
key: Optional[str] = None # Field path for data (e.g., "extra.preview" or "extra.download_type")
color_hint: Optional[ColumnColorHint] = None # For badge type - maps values to colors
uppercase: bool = False # Force uppercase for badge text
key: str | None = None # Field path for data (e.g., "extra.preview" or "extra.download_type")
color_hint: ColumnColorHint | None = None # For badge type - maps values to colors
uppercase: bool = False # Force uppercase for badge text
@dataclass
class SortOption:
"""A sort option that appears in the sort dropdown without being tied to a column."""
label: str # Display label in the sort dropdown
sort_key: str # Field to sort by on the Release object
label: str # Display label in the sort dropdown
sort_key: str # Field to sort by on the Release object
@dataclass
class SourceActionButton:
"""Action button configuration for a release source."""
label: str # Button text (e.g., "Refresh search")
action: str = "expand" # Action type: "expand" triggers expand_search
label: str # Button text (e.g., "Refresh search")
action: str = "expand" # Action type: "expand" triggers expand_search
@dataclass
class ReleaseColumnConfig:
"""Complete column configuration for a release source."""
columns: List[ColumnSchema]
columns: list[ColumnSchema]
grid_template: str = "minmax(0,2fr) 60px 80px 80px" # CSS grid-template-columns
leading_cell: Optional[LeadingCellConfig] = None # Defaults to thumbnail mode if None
online_servers: Optional[List[str]] = None # For IRC: list of currently online server nicks
available_indexers: Optional[List[str]] = None # For Prowlarr: list of all enabled indexer names
default_indexers: Optional[List[str]] = None # For Prowlarr: indexers selected in settings (pre-selected in filter)
cache_ttl_seconds: Optional[int] = None # How long to cache results (default: 5 min)
supported_filters: Optional[List[str]] = None # Which filters this source supports: ["format", "language", "indexer"]
extra_sort_options: Optional[List[SortOption]] = None # Additional sort options not tied to a column
action_button: Optional[SourceActionButton] = None # Custom action button (replaces default expand search)
leading_cell: LeadingCellConfig | None = None # Defaults to thumbnail mode if None
online_servers: list[str] | None = None # For IRC: list of currently online server nicks
available_indexers: list[str] | None = None # For Prowlarr: list of all enabled indexer names
default_indexers: list[str] | None = (
None # For Prowlarr: indexers selected in settings (pre-selected in filter)
)
cache_ttl_seconds: int | None = None # How long to cache results (default: 5 min)
supported_filters: list[str] | None = (
None # Which filters this source supports: ["format", "language", "indexer"]
)
extra_sort_options: list[SortOption] | None = (
None # Additional sort options not tied to a column
)
action_button: SourceActionButton | None = (
None # Custom action button (replaces default expand search)
)
def serialize_column_config(config: ReleaseColumnConfig) -> Dict[str, Any]:
def serialize_column_config(config: ReleaseColumnConfig) -> dict[str, Any]:
"""Serialize column configuration for API response."""
result: Dict[str, Any] = {
result: dict[str, Any] = {
"columns": [
{
"key": col.key,
@@ -185,8 +209,10 @@ def serialize_column_config(config: ReleaseColumnConfig) -> Dict[str, Any]:
"hide_mobile": col.hide_mobile,
"color_hint": {
"type": col.color_hint.type,
"value": col.color_hint.value
} if col.color_hint else None,
"value": col.color_hint.value,
}
if col.color_hint
else None,
"fallback": col.fallback,
"uppercase": col.uppercase,
"sortable": col.sortable,
@@ -204,8 +230,10 @@ def serialize_column_config(config: ReleaseColumnConfig) -> Dict[str, Any]:
"key": config.leading_cell.key,
"color_hint": {
"type": config.leading_cell.color_hint.type,
"value": config.leading_cell.color_hint.value
} if config.leading_cell.color_hint else None,
"value": config.leading_cell.color_hint.value,
}
if config.leading_cell.color_hint
else None,
"uppercase": config.leading_cell.uppercase,
}
@@ -232,8 +260,7 @@ def serialize_column_config(config: ReleaseColumnConfig) -> Dict[str, Any]:
# Include extra sort options (sort entries not tied to a column)
if config.extra_sort_options:
result["extra_sort_options"] = [
{"label": opt.label, "sort_key": opt.sort_key}
for opt in config.extra_sort_options
{"label": opt.label, "sort_key": opt.sort_key} for opt in config.extra_sort_options
]
# Include action button if specified (replaces default expand search)
@@ -286,26 +313,29 @@ def _default_column_config() -> ReleaseColumnConfig:
class ReleaseSource(ABC):
"""Interface for searching a release source."""
name: str # "direct", "prowlarr"
display_name: str # "Direct Download", "Prowlarr"
supported_content_types: List[str] = ["ebook", "audiobook"] # Content types this source supports
can_be_default: bool = True # Whether this source can be selected as default in settings
name: str # "direct", "prowlarr"
display_name: str # "Direct Download", "Prowlarr"
supported_content_types: ClassVar[list[str]] = [
"ebook",
"audiobook",
] # Content types this source supports
can_be_default: bool = True # Whether this source can be selected as default in settings
@abstractmethod
def search(
self,
book: BookMetadata,
plan: "ReleaseSearchPlan",
plan: ReleaseSearchPlan,
*,
expand_search: bool = False,
content_type: str = "ebook"
) -> List[Release]:
content_type: str = "ebook",
) -> list[Release]:
"""Search for releases of a book."""
pass
@abstractmethod
def is_available(self) -> bool:
"""Check if this source is configured and reachable."""
pass
def get_column_config(self) -> ReleaseColumnConfig:
"""Get column configuration for release list UI. Override for custom columns."""
@@ -316,15 +346,16 @@ class ReleaseSource(ABC):
record_id: str,
*,
fetch_download_count: bool = True,
) -> Optional[BrowseRecord]:
) -> BrowseRecord | None:
"""Resolve a source-native record for browse flows."""
raise NotImplementedError(f"{self.display_name} does not support record lookup")
msg = f"{self.display_name} does not support record lookup"
raise NotImplementedError(msg)
def search_results_are_releases(self) -> bool:
"""Whether source-native browse results already represent concrete releases."""
return False
def get_destination_override(self, task: DownloadTask) -> Optional[Path]:
def get_destination_override(self, task: DownloadTask) -> Path | None:
"""Return a source-specific destination override for a queued download."""
return None
@@ -346,12 +377,11 @@ class DownloadHandler(ABC):
task: DownloadTask,
cancel_flag: Event,
progress_callback: Callable[[float], None],
status_callback: Callable[[str, Optional[str]], None]
) -> Optional[str]:
status_callback: Callable[[str, str | None], None],
) -> str | None:
"""Execute download and return a path to the downloaded payload."""
pass
def post_process_cleanup(self, task: DownloadTask, success: bool) -> None:
def post_process_cleanup(self, task: DownloadTask, *, success: bool) -> None:
"""Optional hook called after orchestrator post-processing.
This is primarily used for external download clients, where the handler may need
@@ -362,58 +392,71 @@ class DownloadHandler(ABC):
@abstractmethod
def cancel(self, task_id: str) -> bool:
"""Cancel an in-progress download."""
pass
# --- Registry ---
_SOURCES: Dict[str, Type[ReleaseSource]] = {}
_HANDLERS: Dict[str, Type[DownloadHandler]] = {}
_SOURCES: dict[str, type[ReleaseSource]] = {}
_HANDLERS: dict[str, type[DownloadHandler]] = {}
def register_source(name: str):
def register_source(
name: str,
) -> Callable[[type[ReleaseSource]], type[ReleaseSource]]:
"""Decorator to register a release source."""
def decorator(cls):
def decorator(cls: type[ReleaseSource]) -> type[ReleaseSource]:
_SOURCES[name] = cls
return cls
return decorator
def register_handler(name: str):
def register_handler(
name: str,
) -> Callable[[type[DownloadHandler]], type[DownloadHandler]]:
"""Decorator to register a download handler."""
def decorator(cls):
def decorator(cls: type[DownloadHandler]) -> type[DownloadHandler]:
_HANDLERS[name] = cls
return cls
return decorator
def get_source(name: str) -> ReleaseSource:
"""Get a release source instance by name."""
if name not in _SOURCES:
raise ValueError(f"Unknown release source: {name}")
msg = f"Unknown release source: {name}"
raise ValueError(msg)
return _SOURCES[name]()
def get_handler(name: str) -> DownloadHandler:
"""Get a download handler instance by name."""
if name not in _HANDLERS:
raise ValueError(f"Unknown download handler: {name}")
msg = f"Unknown download handler: {name}"
raise ValueError(msg)
return _HANDLERS[name]()
def list_available_sources() -> List[dict]:
def list_available_sources() -> list[dict]:
"""List all registered sources with their availability status."""
result = []
for name, src_class in _SOURCES.items():
instance = src_class()
result.append({
"name": name,
"display_name": instance.display_name,
"enabled": instance.is_available(),
"supported_content_types": getattr(instance, 'supported_content_types', ["ebook", "audiobook"]),
"browse_results_are_releases": instance.search_results_are_releases(),
"can_be_default": getattr(instance, 'can_be_default', True),
})
result.append(
{
"name": name,
"display_name": instance.display_name,
"enabled": instance.is_available(),
"supported_content_types": getattr(
instance, "supported_content_types", ["ebook", "audiobook"]
),
"browse_results_are_releases": instance.search_results_are_releases(),
"can_be_default": getattr(instance, "can_be_default", True),
}
)
return result
@@ -421,14 +464,14 @@ def get_source_display_name(name: str) -> str:
"""Get display name for a source by its identifier."""
if name in _SOURCES:
return _SOURCES[name]().display_name
return name.replace('_', ' ').title()
return name.replace("_", " ").title()
def browse_record_to_book_metadata(
record: BrowseRecord,
*,
title_override: Optional[str] = None,
author_override: Optional[str] = None,
title_override: str | None = None,
author_override: str | None = None,
) -> BookMetadata:
"""Convert a source-native browse record into generic book metadata."""
resolved_title = title_override or str(record.title or "").strip() or "Unknown title"
@@ -469,7 +512,7 @@ def source_results_are_releases(name: str) -> bool:
# Import source implementations to trigger registration
# These must be imported AFTER the base classes and registry are defined
from shelfmark.release_sources import direct_download # noqa: F401, E402
from shelfmark.release_sources import prowlarr # noqa: F401, E402
from shelfmark.release_sources import irc # noqa: F401, E402
from shelfmark.release_sources import audiobookbay # noqa: F401, E402
from shelfmark.release_sources import audiobookbay as audiobookbay
from shelfmark.release_sources import direct_download as direct_download
from shelfmark.release_sources import irc as irc
from shelfmark.release_sources import prowlarr as prowlarr
shelfmark/release_sources/audiobookbay/__init__.py
+3 -3
View File
@@ -1,6 +1,6 @@
"""AudiobookBay release source - web scraping for audiobook torrents."""
# Import to trigger registration
from shelfmark.release_sources.audiobookbay import source # noqa: F401, E402
from shelfmark.release_sources.audiobookbay import handler # noqa: F401, E402
from shelfmark.release_sources.audiobookbay import settings # noqa: F401, E402
from shelfmark.release_sources.audiobookbay import handler as handler
from shelfmark.release_sources.audiobookbay import settings as settings
from shelfmark.release_sources.audiobookbay import source as source
shelfmark/release_sources/audiobookbay/handler.py
+22 -11
View File
@@ -1,17 +1,28 @@
"""AudiobookBay download handler - resolves magnet links and uses shared client lifecycle."""
from typing import Callable, Optional
from typing import TYPE_CHECKING
from urllib.parse import urlparse
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import DownloadTask
from shelfmark.download.clients import DownloadClient, get_client, list_configured_clients
from shelfmark.download.clients.base_handler import DownloadRequest, ExternalClientHandler
from shelfmark.download.clients import (
DownloadClient,
get_client,
list_configured_clients,
)
from shelfmark.download.clients.base_handler import (
DownloadRequest,
ExternalClientHandler,
)
from shelfmark.release_sources import register_handler
from shelfmark.release_sources.audiobookbay import scraper
from shelfmark.release_sources.audiobookbay.utils import normalize_hostname
if TYPE_CHECKING:
from collections.abc import Callable
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
@@ -20,7 +31,7 @@ class AudiobookBayHandler(ExternalClientHandler):
"""Handler for AudiobookBay downloads via configured torrent client."""
@staticmethod
def _resolve_detail_url(task: DownloadTask) -> Optional[str]:
def _resolve_detail_url(task: DownloadTask) -> str | None:
"""Resolve ABB detail URL from queued task metadata."""
source_url = (task.source_url or "").strip()
if source_url:
@@ -32,7 +43,7 @@ class AudiobookBayHandler(ExternalClientHandler):
return task_id
return None
def _get_client(self, protocol: str) -> Optional[DownloadClient]:
def _get_client(self, protocol: str) -> DownloadClient | None:
"""Compatibility shim so module-level patching still works in tests."""
return get_client(protocol)
@@ -43,13 +54,13 @@ class AudiobookBayHandler(ExternalClientHandler):
def _resolve_download(
self,
task: DownloadTask,
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[DownloadRequest]:
status_callback: Callable[[str, str | None], None],
) -> DownloadRequest | None:
"""Resolve ABB detail page into a magnet-link download request."""
detail_url = self._resolve_detail_url(task)
if not detail_url:
status_callback("error", "Missing AudiobookBay details URL")
logger.warning(f"Missing details URL for AudiobookBay task: {task.task_id}")
logger.warning("Missing details URL for AudiobookBay task: %s", task.task_id)
return None
hostname = normalize_hostname(config.get("ABB_HOSTNAME", ""))
@@ -63,7 +74,7 @@ class AudiobookBayHandler(ExternalClientHandler):
status_callback("error", "Failed to extract magnet link from detail page")
return None
logger.info(f"Extracted magnet link for task {task.task_id}")
logger.info("Extracted magnet link for task %s", task.task_id)
return DownloadRequest(
url=magnet_link,
@@ -79,5 +90,5 @@ class AudiobookBayHandler(ExternalClientHandler):
been sent to the torrent client we do not remove it client-side. Users must
cancel/remove it in their torrent client UI.
"""
logger.debug(f"Cancel requested for AudiobookBay task: {task_id}")
logger.debug("Cancel requested for AudiobookBay task: %s", task_id)
return False
shelfmark/release_sources/audiobookbay/scraper.py
+102 -89
View File
@@ -2,7 +2,6 @@
import re
import time
from typing import List, Optional, Dict
from urllib.parse import quote
import requests
@@ -66,11 +65,15 @@ def _is_homepage_redirect(final_url: str, hostname: str) -> bool:
return normalized_final in {normalized_home, f"{normalized_home}/"}
def _encode_search_query(query: str, exact_phrase: bool) -> str:
def _encode_search_query(query: str, *, exact_phrase: bool) -> str:
"""Encode search query using ABB's space-plus style and optional exact phrase wrapping."""
search_query = query.strip()
if exact_phrase and search_query and not (search_query.startswith('"') and search_query.endswith('"')):
search_query = f"\"{search_query}\""
if (
exact_phrase
and search_query
and not (search_query.startswith('"') and search_query.endswith('"'))
):
search_query = f'"{search_query}"'
# Keep ABB-friendly encoding style (spaces as '+') while percent-encoding quotes.
return search_query.replace('"', "%22").replace(" ", "+")
@@ -110,18 +113,20 @@ def search_audiobookbay(
query: str,
max_pages: int = 1,
hostname: str = "audiobookbay.lu",
*,
exact_phrase: bool = False,
) -> List[Dict[str, str]]:
) -> list[dict[str, str]]:
"""Search AudiobookBay for audiobooks matching the query.
Args:
query: Search query string
max_pages: Maximum number of pages to fetch
hostname: AudiobookBay hostname (e.g., "audiobookbay.lu")
exact_phrase: Wrap query in quotes for exact phrase matching
Returns:
List of dicts with keys: title, link, cover, language, format, bitrate, size, posted_date
"""
results = []
rate_limit_delay = config.get("ABB_RATE_LIMIT_DELAY", 1.0)
@@ -130,12 +135,12 @@ def search_audiobookbay(
# Bootstrap ABB session cookie (PHPSESSID). ABB increasingly serves reliable
# search/detail pages only after session initialization, similar to browsers.
_bootstrap_abb_session(hostname, session, SEARCH_PAGE_RETRY_ATTEMPTS)
# Iterate through pages
for page in range(1, max_pages + 1):
# Construct URL - use + for spaces (matching audiobookbay-automated implementation)
# This avoids aggressive encoding that PHP-based sites may reject.
query_encoded = _encode_search_query(query, exact_phrase)
query_encoded = _encode_search_query(query, exact_phrase=exact_phrase)
# ABB search expects the legacy category query parameter.
primary_url = _build_search_url(
hostname,
@@ -143,7 +148,7 @@ def search_audiobookbay(
query_encoded,
include_legacy_category=True,
)
try:
# Reuse shared HTTP fetch logic (without bypasser)
page_html, final_url = downloader.html_get_page(
@@ -183,84 +188,93 @@ def search_audiobookbay(
)
if not page_html:
logger.warning(f"Failed to fetch page {page}")
logger.warning("Failed to fetch page %s", page)
break
# Check if we were redirected to the homepage (search was rejected/blocked)
if was_home_redirect:
# Search was redirected to homepage - this means the search failed
# This can happen due to geo-blocking, rate limiting, or invalid query format
if page == 1:
logger.warning(f"Search query '{query}' was redirected to homepage - search may be blocked or invalid")
logger.warning(
"Search query '%s' was redirected to homepage - search may be blocked or invalid",
query,
)
break
# Parse HTML
soup = BeautifulSoup(page_html, 'html.parser')
soup = BeautifulSoup(page_html, "html.parser")
# Extract book entries
posts = soup.select('.post')
posts = soup.select(".post")
if not posts:
# No more results
break
for post in posts:
try:
# Extract title
title_elem = post.select_one('.postTitle > h2 > a')
title_elem = post.select_one(".postTitle > h2 > a")
if not title_elem:
continue
title = title_elem.text.strip()
# Extract link (relative, needs hostname prefix)
href = title_elem.get('href', '')
href = title_elem.get("href", "")
if not href:
continue
link = _normalize_result_url(href, hostname)
if not link:
continue
# Extract cover image (try .postContent .center img first, then fallback to any img)
cover = None
cover_elem = post.select_one('.postContent .center img') or post.select_one('img')
cover_elem = post.select_one(".postContent .center img") or post.select_one(
"img"
)
if cover_elem:
cover = _normalize_result_url(cover_elem.get('src', ''), hostname) or None
cover = _normalize_result_url(cover_elem.get("src", ""), hostname) or None
# Extract language from .postInfo
language = None
post_info = post.select_one('.postInfo')
post_info = post.select_one(".postInfo")
if post_info:
info_text = post_info.get_text(separator=' ', strip=True).replace('\xa0', ' ')
info_text = post_info.get_text(separator=" ", strip=True).replace(
"\xa0", " "
)
lang_match = LANGUAGE_PATTERN.search(info_text)
if lang_match:
language = lang_match.group(1).strip()
# Extract format, bitrate, size, and posted date from .postContent
posted_date = None
format_type = None
bitrate = None
size_str = None
post_content = post.select_one('.postContent')
post_content = post.select_one(".postContent")
if post_content:
content_text = post_content.get_text(separator=' ', strip=True).replace('\xa0', ' ')
content_text = post_content.get_text(separator=" ", strip=True).replace(
"\xa0", " "
)
# Extract posted date
posted_match = POSTED_PATTERN.search(content_text)
if posted_match:
posted_date = posted_match.group(1).strip()
# Extract format (e.g., "M4B", "MP3")
format_match = FORMAT_PATTERN.search(content_text)
if format_match:
format_type = format_match.group(1).strip()
# Extract bitrate (e.g., "256 Kbps")
bitrate_match = BITRATE_PATTERN.search(content_text)
if bitrate_match:
bitrate = bitrate_match.group(1).strip()
# Extract file size (e.g., "11.68 GBs" -> normalized to "11.68 GB")
size_match = SIZE_PATTERN.search(content_text)
if size_match:
@@ -270,44 +284,44 @@ def search_audiobookbay(
size_unit = size_unit[:-1]
size_unit = size_unit.upper()
size_str = f"{size_value} {size_unit}"
results.append({
'title': title,
'link': link,
'cover': cover or None,
'language': language,
'format': format_type,
'bitrate': bitrate,
'size': size_str,
'posted_date': posted_date,
})
except Exception as e:
logger.debug(f"Skipping post due to error: {e}")
results.append(
{
"title": title,
"link": link,
"cover": cover or None,
"language": language,
"format": format_type,
"bitrate": bitrate,
"size": size_str,
"posted_date": posted_date,
}
)
except (TypeError, ValueError, AttributeError, IndexError, KeyError) as e:
logger.debug("Skipping post due to error: %s", e)
continue
# Rate limiting delay between pages
if page < max_pages and rate_limit_delay > 0:
time.sleep(rate_limit_delay)
except Exception as e:
logger.error(f"Unexpected error on page {page}: {e}")
except Exception:
logger.exception("Unexpected error on page %s", page)
break
logger.info(f"Found {len(results)} results for query '{query}'")
logger.info("Found %s results for query '%s'", len(results), query)
return results
def extract_magnet_link(
details_url: str,
hostname: str = "audiobookbay.lu"
) -> Optional[str]:
def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") -> str | None:
"""Extract info hash and trackers from book detail page, then construct magnet link.
Args:
details_url: URL of the book's detail page
hostname: AudiobookBay hostname (for logging)
Returns:
Magnet link, or None if extraction fails
"""
try:
session = requests.Session()
@@ -334,65 +348,64 @@ def extract_magnet_link(
success_delay=0,
session=session,
)
if not detail_html:
logger.warning("Failed to fetch details page")
return None
soup = BeautifulSoup(detail_html, 'html.parser')
soup = BeautifulSoup(detail_html, "html.parser")
# 1. Extract Info Hash
# Look for <td>Info Hash</td> and get next sibling value
info_hash = None
info_hash_rows = soup.find_all('td')
info_hash_rows = soup.find_all("td")
for td in info_hash_rows:
if td.text.strip().lower() == 'info hash':
next_td = td.find_next_sibling('td')
if td.text.strip().lower() == "info hash":
next_td = td.find_next_sibling("td")
if next_td:
info_hash = next_td.text.strip()
break
# Alternative: search for text containing "Info Hash" and get next element
if not info_hash:
for elem in soup.find_all(string=INFO_HASH_LABEL_PATTERN):
parent = elem.parent
if parent and parent.name == 'td':
next_td = parent.find_next_sibling('td')
if parent and parent.name == "td":
next_td = parent.find_next_sibling("td")
if next_td:
info_hash = next_td.text.strip()
break
if not info_hash:
logger.warning("Info Hash not found on the page.")
return None
# Clean up info hash (remove whitespace, ensure uppercase)
info_hash = re.sub(r'\s+', '', info_hash).upper()
info_hash = re.sub(r"\s+", "", info_hash).upper()
# 2. Extract Trackers
# Find all <td> containing udp:// or http://
trackers = []
for td in soup.find_all('td'):
for td in soup.find_all("td"):
text = td.text.strip()
if text.startswith(('udp://', 'http://', 'https://')):
if text.startswith(("udp://", "http://", "https://")):
trackers.append(text)
# 3. Use default trackers if none found
if not trackers:
logger.debug("No trackers found on the page. Using default trackers.")
trackers = DEFAULT_TRACKERS
# 4. Construct Magnet Link
# Format: magnet:?xt=urn:btih:{INFO_HASH}&tr={TRACKER1}&tr={TRACKER2}...
tracker_params = "&".join(
f"tr={quote(tracker)}"
for tracker in trackers
)
tracker_params = "&".join(f"tr={quote(tracker)}" for tracker in trackers)
magnet_link = f"magnet:?xt=urn:btih:{info_hash}&{tracker_params}"
logger.debug(f"Generated Magnet Link: {magnet_link[:100]}...")
return magnet_link
except Exception as e:
logger.error(f"Failed to extract magnet link: {e}")
logger.debug("Generated Magnet Link: %s...", magnet_link[:100])
except Exception:
logger.exception("Failed to extract magnet link")
return None
else:
return magnet_link

Some files were not shown because too many files have changed in this diff Show More