mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 22:05:20 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
43e554b8ae | ||
|
|
3be99effe4 | ||
|
|
03c364e375 | ||
|
|
edf25150bd | ||
|
|
a030bca5d3 | ||
|
|
8470095534 | ||
|
|
4e00cf42f6 | ||
|
|
f7375d56e2 | ||
|
|
5a6db5f8a8 | ||
|
|
fd74021594 | ||
|
|
ba906c45df | ||
|
|
0d7a12ca7c |
@@ -229,5 +229,6 @@ pyrightconfig.json
|
||||
/downloaded_files
|
||||
/.local/
|
||||
*.local.*
|
||||
AGENTS.md
|
||||
.claude/
|
||||
.playwright-mcp/
|
||||
|
||||
+1
-1
@@ -152,7 +152,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
# Grant read/execute permissions to others
|
||||
RUN chmod -R o+rx /usr/bin/chromium && \
|
||||
chmod -R o+rx /usr/bin/chromedriver && \
|
||||
chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
|
||||
chmod -R o+rwx /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
|
||||
|
||||
# Default command to run the application entrypoint script
|
||||
CMD ["/app/entrypoint.sh"]
|
||||
|
||||
@@ -2,15 +2,14 @@ services:
|
||||
shelfmark-lite:
|
||||
image: ghcr.io/calibrain/shelfmark-lite:latest
|
||||
environment:
|
||||
# TZ: America/New_York
|
||||
# EXT_BYPASSER_URL: http://flaresolverr:8191 #If using Flaresolverr
|
||||
# PUID: 1000
|
||||
# PGID: 1000
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
ports:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Book destination directory
|
||||
- /path/to/books:/books # Default destination for book downloads
|
||||
- /path/to/config:/config # App configuration
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -4,10 +4,9 @@ services:
|
||||
image: ghcr.io/calibrain/shelfmark:latest
|
||||
environment:
|
||||
FLASK_PORT: 8084
|
||||
# TZ: America/New_York
|
||||
USING_TOR: true
|
||||
# PUID: 1000
|
||||
# PGID: 1000
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
@@ -15,7 +14,7 @@ services:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Book destination directory
|
||||
- /path/to/books:/books # Default destination for book downloads
|
||||
- /path/to/config:/config # App configuration
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -3,14 +3,13 @@ services:
|
||||
image: ghcr.io/calibrain/shelfmark:latest
|
||||
container_name: shelfmark
|
||||
environment:
|
||||
# TZ: America/New_York
|
||||
# PUID: 1000
|
||||
# PGID: 1000
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
ports:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Book destination directory
|
||||
- /path/to/books:/books # Default destination for book downloads
|
||||
- /path/to/config:/config # App configuration
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -1,20 +0,0 @@
|
||||
# Uses external Cloudflare bypasser (FlareSolverr/ByParr) instead of built-in Selenium
|
||||
services:
|
||||
shelfmark-lite:
|
||||
image: ghcr.io/calibrain/shelfmark-lite:dev
|
||||
environment:
|
||||
# TZ: America/New_York
|
||||
EXT_BYPASSER_URL: http://flaresolverr:8191
|
||||
# PUID: 1000
|
||||
# PGID: 1000
|
||||
ports:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Book destination directory
|
||||
- /path/to/config:/config # App configuration
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
|
||||
flaresolverr:
|
||||
image: ghcr.io/flaresolverr/flaresolverr:latest
|
||||
@@ -1,21 +0,0 @@
|
||||
# Routes all traffic through Tor - requires NET_ADMIN capability
|
||||
services:
|
||||
shelfmark-tor:
|
||||
image: ghcr.io/calibrain/shelfmark:dev
|
||||
environment:
|
||||
FLASK_PORT: 8084
|
||||
# TZ: America/New_York
|
||||
USING_TOR: true
|
||||
# PUID: 1000
|
||||
# PGID: 1000
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
ports:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Book destination directory
|
||||
- /path/to/config:/config # App configuration
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -1,16 +0,0 @@
|
||||
services:
|
||||
shelfmark:
|
||||
image: ghcr.io/calibrain/shelfmark:dev
|
||||
container_name: shelfmark
|
||||
environment:
|
||||
# TZ: America/New_York
|
||||
# PUID: 1000
|
||||
# PGID: 1000
|
||||
ports:
|
||||
- 8084:8084
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /path/to/books:/books # Book destination directory
|
||||
- /path/to/config:/config # App configuration
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
@@ -2,7 +2,7 @@
|
||||
services:
|
||||
shelfmark-lite-dev:
|
||||
extends:
|
||||
file: ./compose/edge/docker-compose.extbp.yml
|
||||
file: ./compose/docker-compose.lite.yml
|
||||
service: shelfmark-lite
|
||||
build:
|
||||
context: .
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
- ./.local/books:/books
|
||||
- ./.local/log:/var/log/shelfmark
|
||||
- ./.local/tmp:/tmp/shelfmark
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
|
||||
flaresolverr:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
services:
|
||||
shelfmark-tor-dev:
|
||||
extends:
|
||||
file: ./compose/edge/docker-compose.tor.yml
|
||||
file: ./compose/docker-compose.tor.yml
|
||||
service: shelfmark-tor
|
||||
build:
|
||||
context: .
|
||||
@@ -16,5 +16,5 @@ services:
|
||||
- ./.local/books:/books
|
||||
- ./.local/log:/var/log/shelfmark
|
||||
- ./.local/tmp:/tmp/shelfmark
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
services:
|
||||
shelfmark-dev:
|
||||
extends:
|
||||
file: ./compose/edge/docker-compose.yml
|
||||
file: ./compose/docker-compose.yml
|
||||
service: shelfmark
|
||||
build:
|
||||
context: .
|
||||
@@ -18,5 +18,5 @@ services:
|
||||
- ./.local/log:/var/log/shelfmark
|
||||
- ./.local/tmp:/tmp/shelfmark
|
||||
- ./shelfmark:/app/shelfmark:ro
|
||||
# Download client mount - path must match your torrent/usenet client's volume exactly
|
||||
# Required for torrent / usenet - path must match your download client's volume exactly
|
||||
# - /path/to/downloads:/path/to/downloads
|
||||
|
||||
@@ -11,12 +11,12 @@
|
||||
# - Prowlarr: http://localhost:9696 (no auth by default)
|
||||
# - qBittorrent: http://localhost:8080 (check container logs for temp password)
|
||||
# - Transmission: http://localhost:9091 (admin / admin)
|
||||
# - Deluge: http://localhost:8112 (admin / deluge)
|
||||
# - Deluge: http://localhost:8112 (password: deluge)
|
||||
# - NZBGet: http://localhost:6789 (nzbget / tegbzn6789)
|
||||
# - SABnzbd: http://localhost:8085 (complete setup wizard for API key)
|
||||
# - rTorrent: http://localhost:8000 (admin / admin - if auth enabled)
|
||||
#
|
||||
|
||||
|
||||
services:
|
||||
shelfmark:
|
||||
build:
|
||||
@@ -33,9 +33,10 @@ services:
|
||||
# Use Docker service names for URLs:
|
||||
# - qBittorrent: http://qbittorrent:8080
|
||||
# - Transmission: http://transmission:9091
|
||||
# - Deluge host: deluge (port 58846)
|
||||
# - Deluge Web UI: http://deluge:8112
|
||||
# - NZBGet: http://nzbget:6789
|
||||
# - SABnzbd: http://sabnzbd:8080
|
||||
# - rTorrent: http://rtorrent:80 (XMLRPC via HTTP) or rtorrent (port 5000 for SCGI)
|
||||
ports:
|
||||
- "8084:8084"
|
||||
volumes:
|
||||
@@ -62,6 +63,7 @@ services:
|
||||
- qbittorrent
|
||||
- transmission
|
||||
- deluge
|
||||
- rtorrent
|
||||
restart: unless-stopped
|
||||
|
||||
prowlarr:
|
||||
@@ -159,3 +161,21 @@ services:
|
||||
- "6881:6881"
|
||||
- "6881:6881/udp"
|
||||
restart: unless-stopped
|
||||
|
||||
rtorrent:
|
||||
image: crazymax/rtorrent-rutorrent:latest # linuxserver has deprecated their rtorrent image
|
||||
container_name: test-rtorrent
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
- TZ=UTC
|
||||
volumes:
|
||||
- ./.local/test-clients/rtorrent/config:/config
|
||||
- ./.local/test-clients/downloads:/downloads
|
||||
ports:
|
||||
- "8000:8000" # XMLRPC
|
||||
- "8089:8080" # ruTorrent Web UI
|
||||
- "9000:9000" # SCGI port
|
||||
- "50000:50000" # Incoming connections
|
||||
- "6881:6881/udp"
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# Configuration
|
||||
|
||||
TODO
|
||||
@@ -0,0 +1,3 @@
|
||||
# Developer Documentation
|
||||
|
||||
TODO
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
# Shelfmark Documentation
|
||||
|
||||
TODO
|
||||
@@ -0,0 +1,3 @@
|
||||
# Installation
|
||||
|
||||
TODO
|
||||
@@ -0,0 +1,35 @@
|
||||
# Reverse Proxy & Subpath Hosting
|
||||
|
||||
Shelfmark can run behind a reverse proxy at the root path (recommended) or
|
||||
under a subpath like `/shelfmark`.
|
||||
|
||||
## Subpath setup
|
||||
|
||||
1) Set the base path in Shelfmark:
|
||||
- UI: Settings → Advanced → Base Path
|
||||
- Env var: `URL_BASE=/shelfmark`
|
||||
|
||||
2) Configure your reverse proxy to forward the subpath to Shelfmark and
|
||||
**strip the prefix** before sending to the backend. The proxy must also allow
|
||||
WebSocket upgrades for Socket.IO.
|
||||
|
||||
Example (Nginx-style):
|
||||
|
||||
```
|
||||
location /shelfmark/ {
|
||||
proxy_pass http://shelfmark:8084/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
- Use a trailing slash on the `location` and `proxy_pass` to ensure the
|
||||
`/shelfmark` prefix is removed.
|
||||
- Health checks still work at `/api/health` without the subpath.
|
||||
|
||||
## Root path setup
|
||||
|
||||
If you can serve Shelfmark at the root path (`https://shelfmark.example.com/`),
|
||||
leave `URL_BASE` empty. This is the simplest option.
|
||||
@@ -0,0 +1,3 @@
|
||||
# Troubleshooting
|
||||
|
||||
TODO
|
||||
+77
-12
@@ -1,10 +1,54 @@
|
||||
#!/bin/bash
|
||||
LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
|
||||
mkdir -p $LOG_DIR
|
||||
LOG_FILE=${LOG_DIR}/shelfmark_entrypoint.log
|
||||
|
||||
# Cleanup any existing files or folders in the log directory
|
||||
rm -rf $LOG_DIR/*
|
||||
is_truthy() {
|
||||
case "${1,,}" in
|
||||
true|yes|1|y) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
|
||||
LOG_PIPE_DIR=""
|
||||
LOG_PIPE=""
|
||||
TEE_PID=""
|
||||
|
||||
start_file_logging() {
|
||||
local logfile="$1"
|
||||
|
||||
LOG_PIPE_DIR="$(mktemp -d)"
|
||||
LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
|
||||
mkfifo "$LOG_PIPE"
|
||||
|
||||
tee -a "$logfile" < "$LOG_PIPE" &
|
||||
TEE_PID=$!
|
||||
|
||||
exec 3>&1 4>&2
|
||||
exec > "$LOG_PIPE" 2>&1
|
||||
}
|
||||
|
||||
stop_file_logging() {
|
||||
if [ -z "${TEE_PID:-}" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
exec 1>&3 2>&4
|
||||
exec 3>&- 4>&-
|
||||
|
||||
rm -f "$LOG_PIPE"
|
||||
rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
|
||||
|
||||
wait "$TEE_PID" 2>/dev/null || true
|
||||
TEE_PID=""
|
||||
}
|
||||
|
||||
if is_truthy "$ENABLE_LOGGING_VALUE"; then
|
||||
LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
|
||||
mkdir -p "$LOG_DIR"
|
||||
LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
|
||||
|
||||
# Cleanup any existing files or folders in the log directory
|
||||
rm -rf "$LOG_DIR"/*
|
||||
fi
|
||||
|
||||
(
|
||||
if [ "$USING_TOR" = "true" ]; then
|
||||
@@ -12,10 +56,16 @@ rm -rf $LOG_DIR/*
|
||||
fi
|
||||
)
|
||||
|
||||
exec 3>&1 4>&2
|
||||
exec > >(tee -a $LOG_FILE) 2>&1
|
||||
if is_truthy "$ENABLE_LOGGING_VALUE"; then
|
||||
start_file_logging "$LOG_FILE"
|
||||
fi
|
||||
|
||||
echo "Starting entrypoint script"
|
||||
echo "Log file: $LOG_FILE"
|
||||
if is_truthy "$ENABLE_LOGGING_VALUE"; then
|
||||
echo "Log file: $LOG_FILE"
|
||||
else
|
||||
echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
|
||||
fi
|
||||
set -e
|
||||
|
||||
# Print build version
|
||||
@@ -134,6 +184,24 @@ change_ownership /app
|
||||
change_ownership /var/log/shelfmark
|
||||
change_ownership /tmp/shelfmark
|
||||
|
||||
# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
|
||||
# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
|
||||
# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
|
||||
if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
|
||||
set +e
|
||||
SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
|
||||
set -e
|
||||
|
||||
if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
|
||||
change_ownership "$SELENIUMBASE_DRIVERS_DIR"
|
||||
|
||||
# If the driver already exists, ensure it's executable for the runtime user.
|
||||
if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
|
||||
chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Test write to all folders
|
||||
make_writable ${CONFIG_DIR:-/config}
|
||||
make_writable ${INGEST_DIR:-/books}
|
||||
@@ -263,8 +331,5 @@ UMASK_VALUE=${UMASK:-0022}
|
||||
echo "Setting umask to $UMASK_VALUE"
|
||||
umask $UMASK_VALUE
|
||||
|
||||
# Stop logging
|
||||
exec 1>&3 2>&4
|
||||
exec 3>&- 4>&-
|
||||
|
||||
stop_file_logging
|
||||
exec sudo -E -u "$USERNAME" HOME=/app $command
|
||||
|
||||
@@ -41,9 +41,9 @@ Shelfmark is a unified web interface for searching and aggregating books and aud
|
||||
|
||||
### Installation
|
||||
|
||||
1. Download the docker-compose file:
|
||||
1. Download the [docker-compose file](compose/docker-compose.yml):
|
||||
```bash
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/stable/docker-compose.yml
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
|
||||
```
|
||||
|
||||
2. Start the service:
|
||||
@@ -51,8 +51,6 @@ Shelfmark is a unified web interface for searching and aggregating books and aud
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
> **Edge users**: If you're tracking the main branch (`:dev` tag), use compose files from `compose/edge/` instead.
|
||||
|
||||
3. Open `http://localhost:8084`
|
||||
|
||||
That's it! Configure settings through the web interface as needed.
|
||||
@@ -98,6 +96,8 @@ Environment variables work for initial setup and Docker deployments. They serve
|
||||
| `SEARCH_MODE` | `direct` or `universal` | `direct` |
|
||||
| `USING_TOR` | Enable Tor routing (requires `NET_ADMIN` capability) | `false` |
|
||||
|
||||
See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
|
||||
|
||||
Some of the additional options available in Settings:
|
||||
- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
|
||||
- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
|
||||
@@ -120,7 +120,7 @@ The full-featured image with built-in Cloudflare bypass.
|
||||
#### Enable Tor Routing
|
||||
Routes all traffic through Tor for enhanced privacy:
|
||||
```bash
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/stable/docker-compose.tor.yml
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
|
||||
docker compose -f docker-compose.tor.yml up -d
|
||||
```
|
||||
|
||||
@@ -138,7 +138,7 @@ A smaller image without the built-in Cloudflare bypasser. Ideal for:
|
||||
- **Audiobooks** - Using Shelfmark exclusively for audiobooks
|
||||
|
||||
```bash
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/stable/docker-compose.lite.yml
|
||||
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
|
||||
docker compose -f docker-compose.lite.yml up -d
|
||||
```
|
||||
|
||||
@@ -146,9 +146,17 @@ If you need Cloudflare bypass with the Lite image, configure an external resolve
|
||||
|
||||
## 🔐 Authentication
|
||||
|
||||
Authentication is optional but recommended for shared or exposed instances. Enable in Settings.
|
||||
Authentication is optional but recommended for shared or exposed instances. Three authentication methods are available in Settings:
|
||||
|
||||
**Alternative**: If you're running Calibre-Web, you can reuse its user database by mounting it:
|
||||
**1. Single Username/Password**
|
||||
|
||||
**2. Proxy (Forward) Authentication**
|
||||
|
||||
Proxy auth trusts headers set by your reverse proxy (e.g. `X-Auth-User`). Ensure Shelfmark is not directly exposed, and configure your proxy to strip/overwrite these headers for all inbound requests.
|
||||
|
||||
**3. Calibre-Web Database**
|
||||
|
||||
If you're running Calibre-Web, you can reuse its user database by mounting it:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
|
||||
@@ -14,4 +14,3 @@ emoji
|
||||
rarfile
|
||||
qbittorrent-api
|
||||
transmission-rpc
|
||||
deluge-client
|
||||
|
||||
Executable
+433
@@ -0,0 +1,433 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate markdown documentation for environment variables from the settings registry.
|
||||
|
||||
This script extracts all settings that support environment variable configuration
|
||||
and generates a comprehensive markdown file documenting each option.
|
||||
|
||||
Usage:
|
||||
python scripts/generate_env_docs.py [--output path/to/output.md]
|
||||
|
||||
The generated documentation includes:
|
||||
- Environment variable name
|
||||
- Description
|
||||
- Type (string, number, boolean, etc.)
|
||||
- Default value
|
||||
- Organizational grouping by settings tab/group
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
|
||||
def get_field_type_name(field) -> str:
|
||||
"""Get a human-readable type name for a field."""
|
||||
from shelfmark.core.settings_registry import (
|
||||
CheckboxField,
|
||||
MultiSelectField,
|
||||
NumberField,
|
||||
OrderableListField,
|
||||
PasswordField,
|
||||
SelectField,
|
||||
TextField,
|
||||
)
|
||||
|
||||
if isinstance(field, CheckboxField):
|
||||
return "boolean"
|
||||
elif isinstance(field, NumberField):
|
||||
return "number"
|
||||
elif isinstance(field, SelectField):
|
||||
return "string (choice)"
|
||||
elif isinstance(field, MultiSelectField):
|
||||
return "string (comma-separated)"
|
||||
elif isinstance(field, OrderableListField):
|
||||
return "JSON array"
|
||||
elif isinstance(field, PasswordField):
|
||||
return "string (secret)"
|
||||
elif isinstance(field, TextField):
|
||||
return "string"
|
||||
else:
|
||||
return "string"
|
||||
|
||||
|
||||
def format_default_value(field) -> str:
|
||||
"""Format the default value for display."""
|
||||
default = field.default
|
||||
|
||||
if default is None:
|
||||
return "_none_"
|
||||
elif isinstance(default, bool):
|
||||
return f"`{str(default).lower()}`"
|
||||
elif isinstance(default, (int, float)):
|
||||
return f"`{default}`"
|
||||
elif isinstance(default, str):
|
||||
if default == "":
|
||||
return "_empty string_"
|
||||
return f"`{default}`"
|
||||
elif isinstance(default, list):
|
||||
if not default:
|
||||
return "_empty list_"
|
||||
# For simple lists, show comma-separated values
|
||||
if all(isinstance(item, str) for item in default):
|
||||
return f"`{','.join(default)}`"
|
||||
# For complex lists (e.g., OrderableListField defaults), summarize
|
||||
return f"_see UI for defaults_"
|
||||
else:
|
||||
return f"`{default}`"
|
||||
|
||||
|
||||
def get_select_options(field) -> Optional[List[str]]:
|
||||
"""Get the available options for a SelectField.
|
||||
|
||||
Returns options formatted as 'value (label)' or just 'value' if they match,
|
||||
so users know the actual values to use in environment variables.
|
||||
"""
|
||||
from shelfmark.core.settings_registry import SelectField
|
||||
|
||||
if not isinstance(field, SelectField):
|
||||
return None
|
||||
|
||||
options = field.options
|
||||
if callable(options):
|
||||
try:
|
||||
options = options()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if not options:
|
||||
return None
|
||||
|
||||
result = []
|
||||
for opt in options:
|
||||
value = opt.get("value", "")
|
||||
label = opt.get("label", "")
|
||||
|
||||
# Format as "value (label)" unless they're the same or value is empty
|
||||
if value == "":
|
||||
result.append(f'`""` ({label})')
|
||||
elif value == label or not label:
|
||||
result.append(f"`{value}`")
|
||||
else:
|
||||
result.append(f"`{value}` ({label})")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _generate_bootstrap_env_docs() -> List[str]:
|
||||
"""Generate documentation for bootstrap environment variables from env.py."""
|
||||
# These are environment variables defined in env.py that are used before
|
||||
# the settings registry is available
|
||||
bootstrap_vars = [
|
||||
{
|
||||
"name": "CONFIG_DIR",
|
||||
"description": "Directory for storing configuration files and plugin settings.",
|
||||
"type": "string (path)",
|
||||
"default": "/config",
|
||||
},
|
||||
{
|
||||
"name": "LOG_ROOT",
|
||||
"description": "Root directory for log files.",
|
||||
"type": "string (path)",
|
||||
"default": "/var/log/",
|
||||
},
|
||||
{
|
||||
"name": "TMP_DIR",
|
||||
"description": "Staging directory for downloads before moving to destination.",
|
||||
"type": "string (path)",
|
||||
"default": "/tmp/shelfmark",
|
||||
},
|
||||
{
|
||||
"name": "ENABLE_LOGGING",
|
||||
"description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
|
||||
"type": "boolean",
|
||||
"default": "true",
|
||||
},
|
||||
{
|
||||
"name": "FLASK_HOST",
|
||||
"description": "Host address for the Flask web server.",
|
||||
"type": "string",
|
||||
"default": "0.0.0.0",
|
||||
},
|
||||
{
|
||||
"name": "FLASK_PORT",
|
||||
"description": "Port number for the Flask web server.",
|
||||
"type": "number",
|
||||
"default": "8084",
|
||||
},
|
||||
{
|
||||
"name": "SESSION_COOKIE_SECURE",
|
||||
"description": "Enable secure cookies (requires HTTPS).",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
{
|
||||
"name": "CWA_DB_PATH",
|
||||
"description": "Path to the Calibre-Web database for authentication integration.",
|
||||
"type": "string (path)",
|
||||
"default": "/auth/app.db",
|
||||
},
|
||||
{
|
||||
"name": "DOCKERMODE",
|
||||
"description": "Indicates the application is running inside a Docker container.",
|
||||
"type": "boolean",
|
||||
"default": "false",
|
||||
},
|
||||
]
|
||||
|
||||
lines = [
|
||||
"## Bootstrap Configuration",
|
||||
"",
|
||||
"These environment variables are used at startup before the settings system loads. They typically configure paths and server settings.",
|
||||
"",
|
||||
"| Variable | Description | Type | Default |",
|
||||
"|----------|-------------|------|---------|",
|
||||
]
|
||||
|
||||
for var in bootstrap_vars:
|
||||
lines.append(f"| `{var['name']}` | {var['description']} | {var['type']} | `{var['default']}` |")
|
||||
|
||||
lines.append("")
|
||||
lines.append("<details>")
|
||||
lines.append("<summary>Detailed descriptions</summary>")
|
||||
lines.append("")
|
||||
|
||||
for var in bootstrap_vars:
|
||||
lines.append(f"#### `{var['name']}`")
|
||||
lines.append("")
|
||||
lines.append(var["description"])
|
||||
lines.append("")
|
||||
lines.append(f"- **Type:** {var['type']}")
|
||||
lines.append(f"- **Default:** `{var['default']}`")
|
||||
lines.append("")
|
||||
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def generate_env_docs() -> str:
|
||||
"""Generate markdown documentation for all environment variables."""
|
||||
# Import settings modules to ensure all settings are registered
|
||||
import shelfmark.config.settings # noqa: F401
|
||||
import shelfmark.release_sources.irc.settings # noqa: F401
|
||||
import shelfmark.release_sources.prowlarr.settings # noqa: F401
|
||||
import shelfmark.metadata_providers.hardcover # noqa: F401
|
||||
import shelfmark.metadata_providers.openlibrary # noqa: F401
|
||||
import shelfmark.metadata_providers.googlebooks # noqa: F401
|
||||
|
||||
from shelfmark.core.settings_registry import (
|
||||
ActionButton,
|
||||
HeadingField,
|
||||
get_all_groups,
|
||||
get_all_settings_tabs,
|
||||
)
|
||||
|
||||
tabs = get_all_settings_tabs()
|
||||
groups = {g.name: g for g in get_all_groups()}
|
||||
|
||||
# Organize tabs by group
|
||||
grouped_tabs: Dict[Optional[str], List] = {None: []}
|
||||
for group_name in groups:
|
||||
grouped_tabs[group_name] = []
|
||||
|
||||
for tab in tabs:
|
||||
group_name = tab.group
|
||||
if group_name not in grouped_tabs:
|
||||
grouped_tabs[group_name] = []
|
||||
grouped_tabs[group_name].append(tab)
|
||||
|
||||
# Build markdown output
|
||||
lines = [
|
||||
"# Environment Variables",
|
||||
"",
|
||||
"This document lists all configuration options that can be set via environment variables.",
|
||||
"",
|
||||
"> **Auto-generated** - Do not edit manually. Run `python scripts/generate_env_docs.py` to regenerate.",
|
||||
"",
|
||||
"## Table of Contents",
|
||||
"",
|
||||
]
|
||||
|
||||
# Generate TOC
|
||||
toc_entries = [
|
||||
"- [Bootstrap Configuration](#bootstrap-configuration)",
|
||||
]
|
||||
|
||||
# Ungrouped tabs first
|
||||
for tab in grouped_tabs.get(None, []):
|
||||
anchor = tab.display_name.lower().replace(" ", "-")
|
||||
toc_entries.append(f"- [{tab.display_name}](#{anchor})")
|
||||
|
||||
# Then grouped tabs
|
||||
for group_name, group in groups.items():
|
||||
group_tabs = grouped_tabs.get(group_name, [])
|
||||
if group_tabs:
|
||||
anchor = group.display_name.lower().replace(" ", "-")
|
||||
toc_entries.append(f"- [{group.display_name}](#{anchor})")
|
||||
for tab in group_tabs:
|
||||
sub_anchor = f"{group.display_name}-{tab.display_name}".lower().replace(" ", "-")
|
||||
toc_entries.append(f" - [{tab.display_name}](#{sub_anchor})")
|
||||
|
||||
lines.extend(toc_entries)
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
|
||||
# Add bootstrap environment variables documentation
|
||||
lines.extend(_generate_bootstrap_env_docs())
|
||||
|
||||
# Generate documentation for ungrouped tabs
|
||||
for tab in grouped_tabs.get(None, []):
|
||||
lines.extend(_generate_tab_docs(tab))
|
||||
|
||||
# Generate documentation for grouped tabs
|
||||
for group_name, group in groups.items():
|
||||
group_tabs = grouped_tabs.get(group_name, [])
|
||||
if not group_tabs:
|
||||
continue
|
||||
|
||||
lines.append(f"## {group.display_name}")
|
||||
lines.append("")
|
||||
|
||||
for tab in group_tabs:
|
||||
lines.extend(_generate_tab_docs(tab, group_prefix=group.display_name))
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _generate_tab_docs(tab, group_prefix: Optional[str] = None) -> List[str]:
|
||||
"""Generate documentation for a single settings tab."""
|
||||
from shelfmark.core.settings_registry import ActionButton, HeadingField
|
||||
|
||||
lines = []
|
||||
|
||||
# Section header
|
||||
if group_prefix:
|
||||
lines.append(f"### {group_prefix}: {tab.display_name}")
|
||||
anchor_id = f"{group_prefix}-{tab.display_name}".lower().replace(" ", "-")
|
||||
else:
|
||||
lines.append(f"## {tab.display_name}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Collect env-supported fields
|
||||
env_fields = []
|
||||
for field in tab.fields:
|
||||
# Skip non-value fields
|
||||
if isinstance(field, (ActionButton, HeadingField)):
|
||||
continue
|
||||
|
||||
# Skip fields that don't support ENV vars
|
||||
if not getattr(field, "env_supported", True):
|
||||
continue
|
||||
|
||||
env_fields.append(field)
|
||||
|
||||
if not env_fields:
|
||||
lines.append("_No environment variables for this section._")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
# Generate table
|
||||
lines.append("| Variable | Description | Type | Default |")
|
||||
lines.append("|----------|-------------|------|---------|")
|
||||
|
||||
for field in env_fields:
|
||||
env_var = field.get_env_var_name()
|
||||
description = field.description or field.label
|
||||
# Clean up description for table (remove newlines, escape pipes)
|
||||
description = description.replace("\n", " ").replace("|", "\\|").strip()
|
||||
|
||||
field_type = get_field_type_name(field)
|
||||
default = format_default_value(field)
|
||||
|
||||
lines.append(f"| `{env_var}` | {description} | {field_type} | {default} |")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Add detailed documentation for each field
|
||||
lines.append("<details>")
|
||||
lines.append("<summary>Detailed descriptions</summary>")
|
||||
lines.append("")
|
||||
|
||||
for field in env_fields:
|
||||
env_var = field.get_env_var_name()
|
||||
lines.append(f"#### `{env_var}`")
|
||||
lines.append("")
|
||||
lines.append(f"**{field.label}**")
|
||||
lines.append("")
|
||||
|
||||
if field.description:
|
||||
lines.append(field.description)
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"- **Type:** {get_field_type_name(field)}")
|
||||
lines.append(f"- **Default:** {format_default_value(field)}")
|
||||
|
||||
if getattr(field, "required", False):
|
||||
lines.append("- **Required:** Yes")
|
||||
|
||||
if getattr(field, "requires_restart", False):
|
||||
lines.append("- **Requires restart:** Yes")
|
||||
|
||||
# Show options for SelectField
|
||||
options = get_select_options(field)
|
||||
if options:
|
||||
lines.append(f"- **Options:** {', '.join(options)}")
|
||||
|
||||
# Show constraints for NumberField
|
||||
from shelfmark.core.settings_registry import NumberField
|
||||
if isinstance(field, NumberField):
|
||||
constraints = []
|
||||
if field.min_value is not None:
|
||||
constraints.append(f"min: {field.min_value}")
|
||||
if field.max_value is not None:
|
||||
constraints.append(f"max: {field.max_value}")
|
||||
if constraints:
|
||||
lines.append(f"- **Constraints:** {', '.join(constraints)}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate markdown documentation for environment variables"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=Path,
|
||||
default=project_root / "docs" / "environment-variables.md",
|
||||
help="Output file path (default: docs/environment-variables.md)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stdout",
|
||||
action="store_true",
|
||||
help="Print to stdout instead of file",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
docs = generate_env_docs()
|
||||
|
||||
if args.stdout:
|
||||
print(docs)
|
||||
else:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(docs)
|
||||
print(f"Generated: {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+145
-39
@@ -23,9 +23,10 @@ Web UIs:
|
||||
- Deluge: http://localhost:8112
|
||||
- NZBGet: http://localhost:6789
|
||||
- SABnzbd: http://localhost:8085
|
||||
- rTorrent: http://localhost:8000 (web ui http://localhost:8089 via ruTorrent)
|
||||
|
||||
Prerequisites (for running this script locally):
|
||||
pip install requests transmission-rpc deluge-client qbittorrent-api
|
||||
pip install requests transmission-rpc qbittorrent-api
|
||||
|
||||
First-Time Setup:
|
||||
qBittorrent:
|
||||
@@ -37,10 +38,7 @@ First-Time Setup:
|
||||
- No setup needed, credentials pre-configured (admin/admin)
|
||||
|
||||
Deluge:
|
||||
1. Access Web UI at http://localhost:8112 (default password: deluge)
|
||||
2. Add auth line to .local/test-clients/deluge/config/auth:
|
||||
echo "admin:admin:10" >> .local/test-clients/deluge/config/auth
|
||||
3. Restart: docker restart test-deluge
|
||||
- Access Web UI at http://localhost:8112 (default password: deluge)
|
||||
|
||||
NZBGet:
|
||||
- No setup needed, credentials pre-configured (admin/admin)
|
||||
@@ -53,6 +51,7 @@ First-Time Setup:
|
||||
|
||||
import sys
|
||||
import time
|
||||
from xmlrpc import client
|
||||
|
||||
# Test configuration - matches docker-compose.test-clients.yml
|
||||
CONFIG = {
|
||||
@@ -78,10 +77,11 @@ CONFIG = {
|
||||
"password": "admin",
|
||||
},
|
||||
"deluge": {
|
||||
"host": "localhost",
|
||||
"port": 58846,
|
||||
"username": "admin",
|
||||
"password": "admin",
|
||||
"url": "http://localhost:8112",
|
||||
"password": "deluge",
|
||||
},
|
||||
"rtorrent": {
|
||||
"url": "http://localhost:8000/RPC2",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -325,46 +325,79 @@ def test_transmission():
|
||||
|
||||
|
||||
def test_deluge():
|
||||
"""Test Deluge connection."""
|
||||
"""Test Deluge Web UI (JSON-RPC) connection."""
|
||||
import requests
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("Testing Deluge")
|
||||
print("=" * 50)
|
||||
|
||||
base_url = CONFIG["deluge"]["url"].rstrip("/")
|
||||
password = CONFIG["deluge"]["password"]
|
||||
rpc_url = f"{base_url}/json"
|
||||
|
||||
def rpc_call(session: requests.Session, rpc_id: int, method: str, *params):
|
||||
payload = {"id": rpc_id, "method": method, "params": list(params)}
|
||||
resp = session.post(rpc_url, json=payload, timeout=10)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("error"):
|
||||
err = data["error"]
|
||||
if isinstance(err, dict):
|
||||
raise Exception(err.get("message") or str(err))
|
||||
raise Exception(str(err))
|
||||
return data.get("result")
|
||||
|
||||
try:
|
||||
from deluge_client import DelugeRPCClient
|
||||
session = requests.Session()
|
||||
|
||||
client = DelugeRPCClient(
|
||||
host=CONFIG["deluge"]["host"],
|
||||
port=CONFIG["deluge"]["port"],
|
||||
username=CONFIG["deluge"]["username"],
|
||||
password=CONFIG["deluge"]["password"],
|
||||
)
|
||||
# Authenticate to Deluge Web
|
||||
if rpc_call(session, 1, "auth.login", password) is not True:
|
||||
raise Exception("Authentication failed (check Deluge Web UI password)")
|
||||
|
||||
# Test connection
|
||||
client.connect()
|
||||
version = client.call("daemon.info")
|
||||
# Ensure Deluge Web is connected to a daemon
|
||||
if rpc_call(session, 2, "web.connected") is not True:
|
||||
hosts = rpc_call(session, 3, "web.get_hosts") or []
|
||||
if not hosts:
|
||||
raise Exception(
|
||||
"Deluge Web UI isn't connected to Deluge core (no hosts configured). "
|
||||
"Add/connect a daemon in Deluge Web UI → Connection Manager."
|
||||
)
|
||||
|
||||
host_id = hosts[0][0]
|
||||
for entry in hosts:
|
||||
if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
|
||||
host_id = entry[0]
|
||||
break
|
||||
|
||||
rpc_call(session, 4, "web.connect", host_id)
|
||||
|
||||
if rpc_call(session, 5, "web.connected") is not True:
|
||||
raise Exception(
|
||||
"Deluge Web UI couldn't connect to Deluge core. "
|
||||
"Check Deluge Web UI → Connection Manager."
|
||||
)
|
||||
|
||||
version = rpc_call(session, 6, "daemon.info")
|
||||
print(f" Connected to Deluge {version}")
|
||||
|
||||
# Get torrent list
|
||||
torrents = client.call("core.get_torrents_status", {}, ["name"])
|
||||
torrents = rpc_call(session, 7, "core.get_torrents_status", {}, ["name"]) or {}
|
||||
print(f" Active torrents: {len(torrents)}")
|
||||
|
||||
# Test adding a torrent (then remove it)
|
||||
print(" Testing add/remove torrent...")
|
||||
torrent_id = client.call("core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
|
||||
torrent_id = rpc_call(session, 8, "core.add_torrent_magnet", TEST_MAGNET, {"add_paused": True})
|
||||
|
||||
if torrent_id:
|
||||
torrent_id = str(torrent_id)
|
||||
print(f" Added test torrent: {torrent_id[:20]}...")
|
||||
|
||||
# Get status
|
||||
status = client.call("core.get_torrent_status", torrent_id, ["state", "progress"])
|
||||
state = status.get(b"state", b"unknown")
|
||||
if isinstance(state, bytes):
|
||||
state = state.decode()
|
||||
print(f" Status: {state}")
|
||||
status = rpc_call(session, 9, "core.get_torrent_status", torrent_id, ["state", "progress"]) or {}
|
||||
state = status.get("state", "unknown") if isinstance(status, dict) else "unknown"
|
||||
progress = status.get("progress", 0) if isinstance(status, dict) else 0
|
||||
print(f" Status: {state} ({progress:.1f}%)")
|
||||
|
||||
# Remove it
|
||||
client.call("core.remove_torrent", torrent_id, True)
|
||||
rpc_call(session, 10, "core.remove_torrent", torrent_id, True)
|
||||
print(" Removed test torrent")
|
||||
else:
|
||||
print(" WARNING: Could not add test torrent")
|
||||
@@ -372,19 +405,91 @@ def test_deluge():
|
||||
print(" SUCCESS: Deluge is working!")
|
||||
return True
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
print(" ERROR: Could not connect to Deluge Web UI")
|
||||
print(" Is the container running? docker ps | grep deluge")
|
||||
return False
|
||||
except requests.exceptions.Timeout:
|
||||
print(" ERROR: Deluge Web UI connection timed out")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}")
|
||||
if "auth" in str(e).lower() or "login" in str(e).lower():
|
||||
print(" Check Deluge Web UI password (default: deluge)")
|
||||
return False
|
||||
|
||||
def test_rtorrent():
|
||||
"""Test rTorrent connection."""
|
||||
print("\n" + "=" * 50)
|
||||
print("Testing rTorrent")
|
||||
print("=" * 50)
|
||||
|
||||
try:
|
||||
import xmlrpc.client
|
||||
|
||||
url = "http://localhost:8000/RPC2"
|
||||
client = xmlrpc.client.ServerProxy(url)
|
||||
|
||||
# Test connection
|
||||
version = client.system.library_version()
|
||||
print(f" Connected to rTorrent {version}")
|
||||
|
||||
# Get torrent list
|
||||
torrents = client.download_list()
|
||||
print(f" Active torrents: {len(torrents)}")
|
||||
|
||||
# Test adding a torrent (then remove it)
|
||||
print(" Testing add/remove torrent...")
|
||||
|
||||
label = "automated"
|
||||
|
||||
commands = []
|
||||
if label:
|
||||
commands.append(f"d.custom1.set={label}")
|
||||
|
||||
download_dir = "/downloads"
|
||||
if download_dir:
|
||||
commands.append(f"d.directory_base.set={download_dir}")
|
||||
|
||||
# rtorrent is weird in that it doesn't return the torrent ID/hash on add
|
||||
client.load.start("", TEST_MAGNET, ";".join(commands))
|
||||
|
||||
# but we know that it is 3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0 from the magnet link
|
||||
torrent_id = "3B245504CF5F11BBDBE1201CEA6A6BF45AEE1BC0" # rtorrent uses uppercase hashes
|
||||
print(f" Added test torrent: {torrent_id}")
|
||||
|
||||
torrent_list = client.d.multicall.filtered(
|
||||
"",
|
||||
"default",
|
||||
f"equal=d.hash=,cat={torrent_id}",
|
||||
"d.hash=",
|
||||
"d.state=",
|
||||
"d.completed_bytes=",
|
||||
"d.size_bytes=",
|
||||
"d.down.rate=",
|
||||
"d.up.rate=",
|
||||
"d.custom1=",
|
||||
"d.complete=",
|
||||
)
|
||||
torrent = torrent_list[0]
|
||||
if not torrent:
|
||||
print(" ERROR: Could not find added torrent in list")
|
||||
return False
|
||||
|
||||
|
||||
client.d.erase(torrent_id)
|
||||
print(" Removed test torrent")
|
||||
|
||||
print(" SUCCESS: rTorrent is working!")
|
||||
return True
|
||||
|
||||
except ImportError:
|
||||
print(" ERROR: deluge-client not installed")
|
||||
print(" Run: pip install deluge-client")
|
||||
print(" ERROR: xmlrpc.client not available")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}")
|
||||
if "Connection refused" in str(e):
|
||||
print(" Is the container running? docker ps | grep deluge")
|
||||
elif "Bad login" in str(e) or "auth" in str(e).lower():
|
||||
print("\n Deluge auth setup required:")
|
||||
print(" 1. Add 'admin:admin:10' to .local/test-clients/deluge/config/auth")
|
||||
print(" 2. Restart: docker restart test-deluge")
|
||||
print(" 3. Or access Web UI at http://localhost:8112 (password: deluge)")
|
||||
print(" Is the container running? docker ps | grep rtorrent")
|
||||
return False
|
||||
|
||||
|
||||
@@ -410,6 +515,7 @@ def main():
|
||||
results["qbittorrent"] = test_qbittorrent()
|
||||
results["transmission"] = test_transmission()
|
||||
results["deluge"] = test_deluge()
|
||||
results["rtorrent"] = test_rtorrent()
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 50)
|
||||
|
||||
@@ -10,6 +10,7 @@ import requests
|
||||
from shelfmark.bypass import BypassCancelledException
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.download import network
|
||||
@@ -29,10 +30,11 @@ BACKOFF_CAP = 10.0
|
||||
|
||||
def _fetch_via_bypasser(target_url: str) -> Optional[str]:
|
||||
"""Make a single request to the external bypasser service. Returns HTML or None."""
|
||||
bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
|
||||
raw_bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
|
||||
bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
|
||||
bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
|
||||
|
||||
bypasser_url = normalize_http_url(raw_bypasser_url)
|
||||
if not bypasser_url or not bypasser_path:
|
||||
logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.download.outputs.booklore import (
|
||||
BookloreConfig,
|
||||
BookloreError,
|
||||
booklore_list_libraries,
|
||||
booklore_login,
|
||||
)
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
_BOOKLORE_OPTIONS_CACHE: dict[str, Any] = {
|
||||
"key": None,
|
||||
"library_options": [],
|
||||
"path_options": [],
|
||||
}
|
||||
|
||||
|
||||
def _get_booklore_cache_key(base_url: str, username: str, password: str) -> str:
|
||||
return f"{base_url}|{username}|{hash(password)}"
|
||||
|
||||
|
||||
def _get_booklore_select_options(
|
||||
base_url: str,
|
||||
username: str,
|
||||
password: str,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
# library_id/path_id are not used for login/library listing
|
||||
booklore_config = BookloreConfig(
|
||||
base_url=base_url.rstrip("/"),
|
||||
username=username,
|
||||
password=password,
|
||||
library_id=1,
|
||||
path_id=1,
|
||||
verify_tls=True,
|
||||
refresh_after_upload=True,
|
||||
)
|
||||
|
||||
token = booklore_login(booklore_config)
|
||||
libraries = booklore_list_libraries(booklore_config, token) or []
|
||||
logger.debug("Booklore libraries response: %s", libraries)
|
||||
|
||||
library_options: list[dict[str, Any]] = []
|
||||
path_options: list[dict[str, Any]] = []
|
||||
|
||||
for library in libraries:
|
||||
if not isinstance(library, dict):
|
||||
continue
|
||||
|
||||
library_id = library.get("id")
|
||||
if library_id is None:
|
||||
continue
|
||||
|
||||
library_name = str(library.get("name") or f"Library {library_id}")
|
||||
library_id_str = str(library_id)
|
||||
|
||||
library_options.append({"value": library_id_str, "label": library_name})
|
||||
|
||||
paths = library.get("paths") or []
|
||||
if not isinstance(paths, list):
|
||||
continue
|
||||
|
||||
for path in paths:
|
||||
if not isinstance(path, dict):
|
||||
continue
|
||||
|
||||
path_id = path.get("id")
|
||||
if path_id is None:
|
||||
continue
|
||||
|
||||
path_label = str(path.get("path") or f"Path {path_id}")
|
||||
path_options.append(
|
||||
{
|
||||
"value": str(path_id),
|
||||
"label": f"{library_name}: {path_label}",
|
||||
"childOf": library_id_str,
|
||||
}
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Booklore options built: libraries=%d paths=%d",
|
||||
len(library_options),
|
||||
len(path_options),
|
||||
)
|
||||
|
||||
cache_key = _get_booklore_cache_key(base_url, username, password)
|
||||
_BOOKLORE_OPTIONS_CACHE.update(
|
||||
{
|
||||
"key": cache_key,
|
||||
"library_options": library_options,
|
||||
"path_options": path_options,
|
||||
}
|
||||
)
|
||||
|
||||
return library_options, path_options
|
||||
|
||||
|
||||
def _get_booklore_cached_options(
|
||||
base_url: str,
|
||||
username: str,
|
||||
password: str,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
cache_key = _get_booklore_cache_key(base_url, username, password)
|
||||
if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
|
||||
return (
|
||||
_BOOKLORE_OPTIONS_CACHE.get("library_options", []),
|
||||
_BOOKLORE_OPTIONS_CACHE.get("path_options", []),
|
||||
)
|
||||
|
||||
return _get_booklore_select_options(base_url, username, password)
|
||||
|
||||
|
||||
def get_booklore_library_options() -> list[dict[str, Any]]:
|
||||
"""Build Booklore library options dynamically from config."""
|
||||
if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
|
||||
return []
|
||||
|
||||
base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
|
||||
username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
|
||||
password = config.get("BOOKLORE_PASSWORD", "") or ""
|
||||
|
||||
if not base_url or not username or not password:
|
||||
return []
|
||||
|
||||
cache_key = _get_booklore_cache_key(base_url, username, password)
|
||||
|
||||
try:
|
||||
library_options, _ = _get_booklore_cached_options(base_url, username, password)
|
||||
return library_options
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to fetch Booklore libraries: {exc}")
|
||||
if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
|
||||
return _BOOKLORE_OPTIONS_CACHE.get("library_options", [])
|
||||
return []
|
||||
|
||||
|
||||
def get_booklore_path_options() -> list[dict[str, Any]]:
|
||||
"""Build Booklore path options dynamically from config."""
|
||||
if config.get("BOOKS_OUTPUT_MODE", "folder") != "booklore":
|
||||
return []
|
||||
|
||||
base_url = str(config.get("BOOKLORE_HOST", "") or "").strip().rstrip("/")
|
||||
username = str(config.get("BOOKLORE_USERNAME", "") or "").strip()
|
||||
password = config.get("BOOKLORE_PASSWORD", "") or ""
|
||||
|
||||
if not base_url or not username or not password:
|
||||
return []
|
||||
|
||||
cache_key = _get_booklore_cache_key(base_url, username, password)
|
||||
|
||||
try:
|
||||
_, path_options = _get_booklore_cached_options(base_url, username, password)
|
||||
return path_options
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to fetch Booklore paths: {exc}")
|
||||
if _BOOKLORE_OPTIONS_CACHE.get("key") == cache_key:
|
||||
return _BOOKLORE_OPTIONS_CACHE.get("path_options", [])
|
||||
return []
|
||||
|
||||
|
||||
def test_booklore_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Test the Booklore connection using current form values."""
|
||||
current_values = current_values or {}
|
||||
|
||||
def _get_value(key: str, default: Any = None) -> Any:
|
||||
value = current_values.get(key)
|
||||
if value not in (None, ""):
|
||||
return value
|
||||
if default is None:
|
||||
return config.get(key)
|
||||
return config.get(key, default)
|
||||
|
||||
base_url = str(_get_value("BOOKLORE_HOST", "") or "").strip().rstrip("/")
|
||||
username = str(_get_value("BOOKLORE_USERNAME", "") or "").strip()
|
||||
password = _get_value("BOOKLORE_PASSWORD", "") or ""
|
||||
|
||||
if not base_url:
|
||||
return {"success": False, "message": "Booklore URL is required"}
|
||||
if not username:
|
||||
return {"success": False, "message": "Booklore username is required"}
|
||||
if not password:
|
||||
return {"success": False, "message": "Booklore password is required"}
|
||||
|
||||
try:
|
||||
library_options, _ = _get_booklore_select_options(base_url, username, password)
|
||||
|
||||
message = "Connected to Booklore"
|
||||
if library_options:
|
||||
message = f"Connected to Booklore ({len(library_options)} libraries)"
|
||||
|
||||
return {"success": True, "message": message}
|
||||
except BookloreError as exc:
|
||||
return {"success": False, "message": str(exc)}
|
||||
+137
-13
@@ -10,6 +10,7 @@ from shelfmark.core.settings_registry import (
|
||||
register_on_save,
|
||||
load_config_file,
|
||||
TextField,
|
||||
SelectField,
|
||||
PasswordField,
|
||||
CheckboxField,
|
||||
ActionButton,
|
||||
@@ -18,6 +19,65 @@ from shelfmark.core.settings_registry import (
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def _migrate_security_settings() -> None:
|
||||
import json
|
||||
from shelfmark.core.settings_registry import _get_config_file_path, _ensure_config_dir
|
||||
|
||||
try:
|
||||
config = load_config_file("security")
|
||||
migrated = False
|
||||
|
||||
# Migrate USE_CWA_AUTH to AUTH_METHOD
|
||||
if "USE_CWA_AUTH" in config:
|
||||
old_value = config.pop("USE_CWA_AUTH")
|
||||
|
||||
# Only set AUTH_METHOD if it doesn't already exist
|
||||
if "AUTH_METHOD" not in config:
|
||||
if old_value:
|
||||
config["AUTH_METHOD"] = "cwa"
|
||||
logger.info("Migrated USE_CWA_AUTH=True to AUTH_METHOD='cwa'")
|
||||
else:
|
||||
# If USE_CWA_AUTH was False, determine auth method from credentials
|
||||
if config.get("BUILTIN_USERNAME") and config.get("BUILTIN_PASSWORD_HASH"):
|
||||
config["AUTH_METHOD"] = "builtin"
|
||||
logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='builtin'")
|
||||
else:
|
||||
config["AUTH_METHOD"] = "none"
|
||||
logger.info("Migrated USE_CWA_AUTH=False to AUTH_METHOD='none'")
|
||||
migrated = True
|
||||
else:
|
||||
logger.info("Removed deprecated USE_CWA_AUTH setting (AUTH_METHOD already exists)")
|
||||
migrated = True
|
||||
|
||||
# Migrate RESTRICT_SETTINGS_TO_ADMIN to CWA_RESTRICT_SETTINGS_TO_ADMIN
|
||||
if "RESTRICT_SETTINGS_TO_ADMIN" in config:
|
||||
old_value = config.pop("RESTRICT_SETTINGS_TO_ADMIN")
|
||||
|
||||
# Only migrate if new key doesn't exist
|
||||
if "CWA_RESTRICT_SETTINGS_TO_ADMIN" not in config:
|
||||
config["CWA_RESTRICT_SETTINGS_TO_ADMIN"] = old_value
|
||||
logger.info(f"Migrated RESTRICT_SETTINGS_TO_ADMIN={old_value} to CWA_RESTRICT_SETTINGS_TO_ADMIN={old_value}")
|
||||
migrated = True
|
||||
else:
|
||||
logger.info("Removed deprecated RESTRICT_SETTINGS_TO_ADMIN setting (CWA_RESTRICT_SETTINGS_TO_ADMIN already exists)")
|
||||
migrated = True
|
||||
|
||||
# Save config if any migrations occurred
|
||||
if migrated:
|
||||
_ensure_config_dir("security")
|
||||
config_path = _get_config_file_path("security")
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
logger.info("Security settings migration completed successfully")
|
||||
else:
|
||||
logger.debug("No security settings migration needed")
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.debug("No existing security config file found - nothing to migrate")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to migrate security settings: {e}")
|
||||
|
||||
|
||||
def _clear_builtin_credentials() -> Dict[str, Any]:
|
||||
"""Clear built-in credentials to allow public access."""
|
||||
import json
|
||||
@@ -98,20 +158,40 @@ def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
|
||||
|
||||
@register_settings("security", "Security", icon="shield", order=5)
|
||||
def security_settings():
|
||||
def security_settings():
|
||||
"""Security and authentication settings."""
|
||||
from shelfmark.config.env import CWA_DB_PATH
|
||||
|
||||
cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
|
||||
|
||||
auth_method_options = [
|
||||
{"label": "No Authentication", "value": "none"},
|
||||
{"label": "Username/Password", "value": "builtin"},
|
||||
{"label": "Proxy Authentication", "value": "proxy"},
|
||||
]
|
||||
if cwa_db_available:
|
||||
auth_method_options.append({"label": "Calibre-Web Database", "value": "cwa"})
|
||||
|
||||
auth_method_description = "Select the authentication method for accessing Shelfmark."
|
||||
if not cwa_db_available:
|
||||
auth_method_description += " Calibre-Web database option requires mounting your Calibre-Web app.db to /auth/app.db."
|
||||
|
||||
fields = [
|
||||
SelectField(
|
||||
key="AUTH_METHOD",
|
||||
label="Authentication Method",
|
||||
description=auth_method_description,
|
||||
options=auth_method_options,
|
||||
default="none",
|
||||
env_supported=False,
|
||||
),
|
||||
TextField(
|
||||
key="BUILTIN_USERNAME",
|
||||
label="Username",
|
||||
description="Set a username and password to require login. Leave both empty for public access.",
|
||||
placeholder="Enter username",
|
||||
env_supported=False,
|
||||
disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
|
||||
show_when={"field": "AUTH_METHOD", "value": "builtin"},
|
||||
),
|
||||
PasswordField(
|
||||
key="BUILTIN_PASSWORD",
|
||||
@@ -119,14 +199,14 @@ def security_settings():
|
||||
description="Fill in to set or change the password.",
|
||||
placeholder="Enter new password",
|
||||
env_supported=False,
|
||||
disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
|
||||
show_when={"field": "AUTH_METHOD", "value": "builtin"},
|
||||
),
|
||||
PasswordField(
|
||||
key="BUILTIN_PASSWORD_CONFIRM",
|
||||
label="Confirm Password",
|
||||
placeholder="Confirm new password",
|
||||
env_supported=False,
|
||||
disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
|
||||
show_when={"field": "AUTH_METHOD", "value": "builtin"},
|
||||
),
|
||||
ActionButton(
|
||||
key="clear_credentials",
|
||||
@@ -134,28 +214,72 @@ def security_settings():
|
||||
description="Remove login requirement and make the app publicly accessible.",
|
||||
style="danger",
|
||||
callback=_clear_builtin_credentials,
|
||||
disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."},
|
||||
show_when={"field": "AUTH_METHOD", "value": "builtin"},
|
||||
),
|
||||
TextField(
|
||||
key="PROXY_AUTH_USER_HEADER",
|
||||
label="Proxy Auth User Header",
|
||||
description=(
|
||||
"The HTTP header your proxy uses to pass the authenticated username."
|
||||
),
|
||||
placeholder="e.g. X-Auth-User",
|
||||
default="X-Auth-User",
|
||||
env_supported=False,
|
||||
show_when={"field": "AUTH_METHOD", "value": "proxy"},
|
||||
),
|
||||
TextField(
|
||||
key="PROXY_AUTH_LOGOUT_URL",
|
||||
label="Proxy Auth Logout URL",
|
||||
description=(
|
||||
"The URL to redirect users to for logging out."
|
||||
" Leave empty to disable logout functionality."
|
||||
),
|
||||
placeholder="https://myauth.example.com/logout",
|
||||
default="",
|
||||
env_supported=False,
|
||||
show_when={"field": "AUTH_METHOD", "value": "proxy"},
|
||||
),
|
||||
CheckboxField(
|
||||
key="USE_CWA_AUTH",
|
||||
label="Use Calibre-Web Database",
|
||||
key="PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN",
|
||||
label="Restrict Settings to Admins authenticated via Proxy",
|
||||
description=(
|
||||
"Use your existing Calibre-Web user credentials for authentication."
|
||||
"Only users in the admin group can access settings."
|
||||
),
|
||||
default=False,
|
||||
env_supported=False,
|
||||
disabled=not cwa_db_available,
|
||||
disabled_reason="Mount your Calibre-Web app.db to /auth/app.db in docker compose to enable.",
|
||||
show_when={"field": "AUTH_METHOD", "value": "proxy"},
|
||||
),
|
||||
TextField(
|
||||
key="PROXY_AUTH_ADMIN_GROUP_HEADER",
|
||||
label="Proxy Auth Admin Group Header",
|
||||
description=(
|
||||
"The HTTP header your proxy uses to pass the user's groups/roles."
|
||||
),
|
||||
placeholder="e.g. X-Auth-Groups",
|
||||
default="X-Auth-Groups",
|
||||
env_supported=False,
|
||||
show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
|
||||
),
|
||||
TextField(
|
||||
key="PROXY_AUTH_ADMIN_GROUP_NAME",
|
||||
label="Proxy Auth Admin Group Name",
|
||||
description=(
|
||||
"The name of the group/role that should have admin access."
|
||||
),
|
||||
placeholder="e.g. admins",
|
||||
default="admins",
|
||||
env_supported=False,
|
||||
show_when={"field": "PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", "value": True},
|
||||
),
|
||||
CheckboxField(
|
||||
key="RESTRICT_SETTINGS_TO_ADMIN",
|
||||
label="Restrict Settings to Admins",
|
||||
key="CWA_RESTRICT_SETTINGS_TO_ADMIN",
|
||||
label="Restrict Settings to Admins authenticated via Calibre-Web",
|
||||
description=(
|
||||
"Only users with admin role in Calibre-Web can access settings."
|
||||
),
|
||||
default=False,
|
||||
env_supported=False,
|
||||
show_when={"field": "USE_CWA_AUTH", "value": True},
|
||||
show_when={"field": "AUTH_METHOD", "value": "cwa"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
+268
-19
@@ -3,8 +3,72 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
import json
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def _on_save_advanced(values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Validate advanced settings before persisting."""
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
mappings = values.get("PROWLARR_REMOTE_PATH_MAPPINGS")
|
||||
if mappings is None:
|
||||
return {"error": False, "values": values}
|
||||
|
||||
if not isinstance(mappings, list):
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Remote path mappings must be a list",
|
||||
"values": values,
|
||||
}
|
||||
|
||||
logger.debug("Processing %d remote path mapping entries", len(mappings))
|
||||
|
||||
cleaned = []
|
||||
for i, entry in enumerate(mappings):
|
||||
if not isinstance(entry, dict):
|
||||
logger.debug("Skipping entry %d: not a dict", i)
|
||||
continue
|
||||
|
||||
host = str(entry.get("host", "") or "").strip().lower()
|
||||
remote_path = str(entry.get("remotePath", "") or "").strip()
|
||||
local_path = str(entry.get("localPath", "") or "").strip()
|
||||
|
||||
if not host or not remote_path or not local_path:
|
||||
logger.debug(
|
||||
"Skipping entry %d: missing field(s) - host=%r, remotePath=%r, localPath=%r",
|
||||
i,
|
||||
host,
|
||||
remote_path,
|
||||
local_path,
|
||||
)
|
||||
continue
|
||||
|
||||
if not local_path.startswith("/"):
|
||||
return {
|
||||
"error": True,
|
||||
"message": f"Local Path must be an absolute path (got: {local_path})",
|
||||
"values": values,
|
||||
}
|
||||
|
||||
cleaned.append({"host": host, "remotePath": remote_path, "localPath": local_path})
|
||||
|
||||
logger.info("Saved %d remote path mapping(s)", len(cleaned))
|
||||
if cleaned:
|
||||
for m in cleaned:
|
||||
logger.debug(" Mapping: %s -> %s (client: %s)", m["remotePath"], m["localPath"], m["host"])
|
||||
|
||||
values["PROWLARR_REMOTE_PATH_MAPPINGS"] = cleaned
|
||||
return {"error": False, "values": values}
|
||||
|
||||
|
||||
from shelfmark.config import env
|
||||
from shelfmark.config.booklore_settings import (
|
||||
get_booklore_library_options,
|
||||
get_booklore_path_options,
|
||||
test_booklore_connection,
|
||||
)
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -53,6 +117,8 @@ def _log_external_bypasser_warning() -> None:
|
||||
from shelfmark.core.settings_registry import (
|
||||
register_settings,
|
||||
register_group,
|
||||
register_on_save,
|
||||
load_config_file,
|
||||
TextField,
|
||||
PasswordField,
|
||||
NumberField,
|
||||
@@ -60,6 +126,7 @@ from shelfmark.core.settings_registry import (
|
||||
SelectField,
|
||||
MultiSelectField,
|
||||
OrderableListField,
|
||||
TableField,
|
||||
HeadingField,
|
||||
ActionButton,
|
||||
)
|
||||
@@ -150,6 +217,8 @@ def _get_release_source_options():
|
||||
if source.get("can_be_default", True)
|
||||
]
|
||||
|
||||
|
||||
|
||||
_LANGUAGE_OPTIONS = [{"value": lang["code"], "label": lang["language"]} for lang in _SUPPORTED_BOOK_LANGUAGE]
|
||||
|
||||
def _get_aa_base_url_options():
|
||||
@@ -277,7 +346,6 @@ def general_settings():
|
||||
label="Audiobook Library URL",
|
||||
description="Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text.",
|
||||
placeholder="http://audiobookshelf:8080",
|
||||
env_supported=False,
|
||||
),
|
||||
HeadingField(
|
||||
key="search_defaults_heading",
|
||||
@@ -341,7 +409,6 @@ def search_mode_settings():
|
||||
description="Default sort order for search results.",
|
||||
options=_AA_SORT_OPTIONS,
|
||||
default="relevance",
|
||||
env_supported=False, # UI-only setting
|
||||
show_when={"field": "SEARCH_MODE", "value": "direct"},
|
||||
),
|
||||
HeadingField(
|
||||
@@ -372,7 +439,6 @@ def search_mode_settings():
|
||||
description="The release source tab to open by default in the release modal.",
|
||||
options=_get_release_source_options, # Callable - evaluated lazily to avoid circular imports
|
||||
default="direct_download",
|
||||
env_supported=False, # UI-only setting, not configurable via ENV
|
||||
show_when={"field": "SEARCH_MODE", "value": "universal"},
|
||||
),
|
||||
]
|
||||
@@ -511,6 +577,40 @@ def network_settings():
|
||||
]
|
||||
|
||||
|
||||
def _contains_path_separators(value: Any) -> bool:
|
||||
return isinstance(value, str) and ("/" in value or "\\" in value)
|
||||
|
||||
|
||||
def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate download settings before persisting."""
|
||||
existing = load_config_file("downloads")
|
||||
effective: dict[str, Any] = dict(existing)
|
||||
effective.update(values)
|
||||
|
||||
# Books: only validate templates when saving to a folder.
|
||||
books_output_mode = effective.get("BOOKS_OUTPUT_MODE", "folder")
|
||||
if books_output_mode == "folder" and effective.get("FILE_ORGANIZATION", "rename") == "rename":
|
||||
template = effective.get("TEMPLATE_RENAME", "")
|
||||
if _contains_path_separators(template):
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Books Naming Template cannot contain '/' or '\\' in Rename mode. Use Organize mode to create folders.",
|
||||
"values": values,
|
||||
}
|
||||
|
||||
# Audiobooks are always folder output.
|
||||
if effective.get("FILE_ORGANIZATION_AUDIOBOOK", "rename") == "rename":
|
||||
template = effective.get("TEMPLATE_AUDIOBOOK_RENAME", "")
|
||||
if _contains_path_separators(template):
|
||||
return {
|
||||
"error": True,
|
||||
"message": "Audiobooks Naming Template cannot contain '/' or '\\' in Rename mode. Use Organize mode to create folders.",
|
||||
"values": values,
|
||||
}
|
||||
|
||||
return {"error": False, "values": values}
|
||||
|
||||
|
||||
@register_settings("downloads", "Downloads", icon="folder", order=5)
|
||||
def download_settings():
|
||||
"""Configure download behavior and file locations."""
|
||||
@@ -522,6 +622,24 @@ def download_settings():
|
||||
title="Books",
|
||||
description="Configure where ebooks, comics, and magazines are saved.",
|
||||
),
|
||||
SelectField(
|
||||
key="BOOKS_OUTPUT_MODE",
|
||||
label="Output Mode",
|
||||
description="Choose where completed book files are sent.",
|
||||
options=[
|
||||
{
|
||||
"value": "folder",
|
||||
"label": "Folder",
|
||||
"description": "Save files to the destination folder",
|
||||
},
|
||||
{
|
||||
"value": "booklore",
|
||||
"label": "Booklore (API)",
|
||||
"description": "Upload files directly to Booklore",
|
||||
},
|
||||
],
|
||||
default="folder",
|
||||
),
|
||||
TextField(
|
||||
key="DESTINATION",
|
||||
label="Destination",
|
||||
@@ -529,6 +647,10 @@ def download_settings():
|
||||
default="/books",
|
||||
required=True,
|
||||
env_var="INGEST_DIR", # Legacy env var name for backwards compatibility
|
||||
show_when={
|
||||
"field": "BOOKS_OUTPUT_MODE",
|
||||
"value": "folder",
|
||||
},
|
||||
),
|
||||
SelectField(
|
||||
key="FILE_ORGANIZATION",
|
||||
@@ -542,25 +664,32 @@ def download_settings():
|
||||
},
|
||||
{
|
||||
"value": "rename",
|
||||
"label": "Rename",
|
||||
"label": "Rename Only",
|
||||
"description": "Rename files using a template"
|
||||
},
|
||||
{
|
||||
"value": "organize",
|
||||
"label": "Organize",
|
||||
"label": "Rename and Organize",
|
||||
"description": "Create folders and rename files using a template. Do not use with ingest folders."
|
||||
},
|
||||
],
|
||||
default="rename",
|
||||
show_when={
|
||||
"field": "BOOKS_OUTPUT_MODE",
|
||||
"value": "folder",
|
||||
},
|
||||
),
|
||||
# Rename mode template - filename only
|
||||
TextField(
|
||||
key="TEMPLATE_RENAME",
|
||||
label="Naming Template",
|
||||
description="Variables: {Author}, {Title}, {Year}. Universal adds: {Series}, {SeriesPosition}, {Subtitle}",
|
||||
description="Variables: {Author}, {Title}, {Year}. Universal adds: {Series}, {SeriesPosition}, {Subtitle}. Rename templates are filename-only (no '/' or '\\'); use Organize for folders.",
|
||||
default="{Author} - {Title} ({Year})",
|
||||
placeholder="{Author} - {Title} ({Year})",
|
||||
show_when={"field": "FILE_ORGANIZATION", "value": "rename"},
|
||||
show_when=[
|
||||
{"field": "BOOKS_OUTPUT_MODE", "value": "folder"},
|
||||
{"field": "FILE_ORGANIZATION", "value": "rename"},
|
||||
],
|
||||
),
|
||||
# Organize mode template - folders allowed
|
||||
TextField(
|
||||
@@ -569,7 +698,10 @@ def download_settings():
|
||||
description="Use / to create folders. Variables: {Author}, {Title}, {Year}. Universal adds: {Series}, {SeriesPosition}, {Subtitle}",
|
||||
default="{Author}/{Title} ({Year})",
|
||||
placeholder="{Author}/{Series/}{Title} ({Year})",
|
||||
show_when={"field": "FILE_ORGANIZATION", "value": "organize"},
|
||||
show_when=[
|
||||
{"field": "BOOKS_OUTPUT_MODE", "value": "folder"},
|
||||
{"field": "FILE_ORGANIZATION", "value": "organize"},
|
||||
],
|
||||
),
|
||||
CheckboxField(
|
||||
key="HARDLINK_TORRENTS",
|
||||
@@ -577,6 +709,63 @@ def download_settings():
|
||||
description="Create hardlinks instead of copying. Preserves seeding but archives won't be extracted. Don't use if destination is a library ingest folder.",
|
||||
default=False,
|
||||
universal_only=True,
|
||||
show_when={
|
||||
"field": "BOOKS_OUTPUT_MODE",
|
||||
"value": "folder",
|
||||
},
|
||||
),
|
||||
HeadingField(
|
||||
key="booklore_heading",
|
||||
title="Booklore",
|
||||
description="Upload books directly to Booklore via API. Audiobooks always use folder mode.",
|
||||
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
|
||||
),
|
||||
TextField(
|
||||
key="BOOKLORE_HOST",
|
||||
label="Booklore URL",
|
||||
description="Base URL of your Booklore instance",
|
||||
placeholder="http://booklore:6060",
|
||||
required=True,
|
||||
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
|
||||
),
|
||||
TextField(
|
||||
key="BOOKLORE_USERNAME",
|
||||
label="Username",
|
||||
description="Booklore account username",
|
||||
required=True,
|
||||
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
|
||||
),
|
||||
PasswordField(
|
||||
key="BOOKLORE_PASSWORD",
|
||||
label="Password",
|
||||
description="Booklore account password",
|
||||
required=True,
|
||||
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
|
||||
),
|
||||
SelectField(
|
||||
key="BOOKLORE_LIBRARY_ID",
|
||||
label="Library",
|
||||
description="Booklore library to upload into.",
|
||||
options=get_booklore_library_options,
|
||||
required=True,
|
||||
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
|
||||
),
|
||||
SelectField(
|
||||
key="BOOKLORE_PATH_ID",
|
||||
label="Path",
|
||||
description="Booklore library path for uploads.",
|
||||
options=get_booklore_path_options,
|
||||
required=True,
|
||||
filter_by_field="BOOKLORE_LIBRARY_ID",
|
||||
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
|
||||
),
|
||||
ActionButton(
|
||||
key="test_booklore",
|
||||
label="Test Connection",
|
||||
description="Verify your Booklore configuration",
|
||||
style="primary",
|
||||
callback=test_booklore_connection,
|
||||
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
|
||||
),
|
||||
|
||||
# === AUDIOBOOKS SECTION ===
|
||||
@@ -600,8 +789,8 @@ def download_settings():
|
||||
description="Choose how downloaded audiobook files are named and organized.",
|
||||
options=[
|
||||
{"value": "none", "label": "None", "description": "Keep original filename from source"},
|
||||
{"value": "rename", "label": "Rename", "description": "Rename files using a template"},
|
||||
{"value": "organize", "label": "Organize", "description": "Create folders and rename files using a template. Recommended for Audiobookshelf. Do not use with ingest folders."},
|
||||
{"value": "rename", "label": "Rename Only", "description": "Rename files using a template"},
|
||||
{"value": "organize", "label": "Rename and Organize", "description": "Create folders and rename files using a template. Recommended for Audiobookshelf. Do not use with ingest folders."},
|
||||
],
|
||||
default="rename",
|
||||
universal_only=True,
|
||||
@@ -610,7 +799,7 @@ def download_settings():
|
||||
TextField(
|
||||
key="TEMPLATE_AUDIOBOOK_RENAME",
|
||||
label="Naming Template",
|
||||
description="Variables: {Author}, {Title}, {Year}, {Series}, {SeriesPosition}, {Subtitle}, {PartNumber}",
|
||||
description="Variables: {Author}, {Title}, {Year}, {Series}, {SeriesPosition}, {Subtitle}, {PartNumber}. Rename templates are filename-only (no '/' or '\\'); use Organize for folders.",
|
||||
default="{Author} - {Title}",
|
||||
placeholder="{Author} - {Title}{ - Part }{PartNumber}",
|
||||
show_when={"field": "FILE_ORGANIZATION_AUDIOBOOK", "value": "rename"},
|
||||
@@ -644,14 +833,12 @@ def download_settings():
|
||||
label="Auto-Open Downloads Sidebar",
|
||||
description="Automatically open the downloads sidebar when a new download is queued.",
|
||||
default=False,
|
||||
env_supported=False, # UI-only setting
|
||||
),
|
||||
CheckboxField(
|
||||
key="DOWNLOAD_TO_BROWSER",
|
||||
label="Download to Browser",
|
||||
description="Automatically download completed files to your browser.",
|
||||
default=False,
|
||||
env_supported=False, # UI-only setting
|
||||
),
|
||||
NumberField(
|
||||
key="MAX_CONCURRENT_DOWNLOADS",
|
||||
@@ -673,8 +860,12 @@ def download_settings():
|
||||
]
|
||||
|
||||
|
||||
# Register the on_save handler for this tab
|
||||
register_on_save("downloads", _on_save_downloads)
|
||||
|
||||
|
||||
def _get_fast_source_options():
|
||||
"""Fast download sources - display only, not configurable."""
|
||||
"""Fast download sources - configurable list shown in settings."""
|
||||
from shelfmark.core.config import config
|
||||
|
||||
has_donator_key = bool(config.get("AA_DONATOR_KEY", ""))
|
||||
@@ -777,7 +968,6 @@ def download_source_settings():
|
||||
description="Always tried first, no waiting or bypass required.",
|
||||
options=_get_fast_source_options,
|
||||
default=_get_fast_source_defaults(),
|
||||
env_supported=False,
|
||||
),
|
||||
OrderableListField(
|
||||
key="SOURCE_PRIORITY",
|
||||
@@ -995,10 +1185,11 @@ def advanced_settings():
|
||||
"""Advanced settings for power users."""
|
||||
return [
|
||||
TextField(
|
||||
key="CUSTOM_SCRIPT",
|
||||
label="Custom Script Path",
|
||||
description="Path to a script to run after each successful download. Must be executable.",
|
||||
placeholder="/path/to/script.sh",
|
||||
key="URL_BASE",
|
||||
label="Base Path",
|
||||
description="Optional URL path prefix. Use a path like /shelfmark (no hostname). Leave blank for root.",
|
||||
placeholder="/shelfmark",
|
||||
requires_restart=True,
|
||||
),
|
||||
CheckboxField(
|
||||
key="DEBUG",
|
||||
@@ -1025,6 +1216,61 @@ def advanced_settings():
|
||||
max_value=10,
|
||||
requires_restart=True,
|
||||
),
|
||||
TextField(
|
||||
key="CUSTOM_SCRIPT",
|
||||
label="Custom Script Path",
|
||||
description="Path to a script to run after each successful download. Must be executable.",
|
||||
placeholder="/path/to/script.sh",
|
||||
),
|
||||
SelectField(
|
||||
key="CUSTOM_SCRIPT_PATH_MODE",
|
||||
label="Custom Script Path Mode",
|
||||
description="Pass the path to the custom script as an absolute path or relative to the destination folder.",
|
||||
options=[
|
||||
{"value": "absolute", "label": "Absolute", "description": "Pass the full destination path (default)."},
|
||||
{"value": "relative", "label": "Relative", "description": "Pass the path relative to the destination folder."},
|
||||
],
|
||||
default="absolute",
|
||||
),
|
||||
HeadingField(
|
||||
key="remote_path_mappings_heading",
|
||||
title="Remote Path Mappings",
|
||||
description="Map download client paths to paths inside Shelfmark. Needed when volume mounts differ between containers.",
|
||||
),
|
||||
TableField(
|
||||
key="PROWLARR_REMOTE_PATH_MAPPINGS",
|
||||
label="Path Mappings",
|
||||
columns=[
|
||||
{
|
||||
"key": "host",
|
||||
"label": "Client",
|
||||
"type": "select",
|
||||
"options": [
|
||||
{"value": "qbittorrent", "label": "qBittorrent"},
|
||||
{"value": "transmission", "label": "Transmission"},
|
||||
{"value": "deluge", "label": "Deluge"},
|
||||
{"value": "rtorrent", "label": "rTorrent"},
|
||||
{"value": "nzbget", "label": "NZBGet"},
|
||||
{"value": "sabnzbd", "label": "SABnzbd"},
|
||||
],
|
||||
"defaultValue": "qbittorrent",
|
||||
},
|
||||
{
|
||||
"key": "remotePath",
|
||||
"label": "Remote Path",
|
||||
"type": "path",
|
||||
},
|
||||
{
|
||||
"key": "localPath",
|
||||
"label": "Local Path",
|
||||
"type": "path",
|
||||
},
|
||||
],
|
||||
default=[],
|
||||
add_label="Add Mapping",
|
||||
empty_message="No mappings configured.",
|
||||
env_supported=False,
|
||||
),
|
||||
HeadingField(
|
||||
key="covers_cache_heading",
|
||||
title="Cover Image Cache",
|
||||
@@ -1096,3 +1342,6 @@ def advanced_settings():
|
||||
callback=_clear_metadata_cache,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
register_on_save("advanced", _on_save_advanced)
|
||||
|
||||
+45
-27
@@ -2,6 +2,8 @@
|
||||
|
||||
from typing import List
|
||||
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
|
||||
# Lazy import to avoid circular imports
|
||||
_config_module = None
|
||||
|
||||
@@ -44,6 +46,10 @@ DEFAULT_WELIB_MIRRORS = [
|
||||
]
|
||||
|
||||
|
||||
def _normalize_mirror_url(url: str) -> str:
|
||||
return normalize_http_url(url, default_scheme="https")
|
||||
|
||||
|
||||
def get_aa_mirrors() -> List[str]:
|
||||
"""
|
||||
Get Anna's Archive mirrors from config + defaults.
|
||||
@@ -51,15 +57,16 @@ def get_aa_mirrors() -> List[str]:
|
||||
Returns:
|
||||
List of AA mirror URLs, starting with defaults then custom additions.
|
||||
"""
|
||||
mirrors = list(DEFAULT_AA_MIRRORS)
|
||||
mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
|
||||
mirrors = [url for url in mirrors if url]
|
||||
config = _get_config()
|
||||
|
||||
additional = config.get("AA_ADDITIONAL_URLS", "")
|
||||
if additional:
|
||||
for url in additional.split(","):
|
||||
url = url.strip()
|
||||
if url and url not in mirrors:
|
||||
mirrors.append(url)
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized and normalized not in mirrors:
|
||||
mirrors.append(normalized)
|
||||
|
||||
return mirrors
|
||||
|
||||
@@ -71,15 +78,16 @@ def get_libgen_mirrors() -> List[str]:
|
||||
Returns:
|
||||
List of LibGen mirror URLs (defaults first, then custom additions).
|
||||
"""
|
||||
mirrors = list(DEFAULT_LIBGEN_MIRRORS)
|
||||
mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
|
||||
mirrors = [url for url in mirrors if url]
|
||||
config = _get_config()
|
||||
|
||||
additional = config.get("LIBGEN_ADDITIONAL_URLS", "")
|
||||
if additional:
|
||||
for url in additional.split(","):
|
||||
url = url.strip()
|
||||
if url and url not in mirrors:
|
||||
mirrors.append(url)
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized and normalized not in mirrors:
|
||||
mirrors.append(normalized)
|
||||
|
||||
return mirrors
|
||||
|
||||
@@ -93,21 +101,24 @@ def get_zlib_mirrors() -> List[str]:
|
||||
"""
|
||||
config = _get_config()
|
||||
|
||||
primary = config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0])
|
||||
primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
|
||||
if not primary:
|
||||
primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
|
||||
mirrors = [primary]
|
||||
|
||||
# Add other defaults (excluding primary)
|
||||
for url in DEFAULT_ZLIB_MIRRORS:
|
||||
if url != primary:
|
||||
mirrors.append(url)
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized and normalized != primary:
|
||||
mirrors.append(normalized)
|
||||
|
||||
# Add custom mirrors
|
||||
additional = config.get("ZLIB_ADDITIONAL_URLS", "")
|
||||
if additional:
|
||||
for url in additional.split(","):
|
||||
url = url.strip()
|
||||
if url and url not in mirrors:
|
||||
mirrors.append(url)
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized and normalized not in mirrors:
|
||||
mirrors.append(normalized)
|
||||
|
||||
return mirrors
|
||||
|
||||
@@ -120,7 +131,8 @@ def get_zlib_primary_url() -> str:
|
||||
Primary Z-Library mirror URL.
|
||||
"""
|
||||
config = _get_config()
|
||||
return config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0])
|
||||
primary = _normalize_mirror_url(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
|
||||
return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
|
||||
|
||||
|
||||
def get_zlib_url_template() -> str:
|
||||
@@ -143,21 +155,24 @@ def get_welib_mirrors() -> List[str]:
|
||||
"""
|
||||
config = _get_config()
|
||||
|
||||
primary = config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0])
|
||||
primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
|
||||
if not primary:
|
||||
primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
|
||||
mirrors = [primary]
|
||||
|
||||
# Add other defaults (excluding primary)
|
||||
for url in DEFAULT_WELIB_MIRRORS:
|
||||
if url != primary:
|
||||
mirrors.append(url)
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized and normalized != primary:
|
||||
mirrors.append(normalized)
|
||||
|
||||
# Add custom mirrors
|
||||
additional = config.get("WELIB_ADDITIONAL_URLS", "")
|
||||
if additional:
|
||||
for url in additional.split(","):
|
||||
url = url.strip()
|
||||
if url and url not in mirrors:
|
||||
mirrors.append(url)
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized and normalized not in mirrors:
|
||||
mirrors.append(normalized)
|
||||
|
||||
return mirrors
|
||||
|
||||
@@ -170,7 +185,8 @@ def get_welib_primary_url() -> str:
|
||||
Primary Welib mirror URL.
|
||||
"""
|
||||
config = _get_config()
|
||||
return config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0])
|
||||
primary = _normalize_mirror_url(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
|
||||
return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
|
||||
|
||||
|
||||
def get_welib_url_template() -> str:
|
||||
@@ -197,17 +213,19 @@ def get_zlib_cookie_domains() -> set:
|
||||
|
||||
# Add all default domains
|
||||
for url in DEFAULT_ZLIB_MIRRORS:
|
||||
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
|
||||
domains.add(domain)
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized:
|
||||
domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
|
||||
domains.add(domain)
|
||||
|
||||
# Add custom domains
|
||||
config = _get_config()
|
||||
additional = config.get("ZLIB_ADDITIONAL_URLS", "")
|
||||
if additional:
|
||||
for url in additional.split(","):
|
||||
url = url.strip()
|
||||
if url:
|
||||
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
|
||||
normalized = _normalize_mirror_url(url)
|
||||
if normalized:
|
||||
domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
|
||||
domains.add(domain)
|
||||
|
||||
return domains
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Union
|
||||
from typing import Dict, Optional, Union, Mapping
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
@@ -17,10 +17,10 @@ TOKEN_PATTERN = re.compile(
|
||||
)
|
||||
|
||||
# Characters that are invalid in filenames on various filesystems
|
||||
INVALID_CHARS = re.compile(r'[\\:*?"<>|]')
|
||||
INVALID_CHARS = re.compile(r'[\\/:*?"<>|]')
|
||||
|
||||
|
||||
def _sanitize(name: str, max_length: int = 245) -> str:
|
||||
def _sanitize(name: Optional[str], max_length: int = 245) -> str:
|
||||
"""Sanitize a string for filesystem use."""
|
||||
if not name:
|
||||
return ""
|
||||
@@ -31,7 +31,7 @@ def _sanitize(name: str, max_length: int = 245) -> str:
|
||||
return sanitized[:max_length]
|
||||
|
||||
|
||||
def sanitize_filename(name: str, max_length: int = 245) -> str:
|
||||
def sanitize_filename(name: Optional[str], max_length: int = 245) -> str:
|
||||
"""Sanitize a string for use as a filename or path component."""
|
||||
return _sanitize(name, max_length)
|
||||
|
||||
@@ -40,7 +40,7 @@ def sanitize_filename(name: str, max_length: int = 245) -> str:
|
||||
sanitize_path_component = sanitize_filename
|
||||
|
||||
|
||||
def format_series_position(position: Optional[Union[int, float]]) -> str:
|
||||
def format_series_position(position: Optional[Union[str, int, float]]) -> str:
|
||||
if position is None:
|
||||
return ""
|
||||
|
||||
@@ -78,7 +78,9 @@ def assign_part_numbers(
|
||||
|
||||
def parse_naming_template(
|
||||
template: str,
|
||||
metadata: Dict[str, Optional[Union[str, int, float]]],
|
||||
metadata: Mapping[str, Optional[Union[str, int, float]]],
|
||||
*,
|
||||
allow_path_separators: bool = True,
|
||||
) -> str:
|
||||
if not template:
|
||||
return ""
|
||||
@@ -108,6 +110,8 @@ def parse_naming_template(
|
||||
if not value:
|
||||
return ""
|
||||
|
||||
if not allow_path_separators:
|
||||
value = value.replace("/", "_")
|
||||
# Sanitize the value
|
||||
value = sanitize_filename(value)
|
||||
|
||||
@@ -140,10 +144,10 @@ def parse_naming_template(
|
||||
def build_library_path(
|
||||
base_path: str,
|
||||
template: str,
|
||||
metadata: Dict[str, Optional[Union[str, int, float]]],
|
||||
metadata: Mapping[str, Optional[Union[str, int, float]]],
|
||||
extension: Optional[str] = None,
|
||||
) -> Path:
|
||||
relative = parse_naming_template(template, metadata)
|
||||
relative = parse_naming_template(template, metadata, allow_path_separators=True)
|
||||
|
||||
if not relative:
|
||||
# Fallback to title if template produces empty result
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Remote path mapping utilities.
|
||||
|
||||
Used when an external download client reports a completed download path that does
|
||||
not exist inside the Shelfmark runtime environment (commonly different Docker
|
||||
volume mounts).
|
||||
|
||||
A mapping rewrites a remote path prefix into a local path prefix.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RemotePathMapping:
|
||||
host: str
|
||||
remote_path: str
|
||||
local_path: str
|
||||
|
||||
|
||||
def _normalize_prefix(path: str) -> str:
|
||||
normalized = str(path or "").strip()
|
||||
if not normalized:
|
||||
return ""
|
||||
|
||||
normalized = normalized.replace("\\", "/")
|
||||
|
||||
if normalized != "/":
|
||||
normalized = normalized.rstrip("/")
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _is_windows_path(path: str) -> bool:
|
||||
"""Check if a path looks like a Windows path (has a drive letter like C:/)."""
|
||||
return len(path) >= 2 and path[1] == ":" and path[0].isalpha()
|
||||
|
||||
|
||||
def _normalize_host(host: str) -> str:
|
||||
return str(host or "").strip().lower()
|
||||
|
||||
|
||||
def parse_remote_path_mappings(value: Any) -> list[RemotePathMapping]:
|
||||
if not value or not isinstance(value, list):
|
||||
return []
|
||||
|
||||
mappings: list[RemotePathMapping] = []
|
||||
|
||||
for row in value:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
|
||||
host = _normalize_host(row.get("host", ""))
|
||||
remote_path = _normalize_prefix(row.get("remotePath", ""))
|
||||
local_path = _normalize_prefix(row.get("localPath", ""))
|
||||
|
||||
if not host or not remote_path or not local_path:
|
||||
continue
|
||||
|
||||
mappings.append(RemotePathMapping(host=host, remote_path=remote_path, local_path=local_path))
|
||||
|
||||
mappings.sort(key=lambda m: len(m.remote_path), reverse=True)
|
||||
return mappings
|
||||
|
||||
|
||||
def remap_remote_to_local_with_match(
|
||||
*,
|
||||
mappings: Iterable[RemotePathMapping],
|
||||
host: str,
|
||||
remote_path: str | Path,
|
||||
) -> tuple[Path, bool]:
|
||||
host_normalized = _normalize_host(host)
|
||||
remote_normalized = _normalize_prefix(str(remote_path))
|
||||
|
||||
if not remote_normalized:
|
||||
return Path(str(remote_path)), False
|
||||
|
||||
# Windows paths are case-insensitive, so we need case-insensitive matching
|
||||
# for paths that look like Windows paths (e.g., D:/Torrents)
|
||||
is_windows = _is_windows_path(remote_normalized)
|
||||
|
||||
for mapping in mappings:
|
||||
if _normalize_host(mapping.host) != host_normalized:
|
||||
continue
|
||||
|
||||
remote_prefix = _normalize_prefix(mapping.remote_path)
|
||||
if not remote_prefix:
|
||||
continue
|
||||
|
||||
# For Windows paths, do case-insensitive prefix matching
|
||||
if is_windows:
|
||||
remote_lower = remote_normalized.lower()
|
||||
prefix_lower = remote_prefix.lower()
|
||||
matches = remote_lower == prefix_lower or remote_lower.startswith(prefix_lower + "/")
|
||||
else:
|
||||
matches = remote_normalized == remote_prefix or remote_normalized.startswith(remote_prefix + "/")
|
||||
|
||||
if matches:
|
||||
# Use the length of the original prefix to extract remainder
|
||||
# This preserves the original case in folder names
|
||||
remainder = remote_normalized[len(remote_prefix):]
|
||||
local_prefix = _normalize_prefix(mapping.local_path)
|
||||
|
||||
if remainder.startswith("/"):
|
||||
remainder = remainder[1:]
|
||||
|
||||
remapped = Path(local_prefix) / remainder if remainder else Path(local_prefix)
|
||||
return remapped, True
|
||||
|
||||
return Path(remote_normalized), False
|
||||
|
||||
|
||||
def remap_remote_to_local(*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path) -> Path:
|
||||
remapped, _ = remap_remote_to_local_with_match(
|
||||
mappings=mappings,
|
||||
host=host,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
return remapped
|
||||
|
||||
|
||||
def get_client_host_identifier(client: Any) -> Optional[str]:
|
||||
"""Return a stable identifier used by the mapping UI.
|
||||
|
||||
Sonarr uses the download client's configured host. Shelfmark currently uses
|
||||
the download client 'name' (e.g. qbittorrent, sabnzbd).
|
||||
"""
|
||||
|
||||
name = getattr(client, "name", None)
|
||||
if isinstance(name, str) and name.strip():
|
||||
return name.strip().lower()
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,31 @@
|
||||
"""WSGI middleware for hosting Shelfmark under a URL prefix."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Optional
|
||||
|
||||
|
||||
class PrefixMiddleware:
|
||||
"""Strip a configured URL prefix from PATH_INFO before routing."""
|
||||
|
||||
def __init__(self, app, prefix: str, bypass_paths: Optional[Iterable[str]] = None) -> None:
|
||||
self.app = app
|
||||
self.prefix = prefix.rstrip("/")
|
||||
self.bypass_paths = set(bypass_paths or [])
|
||||
|
||||
def __call__(self, environ, start_response):
|
||||
path = environ.get("PATH_INFO", "") or ""
|
||||
|
||||
if path in self.bypass_paths:
|
||||
return self.app(environ, start_response)
|
||||
|
||||
if not self.prefix:
|
||||
return self.app(environ, start_response)
|
||||
|
||||
if path == self.prefix or path.startswith(self.prefix + "/"):
|
||||
environ["SCRIPT_NAME"] = self.prefix
|
||||
environ["PATH_INFO"] = path[len(self.prefix):] or "/"
|
||||
return self.app(environ, start_response)
|
||||
|
||||
start_response("404 Not Found", [("Content-Type", "text/plain")])
|
||||
return [b"Not Found"]
|
||||
@@ -0,0 +1,164 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
|
||||
MANUAL_QUERY_MAX_LEN = 256
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.metadata_providers import (
|
||||
BookMetadata,
|
||||
group_languages_by_localized_title,
|
||||
build_localized_search_titles,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseSearchVariant:
|
||||
"""A single search variant (title + author) associated with languages."""
|
||||
|
||||
title: str
|
||||
author: str
|
||||
languages: Optional[List[str]] = None
|
||||
|
||||
@property
|
||||
def query(self) -> str:
|
||||
return " ".join(part for part in [self.title, self.author] if part).strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseSearchPlan:
|
||||
"""Pre-computed search inputs shared across release sources."""
|
||||
|
||||
languages: Optional[List[str]]
|
||||
isbn_candidates: List[str]
|
||||
author: str
|
||||
title_variants: List[ReleaseSearchVariant]
|
||||
grouped_title_variants: List[ReleaseSearchVariant]
|
||||
manual_query: Optional[str] = None
|
||||
|
||||
@property
|
||||
def primary_query(self) -> str:
|
||||
return self.title_variants[0].query if self.title_variants else ""
|
||||
|
||||
|
||||
def _normalize_languages(languages: Optional[List[str]]) -> Optional[List[str]]:
|
||||
if not languages:
|
||||
default = config.BOOK_LANGUAGE
|
||||
if not default:
|
||||
return None
|
||||
return [str(lang).strip() for lang in default if str(lang).strip()]
|
||||
|
||||
normalized: List[str] = []
|
||||
for lang in languages:
|
||||
if not lang:
|
||||
continue
|
||||
s = str(lang).strip()
|
||||
if not s:
|
||||
continue
|
||||
normalized.append(s)
|
||||
|
||||
if any(lang.lower() == "all" for lang in normalized):
|
||||
return None
|
||||
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _pick_search_author(book: BookMetadata) -> str:
|
||||
if book.search_author:
|
||||
return book.search_author
|
||||
|
||||
if not book.authors:
|
||||
return ""
|
||||
|
||||
first = book.authors[0]
|
||||
if "," in first:
|
||||
first = first.split(",")[0].strip()
|
||||
|
||||
return first
|
||||
|
||||
|
||||
def _pick_search_title(book: BookMetadata) -> str:
|
||||
return book.search_title or book.title
|
||||
|
||||
|
||||
def build_release_search_plan(
|
||||
book: BookMetadata,
|
||||
languages: Optional[List[str]] = None,
|
||||
manual_query: Optional[str] = None,
|
||||
) -> ReleaseSearchPlan:
|
||||
resolved_languages = _normalize_languages(languages)
|
||||
|
||||
resolved_manual_query = None
|
||||
if manual_query:
|
||||
resolved_manual_query = manual_query.strip()[:MANUAL_QUERY_MAX_LEN] or None
|
||||
|
||||
author = _pick_search_author(book)
|
||||
base_title = _pick_search_title(book)
|
||||
|
||||
if resolved_manual_query:
|
||||
# Manual override: use the raw query as-is (no language/title expansion).
|
||||
variant = ReleaseSearchVariant(title=resolved_manual_query, author="", languages=None)
|
||||
return ReleaseSearchPlan(
|
||||
languages=resolved_languages,
|
||||
isbn_candidates=[],
|
||||
author="",
|
||||
title_variants=[variant],
|
||||
grouped_title_variants=[variant],
|
||||
manual_query=resolved_manual_query,
|
||||
)
|
||||
|
||||
isbn_candidates: List[str] = []
|
||||
if book.isbn_13:
|
||||
isbn_candidates.append(book.isbn_13)
|
||||
if book.isbn_10 and book.isbn_10 not in isbn_candidates:
|
||||
isbn_candidates.append(book.isbn_10)
|
||||
|
||||
titles_by_language = book.titles_by_language or None
|
||||
if book.search_title and titles_by_language:
|
||||
titles_by_language = {
|
||||
k: v
|
||||
for k, v in titles_by_language.items()
|
||||
if str(k).strip().lower() not in {"en", "eng", "english"}
|
||||
}
|
||||
|
||||
grouped = group_languages_by_localized_title(
|
||||
base_title=base_title,
|
||||
languages=resolved_languages,
|
||||
titles_by_language=titles_by_language,
|
||||
)
|
||||
|
||||
grouped_variants: List[ReleaseSearchVariant] = [
|
||||
ReleaseSearchVariant(title=title, author=author, languages=langs)
|
||||
for title, langs in grouped
|
||||
if title
|
||||
]
|
||||
|
||||
expanded_titles = build_localized_search_titles(
|
||||
base_title=base_title,
|
||||
languages=resolved_languages,
|
||||
titles_by_language=titles_by_language,
|
||||
excluded_languages={"en", "eng", "english"},
|
||||
)
|
||||
|
||||
title_variants: List[ReleaseSearchVariant] = [
|
||||
ReleaseSearchVariant(title=title, author=author, languages=None)
|
||||
for title in expanded_titles
|
||||
if title
|
||||
]
|
||||
|
||||
# If no titles could be built, fall back to ISBN queries.
|
||||
if not title_variants and isbn_candidates:
|
||||
title_variants = [
|
||||
ReleaseSearchVariant(title=isbn, author="", languages=None)
|
||||
for isbn in isbn_candidates
|
||||
]
|
||||
|
||||
return ReleaseSearchPlan(
|
||||
languages=resolved_languages,
|
||||
isbn_candidates=isbn_candidates,
|
||||
author=author,
|
||||
title_variants=title_variants,
|
||||
grouped_title_variants=grouped_variants,
|
||||
manual_query=None,
|
||||
)
|
||||
@@ -24,7 +24,7 @@ class FieldBase:
|
||||
env_supported: bool = True # Whether this setting can be set via ENV var (False = UI-only)
|
||||
disabled: bool = False # Whether field is disabled/greyed out
|
||||
disabled_reason: str = "" # Explanation shown when disabled
|
||||
show_when: Optional[Dict[str, Any]] = None # Conditional visibility: {"field": "key", "value": "expected"} or {"field": "key", "notEmpty": True}
|
||||
show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
|
||||
disabled_when: Optional[Dict[str, Any]] = None # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
|
||||
requires_restart: bool = False # Whether changing this setting requires a container restart
|
||||
universal_only: bool = False # Only show in Universal search mode (hide in Direct mode)
|
||||
@@ -71,6 +71,7 @@ class SelectField(FieldBase):
|
||||
"""Single-choice dropdown."""
|
||||
# Options can be a list or a callable that returns a list (for lazy evaluation)
|
||||
options: Any = field(default_factory=list) # [{value: "", label: ""}] or callable
|
||||
filter_by_field: Optional[str] = None # Field key whose value filters options via childOf property
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -93,16 +94,30 @@ class OrderableListField(FieldBase):
|
||||
default: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableField(FieldBase):
|
||||
"""Editable table of structured rows."""
|
||||
|
||||
# Column definitions: [{key, label, type, placeholder?, options?, defaultValue?}, ...]
|
||||
columns: Any = field(default_factory=list) # list or callable
|
||||
|
||||
# Value format: list of objects
|
||||
default: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
add_label: str = "Add"
|
||||
empty_message: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionButton:
|
||||
key: str # Action identifier
|
||||
label: str # Button text
|
||||
description: str = "" # Help text
|
||||
style: str = "default" # "default", "primary", "danger"
|
||||
callback: Optional[Callable[[], Dict[str, Any]]] = None # Returns {"success": bool, "message": str}
|
||||
callback: Optional[Callable[..., Dict[str, Any]]] = None # Returns {"success": bool, "message": str}
|
||||
disabled: bool = False # Whether button is disabled/greyed out
|
||||
disabled_reason: str = "" # Explanation shown when disabled
|
||||
show_when: Optional[Dict[str, Any]] = None # Conditional visibility: {"field": "key", "value": "expected"} or {"field": "key", "notEmpty": True}
|
||||
show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
|
||||
disabled_when: Optional[Dict[str, Any]] = None # Conditional disable: {"field": "key", "value": "expected", "reason": "..."}
|
||||
|
||||
def get_field_type(self) -> str:
|
||||
@@ -122,7 +137,7 @@ class HeadingField:
|
||||
description: str = "" # Description text (supports markdown-style links)
|
||||
link_url: str = "" # Optional URL for a link
|
||||
link_text: str = "" # Text for the link (defaults to URL if not provided)
|
||||
show_when: Optional[Dict[str, Any]] = None # Conditional visibility: {"field": "key", "value": "expected"} or {"field": "key", "notEmpty": True}
|
||||
show_when: Optional[Dict[str, Any] | List[Dict[str, Any]]] = None # Conditional visibility: {"field": "key", "value": "expected"} or list of conditions
|
||||
universal_only: bool = False # Only show in Universal search mode (hide in Direct mode)
|
||||
|
||||
def get_field_type(self) -> str:
|
||||
@@ -534,6 +549,14 @@ def _parse_env_value(value: str, field: SettingsField) -> Any:
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"Invalid JSON for {field.key}, using default")
|
||||
return field.default
|
||||
elif isinstance(field, TableField):
|
||||
# Parse JSON array: [{"col": "value"}, ...]
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
return parsed if isinstance(parsed, list) else field.default
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"Invalid JSON for {field.key}, using default")
|
||||
return field.default
|
||||
else:
|
||||
return value
|
||||
|
||||
@@ -562,7 +585,7 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
|
||||
"""
|
||||
# HeadingField has a different structure - handle separately
|
||||
if isinstance(field, HeadingField):
|
||||
result = {
|
||||
result: Dict[str, Any] = {
|
||||
"key": field.key,
|
||||
"type": field.get_field_type(),
|
||||
"title": field.title,
|
||||
@@ -577,7 +600,7 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
|
||||
result["universalOnly"] = True
|
||||
return result
|
||||
|
||||
result = {
|
||||
result: Dict[str, Any] = {
|
||||
"key": field.key,
|
||||
"label": field.label,
|
||||
"type": field.get_field_type(),
|
||||
@@ -613,6 +636,8 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
|
||||
result["options"] = options
|
||||
if field.default is not None:
|
||||
result["default"] = field.default
|
||||
if field.filter_by_field:
|
||||
result["filterByField"] = field.filter_by_field
|
||||
elif isinstance(field, MultiSelectField):
|
||||
# Support callable options for lazy evaluation (avoids circular imports)
|
||||
options = field.options() if callable(field.options) else field.options
|
||||
@@ -622,12 +647,39 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
|
||||
# Support callable options for lazy evaluation (avoids circular imports)
|
||||
options = field.options() if callable(field.options) else field.options
|
||||
result["options"] = options
|
||||
elif isinstance(field, TableField):
|
||||
columns = field.columns() if callable(field.columns) else field.columns
|
||||
result["columns"] = columns
|
||||
result["addLabel"] = field.add_label
|
||||
result["emptyMessage"] = field.empty_message
|
||||
elif isinstance(field, ActionButton):
|
||||
result["style"] = field.style
|
||||
result["description"] = field.description
|
||||
|
||||
if include_value and not isinstance(field, (ActionButton, HeadingField)):
|
||||
value = get_setting_value(field, tab_name)
|
||||
|
||||
# Ensure select values are serialized as strings so the frontend can
|
||||
# reliably match against string option values.
|
||||
if isinstance(field, SelectField) and value is not None:
|
||||
value = str(value)
|
||||
elif isinstance(field, MultiSelectField):
|
||||
if value is None:
|
||||
value = []
|
||||
elif isinstance(value, list):
|
||||
value = [str(v) for v in value]
|
||||
elif isinstance(value, str):
|
||||
# Support legacy/manual configs where MultiSelect values were saved
|
||||
# as comma-separated strings.
|
||||
value = [v.strip() for v in value.split(",") if v.strip()]
|
||||
else:
|
||||
value = []
|
||||
elif isinstance(field, TableField):
|
||||
if value is None:
|
||||
value = []
|
||||
elif not isinstance(value, list):
|
||||
value = []
|
||||
|
||||
result["value"] = value if value is not None else ""
|
||||
result["fromEnv"] = is_value_from_env(field)
|
||||
|
||||
@@ -696,7 +748,7 @@ def execute_action(tab_name: str, action_key: str, current_values: Optional[Dict
|
||||
try:
|
||||
# Check if callback accepts current_values parameter
|
||||
sig = inspect.signature(field.callback)
|
||||
if 'current_values' in sig.parameters:
|
||||
if "current_values" in sig.parameters:
|
||||
return field.callback(current_values=current_values or {})
|
||||
else:
|
||||
return field.callback()
|
||||
@@ -816,16 +868,22 @@ def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# Save to config file
|
||||
if save_config_file(tab_name, values_to_save):
|
||||
# Refresh the config singleton so live settings take effect immediately
|
||||
config_obj = None
|
||||
try:
|
||||
from shelfmark.core.config import config
|
||||
config.refresh()
|
||||
from shelfmark.core.config import config as config_obj
|
||||
|
||||
config_obj.refresh()
|
||||
except ImportError:
|
||||
pass # Config module not yet available during initial setup
|
||||
config_obj = None # Config module not yet available during initial setup
|
||||
|
||||
# Apply DNS settings changes live (network tab)
|
||||
dns_keys = {"CUSTOM_DNS", "CUSTOM_DNS_MANUAL", "USE_DOH"}
|
||||
if tab_name == "network" and dns_keys.intersection(values_to_save.keys()):
|
||||
_apply_dns_settings(config)
|
||||
if (
|
||||
config_obj is not None
|
||||
and tab_name == "network"
|
||||
and dns_keys.intersection(values_to_save.keys())
|
||||
):
|
||||
_apply_dns_settings(config_obj)
|
||||
|
||||
# Sync metadata provider selection when a provider's enabled state changes
|
||||
tab = get_settings_tab(tab_name)
|
||||
|
||||
@@ -3,6 +3,75 @@
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def normalize_http_url(
|
||||
url: Optional[str],
|
||||
*,
|
||||
default_scheme: str = "http",
|
||||
strip_trailing_slash: bool = True,
|
||||
allow_special: tuple[str, ...] = (),
|
||||
) -> str:
|
||||
"""Normalize a configured HTTP URL for requests and links."""
|
||||
if not isinstance(url, str):
|
||||
return ""
|
||||
|
||||
normalized = url.strip()
|
||||
if not normalized:
|
||||
return ""
|
||||
|
||||
if (normalized.startswith("\"") and normalized.endswith("\"")) or (
|
||||
normalized.startswith("'") and normalized.endswith("'")
|
||||
):
|
||||
normalized = normalized[1:-1].strip()
|
||||
if not normalized:
|
||||
return ""
|
||||
|
||||
if allow_special:
|
||||
special_map = {
|
||||
value.lower(): value
|
||||
for value in allow_special
|
||||
if isinstance(value, str)
|
||||
}
|
||||
special_match = special_map.get(normalized.lower())
|
||||
if special_match is not None:
|
||||
return special_match
|
||||
|
||||
if normalized.startswith(("/", "./", "../")):
|
||||
return normalized
|
||||
|
||||
if "://" not in normalized:
|
||||
scheme = default_scheme.strip().rstrip(":/")
|
||||
if scheme:
|
||||
normalized = f"{scheme}://{normalized}"
|
||||
|
||||
if strip_trailing_slash:
|
||||
normalized = normalized.rstrip("/")
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_base_path(value: Optional[str]) -> str:
|
||||
"""Normalize a URL base path for reverse proxy subpath deployments."""
|
||||
if not isinstance(value, str):
|
||||
return ""
|
||||
|
||||
path = value.strip()
|
||||
if not path:
|
||||
return ""
|
||||
|
||||
if "://" in path:
|
||||
parsed = urlparse(path)
|
||||
path = parsed.path or ""
|
||||
|
||||
if not path or path == "/":
|
||||
return ""
|
||||
|
||||
if not path.startswith("/"):
|
||||
path = "/" + path
|
||||
|
||||
return path.rstrip("/")
|
||||
|
||||
|
||||
def is_audiobook(content_type: Optional[str]) -> bool:
|
||||
|
||||
+22
-232
@@ -3,107 +3,20 @@
|
||||
import os
|
||||
import shutil
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.naming import parse_naming_template, sanitize_filename
|
||||
from shelfmark.download.postprocess.policy import (
|
||||
get_supported_audiobook_formats,
|
||||
get_supported_formats,
|
||||
)
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.fs import atomic_write, atomic_move
|
||||
from shelfmark.download.fs import atomic_write
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def _get_supported_formats() -> List[str]:
|
||||
"""Get current supported formats from config singleton."""
|
||||
formats = config.get("SUPPORTED_FORMATS", ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"])
|
||||
# Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
|
||||
if isinstance(formats, str):
|
||||
return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
|
||||
return [fmt.lower() for fmt in formats]
|
||||
|
||||
|
||||
def _get_supported_audiobook_formats() -> List[str]:
|
||||
"""Get current supported audiobook formats from config singleton."""
|
||||
formats = config.get("SUPPORTED_AUDIOBOOK_FORMATS", ["m4b", "mp3"])
|
||||
# Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
|
||||
if isinstance(formats, str):
|
||||
return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
|
||||
return [fmt.lower() for fmt in formats]
|
||||
|
||||
|
||||
def _get_file_organization(is_audiobook: bool) -> str:
|
||||
"""Get the file organization mode for the content type."""
|
||||
key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
|
||||
mode = config.get(key, "rename")
|
||||
|
||||
# Handle legacy settings migration
|
||||
if mode not in ("none", "rename", "organize"):
|
||||
legacy_key = "PROCESSING_MODE_AUDIOBOOK" if is_audiobook else "PROCESSING_MODE"
|
||||
legacy_mode = config.get(legacy_key, "ingest")
|
||||
if legacy_mode == "library":
|
||||
return "organize"
|
||||
if config.get("USE_BOOK_TITLE", True):
|
||||
return "rename"
|
||||
return "none"
|
||||
|
||||
return mode
|
||||
|
||||
|
||||
def _get_template(is_audiobook: bool, organization_mode: str) -> str:
|
||||
"""Get the template for the content type and organization mode."""
|
||||
# Determine the correct key based on content type and organization mode
|
||||
if is_audiobook:
|
||||
if organization_mode == "organize":
|
||||
key = "TEMPLATE_AUDIOBOOK_ORGANIZE"
|
||||
else:
|
||||
key = "TEMPLATE_AUDIOBOOK_RENAME"
|
||||
else:
|
||||
if organization_mode == "organize":
|
||||
key = "TEMPLATE_ORGANIZE"
|
||||
else:
|
||||
key = "TEMPLATE_RENAME"
|
||||
|
||||
template = config.get(key, "")
|
||||
|
||||
# Fallback to legacy keys if new keys are empty
|
||||
if not template:
|
||||
legacy_key = "TEMPLATE_AUDIOBOOK" if is_audiobook else "TEMPLATE"
|
||||
template = config.get(legacy_key, "")
|
||||
|
||||
if not template:
|
||||
legacy_key = "LIBRARY_TEMPLATE_AUDIOBOOK" if is_audiobook else "LIBRARY_TEMPLATE"
|
||||
template = config.get(legacy_key, "")
|
||||
|
||||
if not template:
|
||||
if organization_mode == "organize":
|
||||
return "{Author}/{Title} ({Year})"
|
||||
return "{Author} - {Title} ({Year})"
|
||||
|
||||
return template
|
||||
|
||||
|
||||
def _build_filename_from_task(task, extension: str, organization_mode: str) -> str:
|
||||
"""Build a filename from task metadata using the configured template."""
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
|
||||
template = _get_template(is_audiobook, organization_mode)
|
||||
metadata = {
|
||||
"Author": task.author,
|
||||
"Title": task.title,
|
||||
"Subtitle": getattr(task, 'subtitle', None),
|
||||
"Year": task.year,
|
||||
"Series": getattr(task, 'series_name', None),
|
||||
"SeriesPosition": getattr(task, 'series_position', None),
|
||||
}
|
||||
|
||||
filename = parse_naming_template(template, metadata)
|
||||
if filename:
|
||||
return f"{sanitize_filename(filename)}.{extension}"
|
||||
return ""
|
||||
|
||||
# Check for rarfile availability at module load
|
||||
try:
|
||||
import rarfile
|
||||
@@ -142,9 +55,9 @@ def _is_supported_file(file_path: Path, content_type: Optional[str] = None) -> b
|
||||
"""Check if file matches user's supported formats setting based on content type."""
|
||||
ext = file_path.suffix.lower().lstrip(".")
|
||||
if check_audiobook(content_type):
|
||||
supported_formats = _get_supported_audiobook_formats()
|
||||
supported_formats = get_supported_audiobook_formats()
|
||||
else:
|
||||
supported_formats = _get_supported_formats()
|
||||
supported_formats = get_supported_formats()
|
||||
return ext in supported_formats
|
||||
|
||||
|
||||
@@ -225,6 +138,21 @@ def extract_archive(
|
||||
return matched_files, warnings, rejected_files
|
||||
|
||||
|
||||
def extract_archive_raw(
|
||||
archive_path: Path,
|
||||
output_dir: Path,
|
||||
) -> Tuple[List[Path], List[str]]:
|
||||
"""Extract archive without filtering (returns all extracted files)."""
|
||||
suffix = archive_path.suffix.lower().lstrip(".")
|
||||
|
||||
if suffix == "zip":
|
||||
return _extract_zip(archive_path, output_dir)
|
||||
if suffix == "rar":
|
||||
return _extract_rar(archive_path, output_dir)
|
||||
|
||||
raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
|
||||
|
||||
|
||||
def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
|
||||
"""Extract files from ZipFile or RarFile to output_dir with security checks."""
|
||||
extracted_files = []
|
||||
@@ -309,141 +237,3 @@ def _extract_rar(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List
|
||||
raise ArchiveExtractionError(f"Permission denied: {e}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchiveResult:
|
||||
"""Result of archive processing."""
|
||||
|
||||
success: bool
|
||||
final_paths: List[Path]
|
||||
message: str
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def process_archive(
|
||||
archive_path: Path,
|
||||
temp_dir: Path,
|
||||
ingest_dir: Path,
|
||||
archive_id: str,
|
||||
task: Optional["DownloadTask"] = None,
|
||||
) -> ArchiveResult:
|
||||
"""Extract archive, filter to supported formats, move to ingest directory."""
|
||||
extract_dir = temp_dir / f"extract_{archive_id}"
|
||||
content_type = task.content_type if task else None
|
||||
is_audiobook = check_audiobook(content_type)
|
||||
file_type_label = "audiobook" if is_audiobook else "book"
|
||||
|
||||
try:
|
||||
# Create temp extraction directory
|
||||
os.makedirs(extract_dir, exist_ok=True)
|
||||
os.makedirs(ingest_dir, exist_ok=True)
|
||||
|
||||
# Extract to temp directory (filters based on content type)
|
||||
extracted_files, warnings, rejected_files = extract_archive(archive_path, extract_dir, content_type)
|
||||
|
||||
if not extracted_files:
|
||||
# Clean up and return error
|
||||
shutil.rmtree(extract_dir, ignore_errors=True)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
|
||||
if rejected_files:
|
||||
# Found files but they weren't in supported formats
|
||||
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
|
||||
rejected_list = ", ".join(rejected_exts)
|
||||
supported_formats = _get_supported_audiobook_formats() if is_audiobook else _get_supported_formats()
|
||||
logger.warning(
|
||||
f"Found {len(rejected_files)} {file_type_label}(s) in archive but format not supported. "
|
||||
f"Rejected: {rejected_list}. Supported: {', '.join(sorted(supported_formats))}"
|
||||
)
|
||||
return ArchiveResult(
|
||||
success=False,
|
||||
final_paths=[],
|
||||
message="",
|
||||
error=f"Found {len(rejected_files)} {file_type_label}(s) but format not supported ({rejected_list}). Enable in Settings > Formats.",
|
||||
)
|
||||
|
||||
return ArchiveResult(
|
||||
success=False,
|
||||
final_paths=[],
|
||||
message="",
|
||||
error=f"No {file_type_label} files found in archive",
|
||||
)
|
||||
|
||||
for warning in warnings:
|
||||
logger.debug(warning)
|
||||
|
||||
logger.info(f"Extracted {len(extracted_files)} {file_type_label} file(s) from archive")
|
||||
|
||||
# Move book files to ingest folder
|
||||
final_paths = []
|
||||
|
||||
# Determine file organization mode
|
||||
is_audiobook = check_audiobook(task.content_type) if task else False
|
||||
organization_mode = _get_file_organization(is_audiobook) if task else "none"
|
||||
|
||||
for extracted_file in extracted_files:
|
||||
# For multi-file archives (book packs, series), always preserve original filenames
|
||||
# since metadata title only applies to the searched book, not the whole pack.
|
||||
# For single files, respect FILE_ORGANIZATION setting.
|
||||
if len(extracted_files) == 1 and organization_mode != "none" and task:
|
||||
# Use the extracted file's actual extension, not the archive's extension
|
||||
extracted_format = extracted_file.suffix.lower().lstrip('.')
|
||||
filename = _build_filename_from_task(task, extracted_format, organization_mode)
|
||||
if not filename:
|
||||
filename = extracted_file.name
|
||||
else:
|
||||
filename = extracted_file.name
|
||||
|
||||
dest_path = ingest_dir / filename
|
||||
final_path = atomic_move(extracted_file, dest_path)
|
||||
final_paths.append(final_path)
|
||||
logger.debug(f"Moved to ingest: {final_path.name}")
|
||||
|
||||
# Clean up temp extraction directory and archive
|
||||
shutil.rmtree(extract_dir, ignore_errors=True)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
|
||||
# Build success message with format info
|
||||
formats = [p.suffix.lstrip(".").upper() for p in final_paths]
|
||||
if len(formats) == 1:
|
||||
message = f"Complete ({formats[0]})"
|
||||
else:
|
||||
message = f"Complete ({len(formats)} files)"
|
||||
|
||||
return ArchiveResult(
|
||||
success=True,
|
||||
final_paths=final_paths,
|
||||
message=message,
|
||||
)
|
||||
|
||||
except PasswordProtectedError:
|
||||
logger.error(f"Password-protected archive: {archive_path.name}")
|
||||
shutil.rmtree(extract_dir, ignore_errors=True)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return ArchiveResult(
|
||||
success=False,
|
||||
final_paths=[],
|
||||
message="",
|
||||
error="Archive is password protected",
|
||||
)
|
||||
|
||||
except CorruptedArchiveError as e:
|
||||
logger.error(f"Corrupted archive: {e}")
|
||||
shutil.rmtree(extract_dir, ignore_errors=True)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return ArchiveResult(
|
||||
success=False,
|
||||
final_paths=[],
|
||||
message="",
|
||||
error=f"Corrupted archive: {e}",
|
||||
)
|
||||
|
||||
except ArchiveExtractionError as e:
|
||||
logger.error(f"Archive extraction failed: {e}")
|
||||
shutil.rmtree(extract_dir, ignore_errors=True)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return ArchiveResult(
|
||||
success=False,
|
||||
final_paths=[],
|
||||
message="",
|
||||
error=f"Extraction failed: {e}",
|
||||
)
|
||||
|
||||
+172
-32
@@ -8,13 +8,47 @@ import errno
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.download.permissions_debug import log_transfer_permission_context
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
|
||||
_VERIFY_IO_WAIT_SECONDS = 3.0
|
||||
|
||||
|
||||
def _verify_transfer_size(
|
||||
dest: Path,
|
||||
expected_size: int,
|
||||
action: str,
|
||||
) -> None:
|
||||
"""Verify file transfer completed successfully.
|
||||
|
||||
Some filesystems (especially remote NAS/CIFS/NFS) can report stale sizes briefly
|
||||
after large writes. Do a second stat after a short delay before declaring failure.
|
||||
"""
|
||||
actual_size = dest.stat().st_size
|
||||
if actual_size == expected_size:
|
||||
return
|
||||
|
||||
logger.debug(
|
||||
f"File {action} size mismatch, waiting for filesystem sync: {dest} "
|
||||
f"({actual_size} != {expected_size})"
|
||||
)
|
||||
time.sleep(_VERIFY_IO_WAIT_SECONDS)
|
||||
|
||||
actual_size = dest.stat().st_size
|
||||
if actual_size != expected_size:
|
||||
raise IOError(
|
||||
f"File {action} incomplete, data loss may have occurred. "
|
||||
f"'{dest}' was {actual_size} bytes instead of expected {expected_size}."
|
||||
)
|
||||
|
||||
|
||||
def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
|
||||
"""Write data to a file with atomic collision detection.
|
||||
|
||||
@@ -40,7 +74,7 @@ def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
|
||||
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
|
||||
try:
|
||||
# O_CREAT | O_EXCL fails atomically if file exists
|
||||
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
|
||||
try:
|
||||
os.write(fd, data)
|
||||
finally:
|
||||
@@ -61,7 +95,7 @@ def _is_permission_error(e: Exception) -> bool:
|
||||
|
||||
def _system_op(op: str, source: Path, dest: Path) -> None:
|
||||
"""Execute system command (mv or cp) as final fallback."""
|
||||
logger.info(f"Attempting system {op} as final fallback: {source} -> {dest}")
|
||||
logger.warning("Attempting system %s as final fallback: %s -> %s", op, source, dest)
|
||||
subprocess.run(
|
||||
[op, "-f", str(source), str(dest)],
|
||||
check=True,
|
||||
@@ -71,36 +105,56 @@ def _system_op(op: str, source: Path, dest: Path) -> None:
|
||||
|
||||
|
||||
def _perform_nfs_fallback(source: Path, dest: Path, is_move: bool) -> None:
|
||||
"""Handle NFS permission errors by falling back to copyfile -> system op."""
|
||||
"""Handle NFS/SMB permission errors by falling back to copyfile -> system op."""
|
||||
expected_size = source.stat().st_size
|
||||
|
||||
try:
|
||||
# Fallback 1: copy content only
|
||||
shutil.copyfile(str(source), str(dest))
|
||||
|
||||
_verify_transfer_size(dest, expected_size, "copy")
|
||||
|
||||
if is_move:
|
||||
# Verify copy success before removing source
|
||||
if dest.exists() and dest.stat().st_size == source.stat().st_size:
|
||||
source.unlink()
|
||||
return
|
||||
else:
|
||||
raise IOError(f"Copy verification failed for {source} -> {dest}")
|
||||
source.unlink()
|
||||
return
|
||||
|
||||
except Exception as copy_error:
|
||||
# Clean up failed copy attempt if it exists
|
||||
if dest.exists():
|
||||
dest.unlink(missing_ok=True)
|
||||
|
||||
logger.error(f"Fallback copyfile failed: {copy_error}")
|
||||
|
||||
dest.unlink(missing_ok=True)
|
||||
|
||||
if _is_permission_error(copy_error):
|
||||
log_transfer_permission_context("nfs_fallback_copyfile", source=source, dest=dest, error=copy_error)
|
||||
logger.error("Fallback copyfile failed (%s -> %s): %s", source, dest, copy_error)
|
||||
|
||||
# Fallback 2: system command
|
||||
op = "mv" if is_move else "cp"
|
||||
try:
|
||||
_system_op(op, source, dest)
|
||||
# Best-effort verify after external command.
|
||||
if dest.exists():
|
||||
_verify_transfer_size(dest, expected_size, op)
|
||||
if is_move:
|
||||
source.unlink(missing_ok=True)
|
||||
except subprocess.CalledProcessError as sys_error:
|
||||
logger.error(f"System {op} failed: {sys_error.stderr}")
|
||||
log_transfer_permission_context("nfs_fallback_system", source=source, dest=dest, error=sys_error)
|
||||
logger.error("System %s failed (%s -> %s): %s", op, source, dest, sys_error.stderr)
|
||||
dest.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def _claim_destination(path: Path) -> bool:
|
||||
"""Atomically claim a destination path by creating a placeholder file.
|
||||
|
||||
Returns True if the placeholder was created. Caller must replace or unlink it.
|
||||
"""
|
||||
try:
|
||||
fd = os.open(str(path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
|
||||
except FileExistsError:
|
||||
return False
|
||||
else:
|
||||
os.close(fd)
|
||||
return True
|
||||
|
||||
|
||||
def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
|
||||
"""Move a file with collision detection.
|
||||
|
||||
@@ -130,49 +184,101 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
|
||||
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
|
||||
|
||||
# Check for existing file (os.rename would overwrite on Unix)
|
||||
claimed = False
|
||||
if try_path.exists():
|
||||
continue
|
||||
# Some filesystems can report false positives for exists() with
|
||||
# special characters. Probe with O_EXCL to confirm.
|
||||
claimed = _claim_destination(try_path)
|
||||
if not claimed:
|
||||
continue
|
||||
|
||||
try:
|
||||
# os.rename is atomic on same filesystem and triggers inotify events
|
||||
os.rename(str(source_path), str(try_path))
|
||||
if claimed:
|
||||
os.replace(str(source_path), str(try_path))
|
||||
else:
|
||||
os.rename(str(source_path), str(try_path))
|
||||
if attempt > 0:
|
||||
logger.info(f"File collision resolved: {try_path.name}")
|
||||
return try_path
|
||||
except FileExistsError:
|
||||
# Race condition: file created between exists() check and rename()
|
||||
if claimed:
|
||||
try_path.unlink(missing_ok=True)
|
||||
continue
|
||||
except OSError as e:
|
||||
# Cross-filesystem - fall back to exclusive create + move
|
||||
# Cross-filesystem - fall back to exclusive create + verified copy + delete.
|
||||
if e.errno != errno.EXDEV:
|
||||
if claimed:
|
||||
try_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
expected_size = source_path.stat().st_size
|
||||
|
||||
try:
|
||||
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
os.close(fd)
|
||||
if not claimed:
|
||||
# Claim destination path atomically.
|
||||
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
|
||||
os.close(fd)
|
||||
|
||||
# Copy to a temp file first, then replace to avoid partial files.
|
||||
temp_path = try_path.parent / f".{try_path.name}.tmp"
|
||||
try:
|
||||
shutil.move(str(source_path), str(try_path))
|
||||
try:
|
||||
shutil.copy2(str(source_path), str(temp_path))
|
||||
except (PermissionError, OSError) as copy_error:
|
||||
if _is_permission_error(copy_error):
|
||||
logger.debug(
|
||||
"Permission error during move-copy, falling back to copyfile (%s -> %s): %s",
|
||||
source_path,
|
||||
temp_path,
|
||||
copy_error,
|
||||
)
|
||||
_perform_nfs_fallback(source_path, temp_path, is_move=False)
|
||||
else:
|
||||
raise
|
||||
|
||||
temp_path.replace(try_path)
|
||||
_verify_transfer_size(try_path, expected_size, "move")
|
||||
source_path.unlink()
|
||||
|
||||
if attempt > 0:
|
||||
logger.info(f"File collision resolved: {try_path.name}")
|
||||
return try_path
|
||||
|
||||
except Exception:
|
||||
# Clean up the placeholder if move failed
|
||||
if try_path.exists() and try_path.stat().st_size == 0:
|
||||
try_path.unlink(missing_ok=True)
|
||||
try_path.unlink(missing_ok=True)
|
||||
temp_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
except FileExistsError:
|
||||
continue
|
||||
except (PermissionError, OSError) as e:
|
||||
# Handle NFS permission errors (e.g. inability to set metadata)
|
||||
if _is_permission_error(e):
|
||||
logger.debug(f"Permission error during move, falling back to copyfile: {e}")
|
||||
log_transfer_permission_context(
|
||||
"atomic_move",
|
||||
source=source_path,
|
||||
dest=try_path,
|
||||
error=e,
|
||||
)
|
||||
logger.debug(
|
||||
"Permission error during move, falling back to copyfile (%s -> %s): %s",
|
||||
source_path,
|
||||
try_path,
|
||||
e,
|
||||
)
|
||||
try:
|
||||
_perform_nfs_fallback(source_path, try_path, is_move=True)
|
||||
if attempt > 0:
|
||||
logger.info(f"File collision resolved (fallback): {try_path.name}")
|
||||
return try_path
|
||||
except Exception as fallback_error:
|
||||
# Fallback failed, chain exceptions for better debugging
|
||||
logger.error(f"NFS fallback also failed: {fallback_error}")
|
||||
logger.error(
|
||||
"NFS fallback also failed (%s -> %s): %s",
|
||||
source_path,
|
||||
try_path,
|
||||
fallback_error,
|
||||
)
|
||||
raise e from fallback_error
|
||||
raise
|
||||
|
||||
@@ -206,6 +312,23 @@ def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100)
|
||||
return try_path
|
||||
except FileExistsError:
|
||||
continue
|
||||
except OSError as e:
|
||||
if _is_permission_error(e) or e.errno in (errno.EXDEV, errno.EMLINK):
|
||||
if _is_permission_error(e):
|
||||
log_transfer_permission_context(
|
||||
"atomic_hardlink",
|
||||
source=source_path,
|
||||
dest=try_path,
|
||||
error=e,
|
||||
)
|
||||
logger.debug(
|
||||
"Hardlink failed (%s), falling back to copy: %s -> %s",
|
||||
e,
|
||||
source_path,
|
||||
dest_path,
|
||||
)
|
||||
return atomic_copy(source_path, dest_path, max_attempts=max_attempts)
|
||||
raise
|
||||
|
||||
raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
|
||||
|
||||
@@ -235,7 +358,7 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
|
||||
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
|
||||
try:
|
||||
# Atomically claim the destination by creating an exclusive file
|
||||
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
|
||||
os.close(fd)
|
||||
|
||||
# Copy to temp file first, then replace to avoid partial files
|
||||
@@ -246,16 +369,33 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
|
||||
except (PermissionError, OSError) as e:
|
||||
# Handle NFS permission errors immediately here
|
||||
if _is_permission_error(e):
|
||||
logger.debug(f"Permission error during copy, falling back to copyfile: {e}")
|
||||
log_transfer_permission_context(
|
||||
"atomic_copy",
|
||||
source=source_path,
|
||||
dest=temp_path,
|
||||
error=e,
|
||||
)
|
||||
logger.debug(
|
||||
"Permission error during copy, falling back to copyfile (%s -> %s): %s",
|
||||
source_path,
|
||||
temp_path,
|
||||
e,
|
||||
)
|
||||
try:
|
||||
_perform_nfs_fallback(source_path, temp_path, is_move=False)
|
||||
except Exception as fallback_error:
|
||||
logger.error(f"NFS fallback also failed: {fallback_error}")
|
||||
logger.error(
|
||||
"NFS fallback also failed (%s -> %s): %s",
|
||||
source_path,
|
||||
temp_path,
|
||||
fallback_error,
|
||||
)
|
||||
raise e from fallback_error
|
||||
else:
|
||||
raise
|
||||
|
||||
temp_path.replace(try_path)
|
||||
_verify_transfer_size(try_path, source_path.stat().st_size, "copy")
|
||||
if attempt > 0:
|
||||
logger.info(f"File collision resolved: {try_path.name}")
|
||||
return try_path
|
||||
|
||||
@@ -176,8 +176,14 @@ def html_get_page(
|
||||
selector: Optional[network.AAMirrorSelector] = None,
|
||||
cancel_flag: Optional[Event] = None,
|
||||
status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
|
||||
allow_bypasser_fallback: bool = True,
|
||||
) -> str:
|
||||
"""Fetch HTML content from a URL with retry mechanism."""
|
||||
"""Fetch HTML content from a URL with retry mechanism.
|
||||
|
||||
Args:
|
||||
allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
|
||||
instead of switching to the bypasser. Use for search operations.
|
||||
"""
|
||||
retry = retry if retry is not None else app_config.MAX_RETRY
|
||||
selector = selector or network.AAMirrorSelector()
|
||||
original_url = url
|
||||
@@ -216,6 +222,15 @@ def html_get_page(
|
||||
|
||||
# 403 = Cloudflare/DDoS-Guard protection
|
||||
if status == 403:
|
||||
# If bypasser fallback is disabled, try mirrors instead
|
||||
if not allow_bypasser_fallback:
|
||||
new_url = _try_rotation(original_url, current_url, selector)
|
||||
if new_url:
|
||||
current_url = new_url
|
||||
continue
|
||||
logger.warning(f"403 error, mirrors exhausted: {current_url}")
|
||||
return ""
|
||||
|
||||
if _is_cf_bypass_enabled() and not use_bypasser_now:
|
||||
# Before switching to bypasser, check if cookies have become available
|
||||
# (another concurrent download may have completed bypass and extracted cookies)
|
||||
|
||||
@@ -12,6 +12,7 @@ import ipaddress
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
@@ -279,9 +280,33 @@ def _decode_port(port: Union[str, bytes, int, None]) -> int:
|
||||
return int(port)
|
||||
|
||||
def _is_local_address(host_str: str) -> bool:
|
||||
"""Check if an address is local or private and should bypass custom DNS."""
|
||||
if host_str == 'localhost':
|
||||
"""Check if an address is local/private and should bypass custom DNS.
|
||||
|
||||
Returns True for:
|
||||
- 'localhost'
|
||||
- Private/loopback/link-local IP addresses
|
||||
- Simple hostnames without a dot (e.g., 'booklore', 'prowlarr') - likely Docker service names
|
||||
- Hostnames ending in common internal TLDs (.local, .internal, .lan, .home, .docker)
|
||||
"""
|
||||
if not host_str:
|
||||
return False
|
||||
|
||||
host_lower = host_str.lower()
|
||||
|
||||
# Check for localhost
|
||||
if host_lower == 'localhost':
|
||||
return True
|
||||
|
||||
# Check for simple hostnames (no dot = likely internal Docker/container name)
|
||||
if '.' not in host_str:
|
||||
return True
|
||||
|
||||
# Check for common internal TLDs
|
||||
internal_tlds = ('.local', '.internal', '.lan', '.home', '.docker', '.localdomain')
|
||||
if any(host_lower.endswith(tld) for tld in internal_tlds):
|
||||
return True
|
||||
|
||||
# Check for private/loopback/link-local IP addresses
|
||||
try:
|
||||
addr = ipaddress.ip_address(host_str)
|
||||
return addr.is_private or addr.is_loopback or addr.is_link_local
|
||||
@@ -705,7 +730,14 @@ def rotate_dns_and_reset_aa() -> bool:
|
||||
return False
|
||||
# Reset AA URL to first available auto option if using auto AA
|
||||
global _aa_base_url, _current_aa_url_index
|
||||
configured_url = app_config.get("AA_BASE_URL", "auto")
|
||||
configured_url = normalize_http_url(
|
||||
app_config.get("AA_BASE_URL", "auto"),
|
||||
default_scheme="https",
|
||||
allow_special=("auto",),
|
||||
)
|
||||
if not configured_url:
|
||||
configured_url = "auto"
|
||||
|
||||
if configured_url == "auto" or _aa_base_url in _aa_urls:
|
||||
_current_aa_url_index = 0
|
||||
_aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.se"
|
||||
@@ -876,7 +908,13 @@ def _initialize_aa_state() -> None:
|
||||
_aa_urls = _build_aa_urls()
|
||||
|
||||
# Get configured base URL from config
|
||||
configured_url = app_config.get("AA_BASE_URL", "auto")
|
||||
configured_url = normalize_http_url(
|
||||
app_config.get("AA_BASE_URL", "auto"),
|
||||
default_scheme="https",
|
||||
allow_special=("auto",),
|
||||
)
|
||||
if not configured_url:
|
||||
configured_url = "auto"
|
||||
|
||||
if configured_url == "auto":
|
||||
if state.get('aa_base_url') and state['aa_base_url'] in _aa_urls:
|
||||
|
||||
@@ -4,11 +4,8 @@ Two-stage architecture: handlers stage to TMP_DIR, orchestrator moves to INGEST_
|
||||
with archive extraction and custom script support.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
@@ -16,261 +13,25 @@ from pathlib import Path
|
||||
from threading import Event, Lock
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from shelfmark.release_sources import direct_download
|
||||
from shelfmark.release_sources.direct_download import SearchUnavailable
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.config.env import TMP_DIR
|
||||
from shelfmark.core.utils import get_ingest_dir, get_destination, get_aa_content_type_dir, is_audiobook as check_audiobook, transform_cover_url
|
||||
from shelfmark.core.naming import build_library_path, same_filesystem, assign_part_numbers, parse_naming_template, sanitize_filename
|
||||
from shelfmark.download.archive import (
|
||||
is_archive,
|
||||
process_archive,
|
||||
_get_file_organization,
|
||||
_get_template,
|
||||
_get_supported_formats as _get_book_formats,
|
||||
_get_supported_audiobook_formats,
|
||||
)
|
||||
from shelfmark.release_sources import get_handler, get_source_display_name
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import BookInfo, DownloadTask, QueueStatus, SearchFilters, SearchMode
|
||||
from shelfmark.core.queue import book_queue
|
||||
from shelfmark.core.utils import transform_cover_url
|
||||
from shelfmark.download.postprocess.pipeline import is_torrent_source, safe_cleanup_path
|
||||
from shelfmark.download.postprocess.router import post_process_download
|
||||
from shelfmark.release_sources import direct_download, get_handler, get_source_display_name
|
||||
from shelfmark.release_sources.direct_download import SearchUnavailable
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Staging Directory Helpers
|
||||
# Task Download and Processing
|
||||
# =============================================================================
|
||||
# Handlers should use these to get paths in the staging area.
|
||||
# The orchestrator handles moving staged files to the ingest folder.
|
||||
|
||||
def get_staging_dir() -> Path:
|
||||
"""Get the staging directory for downloads."""
|
||||
TMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
return TMP_DIR
|
||||
|
||||
|
||||
def get_staging_path(task_id: str, extension: str) -> Path:
|
||||
"""Get a staging path for a download."""
|
||||
staging_dir = get_staging_dir()
|
||||
# Hash task_id in case it contains invalid filename chars (e.g., Prowlarr URLs)
|
||||
safe_id = hashlib.md5(task_id.encode()).hexdigest()[:16]
|
||||
return staging_dir / f"{safe_id}.{extension.lstrip('.')}"
|
||||
|
||||
|
||||
def stage_file(source_path: Path, task_id: str, copy: bool = False) -> Path:
|
||||
"""Stage a file for ingest processing. Use copy=True for torrents to preserve seeding."""
|
||||
staging_dir = get_staging_dir()
|
||||
# Stage with original filename, add counter suffix if collision
|
||||
staged_path = staging_dir / source_path.name
|
||||
if staged_path.exists():
|
||||
counter = 1
|
||||
while staged_path.exists():
|
||||
staged_path = staging_dir / f"{source_path.stem}_{counter}{source_path.suffix}"
|
||||
counter += 1
|
||||
|
||||
if copy:
|
||||
shutil.copy2(str(source_path), str(staged_path))
|
||||
logger.debug(f"Copied to staging: {source_path} -> {staged_path}")
|
||||
else:
|
||||
shutil.move(str(source_path), str(staged_path))
|
||||
logger.debug(f"Moved to staging: {source_path} -> {staged_path}")
|
||||
|
||||
return staged_path
|
||||
|
||||
|
||||
def _should_hardlink(task: DownloadTask) -> bool:
|
||||
"""Check if hardlinking is enabled for this task (Prowlarr torrents only)."""
|
||||
if task.source != "prowlarr":
|
||||
return False
|
||||
|
||||
if not task.original_download_path:
|
||||
return False
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
key = "HARDLINK_TORRENTS_AUDIOBOOK" if is_audiobook else "HARDLINK_TORRENTS"
|
||||
|
||||
hardlink_enabled = config.get(key)
|
||||
if hardlink_enabled is None:
|
||||
hardlink_enabled = config.get("TORRENT_HARDLINK", False)
|
||||
|
||||
return bool(hardlink_enabled)
|
||||
|
||||
|
||||
def _should_extract_archives(task: DownloadTask) -> bool:
|
||||
"""Check if archives should be extracted (disabled when hardlinking)."""
|
||||
return not _should_hardlink(task)
|
||||
|
||||
|
||||
def _get_final_destination(task: DownloadTask) -> Path:
|
||||
"""Get final destination directory, with content-type routing support."""
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
|
||||
# For Anna's Archive (direct_download), check for content-type routing override
|
||||
if task.source == "direct_download" and not is_audiobook:
|
||||
override = get_aa_content_type_dir(task.content_type)
|
||||
if override:
|
||||
return override
|
||||
|
||||
return get_destination(is_audiobook)
|
||||
|
||||
|
||||
def _build_metadata_dict(task: DownloadTask) -> dict:
|
||||
"""Build metadata dictionary from task for template processing."""
|
||||
return {
|
||||
"Author": task.author,
|
||||
"Title": task.title,
|
||||
"Subtitle": task.subtitle,
|
||||
"Year": task.year,
|
||||
"Series": task.series_name,
|
||||
"SeriesPosition": task.series_position,
|
||||
}
|
||||
|
||||
|
||||
def _get_supported_formats(content_type: str = None) -> List[str]:
|
||||
"""Get current supported formats from config singleton based on content type."""
|
||||
if check_audiobook(content_type):
|
||||
return _get_supported_audiobook_formats()
|
||||
return _get_book_formats()
|
||||
|
||||
|
||||
def _find_book_files_in_directory(directory: Path, content_type: str = None) -> Tuple[List[Path], List[Path]]:
|
||||
"""Find book files matching supported formats. Returns (matches, rejected)."""
|
||||
book_files = []
|
||||
rejected_files = []
|
||||
supported_formats = _get_supported_formats(content_type)
|
||||
supported_exts = {f".{fmt}" for fmt in supported_formats}
|
||||
|
||||
is_audiobook = check_audiobook(content_type)
|
||||
if is_audiobook:
|
||||
trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
|
||||
else:
|
||||
trackable_exts = {'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr', '.doc', '.docx', '.rtf', '.txt'}
|
||||
|
||||
for file_path in directory.rglob("*"):
|
||||
if file_path.is_file():
|
||||
if file_path.suffix.lower() in supported_exts:
|
||||
book_files.append(file_path)
|
||||
elif file_path.suffix.lower() in trackable_exts:
|
||||
rejected_files.append(file_path)
|
||||
|
||||
return book_files, rejected_files
|
||||
|
||||
|
||||
def process_directory(
|
||||
directory: Path,
|
||||
ingest_dir: Path,
|
||||
task: DownloadTask,
|
||||
) -> Tuple[List[Path], Optional[str]]:
|
||||
"""Process staged directory: find book files, extract archives, move to ingest."""
|
||||
try:
|
||||
content_type = task.content_type
|
||||
book_files, rejected_files = _find_book_files_in_directory(directory, content_type)
|
||||
|
||||
# Find archives in directory (ZIP/RAR)
|
||||
archive_files = [f for f in directory.rglob("*") if f.is_file() and is_archive(f)]
|
||||
|
||||
if not book_files:
|
||||
# No direct book files - check for archives to extract
|
||||
if archive_files:
|
||||
logger.info(f"No book files found, extracting {len(archive_files)} archive(s)")
|
||||
all_final_paths = []
|
||||
all_errors = []
|
||||
|
||||
for archive in archive_files:
|
||||
result = process_archive(
|
||||
archive_path=archive,
|
||||
temp_dir=directory,
|
||||
ingest_dir=ingest_dir,
|
||||
archive_id=f"{task.task_id}_{archive.stem}",
|
||||
task=task,
|
||||
)
|
||||
if result.success:
|
||||
all_final_paths.extend(result.final_paths)
|
||||
elif result.error:
|
||||
all_errors.append(f"{archive.name}: {result.error}")
|
||||
|
||||
# Clean up directory after processing archives
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
|
||||
if all_final_paths:
|
||||
return all_final_paths, None
|
||||
elif all_errors:
|
||||
return [], "; ".join(all_errors)
|
||||
else:
|
||||
return [], "No book files found in archives"
|
||||
|
||||
# No book files and no archives
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
|
||||
if rejected_files:
|
||||
# Files were found but didn't match supported formats
|
||||
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
|
||||
rejected_list = ", ".join(rejected_exts)
|
||||
supported_formats = _get_supported_formats(content_type)
|
||||
logger.warning(
|
||||
f"Found {len(rejected_files)} file(s) but none match supported formats. "
|
||||
f"Rejected formats: {rejected_list}. Supported: {', '.join(sorted(supported_formats))}"
|
||||
)
|
||||
return [], f"Found {len(rejected_files)} file(s) but format not supported ({rejected_list}). Enable in Settings > Formats."
|
||||
|
||||
return [], "No book files found in download"
|
||||
|
||||
# We have book files - use them directly, skip any archives
|
||||
if archive_files:
|
||||
logger.debug(f"Ignoring {len(archive_files)} archive(s) - already have {len(book_files)} book file(s)")
|
||||
|
||||
logger.info(f"Found {len(book_files)} book file(s) in directory")
|
||||
|
||||
if rejected_files:
|
||||
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
|
||||
logger.debug(f"Also found {len(rejected_files)} file(s) with unsupported formats: {', '.join(rejected_exts)}")
|
||||
|
||||
# Transfer each book file to destination
|
||||
final_paths = []
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
organization_mode = _get_file_organization(is_audiobook)
|
||||
use_hardlink = _should_hardlink(task)
|
||||
is_torrent = _is_torrent_source(directory, task)
|
||||
|
||||
for book_file in book_files:
|
||||
# For multi-file downloads (book packs, series), always preserve original filenames
|
||||
# since metadata title only applies to the searched book, not the whole pack.
|
||||
# For single files, respect FILE_ORGANIZATION setting.
|
||||
if len(book_files) == 1 and organization_mode != "none":
|
||||
# Update task format from actual file if not already set
|
||||
# (Prowlarr releases may not know the format until download completes)
|
||||
if not task.format:
|
||||
task.format = book_file.suffix.lower().lstrip('.')
|
||||
|
||||
# Apply template to generate filename
|
||||
template = _get_template(is_audiobook, "rename")
|
||||
metadata = _build_metadata_dict(task)
|
||||
extension = book_file.suffix.lstrip('.') or task.format or ""
|
||||
|
||||
filename = parse_naming_template(template, metadata)
|
||||
if filename and extension:
|
||||
filename = f"{sanitize_filename(filename)}.{extension}"
|
||||
else:
|
||||
filename = book_file.name
|
||||
else:
|
||||
filename = book_file.name
|
||||
|
||||
dest_path = ingest_dir / filename
|
||||
final_path, op = _transfer_single_file(book_file, dest_path, use_hardlink, is_torrent)
|
||||
final_paths.append(final_path)
|
||||
logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
|
||||
|
||||
if not is_torrent:
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
|
||||
return final_paths, None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing directory: {e}")
|
||||
if not _is_torrent_source(directory, task):
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
return [], str(e)
|
||||
#
|
||||
# Post-download processing (staging, extraction, transfers, cleanup) lives in
|
||||
# `shelfmark.download.postprocess`.
|
||||
|
||||
|
||||
# WebSocket manager (initialized by app.py)
|
||||
@@ -496,14 +257,22 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
|
||||
try:
|
||||
# Check for cancellation before starting
|
||||
if cancel_flag.is_set():
|
||||
logger.info(f"Download cancelled before starting: {task_id}")
|
||||
logger.info("Task %s: cancelled before starting", task_id)
|
||||
return None
|
||||
|
||||
task = book_queue.get_task(task_id)
|
||||
if not task:
|
||||
logger.error(f"Task not found in queue: {task_id}")
|
||||
logger.error("Task not found in queue: %s", task_id)
|
||||
return None
|
||||
|
||||
title_label = task.title or "Unknown title"
|
||||
logger.info(
|
||||
"Task %s: starting download (%s) - %s",
|
||||
task_id,
|
||||
get_source_display_name(task.source),
|
||||
title_label,
|
||||
)
|
||||
|
||||
def progress_callback(progress: float) -> None:
|
||||
update_download_progress(task_id, progress)
|
||||
|
||||
@@ -530,24 +299,37 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
|
||||
|
||||
# Check cancellation before post-processing
|
||||
if cancel_flag.is_set():
|
||||
logger.info(f"Download cancelled before post-processing: {task_id}")
|
||||
if not _is_torrent_source(temp_file, task):
|
||||
if temp_file.is_dir():
|
||||
shutil.rmtree(temp_file, ignore_errors=True)
|
||||
else:
|
||||
temp_file.unlink(missing_ok=True)
|
||||
logger.info("Task %s: cancelled before post-processing", task_id)
|
||||
if not is_torrent_source(temp_file, task):
|
||||
safe_cleanup_path(temp_file, task)
|
||||
return None
|
||||
|
||||
# Post-processing: archive extraction or direct move to ingest
|
||||
return _post_process_download(
|
||||
temp_file, task, cancel_flag, status_callback
|
||||
)
|
||||
logger.info("Task %s: download finished; starting post-processing", task_id)
|
||||
logger.debug("Task %s: post-processing input path: %s", task_id, temp_file)
|
||||
|
||||
# Post-processing: output routing + file processing pipeline
|
||||
result = post_process_download(temp_file, task, cancel_flag, status_callback)
|
||||
|
||||
if cancel_flag.is_set():
|
||||
logger.info("Task %s: post-processing cancelled", task_id)
|
||||
elif result:
|
||||
logger.info("Task %s: post-processing complete", task_id)
|
||||
logger.debug("Task %s: post-processing result: %s", task_id, result)
|
||||
else:
|
||||
logger.warning("Task %s: post-processing failed", task_id)
|
||||
|
||||
try:
|
||||
handler.post_process_cleanup(task, success=bool(result))
|
||||
except Exception as e:
|
||||
logger.warning("Post-processing cleanup hook failed for %s: %s", task_id, e)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
if cancel_flag.is_set():
|
||||
logger.info(f"Download cancelled during error handling: {task_id}")
|
||||
logger.info("Task %s: cancelled during error handling", task_id)
|
||||
else:
|
||||
logger.error_trace(f"Error downloading: {e}")
|
||||
logger.error_trace("Task %s: error downloading: %s", task_id, e)
|
||||
# Update task status so user sees the failure
|
||||
task = book_queue.get_task(task_id)
|
||||
if task:
|
||||
@@ -559,414 +341,13 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
|
||||
"Destination misconfigured. Go to Settings → Downloads to update."
|
||||
)
|
||||
else:
|
||||
book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}")
|
||||
if isinstance(e, PermissionError):
|
||||
book_queue.update_status_message(task_id, f"Permission denied: {e}")
|
||||
else:
|
||||
book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}")
|
||||
return None
|
||||
|
||||
|
||||
def _process_organize_mode(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
status_callback,
|
||||
) -> Optional[str]:
|
||||
"""Organize files into library folders using template. Supports hardlinking."""
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
|
||||
# Get destination and template
|
||||
destination = _get_final_destination(task)
|
||||
template = _get_template(is_audiobook, "organize")
|
||||
|
||||
# Validate destination path
|
||||
if not destination.is_absolute():
|
||||
logger.warning(f"Destination must be absolute: {destination}, falling back to flat mode")
|
||||
status_callback("resolving", f"Destination must be absolute: {destination}")
|
||||
return None
|
||||
|
||||
if not destination.exists():
|
||||
try:
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
except (OSError, PermissionError) as e:
|
||||
logger.warning(f"Cannot create destination: {e}")
|
||||
status_callback("resolving", f"Cannot create destination: {e}")
|
||||
return None
|
||||
|
||||
if not os.access(destination, os.W_OK):
|
||||
logger.warning(f"Destination not writable: {destination}")
|
||||
status_callback("resolving", f"Destination not writable: {destination}")
|
||||
return None
|
||||
|
||||
# Determine if we should use hardlinking
|
||||
use_hardlink = False
|
||||
source = temp_file
|
||||
|
||||
if _should_hardlink(task):
|
||||
hardlink_source = Path(task.original_download_path)
|
||||
if hardlink_source.exists() and same_filesystem(hardlink_source, destination):
|
||||
use_hardlink = True
|
||||
source = hardlink_source
|
||||
elif hardlink_source.exists():
|
||||
logger.warning(
|
||||
f"Cannot hardlink: {hardlink_source} and {destination} are on different filesystems. "
|
||||
"Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination."
|
||||
)
|
||||
status_callback("resolving", "Cannot hardlink (different filesystems), using copy")
|
||||
|
||||
# Build metadata dict for template
|
||||
metadata = _build_metadata_dict(task)
|
||||
|
||||
try:
|
||||
status_callback("resolving", "Creating hardlinks" if use_hardlink else "Organizing files")
|
||||
|
||||
if source.is_dir():
|
||||
return _transfer_directory_to_library(
|
||||
source, str(destination), template, metadata, task, temp_file, status_callback, use_hardlink
|
||||
)
|
||||
else:
|
||||
return _transfer_file_to_library(
|
||||
source, str(destination), template, metadata, task, temp_file, status_callback, use_hardlink
|
||||
)
|
||||
except PermissionError as e:
|
||||
logger.error(f"Permission denied: {e}")
|
||||
status_callback("error", f"Permission denied: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Organization failed: {e}")
|
||||
status_callback("error", f"Organization failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _is_torrent_source(source_path: Path, task: DownloadTask) -> bool:
|
||||
"""Check if source is the torrent client path (needs copy to preserve seeding)."""
|
||||
if not task.original_download_path:
|
||||
return False
|
||||
try:
|
||||
return source_path.resolve() == Path(task.original_download_path).resolve()
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _stage_torrent_path(source: Path) -> Path:
|
||||
"""Copy torrent source to staging directory to preserve seeding."""
|
||||
staging_dir = get_staging_dir()
|
||||
staged_path = staging_dir / source.name
|
||||
counter = 1
|
||||
|
||||
if source.is_dir():
|
||||
while staged_path.exists():
|
||||
staged_path = staging_dir / f"{source.name}_{counter}"
|
||||
counter += 1
|
||||
shutil.copytree(str(source), str(staged_path))
|
||||
else:
|
||||
while staged_path.exists():
|
||||
staged_path = staging_dir / f"{source.stem}_{counter}{source.suffix}"
|
||||
counter += 1
|
||||
shutil.copy2(str(source), str(staged_path))
|
||||
|
||||
logger.debug(f"Staged torrent {'directory' if source.is_dir() else 'file'}: {staged_path.name}")
|
||||
return staged_path
|
||||
|
||||
|
||||
# Import atomic file operations from shared module
|
||||
# Re-exported here for backwards compatibility with existing tests/imports
|
||||
from shelfmark.download.fs import (
|
||||
atomic_hardlink as _atomic_hardlink,
|
||||
atomic_copy as _atomic_copy,
|
||||
atomic_move as _atomic_move,
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_staged_files(temp_file: Path, source_dir: Optional[Path] = None) -> None:
|
||||
"""Remove staged files. Optionally removes source_dir if empty."""
|
||||
try:
|
||||
if temp_file.is_dir():
|
||||
shutil.rmtree(temp_file)
|
||||
elif temp_file.exists():
|
||||
temp_file.unlink()
|
||||
except (OSError, PermissionError) as e:
|
||||
logger.debug(f"Cleanup failed for {temp_file}: {e}")
|
||||
|
||||
if source_dir and source_dir.is_dir():
|
||||
try:
|
||||
source_dir.rmdir()
|
||||
except OSError:
|
||||
pass # Directory not empty or permission issue
|
||||
|
||||
|
||||
def _transfer_single_file(
|
||||
source_path: Path,
|
||||
dest_path: Path,
|
||||
use_hardlink: bool,
|
||||
is_torrent: bool,
|
||||
) -> Tuple[Path, str]:
|
||||
"""Transfer a file via hardlink, copy, or move. Returns (final_path, operation_name)."""
|
||||
if use_hardlink:
|
||||
return _atomic_hardlink(source_path, dest_path), "hardlink"
|
||||
if is_torrent:
|
||||
return _atomic_copy(source_path, dest_path), "copy"
|
||||
return _atomic_move(source_path, dest_path), "move"
|
||||
|
||||
|
||||
def _transfer_file_to_library(
|
||||
source_path: Path,
|
||||
library_base: str,
|
||||
template: str,
|
||||
metadata: dict,
|
||||
task: DownloadTask,
|
||||
temp_file: Optional[Path],
|
||||
status_callback,
|
||||
use_hardlink: bool,
|
||||
) -> Optional[str]:
|
||||
"""Transfer a single file to the library with template-based naming."""
|
||||
extension = source_path.suffix.lstrip('.') or task.format
|
||||
dest_path = build_library_path(library_base, template, metadata, extension)
|
||||
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
is_torrent = _is_torrent_source(source_path, task)
|
||||
final_path, op = _transfer_single_file(source_path, dest_path, use_hardlink, is_torrent)
|
||||
logger.info(f"Library {op}: {final_path}")
|
||||
|
||||
if use_hardlink:
|
||||
_cleanup_staged_files(temp_file)
|
||||
|
||||
status_callback("complete", "Complete")
|
||||
return str(final_path)
|
||||
|
||||
|
||||
def _transfer_directory_to_library(
|
||||
source_dir: Path,
|
||||
library_base: str,
|
||||
template: str,
|
||||
metadata: dict,
|
||||
task: DownloadTask,
|
||||
temp_file: Optional[Path],
|
||||
status_callback,
|
||||
use_hardlink: bool,
|
||||
) -> Optional[str]:
|
||||
"""Transfer all files from a directory to the library with template-based naming."""
|
||||
content_type = task.content_type.lower() if task.content_type else None
|
||||
supported_formats = _get_supported_formats(content_type)
|
||||
|
||||
source_files = [
|
||||
f for f in source_dir.rglob("*")
|
||||
if f.is_file() and f.suffix.lower().lstrip('.') in supported_formats
|
||||
]
|
||||
|
||||
if not source_files:
|
||||
logger.warning(f"No supported files in {source_dir.name}")
|
||||
status_callback("error", "No supported file formats found")
|
||||
if temp_file:
|
||||
_cleanup_staged_files(temp_file)
|
||||
return None
|
||||
|
||||
base_library_path = build_library_path(library_base, template, metadata, extension=None)
|
||||
base_library_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Check if this is a torrent source that needs copy instead of move
|
||||
is_torrent = _is_torrent_source(source_dir, task)
|
||||
transferred_paths = []
|
||||
|
||||
if len(source_files) == 1:
|
||||
# Single file - no part numbering needed
|
||||
source_file = source_files[0]
|
||||
ext = source_file.suffix.lstrip('.')
|
||||
dest_path = base_library_path.with_suffix(f'.{ext}')
|
||||
|
||||
final_path, op = _transfer_single_file(source_file, dest_path, use_hardlink, is_torrent)
|
||||
logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
|
||||
transferred_paths.append(final_path)
|
||||
else:
|
||||
# Multi-file: natural sort then sequential numbering
|
||||
zero_pad_width = max(len(str(len(source_files))), 2)
|
||||
files_with_parts = assign_part_numbers(source_files, zero_pad_width)
|
||||
|
||||
for source_file, part_number in files_with_parts:
|
||||
ext = source_file.suffix.lstrip('.')
|
||||
|
||||
file_metadata = {**metadata, "PartNumber": part_number}
|
||||
file_path = build_library_path(library_base, template, file_metadata, extension=ext)
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
final_path, op = _transfer_single_file(source_file, file_path, use_hardlink, is_torrent)
|
||||
logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
|
||||
transferred_paths.append(final_path)
|
||||
|
||||
# Get operation name for summary log
|
||||
if use_hardlink:
|
||||
operation = "hardlinks"
|
||||
elif is_torrent:
|
||||
operation = "copies"
|
||||
else:
|
||||
operation = "files"
|
||||
logger.info(f"Created {len(transferred_paths)} library {operation} in {base_library_path.parent}")
|
||||
|
||||
# Cleanup staging (not torrent source - that stays for seeding)
|
||||
if use_hardlink:
|
||||
_cleanup_staged_files(temp_file)
|
||||
elif not is_torrent:
|
||||
_cleanup_staged_files(temp_file, source_dir)
|
||||
|
||||
message = f"Complete ({len(transferred_paths)} files)" if len(transferred_paths) > 1 else "Complete"
|
||||
status_callback("complete", message)
|
||||
|
||||
return str(transferred_paths[0])
|
||||
|
||||
|
||||
def _post_process_download(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
cancel_flag: Event,
|
||||
status_callback,
|
||||
) -> Optional[str]:
|
||||
"""Post-process download: extract archives, apply naming template, move to destination."""
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
|
||||
# Validate search_mode
|
||||
if task.search_mode is None:
|
||||
logger.warning(f"Task {task.task_id} has no search_mode set, defaulting to Direct mode behavior")
|
||||
elif task.search_mode not in (SearchMode.DIRECT, SearchMode.UNIVERSAL):
|
||||
logger.warning(f"Task {task.task_id} has invalid search_mode '{task.search_mode}', defaulting to Direct mode behavior")
|
||||
|
||||
# Get file organization mode and destination
|
||||
organization_mode = _get_file_organization(is_audiobook)
|
||||
destination = _get_final_destination(task)
|
||||
|
||||
logger.debug(f"File organization: mode={organization_mode}, destination={destination}")
|
||||
|
||||
# "Organize" mode with folders uses specialized handler
|
||||
if organization_mode == "organize":
|
||||
result = _process_organize_mode(temp_file, task, status_callback)
|
||||
if result is not None:
|
||||
return result
|
||||
# If organize mode fails, fall through to flat mode
|
||||
logger.warning(
|
||||
f"Organize mode failed for '{task.title}', falling back to flat destination. "
|
||||
"Check destination folder permissions and ensure the path is writable."
|
||||
)
|
||||
status_callback("resolving", "Organization failed, using flat destination")
|
||||
|
||||
# Ensure destination exists
|
||||
os.makedirs(destination, exist_ok=True)
|
||||
|
||||
# For torrents with hardlinking disabled, stage first to preserve seeding
|
||||
# (Torrent handler returns original path, not staged copy)
|
||||
if _is_torrent_source(temp_file, task) and not _should_hardlink(task):
|
||||
status_callback("resolving", "Staging torrent files")
|
||||
temp_file = _stage_torrent_path(temp_file)
|
||||
|
||||
# Handle archive extraction (RAR/ZIP) - only if not hardlinking
|
||||
if is_archive(temp_file) and _should_extract_archives(task):
|
||||
logger.info(f"Archive detected, extracting: {temp_file.name}")
|
||||
status_callback("resolving", "Extracting archive")
|
||||
|
||||
result = process_archive(
|
||||
archive_path=temp_file,
|
||||
temp_dir=TMP_DIR,
|
||||
ingest_dir=destination,
|
||||
archive_id=task.task_id,
|
||||
task=task,
|
||||
)
|
||||
|
||||
if result.success:
|
||||
status_callback("complete", result.message)
|
||||
return str(result.final_paths[0])
|
||||
else:
|
||||
status_callback("error", result.error)
|
||||
return None
|
||||
|
||||
# Handle directory (multi-file torrent/usenet downloads)
|
||||
if temp_file.is_dir():
|
||||
logger.info(f"Directory detected, processing: {temp_file.name}")
|
||||
status_callback("resolving", "Processing download folder")
|
||||
|
||||
final_paths, error = process_directory(
|
||||
directory=temp_file,
|
||||
ingest_dir=destination,
|
||||
task=task,
|
||||
)
|
||||
|
||||
if error:
|
||||
status_callback("error", error)
|
||||
return None
|
||||
|
||||
if not final_paths:
|
||||
status_callback("error", "No book files found")
|
||||
return None
|
||||
|
||||
message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
|
||||
status_callback("complete", message)
|
||||
return str(final_paths[0])
|
||||
|
||||
# Non-archive: run custom script if configured, then move to destination
|
||||
if config.CUSTOM_SCRIPT:
|
||||
logger.info(f"Running custom script: {config.CUSTOM_SCRIPT}")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[config.CUSTOM_SCRIPT, str(temp_file)],
|
||||
check=True,
|
||||
timeout=300, # 5 minute timeout
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.stdout:
|
||||
logger.debug(f"Custom script stdout: {result.stdout.strip()}")
|
||||
except FileNotFoundError:
|
||||
logger.error(f"Custom script not found: {config.CUSTOM_SCRIPT}")
|
||||
status_callback("error", f"Custom script not found: {config.CUSTOM_SCRIPT}")
|
||||
return None
|
||||
except PermissionError:
|
||||
logger.error(f"Custom script not executable: {config.CUSTOM_SCRIPT}")
|
||||
status_callback("error", f"Custom script not executable: {config.CUSTOM_SCRIPT}")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(f"Custom script timed out after 300s: {config.CUSTOM_SCRIPT}")
|
||||
status_callback("error", "Custom script timed out")
|
||||
return None
|
||||
except subprocess.CalledProcessError as e:
|
||||
stderr = e.stderr.strip() if e.stderr else "No error output"
|
||||
logger.error(f"Custom script failed (exit code {e.returncode}): {stderr}")
|
||||
status_callback("error", f"Custom script failed: {stderr[:100]}")
|
||||
return None
|
||||
|
||||
use_hardlink = _should_hardlink(task)
|
||||
is_torrent = _is_torrent_source(temp_file, task)
|
||||
|
||||
if cancel_flag.is_set():
|
||||
logger.info(f"Download cancelled before final transfer: {task.task_id}")
|
||||
if not is_torrent:
|
||||
temp_file.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
# Determine filename based on organization mode
|
||||
if organization_mode == "none":
|
||||
# Keep original filename
|
||||
filename = temp_file.name
|
||||
else:
|
||||
# "rename" mode - apply template to filename
|
||||
template = _get_template(is_audiobook, "rename")
|
||||
metadata = _build_metadata_dict(task)
|
||||
extension = temp_file.suffix.lstrip('.') or task.format or ""
|
||||
|
||||
# Parse template to generate filename
|
||||
filename = parse_naming_template(template, metadata)
|
||||
if filename and extension:
|
||||
filename = f"{sanitize_filename(filename)}.{extension}"
|
||||
elif not filename:
|
||||
# Template produced empty result, fall back to original
|
||||
filename = temp_file.name
|
||||
|
||||
dest_path = destination / filename
|
||||
|
||||
try:
|
||||
final_path, op = _transfer_single_file(temp_file, dest_path, use_hardlink, is_torrent)
|
||||
logger.info(f"Download completed ({op}): {final_path.name}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to transfer file to destination: {e}")
|
||||
status_callback("error", f"Failed to transfer file: {e}")
|
||||
return None
|
||||
|
||||
status_callback("complete", "Complete")
|
||||
|
||||
return str(final_path)
|
||||
|
||||
def update_download_progress(book_id: str, progress: float) -> None:
|
||||
"""Update download progress with throttled WebSocket broadcasts."""
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from typing import Callable, Optional
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
|
||||
StatusCallback = Callable[[str, Optional[str]], None]
|
||||
OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback], Optional[str]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OutputRegistration:
|
||||
mode: str
|
||||
supports_task: Callable[[DownloadTask], bool]
|
||||
handler: OutputHandler
|
||||
priority: int = 0
|
||||
|
||||
|
||||
_OUTPUT_REGISTRY: list[OutputRegistration] = []
|
||||
_OUTPUTS_LOADED = False
|
||||
|
||||
|
||||
def register_output(
|
||||
mode: str,
|
||||
supports_task: Callable[[DownloadTask], bool],
|
||||
priority: int = 0,
|
||||
) -> Callable[[OutputHandler], OutputHandler]:
|
||||
def decorator(handler: OutputHandler) -> OutputHandler:
|
||||
_OUTPUT_REGISTRY.append(
|
||||
OutputRegistration(
|
||||
mode=mode,
|
||||
supports_task=supports_task,
|
||||
handler=handler,
|
||||
priority=priority,
|
||||
)
|
||||
)
|
||||
_OUTPUT_REGISTRY.sort(key=lambda entry: entry.priority, reverse=True)
|
||||
return handler
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def load_output_handlers() -> None:
|
||||
global _OUTPUTS_LOADED
|
||||
if _OUTPUTS_LOADED:
|
||||
return
|
||||
|
||||
from . import booklore # noqa: F401
|
||||
from . import folder # noqa: F401
|
||||
|
||||
_OUTPUTS_LOADED = True
|
||||
|
||||
|
||||
def resolve_output_handler(task: DownloadTask) -> Optional[OutputRegistration]:
|
||||
load_output_handlers()
|
||||
for entry in _OUTPUT_REGISTRY:
|
||||
if entry.supports_task(task):
|
||||
return entry
|
||||
return None
|
||||
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from typing import Any, Dict, List, Mapping, Optional
|
||||
|
||||
import requests
|
||||
|
||||
import shelfmark.core.config as core_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.outputs import register_output
|
||||
from shelfmark.download.staging import STAGE_MOVE, STAGE_NONE, build_staging_dir
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
BOOKLORE_OUTPUT_MODE = "booklore"
|
||||
BOOKLORE_SUPPORTED_EXTENSIONS = {".cb7", ".cbr", ".cbz", ".epub", ".fb2", ".pdf"}
|
||||
BOOKLORE_SUPPORTED_FORMATS_LABEL = ", ".join(
|
||||
ext.lstrip(".").upper() for ext in sorted(BOOKLORE_SUPPORTED_EXTENSIONS)
|
||||
)
|
||||
|
||||
|
||||
class BookloreError(Exception):
|
||||
"""Raised when Booklore integration fails."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BookloreConfig:
|
||||
base_url: str
|
||||
username: str
|
||||
password: str
|
||||
library_id: int
|
||||
path_id: int
|
||||
verify_tls: bool = True
|
||||
refresh_after_upload: bool = False
|
||||
|
||||
|
||||
def _parse_int(value: Any, label: str) -> int:
|
||||
if value is None or value == "":
|
||||
raise BookloreError(f"{label} is required")
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise BookloreError(f"{label} must be a number") from exc
|
||||
|
||||
|
||||
def build_booklore_config(values: Mapping[str, Any]) -> BookloreConfig:
|
||||
base_url = str(values.get("BOOKLORE_HOST", "")).strip()
|
||||
username = str(values.get("BOOKLORE_USERNAME", "")).strip()
|
||||
password = values.get("BOOKLORE_PASSWORD", "") or ""
|
||||
|
||||
if not base_url:
|
||||
raise BookloreError("Booklore URL is required")
|
||||
if not username:
|
||||
raise BookloreError("Booklore username is required")
|
||||
if not password:
|
||||
raise BookloreError("Booklore password is required")
|
||||
|
||||
library_id = _parse_int(values.get("BOOKLORE_LIBRARY_ID"), "Booklore library ID")
|
||||
path_id = _parse_int(values.get("BOOKLORE_PATH_ID"), "Booklore path ID")
|
||||
|
||||
return BookloreConfig(
|
||||
base_url=base_url.rstrip("/"),
|
||||
username=username,
|
||||
password=password,
|
||||
library_id=library_id,
|
||||
path_id=path_id,
|
||||
verify_tls=True,
|
||||
refresh_after_upload=True, # Always refresh library after upload
|
||||
)
|
||||
|
||||
|
||||
def booklore_login(booklore_config: BookloreConfig) -> str:
|
||||
url = f"{booklore_config.base_url}/api/v1/auth/login"
|
||||
payload = {"username": booklore_config.username, "password": booklore_config.password}
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, timeout=30, verify=booklore_config.verify_tls)
|
||||
except requests.exceptions.ConnectionError as exc:
|
||||
raise BookloreError("Could not connect to Booklore") from exc
|
||||
except requests.exceptions.Timeout as exc:
|
||||
raise BookloreError("Booklore connection timed out") from exc
|
||||
except requests.exceptions.RequestException as exc:
|
||||
raise BookloreError(f"Booklore login failed: {exc}") from exc
|
||||
|
||||
if response.status_code in {401, 403}:
|
||||
raise BookloreError("Booklore authentication failed")
|
||||
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.HTTPError as exc:
|
||||
raise BookloreError(f"Booklore login failed ({response.status_code})") from exc
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except ValueError as exc:
|
||||
raise BookloreError("Invalid Booklore login response") from exc
|
||||
|
||||
token = data.get("accessToken")
|
||||
if not token:
|
||||
raise BookloreError("Booklore did not return an access token")
|
||||
|
||||
return token
|
||||
|
||||
|
||||
def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list[dict[str, Any]]:
|
||||
url = f"{booklore_config.base_url}/api/v1/libraries"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.RequestException as exc:
|
||||
raise BookloreError(f"Failed to fetch Booklore libraries: {exc}") from exc
|
||||
|
||||
try:
|
||||
return response.json()
|
||||
except ValueError as exc:
|
||||
raise BookloreError("Invalid Booklore libraries response") from exc
|
||||
|
||||
|
||||
def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: Path) -> None:
|
||||
url = f"{booklore_config.base_url}/api/v1/files/upload"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
params = {"libraryId": booklore_config.library_id, "pathId": booklore_config.path_id}
|
||||
|
||||
response = None
|
||||
|
||||
try:
|
||||
with file_path.open("rb") as handle:
|
||||
response = requests.post(
|
||||
url,
|
||||
headers=headers,
|
||||
params=params,
|
||||
files={"file": (file_path.name, handle)},
|
||||
timeout=60,
|
||||
verify=booklore_config.verify_tls,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.HTTPError as exc:
|
||||
message = response.text.strip() if response is not None else ""
|
||||
if message:
|
||||
message = f": {message[:200]}"
|
||||
status_code = response.status_code if response is not None else "unknown"
|
||||
raise BookloreError(f"Booklore upload failed ({status_code}){message}") from exc
|
||||
except requests.exceptions.ConnectionError as exc:
|
||||
raise BookloreError("Could not connect to Booklore") from exc
|
||||
except requests.exceptions.Timeout as exc:
|
||||
raise BookloreError("Booklore upload timed out") from exc
|
||||
except requests.exceptions.RequestException as exc:
|
||||
raise BookloreError(f"Booklore upload failed: {exc}") from exc
|
||||
|
||||
|
||||
def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> None:
|
||||
url = f"{booklore_config.base_url}/api/v1/libraries/{booklore_config.library_id}/refresh"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
try:
|
||||
response = requests.put(url, headers=headers, timeout=30, verify=booklore_config.verify_tls)
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.RequestException as exc:
|
||||
raise BookloreError(f"Booklore refresh failed: {exc}") from exc
|
||||
|
||||
|
||||
def _supports_booklore(task: DownloadTask) -> bool:
|
||||
if check_audiobook(task.content_type):
|
||||
return False
|
||||
return core_config.config.get("BOOKS_OUTPUT_MODE", "folder") == BOOKLORE_OUTPUT_MODE
|
||||
|
||||
|
||||
def _get_booklore_settings() -> Dict[str, Any]:
|
||||
return {
|
||||
"BOOKLORE_HOST": core_config.config.get("BOOKLORE_HOST", ""),
|
||||
"BOOKLORE_USERNAME": core_config.config.get("BOOKLORE_USERNAME", ""),
|
||||
"BOOKLORE_PASSWORD": core_config.config.get("BOOKLORE_PASSWORD", ""),
|
||||
"BOOKLORE_LIBRARY_ID": core_config.config.get("BOOKLORE_LIBRARY_ID"),
|
||||
"BOOKLORE_PATH_ID": core_config.config.get("BOOKLORE_PATH_ID"),
|
||||
}
|
||||
|
||||
|
||||
def _booklore_format_error(rejected_files: List[Path]) -> str:
|
||||
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
|
||||
rejected_list = ", ".join(rejected_exts)
|
||||
return (
|
||||
f"Booklore does not support {rejected_list}. "
|
||||
f"Supported formats: {BOOKLORE_SUPPORTED_FORMATS_LABEL}"
|
||||
)
|
||||
|
||||
|
||||
def _post_process_booklore(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
cancel_flag: Event,
|
||||
status_callback,
|
||||
) -> Optional[str]:
|
||||
from shelfmark.download.postprocess.pipeline import (
|
||||
OutputPlan,
|
||||
cleanup_output_staging,
|
||||
is_managed_workspace_path,
|
||||
prepare_output_files,
|
||||
)
|
||||
|
||||
if cancel_flag.is_set():
|
||||
logger.info("Task %s: cancelled before Booklore upload", task.task_id)
|
||||
return None
|
||||
|
||||
try:
|
||||
booklore_config = build_booklore_config(_get_booklore_settings())
|
||||
except BookloreError as e:
|
||||
logger.warning("Task %s: Booklore configuration error: %s", task.task_id, e)
|
||||
status_callback("error", str(e))
|
||||
return None
|
||||
|
||||
status_callback("resolving", "Preparing Booklore upload")
|
||||
|
||||
output_plan = OutputPlan(
|
||||
mode=BOOKLORE_OUTPUT_MODE,
|
||||
stage_action=STAGE_MOVE if is_managed_workspace_path(temp_file) else STAGE_NONE,
|
||||
staging_dir=build_staging_dir("booklore", task.task_id),
|
||||
allow_archive_extraction=True,
|
||||
)
|
||||
|
||||
prepared = prepare_output_files(
|
||||
temp_file,
|
||||
task,
|
||||
BOOKLORE_OUTPUT_MODE,
|
||||
status_callback,
|
||||
output_plan=output_plan,
|
||||
)
|
||||
if not prepared:
|
||||
return None
|
||||
|
||||
logger.debug("Task %s: prepared %d file(s) for Booklore upload", task.task_id, len(prepared.files))
|
||||
|
||||
try:
|
||||
unsupported_files = [
|
||||
file_path
|
||||
for file_path in prepared.files
|
||||
if file_path.suffix.lower() not in BOOKLORE_SUPPORTED_EXTENSIONS
|
||||
]
|
||||
if unsupported_files:
|
||||
error_message = _booklore_format_error(unsupported_files)
|
||||
logger.warning("Task %s: %s", task.task_id, error_message)
|
||||
status_callback("error", error_message)
|
||||
return None
|
||||
|
||||
token = booklore_login(booklore_config)
|
||||
logger.info("Task %s: uploading %d file(s) to Booklore", task.task_id, len(prepared.files))
|
||||
|
||||
for index, file_path in enumerate(prepared.files, start=1):
|
||||
if cancel_flag.is_set():
|
||||
logger.info("Task %s: cancelled during Booklore upload", task.task_id)
|
||||
return None
|
||||
status_callback("resolving", f"Uploading to Booklore ({index}/{len(prepared.files)})")
|
||||
booklore_upload_file(booklore_config, token, file_path)
|
||||
|
||||
if booklore_config.refresh_after_upload:
|
||||
try:
|
||||
booklore_refresh_library(booklore_config, token)
|
||||
except BookloreError as e:
|
||||
logger.warning("Task %s: Booklore refresh failed: %s", task.task_id, e)
|
||||
|
||||
logger.info("Task %s: uploaded %d file(s) to Booklore", task.task_id, len(prepared.files))
|
||||
|
||||
message = "Uploaded to Booklore"
|
||||
if len(prepared.files) > 1:
|
||||
message = f"Uploaded to Booklore ({len(prepared.files)} files)"
|
||||
status_callback("complete", message)
|
||||
return f"booklore://{task.task_id}"
|
||||
|
||||
except BookloreError as e:
|
||||
logger.warning("Task %s: Booklore upload failed: %s", task.task_id, e)
|
||||
status_callback("error", str(e))
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error_trace("Task %s: unexpected error uploading to Booklore: %s", task.task_id, e)
|
||||
status_callback("error", f"Booklore upload failed: {e}")
|
||||
return None
|
||||
finally:
|
||||
cleanup_output_staging(
|
||||
prepared.output_plan,
|
||||
prepared.working_path,
|
||||
task,
|
||||
prepared.cleanup_paths,
|
||||
)
|
||||
|
||||
|
||||
@register_output(BOOKLORE_OUTPUT_MODE, supports_task=_supports_booklore, priority=10)
|
||||
def process_booklore_output(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
cancel_flag: Event,
|
||||
status_callback,
|
||||
) -> Optional[str]:
|
||||
return _post_process_booklore(temp_file, task, cancel_flag, status_callback)
|
||||
@@ -0,0 +1,310 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from typing import Any, Optional, List
|
||||
|
||||
import shelfmark.core.config as core_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.archive import is_archive
|
||||
from shelfmark.download.outputs import register_output
|
||||
from shelfmark.download.staging import StageAction, STAGE_NONE
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
FOLDER_OUTPUT_MODE = "folder"
|
||||
|
||||
|
||||
def _resolve_custom_script_target(target_path: Path, destination: Path, path_mode: str) -> Path:
|
||||
mode = (path_mode or "absolute").strip().lower()
|
||||
if mode != "relative":
|
||||
return target_path
|
||||
|
||||
try:
|
||||
return target_path.relative_to(destination)
|
||||
except ValueError:
|
||||
if target_path.is_absolute():
|
||||
return Path(target_path.name)
|
||||
return target_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ProcessingPlan:
|
||||
destination: Path
|
||||
organization_mode: str
|
||||
use_hardlink: bool
|
||||
allow_archive_extraction: bool
|
||||
stage_action: StageAction
|
||||
staging_dir: Path
|
||||
hardlink_source: Optional[Path]
|
||||
output_mode: str = FOLDER_OUTPUT_MODE
|
||||
|
||||
|
||||
def _supports_folder_output(task: DownloadTask) -> bool:
|
||||
if check_audiobook(task.content_type):
|
||||
return True
|
||||
return core_config.config.get("BOOKS_OUTPUT_MODE", FOLDER_OUTPUT_MODE) == FOLDER_OUTPUT_MODE
|
||||
|
||||
|
||||
def _build_processing_plan(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
status_callback,
|
||||
) -> Optional[_ProcessingPlan]:
|
||||
from shelfmark.download.postprocess.pipeline import (
|
||||
build_output_plan,
|
||||
get_final_destination,
|
||||
validate_destination,
|
||||
)
|
||||
from shelfmark.download.postprocess.policy import get_file_organization
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
organization_mode = get_file_organization(is_audiobook)
|
||||
destination = get_final_destination(task)
|
||||
|
||||
if not validate_destination(destination, status_callback):
|
||||
return None
|
||||
|
||||
output_plan = build_output_plan(
|
||||
temp_file,
|
||||
task,
|
||||
output_mode=FOLDER_OUTPUT_MODE,
|
||||
destination=destination,
|
||||
status_callback=status_callback,
|
||||
)
|
||||
if not output_plan.transfer_plan:
|
||||
return None
|
||||
|
||||
transfer_plan = output_plan.transfer_plan
|
||||
hardlink_source = transfer_plan.source_path if transfer_plan.use_hardlink else None
|
||||
|
||||
return _ProcessingPlan(
|
||||
destination=destination,
|
||||
organization_mode=organization_mode,
|
||||
use_hardlink=transfer_plan.use_hardlink,
|
||||
allow_archive_extraction=transfer_plan.allow_archive_extraction,
|
||||
stage_action=output_plan.stage_action,
|
||||
staging_dir=output_plan.staging_dir,
|
||||
hardlink_source=hardlink_source,
|
||||
)
|
||||
|
||||
|
||||
@register_output(FOLDER_OUTPUT_MODE, supports_task=_supports_folder_output, priority=0)
|
||||
def process_folder_output(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
cancel_flag: Event,
|
||||
status_callback,
|
||||
) -> Optional[str]:
|
||||
"""Post-process download to the configured folder destination."""
|
||||
from shelfmark.download.postprocess.pipeline import (
|
||||
cleanup_output_staging,
|
||||
is_torrent_source,
|
||||
log_plan_steps,
|
||||
prepare_output_files,
|
||||
record_step,
|
||||
safe_cleanup_path,
|
||||
transfer_book_files,
|
||||
)
|
||||
|
||||
plan = _build_processing_plan(temp_file, task, status_callback)
|
||||
if not plan:
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"Processing plan for task %s: mode=%s destination=%s hardlink=%s stage_action=%s extract_archives=%s",
|
||||
task.task_id,
|
||||
plan.organization_mode,
|
||||
plan.destination,
|
||||
plan.use_hardlink,
|
||||
plan.stage_action,
|
||||
plan.allow_archive_extraction,
|
||||
)
|
||||
|
||||
prepared = prepare_output_files(
|
||||
temp_file,
|
||||
task,
|
||||
output_mode=plan.output_mode,
|
||||
status_callback=status_callback,
|
||||
destination=plan.destination,
|
||||
)
|
||||
if not prepared:
|
||||
return None
|
||||
|
||||
steps: List[Any] = []
|
||||
if prepared.output_plan.stage_action != STAGE_NONE:
|
||||
step_name = f"stage_{prepared.output_plan.stage_action}"
|
||||
record_step(steps, step_name, source=str(temp_file), dest=str(prepared.output_plan.staging_dir))
|
||||
|
||||
def run_custom_script(script_path: str, target_path: Path, phase: str) -> bool:
|
||||
path_mode = core_config.config.get("CUSTOM_SCRIPT_PATH_MODE", "absolute")
|
||||
script_target = _resolve_custom_script_target(target_path, plan.destination, path_mode)
|
||||
env = {
|
||||
**os.environ,
|
||||
"SHELFMARK_CUSTOM_SCRIPT_TARGET": str(target_path),
|
||||
"SHELFMARK_CUSTOM_SCRIPT_RELATIVE": str(_resolve_custom_script_target(target_path, plan.destination, "relative")),
|
||||
"SHELFMARK_CUSTOM_SCRIPT_DESTINATION": str(plan.destination),
|
||||
"SHELFMARK_CUSTOM_SCRIPT_MODE": str(path_mode),
|
||||
"SHELFMARK_CUSTOM_SCRIPT_PHASE": phase,
|
||||
}
|
||||
record_step(
|
||||
steps,
|
||||
"custom_script",
|
||||
script=str(script_path),
|
||||
target=str(script_target),
|
||||
target_abs=str(target_path),
|
||||
mode=str(path_mode),
|
||||
phase=phase,
|
||||
)
|
||||
log_plan_steps(task.task_id, steps)
|
||||
logger.info(
|
||||
"Task %s: running custom script %s on %s (%s)",
|
||||
task.task_id,
|
||||
script_path,
|
||||
script_target,
|
||||
phase,
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[script_path, str(script_target)],
|
||||
check=True,
|
||||
timeout=300, # 5 minute timeout
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
if result.stdout:
|
||||
logger.debug("Task %s: custom script stdout: %s", task.task_id, result.stdout.strip())
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
logger.error("Task %s: custom script not found: %s", task.task_id, script_path)
|
||||
status_callback("error", f"Custom script not found: {script_path}")
|
||||
return False
|
||||
except PermissionError:
|
||||
logger.error("Task %s: custom script not executable: %s", task.task_id, script_path)
|
||||
status_callback("error", f"Custom script not executable: {script_path}")
|
||||
return False
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("Task %s: custom script timed out after 300s: %s", task.task_id, script_path)
|
||||
status_callback("error", "Custom script timed out")
|
||||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
stderr = e.stderr.strip() if e.stderr else "No error output"
|
||||
logger.error(
|
||||
"Task %s: custom script failed (exit code %s): %s",
|
||||
task.task_id,
|
||||
e.returncode,
|
||||
stderr,
|
||||
)
|
||||
status_callback("error", f"Custom script failed: {stderr[:100]}")
|
||||
return False
|
||||
|
||||
# Custom script is run post-transfer (see below).
|
||||
|
||||
# If we staged a copy into TMP_DIR (e.g. for custom script), transfer from the staged
|
||||
# path and disable hardlinking for this transfer.
|
||||
use_hardlink = plan.use_hardlink and prepared.output_plan.stage_action == STAGE_NONE
|
||||
source_path = plan.hardlink_source if use_hardlink and plan.hardlink_source else prepared.working_path
|
||||
is_torrent = is_torrent_source(source_path, task)
|
||||
|
||||
usenet_action = core_config.config.get("PROWLARR_USENET_ACTION", "move")
|
||||
is_usenet = task.source == "prowlarr" and not task.original_download_path
|
||||
|
||||
# For external usenet downloads, always copy from the client path.
|
||||
# "Move" is implemented as a client-side cleanup after import.
|
||||
preserve_source = is_usenet
|
||||
|
||||
copy_for_label = is_torrent or preserve_source or prepared.output_plan.stage_action != STAGE_NONE
|
||||
|
||||
if cancel_flag.is_set():
|
||||
logger.info("Task %s: cancelled before final transfer", task.task_id)
|
||||
cleanup_output_staging(
|
||||
prepared.output_plan,
|
||||
prepared.working_path,
|
||||
task,
|
||||
prepared.cleanup_paths,
|
||||
)
|
||||
return None
|
||||
|
||||
if use_hardlink:
|
||||
op_label = "Hardlinking"
|
||||
elif is_usenet and usenet_action == "move" and prepared.output_plan.stage_action == STAGE_NONE:
|
||||
# Presented as a move, but implemented as copy + client cleanup.
|
||||
op_label = "Moving"
|
||||
elif copy_for_label:
|
||||
op_label = "Copying"
|
||||
else:
|
||||
op_label = "Moving"
|
||||
|
||||
status_callback("resolving", f"{op_label} file")
|
||||
record_step(
|
||||
steps,
|
||||
"transfer",
|
||||
op=op_label.lower(),
|
||||
source=str(source_path),
|
||||
dest=str(plan.destination),
|
||||
hardlink=use_hardlink,
|
||||
torrent=copy_for_label,
|
||||
)
|
||||
if prepared.output_plan.stage_action != STAGE_NONE:
|
||||
record_step(steps, "cleanup_staging", path=str(prepared.working_path))
|
||||
log_plan_steps(task.task_id, steps)
|
||||
|
||||
final_paths, error = transfer_book_files(
|
||||
prepared.files,
|
||||
destination=plan.destination,
|
||||
task=task,
|
||||
use_hardlink=use_hardlink,
|
||||
is_torrent=is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
organization_mode=plan.organization_mode,
|
||||
)
|
||||
|
||||
if error:
|
||||
logger.warning("Task %s: transfer failed: %s", task.task_id, error)
|
||||
status_callback("error", error)
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
"Task %s: transferred %d file(s) to %s (%s)",
|
||||
task.task_id,
|
||||
len(final_paths),
|
||||
plan.destination,
|
||||
op_label.lower(),
|
||||
)
|
||||
|
||||
# Run custom script once per successful task, after transfer.
|
||||
if core_config.config.CUSTOM_SCRIPT:
|
||||
if len(final_paths) == 1:
|
||||
target_path = final_paths[0]
|
||||
else:
|
||||
try:
|
||||
target_path = Path(os.path.commonpath([str(p.parent) for p in final_paths]))
|
||||
except ValueError:
|
||||
target_path = plan.destination
|
||||
|
||||
if not run_custom_script(core_config.config.CUSTOM_SCRIPT, target_path, phase="post_transfer"):
|
||||
cleanup_output_staging(
|
||||
prepared.output_plan,
|
||||
prepared.working_path,
|
||||
task,
|
||||
prepared.cleanup_paths,
|
||||
)
|
||||
return None
|
||||
|
||||
cleanup_output_staging(
|
||||
prepared.output_plan,
|
||||
prepared.working_path,
|
||||
task,
|
||||
prepared.cleanup_paths,
|
||||
)
|
||||
|
||||
message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
|
||||
status_callback("complete", message)
|
||||
|
||||
return str(final_paths[0])
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Permission/ownership diagnostics for filesystem operations.
|
||||
|
||||
This module centralizes best-effort debug logging used by download post-processing
|
||||
and atomic filesystem operations.
|
||||
|
||||
It is intentionally defensive: failures collecting context should never mask the
|
||||
original error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def _format_uid(uid: int) -> str:
|
||||
try:
|
||||
import pwd
|
||||
|
||||
return pwd.getpwuid(uid).pw_name
|
||||
except Exception:
|
||||
return str(uid)
|
||||
|
||||
|
||||
def _format_gid(gid: int) -> str:
|
||||
try:
|
||||
import grp
|
||||
|
||||
return grp.getgrgid(gid).gr_name
|
||||
except Exception:
|
||||
return str(gid)
|
||||
|
||||
|
||||
def log_path_permission_context(label: str, path: Path) -> None:
|
||||
"""Log useful permission/ownership context for a path.
|
||||
|
||||
Only call this from failure paths.
|
||||
"""
|
||||
|
||||
try:
|
||||
euid = os.geteuid() if hasattr(os, "geteuid") else None
|
||||
egid = os.getegid() if hasattr(os, "getegid") else None
|
||||
groups = os.getgroups() if hasattr(os, "getgroups") else []
|
||||
|
||||
if euid is not None and egid is not None:
|
||||
logger.debug(
|
||||
"Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s",
|
||||
label,
|
||||
_format_uid(euid),
|
||||
euid,
|
||||
_format_gid(egid),
|
||||
egid,
|
||||
[f"{_format_gid(g)}({g})" for g in groups],
|
||||
)
|
||||
|
||||
for probe in [path, path.parent]:
|
||||
try:
|
||||
resolved = probe.resolve()
|
||||
except Exception:
|
||||
resolved = probe
|
||||
|
||||
try:
|
||||
st = probe.stat()
|
||||
logger.debug(
|
||||
"Path permissions (%s): path=%s resolved=%s mode=%s owner=%s(%d) group=%s(%d) dir=%s symlink=%s",
|
||||
label,
|
||||
probe,
|
||||
resolved,
|
||||
oct(st.st_mode & 0o777),
|
||||
_format_uid(st.st_uid),
|
||||
st.st_uid,
|
||||
_format_gid(st.st_gid),
|
||||
st.st_gid,
|
||||
probe.is_dir(),
|
||||
probe.is_symlink(),
|
||||
)
|
||||
except Exception as stat_error:
|
||||
logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
|
||||
except Exception as context_error:
|
||||
logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
|
||||
|
||||
|
||||
def log_transfer_permission_context(label: str, source: Path, dest: Path, error: Exception) -> None:
|
||||
"""Log useful permission/ownership context when a file transfer fails."""
|
||||
|
||||
try:
|
||||
euid = os.geteuid() if hasattr(os, "geteuid") else None
|
||||
egid = os.getegid() if hasattr(os, "getegid") else None
|
||||
groups = os.getgroups() if hasattr(os, "getgroups") else []
|
||||
|
||||
if euid is not None and egid is not None:
|
||||
logger.debug(
|
||||
"Permission context (%s): euid=%s(%d) egid=%s(%d) groups=%s error=%s",
|
||||
label,
|
||||
_format_uid(euid),
|
||||
euid,
|
||||
_format_gid(egid),
|
||||
egid,
|
||||
[f"{_format_gid(g)}({g})" for g in groups],
|
||||
error,
|
||||
)
|
||||
|
||||
for probe in [source, dest, dest.parent]:
|
||||
try:
|
||||
st = probe.stat()
|
||||
logger.debug(
|
||||
"Path permissions (%s): path=%s mode=%s owner=%s(%d) group=%s(%d) exists=%s dir=%s",
|
||||
label,
|
||||
probe,
|
||||
oct(st.st_mode & 0o777),
|
||||
_format_uid(st.st_uid),
|
||||
st.st_uid,
|
||||
_format_gid(st.st_gid),
|
||||
st.st_gid,
|
||||
probe.exists(),
|
||||
probe.is_dir(),
|
||||
)
|
||||
except Exception as stat_error:
|
||||
logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
|
||||
except Exception as context_error:
|
||||
logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Post-download processing pipeline.
|
||||
|
||||
This package contains the post-download processing pipeline (staging, scanning,
|
||||
archive extraction, transfers, and safe cleanup) and the router that selects an
|
||||
output handler.
|
||||
|
||||
Output handlers live in `shelfmark.download.outputs` and should depend on
|
||||
`pipeline` (not `router`) to avoid circular imports.
|
||||
"""
|
||||
|
||||
from .router import post_process_download
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.core.utils import (
|
||||
get_aa_content_type_dir,
|
||||
get_destination,
|
||||
is_audiobook as check_audiobook,
|
||||
)
|
||||
from shelfmark.download.permissions_debug import log_path_permission_context
|
||||
|
||||
logger = setup_logger("shelfmark.download.postprocess.pipeline")
|
||||
|
||||
|
||||
def validate_destination(destination: Path, status_callback) -> bool:
|
||||
"""Validate destination path is absolute, exists, and writable."""
|
||||
|
||||
if not destination.is_absolute():
|
||||
logger.warning(f"Destination must be absolute: {destination}")
|
||||
status_callback("error", f"Destination must be absolute: {destination}")
|
||||
return False
|
||||
|
||||
if destination.exists() and not destination.is_dir():
|
||||
logger.warning(f"Destination is not a directory: {destination}")
|
||||
status_callback("error", f"Destination is not a directory: {destination}")
|
||||
return False
|
||||
|
||||
if not destination.exists():
|
||||
try:
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
except (OSError, PermissionError) as exc:
|
||||
log_path_permission_context("destination_create", destination)
|
||||
logger.warning(f"Cannot create destination: {destination} ({exc})")
|
||||
status_callback("error", f"Cannot create destination: {destination} ({exc})")
|
||||
return False
|
||||
|
||||
test_path = destination / f".shelfmark_write_test_{uuid.uuid4().hex}.tmp"
|
||||
|
||||
try:
|
||||
test_content = (
|
||||
f"This file was created to verify if '{destination}' is writable. "
|
||||
"It should've been automatically deleted. Feel free to delete it.\n"
|
||||
)
|
||||
test_path.write_text(test_content)
|
||||
test_path.unlink(missing_ok=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Destination write probe path: %s", test_path)
|
||||
log_path_permission_context("destination_write_probe", destination)
|
||||
logger.warning(f"Destination not writable: {destination} ({exc})")
|
||||
status_callback("error", f"Destination not writable: {destination} ({exc})")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def get_final_destination(task: DownloadTask) -> Path:
|
||||
"""Get final destination directory, with content-type routing support."""
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
|
||||
if task.source == "direct_download" and not is_audiobook:
|
||||
override = get_aa_content_type_dir(task.content_type)
|
||||
if override:
|
||||
return override
|
||||
|
||||
return get_destination(is_audiobook)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Post-download processing pipeline.
|
||||
|
||||
This module is the public API surface for post-download processing.
|
||||
|
||||
Implementation lives in submodules in this package:
|
||||
|
||||
- `types`: dataclasses used across the pipeline
|
||||
- `workspace`: managed workspace + cleanup rules
|
||||
- `scan`: directory scanning + archive extraction
|
||||
- `transfer`: hardlink/copy/move + naming/organization
|
||||
- `prepare`: staging plan + prepared file selection
|
||||
- `steps`: lightweight plan logging helpers
|
||||
|
||||
Keeping this file as a facade avoids churn in call sites while letting the
|
||||
implementation stay modular.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .destination import get_final_destination, validate_destination
|
||||
from .prepare import build_output_plan, prepare_output_files
|
||||
from .scan import (
|
||||
collect_directory_files,
|
||||
collect_staged_files,
|
||||
extract_archive_files,
|
||||
get_supported_formats,
|
||||
scan_directory_tree,
|
||||
)
|
||||
from .steps import log_plan_steps, record_step
|
||||
from .transfer import (
|
||||
build_metadata_dict,
|
||||
is_torrent_source,
|
||||
process_directory,
|
||||
resolve_hardlink_source,
|
||||
should_hardlink,
|
||||
transfer_book_files,
|
||||
transfer_directory_to_library,
|
||||
transfer_file_to_library,
|
||||
)
|
||||
from .types import OutputPlan, PlanStep, PreparedFiles, TransferPlan
|
||||
from .workspace import (
|
||||
cleanup_output_staging,
|
||||
is_managed_workspace_path,
|
||||
is_within_tmp_dir,
|
||||
safe_cleanup_path,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"OutputPlan",
|
||||
"PlanStep",
|
||||
"PreparedFiles",
|
||||
"TransferPlan",
|
||||
"build_metadata_dict",
|
||||
"build_output_plan",
|
||||
"cleanup_output_staging",
|
||||
"collect_directory_files",
|
||||
"collect_staged_files",
|
||||
"extract_archive_files",
|
||||
"get_final_destination",
|
||||
"get_supported_formats",
|
||||
"is_managed_workspace_path",
|
||||
"is_torrent_source",
|
||||
"is_within_tmp_dir",
|
||||
"log_plan_steps",
|
||||
"prepare_output_files",
|
||||
"process_directory",
|
||||
"record_step",
|
||||
"resolve_hardlink_source",
|
||||
"safe_cleanup_path",
|
||||
"scan_directory_tree",
|
||||
"should_hardlink",
|
||||
"transfer_book_files",
|
||||
"transfer_directory_to_library",
|
||||
"transfer_file_to_library",
|
||||
"validate_destination",
|
||||
]
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Post-download processing policy.
|
||||
|
||||
This module holds configuration-driven *policy* decisions that are shared across
|
||||
post-download processing components, but are not specific to archive extraction.
|
||||
|
||||
Examples:
|
||||
- Which file formats are enabled
|
||||
- How files should be organized (none/rename/organize)
|
||||
- Which naming templates to use
|
||||
|
||||
Implementation note:
|
||||
Keep this module free of dependencies on archive extraction mechanics to avoid
|
||||
circular imports (`archive` is used by the pipeline).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
import shelfmark.core.config as core_config
|
||||
|
||||
|
||||
def get_supported_formats() -> List[str]:
|
||||
"""Get current supported formats from config singleton."""
|
||||
|
||||
formats = core_config.config.get(
|
||||
"SUPPORTED_FORMATS",
|
||||
["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
|
||||
)
|
||||
|
||||
# Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
|
||||
if isinstance(formats, str):
|
||||
return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
|
||||
|
||||
return [fmt.lower() for fmt in formats]
|
||||
|
||||
|
||||
def get_supported_audiobook_formats() -> List[str]:
|
||||
"""Get current supported audiobook formats from config singleton."""
|
||||
|
||||
formats = core_config.config.get("SUPPORTED_AUDIOBOOK_FORMATS", ["m4b", "mp3"])
|
||||
|
||||
# Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
|
||||
if isinstance(formats, str):
|
||||
return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
|
||||
|
||||
return [fmt.lower() for fmt in formats]
|
||||
|
||||
|
||||
def get_file_organization(is_audiobook: bool) -> str:
|
||||
"""Get the file organization mode for the content type."""
|
||||
|
||||
key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
|
||||
mode = core_config.config.get(key, "rename")
|
||||
|
||||
# Handle legacy settings migration
|
||||
if mode not in ("none", "rename", "organize"):
|
||||
legacy_key = "PROCESSING_MODE_AUDIOBOOK" if is_audiobook else "PROCESSING_MODE"
|
||||
legacy_mode = core_config.config.get(legacy_key, "ingest")
|
||||
if legacy_mode == "library":
|
||||
return "organize"
|
||||
if core_config.config.get("USE_BOOK_TITLE", True):
|
||||
return "rename"
|
||||
return "none"
|
||||
|
||||
return mode
|
||||
|
||||
|
||||
def get_template(is_audiobook: bool, organization_mode: str) -> str:
|
||||
"""Get the template for the content type and organization mode."""
|
||||
|
||||
# Determine the correct key based on content type and organization mode
|
||||
if is_audiobook:
|
||||
if organization_mode == "organize":
|
||||
key = "TEMPLATE_AUDIOBOOK_ORGANIZE"
|
||||
else:
|
||||
key = "TEMPLATE_AUDIOBOOK_RENAME"
|
||||
else:
|
||||
if organization_mode == "organize":
|
||||
key = "TEMPLATE_ORGANIZE"
|
||||
else:
|
||||
key = "TEMPLATE_RENAME"
|
||||
|
||||
template = core_config.config.get(key, "")
|
||||
|
||||
# Fallback to legacy keys if new keys are empty
|
||||
if not template:
|
||||
legacy_key = "TEMPLATE_AUDIOBOOK" if is_audiobook else "TEMPLATE"
|
||||
template = core_config.config.get(legacy_key, "")
|
||||
|
||||
if not template:
|
||||
legacy_key = "LIBRARY_TEMPLATE_AUDIOBOOK" if is_audiobook else "LIBRARY_TEMPLATE"
|
||||
template = core_config.config.get(legacy_key, "")
|
||||
|
||||
if not template:
|
||||
if organization_mode == "organize":
|
||||
return "{Author}/{Title} ({Year})"
|
||||
return "{Author} - {Title} ({Year})"
|
||||
|
||||
return template
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import shelfmark.core.config as core_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.download.archive import is_archive
|
||||
from shelfmark.download.staging import STAGE_COPY, STAGE_NONE, get_staging_dir, stage_path
|
||||
|
||||
from .scan import collect_staged_files
|
||||
from .transfer import resolve_hardlink_source
|
||||
from .types import OutputPlan, PreparedFiles
|
||||
from .workspace import cleanup_output_staging, is_managed_workspace_path
|
||||
|
||||
logger = setup_logger("shelfmark.download.postprocess.pipeline")
|
||||
|
||||
|
||||
def build_output_plan(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
output_mode: str,
|
||||
destination: Optional[Path] = None,
|
||||
status_callback=None,
|
||||
) -> OutputPlan:
|
||||
"""Build an output plan that describes staging behavior for file-based outputs."""
|
||||
|
||||
transfer_plan = resolve_hardlink_source(temp_file, task, destination, status_callback)
|
||||
runs_custom_script = bool(core_config.config.CUSTOM_SCRIPT) and temp_file.is_file() and not is_archive(temp_file)
|
||||
|
||||
stage_action = STAGE_COPY if runs_custom_script and not is_managed_workspace_path(temp_file) else STAGE_NONE
|
||||
staging_dir = get_staging_dir()
|
||||
|
||||
return OutputPlan(
|
||||
mode=output_mode,
|
||||
stage_action=stage_action,
|
||||
staging_dir=staging_dir,
|
||||
allow_archive_extraction=transfer_plan.allow_archive_extraction,
|
||||
transfer_plan=transfer_plan,
|
||||
)
|
||||
|
||||
|
||||
def prepare_output_files(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
output_mode: str,
|
||||
status_callback,
|
||||
destination: Optional[Path] = None,
|
||||
output_plan: Optional[OutputPlan] = None,
|
||||
) -> Optional[PreparedFiles]:
|
||||
if output_plan is None:
|
||||
output_plan = build_output_plan(
|
||||
temp_file,
|
||||
task,
|
||||
output_mode=output_mode,
|
||||
destination=destination,
|
||||
status_callback=status_callback,
|
||||
)
|
||||
|
||||
working_path = temp_file
|
||||
if output_plan.stage_action != STAGE_NONE:
|
||||
step_label = "Staging torrent files" if output_plan.stage_action == STAGE_COPY else "Staging files"
|
||||
status_callback("resolving", step_label)
|
||||
working_path = stage_path(working_path, output_plan.staging_dir, output_plan.stage_action)
|
||||
|
||||
can_delete_source_archives = output_plan.stage_action != STAGE_NONE or is_managed_workspace_path(working_path)
|
||||
|
||||
files, rejected_files, cleanup_paths, error = collect_staged_files(
|
||||
working_path=working_path,
|
||||
task=task,
|
||||
allow_archive_extraction=output_plan.allow_archive_extraction,
|
||||
status_callback=status_callback,
|
||||
cleanup_archives=can_delete_source_archives,
|
||||
)
|
||||
|
||||
if error:
|
||||
status_callback("error", error)
|
||||
cleanup_output_staging(output_plan, working_path, task, cleanup_paths)
|
||||
return None
|
||||
|
||||
if output_plan.stage_action == STAGE_NONE and is_managed_workspace_path(working_path):
|
||||
cleanup_paths = [*cleanup_paths, working_path]
|
||||
|
||||
return PreparedFiles(
|
||||
output_plan=output_plan,
|
||||
working_path=working_path,
|
||||
files=files,
|
||||
rejected_files=rejected_files,
|
||||
cleanup_paths=cleanup_paths,
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Output routing for post-download processing.
|
||||
|
||||
This module selects the appropriate output handler and invokes it.
|
||||
|
||||
Keeping this separate from `pipeline.py` avoids circular imports:
|
||||
|
||||
- output handlers depend on `pipeline`
|
||||
- router depends on the output registry
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from typing import Optional
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask, SearchMode
|
||||
from shelfmark.download.outputs import resolve_output_handler
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def post_process_download(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
cancel_flag: Event,
|
||||
status_callback,
|
||||
) -> Optional[str]:
|
||||
"""Post-process download using the selected output handler."""
|
||||
|
||||
if task.search_mode is None:
|
||||
logger.warning(
|
||||
"Task %s: missing search_mode; defaulting to Direct mode behavior",
|
||||
task.task_id,
|
||||
)
|
||||
elif task.search_mode not in (SearchMode.DIRECT, SearchMode.UNIVERSAL):
|
||||
logger.warning(
|
||||
"Task %s: invalid search_mode=%s; defaulting to Direct mode behavior",
|
||||
task.task_id,
|
||||
task.search_mode,
|
||||
)
|
||||
|
||||
output_handler = resolve_output_handler(task)
|
||||
if output_handler:
|
||||
logger.info("Task %s: using output mode %s", task.task_id, output_handler.mode)
|
||||
return output_handler.handler(temp_file, task, cancel_flag, status_callback)
|
||||
|
||||
from shelfmark.download.outputs.folder import process_folder_output
|
||||
|
||||
logger.info("Task %s: using output mode folder", task.task_id)
|
||||
return process_folder_output(temp_file, task, cancel_flag, status_callback)
|
||||
@@ -0,0 +1,318 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive
|
||||
from shelfmark.download.permissions_debug import log_path_permission_context
|
||||
from shelfmark.download.postprocess.policy import (
|
||||
get_supported_audiobook_formats,
|
||||
get_supported_formats as get_book_formats,
|
||||
)
|
||||
from shelfmark.download.staging import build_staging_dir
|
||||
|
||||
logger = setup_logger("shelfmark.download.postprocess.pipeline")
|
||||
|
||||
|
||||
def get_supported_formats(content_type: Optional[str] = None) -> List[str]:
|
||||
if check_audiobook(content_type):
|
||||
return get_supported_audiobook_formats()
|
||||
return get_book_formats()
|
||||
|
||||
|
||||
def _format_not_supported_error(rejected_files: List[Path], task: DownloadTask) -> str:
|
||||
content_type = task.content_type
|
||||
file_type_label = "audiobook" if check_audiobook(content_type) else "book"
|
||||
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
|
||||
rejected_list = ", ".join(rejected_exts)
|
||||
supported_formats = get_supported_formats(content_type)
|
||||
|
||||
logger.warning(
|
||||
"Task %s: found %d %s(s) but none match supported formats. Rejected formats: %s. Supported: %s",
|
||||
task.task_id,
|
||||
len(rejected_files),
|
||||
file_type_label,
|
||||
rejected_list,
|
||||
", ".join(sorted(supported_formats)),
|
||||
)
|
||||
|
||||
return (
|
||||
f"Found {len(rejected_files)} {file_type_label}(s) but format not supported ({rejected_list}). "
|
||||
"Enable in Settings > Formats."
|
||||
)
|
||||
|
||||
|
||||
def extract_archive_files(
|
||||
archive_path: Path,
|
||||
output_dir: Path,
|
||||
task: DownloadTask,
|
||||
cleanup_archive: bool,
|
||||
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
|
||||
content_type = task.content_type
|
||||
|
||||
try:
|
||||
extracted_files, warnings, rejected_files = extract_archive(archive_path, output_dir, content_type)
|
||||
except ArchiveExtractionError as exc:
|
||||
logger.warning(
|
||||
"Task %s: archive extraction failed for %s: %s",
|
||||
task.task_id,
|
||||
archive_path.name,
|
||||
exc,
|
||||
)
|
||||
return [], [], [], str(exc)
|
||||
|
||||
if warnings:
|
||||
logger.debug(
|
||||
"Task %s: archive warnings for %s: %s",
|
||||
task.task_id,
|
||||
archive_path.name,
|
||||
"; ".join(warnings),
|
||||
)
|
||||
|
||||
if cleanup_archive:
|
||||
archive_path.unlink(missing_ok=True)
|
||||
|
||||
cleanup_paths = [output_dir]
|
||||
|
||||
if not extracted_files:
|
||||
if rejected_files:
|
||||
return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
|
||||
file_type_label = "audiobook" if check_audiobook(content_type) else "book"
|
||||
return [], rejected_files, cleanup_paths, f"No {file_type_label} files found in archive"
|
||||
|
||||
logger.debug(
|
||||
"Task %s: extracted %d file(s) from archive %s",
|
||||
task.task_id,
|
||||
len(extracted_files),
|
||||
archive_path.name,
|
||||
)
|
||||
|
||||
return extracted_files, rejected_files, cleanup_paths, None
|
||||
|
||||
|
||||
def scan_directory_tree(
|
||||
directory: Path,
|
||||
content_type: Optional[str],
|
||||
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
|
||||
"""Scan a directory tree for book files, trackable-but-unsupported files, and archives."""
|
||||
|
||||
try:
|
||||
with os.scandir(directory) as it:
|
||||
next(it, None)
|
||||
except PermissionError as exc:
|
||||
log_path_permission_context("scan_directory", directory)
|
||||
logger.warning(f"Permission denied scanning directory: {directory} ({exc})")
|
||||
return [], [], [], f"Permission denied accessing download folder: {directory}"
|
||||
except (FileNotFoundError, NotADirectoryError, OSError) as exc:
|
||||
logger.warning(f"Cannot access download folder: {directory} ({exc})")
|
||||
return [], [], [], f"Cannot access download folder: {directory} ({exc})"
|
||||
|
||||
book_files: List[Path] = []
|
||||
rejected_files: List[Path] = []
|
||||
archive_files: List[Path] = []
|
||||
|
||||
supported_formats = get_supported_formats(content_type)
|
||||
supported_exts = {f".{fmt}" for fmt in supported_formats}
|
||||
|
||||
is_audiobook = check_audiobook(content_type)
|
||||
if is_audiobook:
|
||||
trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
|
||||
else:
|
||||
trackable_exts = {
|
||||
'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
|
||||
'.doc', '.docx', '.rtf', '.txt',
|
||||
}
|
||||
|
||||
logged_walk_permission_context = False
|
||||
|
||||
def onerror(error: OSError) -> None:
|
||||
nonlocal logged_walk_permission_context
|
||||
|
||||
if isinstance(error, PermissionError):
|
||||
if not logged_walk_permission_context:
|
||||
try:
|
||||
error_path = Path(getattr(error, "filename", "") or str(directory))
|
||||
except Exception:
|
||||
error_path = directory
|
||||
|
||||
log_path_permission_context("scan_directory_walk", error_path)
|
||||
logged_walk_permission_context = True
|
||||
|
||||
logger.debug(f"Skipping inaccessible path during scan: {error}")
|
||||
else:
|
||||
logger.debug(f"Error scanning directory tree: {error}")
|
||||
|
||||
for root, _, files in os.walk(directory, onerror=onerror):
|
||||
for filename in files:
|
||||
file_path = Path(root) / filename
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix in supported_exts:
|
||||
book_files.append(file_path)
|
||||
elif suffix in trackable_exts:
|
||||
rejected_files.append(file_path)
|
||||
|
||||
if is_archive(file_path):
|
||||
archive_files.append(file_path)
|
||||
|
||||
return book_files, rejected_files, archive_files, None
|
||||
|
||||
|
||||
def collect_directory_files(
|
||||
directory: Path,
|
||||
task: DownloadTask,
|
||||
allow_archive_extraction: bool,
|
||||
status_callback=None,
|
||||
cleanup_archives: bool = False,
|
||||
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
|
||||
content_type = task.content_type
|
||||
book_files, rejected_files, archive_files, scan_error = scan_directory_tree(directory, content_type)
|
||||
if scan_error:
|
||||
return [], [], [], scan_error
|
||||
|
||||
if book_files:
|
||||
if archive_files:
|
||||
logger.debug(
|
||||
"Task %s: ignoring %d archive(s) - already have %d book file(s)",
|
||||
task.task_id,
|
||||
len(archive_files),
|
||||
len(book_files),
|
||||
)
|
||||
if rejected_files:
|
||||
rejected_exts = sorted(set(f.suffix.lower() for f in rejected_files))
|
||||
logger.debug(
|
||||
"Task %s: also found %d file(s) with unsupported formats: %s",
|
||||
task.task_id,
|
||||
len(rejected_files),
|
||||
", ".join(rejected_exts),
|
||||
)
|
||||
return book_files, rejected_files, [], None
|
||||
|
||||
if archive_files:
|
||||
if not allow_archive_extraction:
|
||||
logger.warning(
|
||||
"Task %s: archive extraction disabled (torrent hardlinking enabled) for %s",
|
||||
task.task_id,
|
||||
directory,
|
||||
)
|
||||
return [], rejected_files, [], "Archive extraction is disabled when torrent hardlinking is enabled"
|
||||
|
||||
if status_callback:
|
||||
status_callback("resolving", "Extracting archives")
|
||||
|
||||
logger.info("Task %s: extracting %d archive(s)", task.task_id, len(archive_files))
|
||||
|
||||
all_files: List[Path] = []
|
||||
all_errors: List[str] = []
|
||||
cleanup_paths: List[Path] = []
|
||||
|
||||
for archive in archive_files:
|
||||
extract_dir = build_staging_dir("extract", task.task_id)
|
||||
extracted_files, archive_rejected, archive_cleanup, error = extract_archive_files(
|
||||
archive_path=archive,
|
||||
output_dir=extract_dir,
|
||||
task=task,
|
||||
cleanup_archive=cleanup_archives,
|
||||
)
|
||||
|
||||
if error:
|
||||
all_errors.append(f"{archive.name}: {error}")
|
||||
if archive_rejected:
|
||||
rejected_files.extend(archive_rejected)
|
||||
if extracted_files:
|
||||
all_files.extend(extracted_files)
|
||||
if archive_cleanup:
|
||||
cleanup_paths.extend(archive_cleanup)
|
||||
|
||||
if all_files:
|
||||
logger.info(
|
||||
"Task %s: extracted %d file(s) from %d archive(s)",
|
||||
task.task_id,
|
||||
len(all_files),
|
||||
len(archive_files),
|
||||
)
|
||||
return all_files, rejected_files, cleanup_paths, None
|
||||
|
||||
if all_errors:
|
||||
return [], rejected_files, cleanup_paths, "; ".join(all_errors)
|
||||
|
||||
if rejected_files:
|
||||
return [], rejected_files, cleanup_paths, _format_not_supported_error(rejected_files, task)
|
||||
|
||||
return [], rejected_files, cleanup_paths, "No book files found in archives"
|
||||
|
||||
if rejected_files:
|
||||
return [], rejected_files, [], _format_not_supported_error(rejected_files, task)
|
||||
|
||||
return [], rejected_files, [], "No book files found in download"
|
||||
|
||||
|
||||
def collect_staged_files(
|
||||
working_path: Path,
|
||||
task: DownloadTask,
|
||||
allow_archive_extraction: bool,
|
||||
status_callback,
|
||||
cleanup_archives: bool,
|
||||
) -> Tuple[List[Path], List[Path], List[Path], Optional[str]]:
|
||||
if working_path.is_dir():
|
||||
if status_callback:
|
||||
status_callback("resolving", "Processing download folder")
|
||||
return collect_directory_files(
|
||||
working_path,
|
||||
task,
|
||||
allow_archive_extraction=allow_archive_extraction,
|
||||
status_callback=status_callback,
|
||||
cleanup_archives=cleanup_archives,
|
||||
)
|
||||
|
||||
if is_archive(working_path) and allow_archive_extraction:
|
||||
if status_callback:
|
||||
status_callback("resolving", "Extracting archive")
|
||||
|
||||
logger.info("Task %s: extracting archive %s", task.task_id, working_path.name)
|
||||
|
||||
extract_dir = build_staging_dir("extract", task.task_id)
|
||||
extracted_files, rejected_files, cleanup_paths, error = extract_archive_files(
|
||||
archive_path=working_path,
|
||||
output_dir=extract_dir,
|
||||
task=task,
|
||||
cleanup_archive=cleanup_archives,
|
||||
)
|
||||
|
||||
if extracted_files:
|
||||
logger.info(
|
||||
"Task %s: extracted %d file(s) from archive %s",
|
||||
task.task_id,
|
||||
len(extracted_files),
|
||||
working_path.name,
|
||||
)
|
||||
|
||||
return extracted_files, rejected_files, cleanup_paths, error
|
||||
|
||||
# Single-file download result (non-archive).
|
||||
# Ensure we respect the user's supported format settings.
|
||||
suffix = working_path.suffix.lower()
|
||||
supported_formats = get_supported_formats(task.content_type)
|
||||
supported_exts = {f".{fmt}" for fmt in supported_formats}
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
if is_audiobook:
|
||||
trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
|
||||
else:
|
||||
trackable_exts = {
|
||||
'.pdf', '.epub', '.mobi', '.azw', '.azw3', '.fb2', '.djvu', '.cbz', '.cbr',
|
||||
'.doc', '.docx', '.rtf', '.txt',
|
||||
}
|
||||
|
||||
if suffix in supported_exts:
|
||||
return [working_path], [], [], None
|
||||
|
||||
if suffix in trackable_exts:
|
||||
return [], [working_path], [], _format_not_supported_error([working_path], task)
|
||||
|
||||
file_type_label = "audiobook" if is_audiobook else "book"
|
||||
return [], [], [], f"Unsupported {file_type_label} file type: {suffix or working_path.name}"
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, List
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
from .types import PlanStep
|
||||
|
||||
logger = setup_logger("shelfmark.download.postprocess.pipeline")
|
||||
|
||||
|
||||
def record_step(steps: List[PlanStep], name: str, **details: Any) -> None:
|
||||
steps.append(PlanStep(name=name, details=details))
|
||||
|
||||
|
||||
def log_plan_steps(task_id: str, steps: List[PlanStep]) -> None:
|
||||
if not steps:
|
||||
return
|
||||
summary = " -> ".join(step.name for step in steps)
|
||||
logger.debug("Processing plan for %s: %s", task_id, summary)
|
||||
@@ -0,0 +1,399 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import shelfmark.core.config as core_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.core.naming import (
|
||||
assign_part_numbers,
|
||||
build_library_path,
|
||||
parse_naming_template,
|
||||
same_filesystem,
|
||||
sanitize_filename,
|
||||
)
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.fs import atomic_copy, atomic_hardlink, atomic_move
|
||||
from shelfmark.download.postprocess.policy import get_file_organization, get_template
|
||||
|
||||
from .scan import collect_directory_files, scan_directory_tree
|
||||
from .types import TransferPlan
|
||||
from .workspace import safe_cleanup_path
|
||||
|
||||
logger = setup_logger("shelfmark.download.postprocess.pipeline")
|
||||
|
||||
|
||||
def should_hardlink(task: DownloadTask) -> bool:
|
||||
"""Check if hardlinking is enabled for this task (Prowlarr torrents only)."""
|
||||
|
||||
if task.source != "prowlarr":
|
||||
return False
|
||||
|
||||
if not task.original_download_path:
|
||||
return False
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
key = "HARDLINK_TORRENTS_AUDIOBOOK" if is_audiobook else "HARDLINK_TORRENTS"
|
||||
|
||||
hardlink_enabled = core_config.config.get(key)
|
||||
if hardlink_enabled is None:
|
||||
hardlink_enabled = core_config.config.get("TORRENT_HARDLINK", False)
|
||||
|
||||
return bool(hardlink_enabled)
|
||||
|
||||
|
||||
|
||||
def build_metadata_dict(task: DownloadTask) -> dict:
|
||||
return {
|
||||
"Author": task.author,
|
||||
"Title": task.title,
|
||||
"Subtitle": task.subtitle,
|
||||
"Year": task.year,
|
||||
"Series": task.series_name,
|
||||
"SeriesPosition": task.series_position,
|
||||
}
|
||||
|
||||
|
||||
def resolve_hardlink_source(
|
||||
temp_file: Path,
|
||||
task: DownloadTask,
|
||||
destination: Optional[Path],
|
||||
status_callback=None,
|
||||
) -> TransferPlan:
|
||||
"""Resolve hardlink eligibility and source path for transfers."""
|
||||
|
||||
use_hardlink = False
|
||||
source_path = temp_file
|
||||
hardlink_enabled = should_hardlink(task)
|
||||
|
||||
if hardlink_enabled and task.original_download_path:
|
||||
hardlink_source = Path(task.original_download_path)
|
||||
if destination and hardlink_source.exists() and same_filesystem(hardlink_source, destination):
|
||||
use_hardlink = True
|
||||
source_path = hardlink_source
|
||||
elif hardlink_source.exists():
|
||||
logger.warning(
|
||||
f"Cannot hardlink: {hardlink_source} and {destination} are on different filesystems. "
|
||||
"Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination."
|
||||
)
|
||||
if status_callback:
|
||||
status_callback("resolving", "Cannot hardlink (different filesystems), using copy")
|
||||
|
||||
return TransferPlan(
|
||||
source_path=source_path,
|
||||
use_hardlink=use_hardlink,
|
||||
allow_archive_extraction=not hardlink_enabled,
|
||||
hardlink_enabled=hardlink_enabled,
|
||||
)
|
||||
|
||||
|
||||
def is_torrent_source(source_path: Path, task: DownloadTask) -> bool:
|
||||
"""Check if source is the torrent client path (needs copy to preserve seeding)."""
|
||||
|
||||
if not task.original_download_path:
|
||||
return False
|
||||
|
||||
original_path = Path(task.original_download_path)
|
||||
try:
|
||||
return source_path.resolve() == original_path.resolve()
|
||||
except (OSError, ValueError):
|
||||
try:
|
||||
return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _max_attempts_for_batch(file_count: int, default: int = 100) -> int:
|
||||
if file_count <= 1:
|
||||
return default
|
||||
return max(default, file_count + default)
|
||||
|
||||
|
||||
def _transfer_single_file(
|
||||
source_path: Path,
|
||||
dest_path: Path,
|
||||
use_hardlink: bool,
|
||||
is_torrent: bool,
|
||||
preserve_source: bool = False,
|
||||
max_attempts: int = 100,
|
||||
) -> Tuple[Path, str]:
|
||||
if use_hardlink:
|
||||
final_path = atomic_hardlink(source_path, dest_path, max_attempts=max_attempts)
|
||||
try:
|
||||
if os.stat(source_path).st_ino == os.stat(final_path).st_ino:
|
||||
return final_path, "hardlink"
|
||||
except OSError:
|
||||
return final_path, "hardlink"
|
||||
return final_path, "copy"
|
||||
|
||||
if is_torrent or preserve_source:
|
||||
return atomic_copy(source_path, dest_path, max_attempts=max_attempts), "copy"
|
||||
|
||||
return atomic_move(source_path, dest_path, max_attempts=max_attempts), "move"
|
||||
|
||||
|
||||
def transfer_book_files(
|
||||
book_files: List[Path],
|
||||
destination: Path,
|
||||
task: DownloadTask,
|
||||
use_hardlink: bool,
|
||||
is_torrent: bool,
|
||||
preserve_source: bool = False,
|
||||
organization_mode: Optional[str] = None,
|
||||
) -> Tuple[List[Path], Optional[str]]:
|
||||
if not book_files:
|
||||
return [], "No book files found"
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
organization_mode = organization_mode or get_file_organization(is_audiobook)
|
||||
max_attempts = _max_attempts_for_batch(len(book_files))
|
||||
|
||||
final_paths: List[Path] = []
|
||||
|
||||
if organization_mode == "organize":
|
||||
template = get_template(is_audiobook, "organize")
|
||||
metadata = build_metadata_dict(task)
|
||||
|
||||
if len(book_files) == 1:
|
||||
source_file = book_files[0]
|
||||
ext = source_file.suffix.lstrip(".") or task.format or ""
|
||||
dest_path = build_library_path(str(destination), template, metadata, extension=ext or None)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
final_path, op = _transfer_single_file(
|
||||
source_file,
|
||||
dest_path,
|
||||
use_hardlink,
|
||||
is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
final_paths.append(final_path)
|
||||
logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
|
||||
else:
|
||||
zero_pad_width = max(len(str(len(book_files))), 2)
|
||||
files_with_parts = assign_part_numbers(book_files, zero_pad_width)
|
||||
|
||||
for source_file, part_number in files_with_parts:
|
||||
ext = source_file.suffix.lstrip(".") or task.format or ""
|
||||
file_metadata = {**metadata, "PartNumber": part_number}
|
||||
dest_path = build_library_path(str(destination), template, file_metadata, extension=ext or None)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
final_path, op = _transfer_single_file(
|
||||
source_file,
|
||||
dest_path,
|
||||
use_hardlink,
|
||||
is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
final_paths.append(final_path)
|
||||
logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
|
||||
|
||||
return final_paths, None
|
||||
|
||||
for book_file in book_files:
|
||||
if len(book_files) == 1 and organization_mode != "none":
|
||||
if not task.format:
|
||||
task.format = book_file.suffix.lower().lstrip(".")
|
||||
|
||||
template = get_template(is_audiobook, "rename")
|
||||
metadata = build_metadata_dict(task)
|
||||
extension = book_file.suffix.lstrip(".") or task.format or ""
|
||||
|
||||
filename = parse_naming_template(template, metadata, allow_path_separators=False)
|
||||
filename = Path(filename).name if filename else ""
|
||||
if filename and extension:
|
||||
filename = f"{sanitize_filename(filename)}.{extension}"
|
||||
else:
|
||||
filename = book_file.name
|
||||
else:
|
||||
filename = book_file.name
|
||||
|
||||
dest_path = destination / filename
|
||||
final_path, op = _transfer_single_file(
|
||||
book_file,
|
||||
dest_path,
|
||||
use_hardlink,
|
||||
is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
final_paths.append(final_path)
|
||||
logger.debug(f"{op.capitalize()} to destination: {final_path.name}")
|
||||
|
||||
return final_paths, None
|
||||
|
||||
|
||||
def process_directory(
|
||||
directory: Path,
|
||||
ingest_dir: Path,
|
||||
task: DownloadTask,
|
||||
allow_archive_extraction: bool = True,
|
||||
use_hardlink: Optional[bool] = None,
|
||||
) -> Tuple[List[Path], Optional[str]]:
|
||||
"""Process staged directory: find book files, extract archives, move to ingest."""
|
||||
|
||||
try:
|
||||
is_torrent = is_torrent_source(directory, task)
|
||||
book_files, _, cleanup_paths, error = collect_directory_files(
|
||||
directory,
|
||||
task,
|
||||
allow_archive_extraction=allow_archive_extraction,
|
||||
status_callback=None,
|
||||
cleanup_archives=not is_torrent,
|
||||
)
|
||||
|
||||
if error:
|
||||
if not is_torrent:
|
||||
safe_cleanup_path(directory, task)
|
||||
for cleanup_path in cleanup_paths:
|
||||
safe_cleanup_path(cleanup_path, task)
|
||||
return [], error
|
||||
|
||||
if use_hardlink is None:
|
||||
use_hardlink = should_hardlink(task)
|
||||
|
||||
final_paths, error = transfer_book_files(
|
||||
book_files,
|
||||
destination=ingest_dir,
|
||||
task=task,
|
||||
use_hardlink=use_hardlink,
|
||||
is_torrent=is_torrent,
|
||||
)
|
||||
|
||||
if error:
|
||||
return [], error
|
||||
|
||||
if not is_torrent:
|
||||
safe_cleanup_path(directory, task)
|
||||
for cleanup_path in cleanup_paths:
|
||||
safe_cleanup_path(cleanup_path, task)
|
||||
|
||||
return final_paths, None
|
||||
|
||||
except Exception as exc:
|
||||
logger.error_trace("Task %s: error processing directory %s: %s", task.task_id, directory, exc)
|
||||
if not is_torrent_source(directory, task):
|
||||
safe_cleanup_path(directory, task)
|
||||
return [], str(exc)
|
||||
|
||||
|
||||
def transfer_file_to_library(
|
||||
source_path: Path,
|
||||
library_base: str,
|
||||
template: str,
|
||||
metadata: dict,
|
||||
task: DownloadTask,
|
||||
temp_file: Optional[Path],
|
||||
status_callback,
|
||||
use_hardlink: bool,
|
||||
) -> Optional[str]:
|
||||
extension = source_path.suffix.lstrip(".") or task.format
|
||||
dest_path = build_library_path(library_base, template, metadata, extension)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
is_torrent = is_torrent_source(source_path, task)
|
||||
final_path, op = _transfer_single_file(
|
||||
source_path,
|
||||
dest_path,
|
||||
use_hardlink,
|
||||
is_torrent,
|
||||
max_attempts=_max_attempts_for_batch(1),
|
||||
)
|
||||
logger.info(f"Library {op}: {final_path}")
|
||||
|
||||
if use_hardlink and temp_file and not is_torrent_source(temp_file, task):
|
||||
safe_cleanup_path(temp_file, task)
|
||||
|
||||
status_callback("complete", "Complete")
|
||||
return str(final_path)
|
||||
|
||||
|
||||
def transfer_directory_to_library(
|
||||
source_dir: Path,
|
||||
library_base: str,
|
||||
template: str,
|
||||
metadata: dict,
|
||||
task: DownloadTask,
|
||||
temp_file: Optional[Path],
|
||||
status_callback,
|
||||
use_hardlink: bool,
|
||||
) -> Optional[str]:
|
||||
content_type = task.content_type.lower() if task.content_type else None
|
||||
source_files, _, _, scan_error = scan_directory_tree(source_dir, content_type)
|
||||
if scan_error:
|
||||
logger.warning(scan_error)
|
||||
status_callback("error", scan_error)
|
||||
if temp_file:
|
||||
safe_cleanup_path(temp_file, task)
|
||||
return None
|
||||
|
||||
if not source_files:
|
||||
logger.warning(f"No supported files in {source_dir.name}")
|
||||
status_callback("error", "No supported file formats found")
|
||||
if temp_file:
|
||||
safe_cleanup_path(temp_file, task)
|
||||
return None
|
||||
|
||||
base_library_path = build_library_path(library_base, template, metadata, extension=None)
|
||||
base_library_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
is_torrent = is_torrent_source(source_dir, task)
|
||||
transferred_paths: List[Path] = []
|
||||
max_attempts = _max_attempts_for_batch(len(source_files))
|
||||
|
||||
if len(source_files) == 1:
|
||||
source_file = source_files[0]
|
||||
ext = source_file.suffix.lstrip(".")
|
||||
dest_path = base_library_path.with_suffix(f".{ext}")
|
||||
final_path, op = _transfer_single_file(
|
||||
source_file,
|
||||
dest_path,
|
||||
use_hardlink,
|
||||
is_torrent,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
|
||||
transferred_paths.append(final_path)
|
||||
else:
|
||||
zero_pad_width = max(len(str(len(source_files))), 2)
|
||||
files_with_parts = assign_part_numbers(source_files, zero_pad_width)
|
||||
|
||||
for source_file, part_number in files_with_parts:
|
||||
ext = source_file.suffix.lstrip(".")
|
||||
file_metadata = {**metadata, "PartNumber": part_number}
|
||||
file_path = build_library_path(library_base, template, file_metadata, extension=ext)
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
final_path, op = _transfer_single_file(
|
||||
source_file,
|
||||
file_path,
|
||||
use_hardlink,
|
||||
is_torrent,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
logger.debug(f"Library {op}: {source_file.name} -> {final_path}")
|
||||
transferred_paths.append(final_path)
|
||||
|
||||
if use_hardlink:
|
||||
operation = "hardlinks"
|
||||
elif is_torrent:
|
||||
operation = "copies"
|
||||
else:
|
||||
operation = "files"
|
||||
logger.info(f"Created {len(transferred_paths)} library {operation} in {base_library_path.parent}")
|
||||
|
||||
if use_hardlink and temp_file and not is_torrent_source(temp_file, task):
|
||||
safe_cleanup_path(temp_file, task)
|
||||
elif not is_torrent:
|
||||
safe_cleanup_path(temp_file, task)
|
||||
safe_cleanup_path(source_dir, task)
|
||||
|
||||
message = f"Complete ({len(transferred_paths)} files)" if len(transferred_paths) > 1 else "Complete"
|
||||
status_callback("complete", message)
|
||||
|
||||
return str(transferred_paths[0])
|
||||
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from shelfmark.download.staging import StageAction
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransferPlan:
|
||||
source_path: Path
|
||||
use_hardlink: bool
|
||||
allow_archive_extraction: bool
|
||||
hardlink_enabled: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OutputPlan:
|
||||
mode: str
|
||||
stage_action: StageAction
|
||||
staging_dir: Path
|
||||
allow_archive_extraction: bool
|
||||
transfer_plan: Optional[TransferPlan] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedFiles:
|
||||
output_plan: OutputPlan
|
||||
working_path: Path
|
||||
files: List[Path]
|
||||
rejected_files: List[Path]
|
||||
cleanup_paths: List[Path]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlanStep:
|
||||
name: str
|
||||
details: Dict[str, Any]
|
||||
@@ -0,0 +1,83 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from shelfmark.config import env as env_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.download.staging import STAGE_NONE
|
||||
|
||||
from .types import OutputPlan
|
||||
|
||||
logger = setup_logger("shelfmark.download.postprocess.pipeline")
|
||||
|
||||
|
||||
def _tmp_dir() -> Path:
|
||||
return env_config.TMP_DIR
|
||||
|
||||
|
||||
def is_within_tmp_dir(path: Path) -> bool:
|
||||
"""Legacy helper: True if path is inside TMP_DIR."""
|
||||
|
||||
try:
|
||||
path.resolve().relative_to(_tmp_dir().resolve())
|
||||
return True
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def is_managed_workspace_path(path: Path) -> bool:
|
||||
"""True if Shelfmark should treat this path as mutable.
|
||||
|
||||
The managed workspace is `TMP_DIR`. Anything outside it should be treated as
|
||||
read-only for safety (e.g. torrent seeding directories).
|
||||
"""
|
||||
|
||||
return is_within_tmp_dir(path)
|
||||
|
||||
|
||||
def _is_original_download(path: Optional[Path], task: DownloadTask) -> bool:
|
||||
if not path or not task.original_download_path:
|
||||
return False
|
||||
try:
|
||||
return path.resolve() == Path(task.original_download_path).resolve()
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def safe_cleanup_path(path: Optional[Path], task: DownloadTask) -> None:
|
||||
"""Remove a temp path only if it is safe and in our managed workspace."""
|
||||
|
||||
if not path or _is_original_download(path, task):
|
||||
return
|
||||
|
||||
if not is_managed_workspace_path(path):
|
||||
logger.debug("Skip cleanup (outside TMP_DIR) for task %s: %s", task.task_id, path)
|
||||
return
|
||||
|
||||
try:
|
||||
if path.is_dir():
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
elif path.exists():
|
||||
path.unlink(missing_ok=True)
|
||||
except (OSError, PermissionError) as exc:
|
||||
logger.warning("Cleanup failed for task %s (%s): %s", task.task_id, path, exc)
|
||||
|
||||
|
||||
def cleanup_output_staging(
|
||||
output_plan: OutputPlan,
|
||||
working_path: Path,
|
||||
task: DownloadTask,
|
||||
cleanup_paths: Optional[List[Path]] = None,
|
||||
) -> None:
|
||||
if output_plan.stage_action != STAGE_NONE:
|
||||
cleanup_target = output_plan.staging_dir
|
||||
if output_plan.staging_dir == _tmp_dir():
|
||||
cleanup_target = working_path
|
||||
safe_cleanup_path(cleanup_target, task)
|
||||
|
||||
if cleanup_paths:
|
||||
for path in cleanup_paths:
|
||||
safe_cleanup_path(path, task)
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from shelfmark.config import env as env_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
StageAction = Literal["none", "copy", "move"]
|
||||
STAGE_NONE: StageAction = "none"
|
||||
STAGE_COPY: StageAction = "copy"
|
||||
STAGE_MOVE: StageAction = "move"
|
||||
|
||||
|
||||
def get_staging_dir() -> Path:
|
||||
"""Get the staging directory for downloads."""
|
||||
tmp_dir = env_config.TMP_DIR
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
return tmp_dir
|
||||
|
||||
|
||||
def get_staging_path(task_id: str, extension: str) -> Path:
|
||||
"""Get a staging path for a download."""
|
||||
staging_dir = get_staging_dir()
|
||||
safe_id = hashlib.md5(task_id.encode()).hexdigest()[:16]
|
||||
return staging_dir / f"{safe_id}.{extension.lstrip('.')}"
|
||||
|
||||
|
||||
def build_staging_dir(prefix: str | None, task_id: str) -> Path:
|
||||
"""Build a dedicated staging directory for output processing."""
|
||||
base_dir = get_staging_dir()
|
||||
if not prefix:
|
||||
return base_dir
|
||||
|
||||
safe_id = hashlib.md5(task_id.encode()).hexdigest()[:8]
|
||||
staging_dir = base_dir / f"{prefix}_{safe_id}"
|
||||
counter = 1
|
||||
|
||||
while staging_dir.exists():
|
||||
staging_dir = base_dir / f"{prefix}_{safe_id}_{counter}"
|
||||
counter += 1
|
||||
|
||||
staging_dir.mkdir(parents=True, exist_ok=True)
|
||||
return staging_dir
|
||||
|
||||
|
||||
def stage_file(source_path: Path, task_id: str, copy: bool = False) -> Path:
|
||||
"""Stage a file for ingest processing. Use copy=True for torrents to preserve seeding."""
|
||||
staging_dir = get_staging_dir()
|
||||
return stage_path(source_path, staging_dir, STAGE_COPY if copy else STAGE_MOVE)
|
||||
|
||||
|
||||
def stage_path(source: Path, staging_dir: Path, action: StageAction) -> Path:
|
||||
"""Stage a file or directory into a staging dir."""
|
||||
if action == STAGE_NONE:
|
||||
return source
|
||||
|
||||
staged_path = staging_dir / source.name
|
||||
counter = 1
|
||||
|
||||
if source.is_dir():
|
||||
while staged_path.exists():
|
||||
staged_path = staging_dir / f"{source.name}_{counter}"
|
||||
counter += 1
|
||||
if action == STAGE_COPY:
|
||||
shutil.copytree(str(source), str(staged_path))
|
||||
else:
|
||||
shutil.move(str(source), str(staged_path))
|
||||
else:
|
||||
while staged_path.exists():
|
||||
staged_path = staging_dir / f"{source.stem}_{counter}{source.suffix}"
|
||||
counter += 1
|
||||
if action == STAGE_COPY:
|
||||
shutil.copy2(str(source), str(staged_path))
|
||||
else:
|
||||
shutil.move(str(source), str(staged_path))
|
||||
|
||||
staged_kind = "directory" if source.is_dir() else "file"
|
||||
logger.debug("Staged %s via %s: %s -> %s", staged_kind, action, source, staged_path)
|
||||
return staged_path
|
||||
+182
-20
@@ -26,6 +26,8 @@ from shelfmark.config.env import (
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import SearchFilters
|
||||
from shelfmark.core.prefix_middleware import PrefixMiddleware
|
||||
from shelfmark.core.utils import normalize_base_path
|
||||
from shelfmark.api.websocket import ws_manager
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -34,10 +36,14 @@ logger = setup_logger(__name__)
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
FRONTEND_DIST = os.path.join(PROJECT_ROOT, 'frontend-dist')
|
||||
|
||||
BASE_PATH = normalize_base_path(app_config.get("URL_BASE", ""))
|
||||
|
||||
app = Flask(__name__)
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore
|
||||
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 # Disable caching
|
||||
app.config['APPLICATION_ROOT'] = '/'
|
||||
app.config['APPLICATION_ROOT'] = BASE_PATH or '/'
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore
|
||||
if BASE_PATH:
|
||||
app.wsgi_app = PrefixMiddleware(app.wsgi_app, BASE_PATH, bypass_paths={"/api/health"})
|
||||
|
||||
# Socket.IO async mode.
|
||||
# We run this app under Gunicorn with a gevent websocket worker (even when DEBUG=true),
|
||||
@@ -77,6 +83,10 @@ try:
|
||||
except ImportError as e:
|
||||
logger.warning(f"Failed to import plugin modules: {e}")
|
||||
|
||||
# Migrate legacy security settings if needed
|
||||
from shelfmark.config.security import _migrate_security_settings
|
||||
_migrate_security_settings()
|
||||
|
||||
# Start download coordinator
|
||||
backend.start()
|
||||
|
||||
@@ -156,12 +166,13 @@ def get_auth_mode() -> str:
|
||||
|
||||
try:
|
||||
security_config = load_config_file("security")
|
||||
# 1. Check for explicit CWA auth (CWA_DB_PATH is pre-validated at startup)
|
||||
if security_config.get("USE_CWA_AUTH") and CWA_DB_PATH:
|
||||
auth_mode = security_config.get("AUTH_METHOD", "none")
|
||||
if auth_mode == "cwa" and CWA_DB_PATH:
|
||||
return "cwa"
|
||||
# 2. Check for built-in credentials
|
||||
if security_config.get("BUILTIN_USERNAME") and security_config.get("BUILTIN_PASSWORD_HASH"):
|
||||
if auth_mode == "builtin" and security_config.get("BUILTIN_USERNAME") and security_config.get("BUILTIN_PASSWORD_HASH"):
|
||||
return "builtin"
|
||||
if auth_mode == "proxy" and security_config.get("PROXY_AUTH_USER_HEADER"):
|
||||
return "proxy"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -237,6 +248,66 @@ app.config.update(
|
||||
|
||||
logger.info(f"Session cookie secure setting: {SESSION_COOKIE_SECURE} (from env: {SESSION_COOKIE_SECURE_ENV})")
|
||||
|
||||
@app.before_request
|
||||
def proxy_auth_middleware():
|
||||
"""
|
||||
Middleware to handle proxy authentication.
|
||||
|
||||
When AUTH_METHOD is set to "proxy", this middleware automatically
|
||||
authenticates users based on headers set by the reverse proxy.
|
||||
"""
|
||||
auth_mode = get_auth_mode()
|
||||
|
||||
# Only run for proxy auth mode
|
||||
if auth_mode != "proxy":
|
||||
return None
|
||||
|
||||
# Skip for public endpoints that don't need auth
|
||||
if request.path == '/api/health':
|
||||
return None
|
||||
|
||||
from shelfmark.core.settings_registry import load_config_file
|
||||
|
||||
try:
|
||||
security_config = load_config_file("security")
|
||||
user_header = security_config.get("PROXY_AUTH_USER_HEADER", "X-Auth-User")
|
||||
|
||||
# Extract username from proxy header
|
||||
username = request.headers.get(user_header)
|
||||
|
||||
if not username:
|
||||
if request.path.startswith('/api/auth/'):
|
||||
return None
|
||||
|
||||
logger.warning(f"Proxy auth enabled but no username found in header '{user_header}'")
|
||||
return jsonify({"error": "Authentication required. Proxy header not set."}), 401
|
||||
|
||||
# Check if settings access should be restricted to admins
|
||||
restrict_to_admin = security_config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", False)
|
||||
is_admin = True # Default to admin if not restricting
|
||||
|
||||
if restrict_to_admin:
|
||||
admin_group_header = security_config.get("PROXY_AUTH_ADMIN_GROUP_HEADER", "X-Auth-Groups")
|
||||
admin_group_name = security_config.get("PROXY_AUTH_ADMIN_GROUP_NAME", "admins")
|
||||
|
||||
# Extract groups from proxy header (can be comma or pipe separated)
|
||||
groups_header = request.headers.get(admin_group_header, "")
|
||||
user_groups_delimiter = "," if "," in groups_header else "|"
|
||||
user_groups = [g.strip() for g in groups_header.split(user_groups_delimiter) if g.strip()]
|
||||
|
||||
is_admin = admin_group_name in user_groups
|
||||
|
||||
# Create or update session
|
||||
session['user_id'] = username
|
||||
session['is_admin'] = is_admin
|
||||
session.permanent = False
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Proxy auth middleware error: {e}")
|
||||
return jsonify({"error": "Authentication error"}), 500
|
||||
|
||||
def login_required(f):
|
||||
@wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
@@ -255,10 +326,53 @@ def login_required(f):
|
||||
if 'user_id' not in session:
|
||||
return jsonify({"error": "Unauthorized"}), 401
|
||||
|
||||
# Check admin access for settings endpoints (proxy and CWA modes)
|
||||
if auth_mode in ("proxy", "cwa") and (request.path.startswith('/api/settings') or request.path.startswith('/api/onboarding')):
|
||||
from shelfmark.core.settings_registry import load_config_file
|
||||
|
||||
try:
|
||||
security_config = load_config_file("security")
|
||||
|
||||
if auth_mode == "proxy":
|
||||
restrict_to_admin = security_config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", False)
|
||||
else:
|
||||
restrict_to_admin = security_config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN", False)
|
||||
|
||||
if restrict_to_admin and not session.get('is_admin', False):
|
||||
return jsonify({"error": "Admin access required"}), 403
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Admin access check error: {e}")
|
||||
return jsonify({"error": "Internal Server Error"}), 500
|
||||
|
||||
return f(*args, **kwargs)
|
||||
return decorated_function
|
||||
|
||||
|
||||
_BASE_TAG = '<base href="/" data-shelfmark-base />'
|
||||
|
||||
|
||||
def _base_href() -> str:
|
||||
if not BASE_PATH:
|
||||
return "/"
|
||||
return f"{BASE_PATH}/"
|
||||
|
||||
|
||||
def _serve_index_html() -> Response:
|
||||
"""Serve index.html with an adjusted base tag for subpath deployments."""
|
||||
index_path = os.path.join(FRONTEND_DIST, 'index.html')
|
||||
try:
|
||||
with open(index_path, 'r', encoding='utf-8') as handle:
|
||||
html = handle.read()
|
||||
except OSError:
|
||||
return send_from_directory(FRONTEND_DIST, 'index.html')
|
||||
|
||||
if BASE_PATH and _BASE_TAG in html:
|
||||
html = html.replace(_BASE_TAG, f'<base href="{_base_href()}" data-shelfmark-base />', 1)
|
||||
|
||||
return Response(html, mimetype='text/html')
|
||||
|
||||
|
||||
# Serve frontend static files
|
||||
@app.route('/assets/<path:filename>')
|
||||
def serve_frontend_assets(filename: str) -> Response:
|
||||
@@ -273,7 +387,7 @@ def index() -> Response:
|
||||
Serve the React frontend application.
|
||||
Authentication is handled by the React app itself.
|
||||
"""
|
||||
return send_from_directory(FRONTEND_DIST, 'index.html')
|
||||
return _serve_index_html()
|
||||
|
||||
@app.route('/logo.png')
|
||||
def logo() -> Response:
|
||||
@@ -660,7 +774,7 @@ def api_cover(cover_id: str) -> Union[Response, Tuple[Response, int]]:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/download/<book_id>/cancel', methods=['DELETE'])
|
||||
@app.route('/api/download/<path:book_id>/cancel', methods=['DELETE'])
|
||||
@login_required
|
||||
def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
@@ -681,7 +795,7 @@ def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]:
|
||||
logger.error_trace(f"Cancel download error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/queue/<book_id>/priority', methods=['PUT'])
|
||||
@app.route('/api/queue/<path:book_id>/priority', methods=['PUT'])
|
||||
@login_required
|
||||
def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
@@ -870,6 +984,10 @@ def api_login() -> Union[Response, Tuple[Response, int]]:
|
||||
if not data:
|
||||
return jsonify({"error": "No data provided"}), 400
|
||||
|
||||
auth_mode = get_auth_mode()
|
||||
if auth_mode == "proxy":
|
||||
return jsonify({"error": "Proxy authentication is enabled"}), 401
|
||||
|
||||
username = data.get('username', '').strip()
|
||||
password = data.get('password', '')
|
||||
remember_me = data.get('remember_me', False)
|
||||
@@ -886,8 +1004,6 @@ def api_login() -> Union[Response, Tuple[Response, int]]:
|
||||
"error": f"Account temporarily locked due to multiple failed login attempts. Try again in {int(remaining_time)} minutes."
|
||||
}), 429
|
||||
|
||||
auth_mode = get_auth_mode()
|
||||
|
||||
# If no authentication is configured, authentication always succeeds
|
||||
if auth_mode == "none":
|
||||
session['user_id'] = username
|
||||
@@ -964,15 +1080,27 @@ def api_login() -> Union[Response, Tuple[Response, int]]:
|
||||
def api_logout() -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
Logout endpoint that clears the session.
|
||||
For proxy auth, returns the logout URL if configured.
|
||||
|
||||
Returns:
|
||||
flask.Response: JSON with success status.
|
||||
flask.Response: JSON with success status and optional logout_url.
|
||||
"""
|
||||
from shelfmark.core.settings_registry import load_config_file
|
||||
|
||||
try:
|
||||
auth_mode = get_auth_mode()
|
||||
ip_address = get_client_ip()
|
||||
username = session.get('user_id', 'unknown')
|
||||
session.clear()
|
||||
logger.info(f"Logout successful for user '{username}' from IP {ip_address}")
|
||||
|
||||
# For proxy auth, include logout URL if configured
|
||||
if auth_mode == "proxy":
|
||||
security_config = load_config_file("security")
|
||||
logout_url = security_config.get("PROXY_AUTH_LOGOUT_URL", "")
|
||||
if logout_url:
|
||||
return jsonify({"success": True, "logout_url": logout_url})
|
||||
|
||||
return jsonify({"success": True})
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Logout error: {e}")
|
||||
@@ -990,6 +1118,7 @@ def api_auth_check() -> Union[Response, Tuple[Response, int]]:
|
||||
from shelfmark.core.settings_registry import load_config_file
|
||||
|
||||
try:
|
||||
security_config = load_config_file("security")
|
||||
auth_mode = get_auth_mode()
|
||||
|
||||
# If no authentication is configured, access is allowed (full admin)
|
||||
@@ -1007,25 +1136,37 @@ def api_auth_check() -> Union[Response, Tuple[Response, int]]:
|
||||
# Determine admin status for settings access
|
||||
# - Built-in auth: single user is always admin
|
||||
# - CWA auth: check RESTRICT_SETTINGS_TO_ADMIN setting
|
||||
# - Proxy auth: check PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN setting
|
||||
if auth_mode == "builtin":
|
||||
is_admin = True
|
||||
elif auth_mode == "cwa":
|
||||
security_config = load_config_file("security")
|
||||
restrict_to_admin = security_config.get("RESTRICT_SETTINGS_TO_ADMIN", False)
|
||||
restrict_to_admin = security_config.get("CWA_RESTRICT_SETTINGS_TO_ADMIN", False)
|
||||
if restrict_to_admin:
|
||||
is_admin = session.get('is_admin', False)
|
||||
else:
|
||||
# All authenticated CWA users can access settings
|
||||
is_admin = True
|
||||
elif auth_mode == "proxy":
|
||||
restrict_to_admin = security_config.get("PROXY_AUTH_RESTRICT_SETTINGS_TO_ADMIN", False)
|
||||
is_admin = session.get('is_admin', not restrict_to_admin)
|
||||
else:
|
||||
is_admin = False
|
||||
|
||||
return jsonify({
|
||||
response_data = {
|
||||
"authenticated": is_authenticated,
|
||||
"auth_required": True,
|
||||
"auth_mode": auth_mode,
|
||||
"is_admin": is_admin if is_authenticated else False
|
||||
})
|
||||
"is_admin": is_admin if is_authenticated else False,
|
||||
"username": session.get('user_id') if is_authenticated else None
|
||||
}
|
||||
|
||||
# Add logout URL for proxy auth if configured
|
||||
if auth_mode == "proxy" and security_config.get("PROXY_AUTH_USER_HEADER"):
|
||||
logout_url = security_config.get("PROXY_AUTH_LOGOUT_URL", "")
|
||||
if logout_url:
|
||||
response_data["logout_url"] = logout_url
|
||||
|
||||
return jsonify(response_data)
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Auth check error: {e}")
|
||||
return jsonify({
|
||||
@@ -1280,6 +1421,8 @@ def api_releases() -> Union[Response, Tuple[Response, int]]:
|
||||
# Content type for audiobook vs ebook search
|
||||
content_type = request.args.get('content_type', 'ebook').strip()
|
||||
|
||||
manual_query = request.args.get('manual_query', '').strip()
|
||||
|
||||
if not provider or not book_id:
|
||||
return jsonify({"error": "Parameters 'provider' and 'book_id' are required"}), 400
|
||||
|
||||
@@ -1317,8 +1460,27 @@ def api_releases() -> Union[Response, Tuple[Response, int]]:
|
||||
try:
|
||||
source = get_source(source_name)
|
||||
source_instances[source_name] = source
|
||||
logger.debug(f"Searching {source_name} for '{book.title}' by {book.authors} (expand={expand_search}, content_type={content_type})")
|
||||
releases = source.search(book, expand_search=expand_search, languages=languages, content_type=content_type)
|
||||
|
||||
from shelfmark.core.search_plan import build_release_search_plan
|
||||
|
||||
plan = build_release_search_plan(book, languages=languages, manual_query=manual_query)
|
||||
|
||||
if plan.manual_query:
|
||||
planned_query = plan.manual_query
|
||||
planned_query_type = "manual"
|
||||
elif not expand_search and plan.isbn_candidates:
|
||||
planned_query = plan.isbn_candidates[0]
|
||||
planned_query_type = "isbn"
|
||||
else:
|
||||
planned_query = plan.primary_query
|
||||
planned_query_type = "title_author"
|
||||
|
||||
logger.debug(
|
||||
f"Searching {source_name}: {planned_query_type}='{planned_query}' "
|
||||
f"(title='{book.title}', authors={book.authors}, expand={expand_search}, content_type={content_type})"
|
||||
)
|
||||
|
||||
releases = source.search(book, plan, expand_search=expand_search, content_type=content_type)
|
||||
all_releases.extend(releases)
|
||||
except ValueError:
|
||||
errors.append(f"Unknown source: {source_name}")
|
||||
@@ -1631,7 +1793,7 @@ def catch_all(path: str) -> Response:
|
||||
if path.startswith('api/') or path.startswith('assets/'):
|
||||
return jsonify({"error": "Resource not found"}), 404
|
||||
# Otherwise serve the React app
|
||||
return send_from_directory(FRONTEND_DIST, 'index.html')
|
||||
return _serve_index_html()
|
||||
|
||||
# WebSocket event handlers
|
||||
@socketio.on('connect')
|
||||
|
||||
@@ -145,6 +145,7 @@ class BookMetadata:
|
||||
source_url: Optional[str] = None # Link to book on provider's site
|
||||
subtitle: Optional[str] = None # Book subtitle, if any
|
||||
search_title: Optional[str] = None # Cleaner title for search queries (provider-specific)
|
||||
search_author: Optional[str] = None # Cleaner author for search queries (provider-specific)
|
||||
|
||||
# Provider-specific display fields for cards/lists
|
||||
display_fields: List[DisplayField] = field(default_factory=list)
|
||||
@@ -159,6 +160,101 @@ class BookMetadata:
|
||||
titles_by_language: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
def group_languages_by_localized_title(
|
||||
base_title: str,
|
||||
languages: Optional[List[str]],
|
||||
titles_by_language: Optional[Dict[str, str]] = None,
|
||||
) -> List[tuple[str, Optional[List[str]]]]:
|
||||
"""Group language codes by localized title.
|
||||
|
||||
Release sources that support language filtering (e.g., Anna's Archive)
|
||||
may want to run separate searches per localized title, while still
|
||||
passing the correct language filters per query.
|
||||
|
||||
Args:
|
||||
base_title: Fallback title when no localized title exists.
|
||||
languages: Requested language codes (e.g., ["en", "hu"]).
|
||||
titles_by_language: Mapping of language identifiers to localized titles.
|
||||
|
||||
Returns:
|
||||
List of (title, languages) tuples. If languages is None/empty, returns
|
||||
[(base_title, None)].
|
||||
"""
|
||||
if not base_title:
|
||||
return []
|
||||
|
||||
if not languages:
|
||||
return [(base_title, None)]
|
||||
|
||||
normalized_langs = [lang.strip() for lang in languages if lang and lang.strip()]
|
||||
if not normalized_langs:
|
||||
return [(base_title, None)]
|
||||
|
||||
if not titles_by_language:
|
||||
return [(base_title, normalized_langs)]
|
||||
|
||||
title_to_langs: Dict[str, List[str]] = {}
|
||||
for lang in normalized_langs:
|
||||
localized_title = titles_by_language.get(lang) or base_title
|
||||
title_to_langs.setdefault(localized_title, []).append(lang)
|
||||
|
||||
return list(title_to_langs.items())
|
||||
|
||||
|
||||
def build_localized_search_titles(
|
||||
base_title: str,
|
||||
languages: Optional[List[str]],
|
||||
titles_by_language: Optional[Dict[str, str]] = None,
|
||||
excluded_languages: Optional[set[str]] = None,
|
||||
) -> List[str]:
|
||||
"""Build a list of titles to search for, including localized editions.
|
||||
|
||||
This is useful for release sources that *can't* pass language filters to
|
||||
an upstream search API (e.g., Prowlarr), but still want to broaden matches
|
||||
by searching for localized edition titles.
|
||||
|
||||
The list always includes base_title first.
|
||||
|
||||
Args:
|
||||
base_title: Primary title to search for.
|
||||
languages: User language preferences (order matters).
|
||||
titles_by_language: Mapping of language identifiers to localized titles.
|
||||
excluded_languages: Optional set of normalized language identifiers to skip.
|
||||
|
||||
Returns:
|
||||
List of unique titles to search for, in priority order.
|
||||
"""
|
||||
if not base_title:
|
||||
return []
|
||||
|
||||
titles: List[str] = [base_title]
|
||||
seen = {base_title}
|
||||
|
||||
if not languages or not titles_by_language:
|
||||
return titles
|
||||
|
||||
excluded = {lang.lower() for lang in (excluded_languages or set())}
|
||||
|
||||
for lang in languages:
|
||||
if not lang:
|
||||
continue
|
||||
normalized_lang = lang.strip()
|
||||
if not normalized_lang:
|
||||
continue
|
||||
if normalized_lang.lower() in excluded:
|
||||
continue
|
||||
|
||||
localized_title = titles_by_language.get(normalized_lang)
|
||||
if not localized_title:
|
||||
continue
|
||||
|
||||
if localized_title not in seen:
|
||||
seen.add(localized_title)
|
||||
titles.append(localized_title)
|
||||
|
||||
return titles
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Result from a metadata search with pagination info."""
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Hardcover.app metadata provider. Requires API key."""
|
||||
|
||||
import re
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
@@ -95,27 +96,159 @@ def _build_source_url(slug: str) -> Optional[str]:
|
||||
return f"https://hardcover.app/books/{slug}" if slug else None
|
||||
|
||||
|
||||
def _compute_search_title(title: str, subtitle: Optional[str]) -> Optional[str]:
|
||||
"""Compute a cleaner search title from title and subtitle.
|
||||
def _is_probably_series_position(subtitle: str) -> bool:
|
||||
normalized = subtitle.strip().lower()
|
||||
|
||||
When Hardcover uses the "Series: Book Title" format, the subtitle contains
|
||||
the actual book title which is better for searching. For example:
|
||||
- title: "Mistborn: The Final Empire"
|
||||
- subtitle: "The Final Empire"
|
||||
- search_title: "The Final Empire" (better for Prowlarr/indexer searches)
|
||||
# Common patterns: "Book One", "Book 1", "Part 2", "Volume III", etc.
|
||||
if re.match(r"^(book|part|volume|vol\.?|episode)\s+([0-9]+|[ivxlcdm]+|one|two|three|four|five|six|seven|eight|nine|ten)\b", normalized):
|
||||
return True
|
||||
|
||||
Skips subtitles that start with series position indicators like "Book One",
|
||||
"Part 1", "Volume 2" as these are descriptors, not the actual title.
|
||||
# e.g. "A Novel", "An Epic Fantasy", etc. These add noise to indexer queries.
|
||||
if normalized in {"a novel", "a novella", "a story", "a memoir"}:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _strip_parenthetical_suffix(title: str) -> str:
|
||||
# Drop trailing qualifiers like "(Unabridged)", "(Illustrated Edition)", etc.
|
||||
return re.sub(r"\s*\([^)]*\)\s*$", "", title).strip()
|
||||
|
||||
|
||||
def _simplify_author_for_search(author: str) -> Optional[str]:
|
||||
"""Return a looser author string for indexer searches.
|
||||
|
||||
Primary goal: reduce mismatch between metadata providers and indexers.
|
||||
|
||||
Heuristics (intentionally conservative):
|
||||
- Remove middle initials (e.g. "Robert R. McCammon" -> "Robert McCammon")
|
||||
- Remove standalone middle names that are just an initial or initial+dot
|
||||
- Preserve suffixes like "Jr."/"Sr."/"III" as they sometimes matter
|
||||
"""
|
||||
if not subtitle or subtitle not in title:
|
||||
if not author:
|
||||
return None
|
||||
|
||||
# Skip if subtitle starts with series position indicators
|
||||
skip_prefixes = ('book ', 'part ', 'volume ')
|
||||
if subtitle.lower().startswith(skip_prefixes):
|
||||
normalized = " ".join(author.split()).strip()
|
||||
if not normalized:
|
||||
return None
|
||||
|
||||
return subtitle
|
||||
# Handle "Last, First ..." -> "First ... Last"
|
||||
if "," in normalized:
|
||||
parts = [p.strip() for p in normalized.split(",") if p.strip()]
|
||||
if len(parts) >= 2:
|
||||
normalized = " ".join(parts[1:] + [parts[0]]).strip()
|
||||
|
||||
tokens = normalized.split(" ")
|
||||
if len(tokens) < 2:
|
||||
return None
|
||||
|
||||
keep_suffixes = {"jr", "jr.", "sr", "sr.", "ii", "iii", "iv", "v"}
|
||||
|
||||
simplified: list[str] = []
|
||||
for idx, token in enumerate(tokens):
|
||||
t = token.strip()
|
||||
if not t:
|
||||
continue
|
||||
|
||||
t_lower = t.lower()
|
||||
is_suffix = (idx == len(tokens) - 1) and (t_lower in keep_suffixes)
|
||||
if is_suffix:
|
||||
simplified.append(t)
|
||||
continue
|
||||
|
||||
# Drop middle initials like "R." or "R"
|
||||
is_initial = re.match(r"^[A-Za-z]\.?$", t) is not None
|
||||
is_middle_token = 0 < idx < (len(tokens) - 1)
|
||||
if is_middle_token and is_initial:
|
||||
continue
|
||||
|
||||
simplified.append(t)
|
||||
|
||||
if len(simplified) < 2:
|
||||
return None
|
||||
|
||||
candidate = " ".join(simplified).strip()
|
||||
if candidate.lower() == normalized.lower():
|
||||
return None
|
||||
|
||||
return candidate
|
||||
|
||||
|
||||
def _compute_search_title(
|
||||
title: str,
|
||||
subtitle: Optional[str],
|
||||
*,
|
||||
series_name: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Compute a provider-specific, *looser* title for indexer searching.
|
||||
|
||||
Goal: produce a string that maximizes recall in downstream sources (Prowlarr,
|
||||
IRC bots, etc.). Being too detailed is counterproductive.
|
||||
|
||||
Hardcover often stores titles in a "Series: Book Title" format and places the
|
||||
standalone book title in `subtitle`. When this appears to be the case, prefer
|
||||
the subtitle (unless it looks like a series position or other noise).
|
||||
|
||||
Additional heuristics:
|
||||
- If Hardcover prefixes the series in the title, remove it.
|
||||
- Drop trailing parenthetical qualifiers.
|
||||
"""
|
||||
if not title:
|
||||
return None
|
||||
|
||||
original_title = " ".join(title.split()).strip()
|
||||
|
||||
normalized_title = _strip_parenthetical_suffix(original_title)
|
||||
|
||||
normalized_subtitle = " ".join(subtitle.split()).strip() if subtitle else ""
|
||||
normalized_subtitle = _strip_parenthetical_suffix(normalized_subtitle) if normalized_subtitle else ""
|
||||
|
||||
if normalized_subtitle and normalized_subtitle.lower() == normalized_title.lower():
|
||||
normalized_subtitle = ""
|
||||
|
||||
# Prefer subtitle when it looks like the real title.
|
||||
if normalized_subtitle and not _is_probably_series_position(normalized_subtitle):
|
||||
match = re.match(r"^(.+?)\s*:\s*(.+)$", normalized_title)
|
||||
if match:
|
||||
prefix = match.group(1).strip()
|
||||
suffix = _strip_parenthetical_suffix(match.group(2).strip())
|
||||
|
||||
prefix_words = len(prefix.split()) if prefix else 0
|
||||
subtitle_words = len(normalized_subtitle.split())
|
||||
|
||||
series_normalized = " ".join(series_name.split()).strip() if series_name else ""
|
||||
if series_normalized and prefix.lower() == series_normalized.lower():
|
||||
return normalized_subtitle
|
||||
|
||||
# If the subtitle is much longer than the prefix, treat it as a descriptive subtitle.
|
||||
if prefix and subtitle_words >= (prefix_words + 4):
|
||||
return prefix
|
||||
|
||||
# Otherwise assume "Series: Book Title" and prefer the subtitle.
|
||||
if normalized_subtitle.lower() == suffix.lower() or normalized_subtitle.lower() in suffix.lower():
|
||||
return normalized_subtitle
|
||||
|
||||
# Fallback: if title contains the subtitle, this is likely "Series: Subtitle".
|
||||
if normalized_subtitle.lower() in normalized_title.lower():
|
||||
return normalized_subtitle
|
||||
|
||||
# If we know the series name (from full book fetch), strip it.
|
||||
if series_name:
|
||||
series_normalized = " ".join(series_name.split()).strip()
|
||||
if series_normalized:
|
||||
# Common Hardcover format: "Series: Book Title".
|
||||
prefix = f"{series_normalized}:"
|
||||
if normalized_title.lower().startswith(prefix.lower()):
|
||||
candidate = normalized_title[len(prefix):].strip()
|
||||
candidate = _strip_parenthetical_suffix(candidate)
|
||||
if candidate and candidate.lower() != normalized_title.lower():
|
||||
return candidate
|
||||
|
||||
# Last resort: return a cleaned version of the title if we removed noise.
|
||||
if normalized_title and normalized_title.lower() != original_title.lower():
|
||||
return normalized_title
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@register_provider_kwargs("hardcover")
|
||||
@@ -390,7 +523,11 @@ class HardcoverProvider(MetadataProvider):
|
||||
primary_books_count
|
||||
}
|
||||
}
|
||||
editions(limit: 20, order_by: {users_count: desc}) {
|
||||
editions(
|
||||
distinct_on: language_id
|
||||
order_by: [{language_id: asc}, {users_count: desc}]
|
||||
limit: 200
|
||||
) {
|
||||
title
|
||||
language {
|
||||
language
|
||||
@@ -553,6 +690,8 @@ class HardcoverProvider(MetadataProvider):
|
||||
# Normalize whitespace in author names (some API data has multiple spaces)
|
||||
authors = [" ".join(name.split()) for name in authors]
|
||||
|
||||
search_author = _simplify_author_for_search(authors[0]) if authors else None
|
||||
|
||||
cover_url = _extract_cover_url(item, "image")
|
||||
publish_year = _extract_publish_year(item)
|
||||
source_url = _build_source_url(item.get("slug", ""))
|
||||
@@ -588,6 +727,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
title=title,
|
||||
subtitle=subtitle,
|
||||
search_title=_compute_search_title(title, subtitle),
|
||||
search_author=search_author,
|
||||
provider_display_name="Hardcover",
|
||||
authors=authors,
|
||||
cover_url=cover_url,
|
||||
@@ -597,12 +737,16 @@ class HardcoverProvider(MetadataProvider):
|
||||
display_fields=display_fields,
|
||||
)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse Hardcover search result: {e}")
|
||||
return None
|
||||
|
||||
def _parse_book(self, book: Dict) -> BookMetadata:
|
||||
"""Parse a book object into BookMetadata."""
|
||||
title = str(book.get("title") or "")
|
||||
subtitle = book.get("subtitle")
|
||||
|
||||
# Extract authors - try contributions first (filtered), fall back to cached_contributors
|
||||
authors = []
|
||||
contributions = book.get("contributions") or []
|
||||
@@ -630,6 +774,8 @@ class HardcoverProvider(MetadataProvider):
|
||||
# Normalize whitespace in author names (some API data has multiple spaces)
|
||||
authors = [" ".join(name.split()) for name in authors]
|
||||
|
||||
search_author = _simplify_author_for_search(authors[0]) if authors else None
|
||||
|
||||
cover_url = _extract_cover_url(book, "cached_image", "image")
|
||||
publish_year = _extract_publish_year(book)
|
||||
|
||||
@@ -705,29 +851,28 @@ class HardcoverProvider(MetadataProvider):
|
||||
if code3 and code3 not in titles_by_language:
|
||||
titles_by_language[code3] = edition_title
|
||||
|
||||
title = book["title"]
|
||||
subtitle = book.get("subtitle")
|
||||
|
||||
return BookMetadata(
|
||||
provider="hardcover",
|
||||
provider_id=str(book["id"]),
|
||||
title=title,
|
||||
subtitle=subtitle,
|
||||
search_title=_compute_search_title(title, subtitle),
|
||||
provider_display_name="Hardcover",
|
||||
authors=authors,
|
||||
isbn_10=isbn_10,
|
||||
isbn_13=isbn_13,
|
||||
cover_url=cover_url,
|
||||
description=full_description,
|
||||
publish_year=publish_year,
|
||||
genres=genres,
|
||||
source_url=source_url,
|
||||
series_name=series_name,
|
||||
series_position=series_position,
|
||||
series_count=series_count,
|
||||
titles_by_language=titles_by_language,
|
||||
)
|
||||
provider="hardcover",
|
||||
provider_id=str(book["id"]),
|
||||
title=title,
|
||||
subtitle=subtitle,
|
||||
search_title=_compute_search_title(title, subtitle, series_name=series_name),
|
||||
search_author=search_author,
|
||||
provider_display_name="Hardcover",
|
||||
authors=authors,
|
||||
isbn_10=isbn_10,
|
||||
isbn_13=isbn_13,
|
||||
cover_url=cover_url,
|
||||
description=full_description,
|
||||
publish_year=publish_year,
|
||||
genres=genres,
|
||||
source_url=source_url,
|
||||
series_name=series_name,
|
||||
series_position=series_position,
|
||||
series_count=series_count,
|
||||
titles_by_language=titles_by_language,
|
||||
)
|
||||
|
||||
|
||||
|
||||
def _test_hardcover_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
@@ -835,7 +980,6 @@ def hardcover_settings():
|
||||
label="API Key",
|
||||
description="Get your API key from hardcover.app/account/api",
|
||||
required=True,
|
||||
env_supported=False, # UI-only setting, no ENV var support
|
||||
),
|
||||
ActionButton(
|
||||
key="test_connection",
|
||||
@@ -850,7 +994,6 @@ def hardcover_settings():
|
||||
description="Default sort order for Hardcover search results.",
|
||||
options=_HARDCOVER_SORT_OPTIONS,
|
||||
default="relevance",
|
||||
env_supported=False, # UI-only setting
|
||||
),
|
||||
CheckboxField(
|
||||
key="HARDCOVER_EXCLUDE_COMPILATIONS",
|
||||
|
||||
@@ -558,6 +558,5 @@ def openlibrary_settings():
|
||||
description="Default sort order for Open Library search results.",
|
||||
options=_OPENLIBRARY_SORT_OPTIONS,
|
||||
default="relevance",
|
||||
env_supported=False, # UI-only setting
|
||||
),
|
||||
]
|
||||
|
||||
@@ -4,7 +4,10 @@ from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from enum import Enum
|
||||
from threading import Event
|
||||
from typing import List, Optional, Dict, Type, Callable, Literal, Any
|
||||
from typing import List, Optional, Dict, Type, Callable, Literal, Any, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.search_plan import ReleaseSearchPlan
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
@@ -233,8 +236,8 @@ class ReleaseSource(ABC):
|
||||
def search(
|
||||
self,
|
||||
book: BookMetadata,
|
||||
plan: "ReleaseSearchPlan",
|
||||
expand_search: bool = False,
|
||||
languages: Optional[List[str]] = None,
|
||||
content_type: str = "ebook"
|
||||
) -> List[Release]:
|
||||
"""Search for releases of a book."""
|
||||
@@ -245,15 +248,20 @@ class ReleaseSource(ABC):
|
||||
"""Check if this source is configured and reachable."""
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def get_column_config(cls) -> ReleaseColumnConfig:
|
||||
def get_column_config(self) -> ReleaseColumnConfig:
|
||||
"""Get column configuration for release list UI. Override for custom columns."""
|
||||
return _default_column_config()
|
||||
|
||||
|
||||
class DownloadHandler(ABC):
|
||||
"""Interface for executing downloads. Handlers stage files to TMP_DIR;
|
||||
orchestrator handles post-processing and move to INGEST_DIR.
|
||||
"""Interface for executing downloads.
|
||||
|
||||
A handler may either:
|
||||
- download directly into ``TMP_DIR`` (managed by Shelfmark), or
|
||||
- return a path owned by an external client (e.g. torrent/usenet).
|
||||
|
||||
The orchestrator is responsible for post-processing (archive extraction, output mode
|
||||
handling) and transferring files into their final destination.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
@@ -264,9 +272,17 @@ class DownloadHandler(ABC):
|
||||
progress_callback: Callable[[float], None],
|
||||
status_callback: Callable[[str, Optional[str]], None]
|
||||
) -> Optional[str]:
|
||||
"""Execute download and return path to staged file in TMP_DIR."""
|
||||
"""Execute download and return a path to the downloaded payload."""
|
||||
pass
|
||||
|
||||
def post_process_cleanup(self, task: DownloadTask, success: bool) -> None:
|
||||
"""Optional hook called after orchestrator post-processing.
|
||||
|
||||
This is primarily used for external download clients, where the handler may need
|
||||
to trigger client-side cleanup only after Shelfmark has safely imported the files.
|
||||
"""
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def cancel(self, task_id: str) -> bool:
|
||||
"""Cancel an in-progress download."""
|
||||
|
||||
@@ -20,7 +20,7 @@ from shelfmark.core.config import config
|
||||
from shelfmark.core.utils import CONTENT_TYPES
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import BookInfo, SearchFilters, DownloadTask
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
from shelfmark.metadata_providers import BookMetadata, group_languages_by_localized_title
|
||||
from shelfmark.release_sources import (
|
||||
Release,
|
||||
ReleaseProtocol,
|
||||
@@ -88,18 +88,25 @@ _LIBGEN_GET_PATTERNS = [
|
||||
def _get_source_priority() -> List[Dict]:
|
||||
"""Get the full source priority list.
|
||||
|
||||
Fast sources (AA Fast, LibGen) are hardcoded first.
|
||||
Fast sources come from user config (FAST_SOURCES_DISPLAY).
|
||||
Slow sources come from user config.
|
||||
"""
|
||||
# Fast sources - always first, hardcoded
|
||||
fast_sources = []
|
||||
# Fast sources - always first, configurable via settings/env
|
||||
fast_sources: List[Dict] = []
|
||||
configured_fast = config.get("FAST_SOURCES_DISPLAY") or []
|
||||
has_donator_key = bool(config.get("AA_DONATOR_KEY"))
|
||||
|
||||
# AA Fast only if donator key is set
|
||||
if config.get("AA_DONATOR_KEY"):
|
||||
fast_sources.append({"id": "aa-fast", "enabled": True})
|
||||
|
||||
# LibGen always available
|
||||
fast_sources.append({"id": "libgen", "enabled": True})
|
||||
if isinstance(configured_fast, list):
|
||||
for item in configured_fast:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
source_id = item.get("id")
|
||||
if source_id not in ("aa-fast", "libgen"):
|
||||
continue
|
||||
enabled = bool(item.get("enabled", True))
|
||||
if source_id == "aa-fast" and not has_donator_key:
|
||||
enabled = False
|
||||
fast_sources.append({"id": source_id, "enabled": enabled})
|
||||
|
||||
# User's configured slow sources (config won't contain fast sources)
|
||||
slow_sources = config.get("SOURCE_PRIORITY") or []
|
||||
@@ -188,7 +195,7 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
|
||||
f"{filters_query}"
|
||||
)
|
||||
|
||||
html = downloader.html_get_page(url, selector=selector)
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
|
||||
if not html:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients
|
||||
raise SearchUnavailable("Unable to reach download source. Network restricted or mirrors are blocked.")
|
||||
@@ -238,7 +245,7 @@ def get_book_info(book_id: str, fetch_download_count: bool = True) -> BookInfo:
|
||||
"""
|
||||
url = f"{network.get_aa_base_url()}/md5/{book_id}"
|
||||
selector = network.AAMirrorSelector()
|
||||
html = downloader.html_get_page(url, selector=selector)
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
|
||||
|
||||
if not html:
|
||||
raise Exception(f"Failed to fetch book info for ID: {book_id}")
|
||||
@@ -391,7 +398,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str, fetch_download_coun
|
||||
if fetch_download_count:
|
||||
try:
|
||||
summary_url = f"{network.get_aa_base_url()}/dyn/md5/summary/{book_id}"
|
||||
summary_response = downloader.html_get_page(summary_url, selector=network.AAMirrorSelector())
|
||||
summary_response = downloader.html_get_page(summary_url, selector=network.AAMirrorSelector(), allow_bypasser_fallback=False)
|
||||
if summary_response:
|
||||
summary_data = json.loads(summary_response)
|
||||
if "downloads_total" in summary_data:
|
||||
@@ -1090,8 +1097,7 @@ class DirectDownloadSource(ReleaseSource):
|
||||
"""Returns the search type used in the last search() call."""
|
||||
return self._last_search_type
|
||||
|
||||
@classmethod
|
||||
def get_column_config(cls) -> ReleaseColumnConfig:
|
||||
def get_column_config(self) -> ReleaseColumnConfig:
|
||||
"""Column configuration for Direct Download source.
|
||||
|
||||
Shows language, format, and size badges for each release.
|
||||
@@ -1135,8 +1141,8 @@ class DirectDownloadSource(ReleaseSource):
|
||||
def search(
|
||||
self,
|
||||
book: BookMetadata,
|
||||
plan: "ReleaseSearchPlan", # noqa: F821
|
||||
expand_search: bool = False,
|
||||
languages: Optional[List[str]] = None,
|
||||
content_type: str = "ebook"
|
||||
) -> List[Release]:
|
||||
"""
|
||||
@@ -1151,17 +1157,19 @@ class DirectDownloadSource(ReleaseSource):
|
||||
languages: Language codes to filter by (overrides book.language/config)
|
||||
content_type: Ignored - Direct download uses format filtering instead
|
||||
"""
|
||||
# Language filter: explicit param > book.language > config default
|
||||
lang_filter = languages or ([book.language] if book.language else config.BOOK_LANGUAGE)
|
||||
lang_filter = plan.languages
|
||||
|
||||
# Reset search type tracking
|
||||
self._last_search_type = "title_author"
|
||||
|
||||
# ISBN search first (unless expand_search requested)
|
||||
if plan.manual_query:
|
||||
expand_search = True
|
||||
|
||||
if not expand_search:
|
||||
isbn = book.isbn_13 or book.isbn_10
|
||||
isbn = plan.isbn_candidates[0] if plan.isbn_candidates else None
|
||||
if isbn:
|
||||
logger.debug(f"Searching by ISBN: {isbn}")
|
||||
logger.debug(f"Searching direct_download: isbn='{isbn}', langs={lang_filter}")
|
||||
filters = SearchFilters(isbn=[isbn])
|
||||
if lang_filter:
|
||||
filters.lang = lang_filter
|
||||
@@ -1178,17 +1186,8 @@ class DirectDownloadSource(ReleaseSource):
|
||||
logger.warning(f"ISBN search failed: {e}")
|
||||
|
||||
# Title + author fallback
|
||||
author = book.authors[0] if book.authors else ""
|
||||
|
||||
# Group languages by localized title to avoid duplicate searches
|
||||
if lang_filter and book.titles_by_language:
|
||||
title_to_langs: Dict[str, List[str]] = {}
|
||||
for lang in lang_filter:
|
||||
title = book.titles_by_language.get(lang, book.title)
|
||||
title_to_langs.setdefault(title, []).append(lang)
|
||||
searches = list(title_to_langs.items())
|
||||
else:
|
||||
searches = [(book.title, lang_filter)]
|
||||
author = plan.author
|
||||
searches = [(v.title, v.languages) for v in plan.grouped_title_variants]
|
||||
|
||||
# Execute searches with deduplication
|
||||
seen_ids: set = set()
|
||||
@@ -1199,7 +1198,7 @@ class DirectDownloadSource(ReleaseSource):
|
||||
if not query:
|
||||
continue
|
||||
|
||||
logger.debug(f"Searching: query='{query}', langs={langs}")
|
||||
logger.debug(f"Searching direct_download: title_author='{query}', langs={langs}")
|
||||
filters = SearchFilters(lang=langs) if langs else SearchFilters()
|
||||
try:
|
||||
for bi in search_books(query, filters):
|
||||
@@ -1300,7 +1299,7 @@ class DirectDownloadHandler(DownloadHandler):
|
||||
handle bypass, move to final location.
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Starting download: {book_info.title}")
|
||||
logger.debug("Starting download: %s", book_info.title)
|
||||
|
||||
# Prepare paths - use descriptive staging filename, orchestrator will rename
|
||||
# based on FILE_ORGANIZATION setting
|
||||
|
||||
@@ -100,7 +100,7 @@ class IRCDownloadHandler(DownloadHandler):
|
||||
ext = Path(offer.filename).suffix.lstrip('.') or task.format or "epub"
|
||||
|
||||
# Stage to temp directory (lazy import to avoid circular import)
|
||||
from shelfmark.download.orchestrator import get_staging_path
|
||||
from shelfmark.download.staging import get_staging_path
|
||||
staging_path = get_staging_path(task.task_id, ext)
|
||||
|
||||
download_dcc(
|
||||
|
||||
@@ -6,7 +6,10 @@ Searches IRC ebook channels for book releases.
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.search_plan import ReleaseSearchPlan
|
||||
|
||||
from shelfmark.api.websocket import ws_manager
|
||||
from shelfmark.core.config import config
|
||||
@@ -74,8 +77,7 @@ class IRCReleaseSource(ReleaseSource):
|
||||
# Track online servers from most recent search
|
||||
self._online_servers: Optional[set[str]] = None
|
||||
|
||||
@classmethod
|
||||
def is_available(cls) -> bool:
|
||||
def is_available(self) -> bool:
|
||||
"""Check if IRC is configured (server, channel, and nick are set)."""
|
||||
server = config.get("IRC_SERVER", "")
|
||||
channel = config.get("IRC_CHANNEL", "")
|
||||
@@ -122,8 +124,8 @@ class IRCReleaseSource(ReleaseSource):
|
||||
def search(
|
||||
self,
|
||||
book: BookMetadata,
|
||||
plan: "ReleaseSearchPlan",
|
||||
expand_search: bool = False,
|
||||
languages: Optional[List[str]] = None,
|
||||
content_type: str = "ebook"
|
||||
) -> List[Release]:
|
||||
"""Search IRC for books matching metadata.
|
||||
@@ -146,7 +148,7 @@ class IRCReleaseSource(ReleaseSource):
|
||||
return cached["releases"]
|
||||
|
||||
# Build search query
|
||||
query = self._build_query(book)
|
||||
query = plan.primary_query or self._build_query(book)
|
||||
if not query:
|
||||
logger.warning("No search query could be built")
|
||||
return []
|
||||
@@ -248,10 +250,12 @@ class IRCReleaseSource(ReleaseSource):
|
||||
"""Build search query from book metadata."""
|
||||
parts = []
|
||||
|
||||
if book.title:
|
||||
parts.append(book.title)
|
||||
if book.search_title or book.title:
|
||||
parts.append(book.search_title or book.title)
|
||||
|
||||
if book.authors:
|
||||
if book.search_author:
|
||||
parts.append(book.search_author)
|
||||
elif book.authors:
|
||||
# Use first author
|
||||
author = book.authors[0] if isinstance(book.authors, list) else book.authors
|
||||
parts.append(author)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
"""Prowlarr API client for connection testing, indexer listing, and search."""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -14,7 +14,7 @@ class ProwlarrClient:
|
||||
"""Client for interacting with the Prowlarr API."""
|
||||
|
||||
def __init__(self, url: str, api_key: str, timeout: int = 30):
|
||||
self.base_url = url.rstrip("/")
|
||||
self.base_url = normalize_http_url(url)
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
self._session = requests.Session()
|
||||
@@ -31,7 +31,7 @@ class ProwlarrClient:
|
||||
json_data: Optional[Dict[str, Any]] = None,
|
||||
) -> Any:
|
||||
"""Make an API request to Prowlarr. Returns parsed JSON response."""
|
||||
url = urljoin(self.base_url, endpoint)
|
||||
url = self.base_url + endpoint
|
||||
logger.debug(f"Prowlarr API: {method} {url}")
|
||||
|
||||
try:
|
||||
|
||||
@@ -11,13 +11,89 @@ Clients register themselves via the @register_client decorator.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional, Tuple, Type, Union
|
||||
from functools import wraps
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast, Any
|
||||
|
||||
import requests
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
# Type variable for generic return type
|
||||
T = TypeVar('T')
|
||||
|
||||
# Exceptions that should trigger a retry
|
||||
RETRYABLE_EXCEPTIONS = (
|
||||
requests.exceptions.ConnectionError,
|
||||
requests.exceptions.Timeout,
|
||||
requests.exceptions.HTTPError,
|
||||
)
|
||||
|
||||
|
||||
def with_retry(
|
||||
max_attempts: int = 3,
|
||||
base_delay: float = 1.0,
|
||||
max_delay: float = 10.0,
|
||||
jitter: float = 0.5,
|
||||
) -> Callable[[Callable[..., T]], Callable[..., T]]:
|
||||
"""
|
||||
Decorator for retrying API calls with exponential backoff.
|
||||
|
||||
Args:
|
||||
max_attempts: Maximum number of attempts (default 3)
|
||||
base_delay: Initial delay in seconds (default 1.0)
|
||||
max_delay: Maximum delay cap in seconds (default 10.0)
|
||||
jitter: Random jitter factor 0-1 to add to delay (default 0.5)
|
||||
|
||||
Retries on:
|
||||
- Connection errors
|
||||
- Timeouts
|
||||
- HTTP 5xx server errors
|
||||
|
||||
Does NOT retry on:
|
||||
- HTTP 4xx client errors (bad request, auth failures)
|
||||
- Other exceptions (programming errors)
|
||||
"""
|
||||
def decorator(func: Callable[..., T]) -> Callable[..., T]:
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs) -> T:
|
||||
last_exception = None
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except requests.exceptions.HTTPError as e:
|
||||
# Only retry on server errors (5xx), not client errors (4xx)
|
||||
if e.response is not None and e.response.status_code < 500:
|
||||
raise
|
||||
last_exception = e
|
||||
except RETRYABLE_EXCEPTIONS as e:
|
||||
last_exception = e
|
||||
|
||||
if attempt < max_attempts:
|
||||
# Calculate delay with exponential backoff
|
||||
delay = min(base_delay * (2 ** (attempt - 1)), max_delay)
|
||||
# Add jitter to prevent thundering herd
|
||||
delay += random.uniform(0, delay * jitter)
|
||||
_logger.debug(
|
||||
f"Retry {attempt}/{max_attempts} for {func.__name__} "
|
||||
f"after {delay:.1f}s (error: {last_exception})"
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
# All retries exhausted
|
||||
if last_exception is None:
|
||||
raise RuntimeError("Retry failed without exception")
|
||||
raise cast(Exception, last_exception)
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
class DownloadState(Enum):
|
||||
"""Valid states for a download."""
|
||||
@@ -97,6 +173,49 @@ class DownloadClient(ABC):
|
||||
protocol: str
|
||||
name: str
|
||||
|
||||
def _log_error(self, method: str, e: Exception, level: str = "error") -> str:
|
||||
"""
|
||||
Log a client error with consistent formatting.
|
||||
|
||||
Args:
|
||||
method: Name of the method that failed (e.g., "get_status")
|
||||
e: The exception that was raised
|
||||
level: Log level - "error" or "debug"
|
||||
|
||||
Returns:
|
||||
Formatted error message string (for use in DownloadStatus.error())
|
||||
"""
|
||||
error_type = type(e).__name__
|
||||
msg = f"{self.name} {method} failed ({error_type}): {e}"
|
||||
if level == "debug":
|
||||
_logger.debug(msg)
|
||||
else:
|
||||
_logger.error(msg)
|
||||
|
||||
# Reset connection state if client tracks it (e.g., Deluge)
|
||||
if hasattr(self, "_connected"):
|
||||
self._connected = False
|
||||
|
||||
return f"{error_type}: {e}"
|
||||
|
||||
def _build_path(self, *components: str) -> Optional[str]:
|
||||
"""
|
||||
Safely build a file path from components.
|
||||
|
||||
Args:
|
||||
*components: Path components to join (e.g., save_path, name)
|
||||
|
||||
Returns:
|
||||
Normalized path string, or None if any component is empty/None.
|
||||
"""
|
||||
# Filter out empty/None components
|
||||
valid = [c for c in components if c]
|
||||
if len(valid) != len(components):
|
||||
return None
|
||||
|
||||
# Join and normalize
|
||||
return os.path.normpath(os.path.join(*valid))
|
||||
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
"""Validate that subclasses define required class attributes."""
|
||||
super().__init_subclass__(**kwargs)
|
||||
@@ -139,14 +258,22 @@ class DownloadClient(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def add_download(self, url: str, name: str, category: str = "cwabd") -> str:
|
||||
"""
|
||||
Add a download to the client.
|
||||
def add_download(
|
||||
self,
|
||||
url: str,
|
||||
name: str,
|
||||
category: Optional[str] = None,
|
||||
expected_hash: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
|
||||
"""Add a download to the client.
|
||||
|
||||
Args:
|
||||
url: Download URL (magnet link, .torrent URL, or NZB URL)
|
||||
name: Display name for the download
|
||||
category: Category/label for organization
|
||||
category: Category/label for organization (None = client default)
|
||||
expected_hash: Optional info_hash hint (torrents only)
|
||||
|
||||
Returns:
|
||||
Client-specific download ID (hash for torrents, ID for NZBGet).
|
||||
@@ -196,7 +323,9 @@ class DownloadClient(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
def find_existing(
|
||||
self, url: str, category: Optional[str] = None
|
||||
) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
"""
|
||||
Check if a download for this URL already exists in the client.
|
||||
|
||||
@@ -205,6 +334,7 @@ class DownloadClient(ABC):
|
||||
|
||||
Args:
|
||||
url: Download URL (magnet link, .torrent URL, or NZB URL)
|
||||
category: Category to filter by (usenet clients only)
|
||||
|
||||
Returns:
|
||||
Tuple of (download_id, status) if found, None if not found.
|
||||
@@ -298,3 +428,4 @@ from shelfmark.release_sources.prowlarr.clients import nzbget # noqa: F401, E40
|
||||
from shelfmark.release_sources.prowlarr.clients import sabnzbd # noqa: F401, E402
|
||||
from shelfmark.release_sources.prowlarr.clients import transmission # noqa: F401, E402
|
||||
from shelfmark.release_sources.prowlarr.clients import deluge # noqa: F401, E402
|
||||
from shelfmark.release_sources.prowlarr.clients import rtorrent # noqa: F401, E402
|
||||
|
||||
@@ -1,17 +1,25 @@
|
||||
"""
|
||||
Deluge download client for Prowlarr integration.
|
||||
"""Deluge download client for Prowlarr integration.
|
||||
|
||||
Uses the deluge-client library to communicate with Deluge's RPC daemon.
|
||||
Note: Deluge uses a custom binary RPC protocol over TCP (default port 58846,
|
||||
configurable via DELUGE_PORT), which requires the daemon to have
|
||||
"Allow Remote Connections" enabled.
|
||||
This implementation talks to Deluge via the Web UI JSON-RPC API (``/json``).
|
||||
|
||||
Why Web UI API instead of daemon RPC (port 58846)?
|
||||
- Matches the approach used by common automation apps
|
||||
- Avoids requiring Deluge daemon ``auth`` file credentials (username/password)
|
||||
|
||||
Requirements:
|
||||
- ``deluge-web`` must be enabled and reachable from Shelfmark
|
||||
- Deluge Web UI must be connected (or connectable) to a Deluge daemon
|
||||
"""
|
||||
|
||||
import base64
|
||||
from typing import Any, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
@@ -24,146 +32,247 @@ from shelfmark.release_sources.prowlarr.clients.torrent_utils import (
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def _decode(value: Any) -> Any:
|
||||
"""Decode bytes to string if needed (Deluge returns bytes for strings)."""
|
||||
return value.decode('utf-8') if isinstance(value, bytes) else value
|
||||
class DelugeRpcError(RuntimeError):
|
||||
def __init__(self, message: str, code: int | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _get_error_message(error: Any) -> Tuple[str, int | None]:
|
||||
if isinstance(error, dict):
|
||||
return str(error.get("message") or error), error.get("code")
|
||||
return str(error), None
|
||||
|
||||
|
||||
@register_client("torrent")
|
||||
class DelugeClient(DownloadClient):
|
||||
"""Deluge download client using deluge-client RPC library."""
|
||||
"""Deluge download client using Deluge Web UI JSON-RPC."""
|
||||
|
||||
protocol = "torrent"
|
||||
name = "deluge"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Deluge client with settings from config."""
|
||||
from deluge_client import DelugeRPCClient
|
||||
raw_host = str(config.get("DELUGE_HOST", "localhost") or "")
|
||||
raw_port = str(config.get("DELUGE_PORT", "8112") or "8112")
|
||||
password = str(config.get("DELUGE_PASSWORD", "") or "")
|
||||
|
||||
host = config.get("DELUGE_HOST", "localhost")
|
||||
password = config.get("DELUGE_PASSWORD", "")
|
||||
|
||||
if not host:
|
||||
if not raw_host:
|
||||
raise ValueError("DELUGE_HOST is required")
|
||||
if not password:
|
||||
raise ValueError("DELUGE_PASSWORD is required")
|
||||
|
||||
port = int(config.get("DELUGE_PORT", "58846"))
|
||||
username = config.get("DELUGE_USERNAME", "")
|
||||
scheme = "http"
|
||||
base_path = ""
|
||||
|
||||
self._client = DelugeRPCClient(
|
||||
host=host,
|
||||
port=port,
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
# Allow DELUGE_HOST to be either a hostname OR a full URL
|
||||
# (useful when Deluge is behind a reverse proxy path).
|
||||
raw_host = normalize_http_url(raw_host, strip_trailing_slash=False) if raw_host else ""
|
||||
if not raw_host:
|
||||
raise ValueError("DELUGE_HOST is invalid")
|
||||
|
||||
host = raw_host
|
||||
port = int(raw_port)
|
||||
|
||||
if raw_host.startswith(("http://", "https://")):
|
||||
parsed = urlparse(raw_host)
|
||||
scheme = parsed.scheme or "http"
|
||||
host = parsed.hostname or "localhost"
|
||||
if parsed.port is not None:
|
||||
port = parsed.port
|
||||
base_path = (parsed.path or "").rstrip("/")
|
||||
else:
|
||||
# Allow "host:port" in DELUGE_HOST for convenience.
|
||||
if ":" in raw_host and raw_host.count(":") == 1:
|
||||
host_part, port_part = raw_host.split(":", 1)
|
||||
if host_part and port_part.isdigit():
|
||||
host = host_part
|
||||
port = int(port_part)
|
||||
|
||||
self._rpc_url = f"{scheme}://{host}:{port}{base_path}/json"
|
||||
self._password = password
|
||||
self._session = requests.Session()
|
||||
|
||||
self._authenticated = False
|
||||
self._connected = False
|
||||
self._category = config.get("DELUGE_CATEGORY", "cwabd")
|
||||
self._rpc_id = 0
|
||||
|
||||
def _ensure_connected(self):
|
||||
"""Ensure we're connected to the Deluge daemon."""
|
||||
if not self._connected:
|
||||
logger.debug("Connecting to Deluge daemon...")
|
||||
self._category = str(config.get("DELUGE_CATEGORY", "books") or "books")
|
||||
|
||||
def _next_rpc_id(self) -> int:
|
||||
self._rpc_id += 1
|
||||
return self._rpc_id
|
||||
|
||||
def _rpc_call(self, method: str, *params: Any, timeout: int = 15) -> Any:
|
||||
payload = {
|
||||
"id": self._next_rpc_id(),
|
||||
"method": method,
|
||||
"params": list(params),
|
||||
}
|
||||
|
||||
response = self._session.post(self._rpc_url, json=payload, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
if data.get("error"):
|
||||
message, code = _get_error_message(data["error"])
|
||||
raise DelugeRpcError(message, code)
|
||||
|
||||
return data.get("result")
|
||||
|
||||
def _login(self) -> None:
|
||||
result = self._rpc_call("auth.login", self._password)
|
||||
if result is not True:
|
||||
raise DelugeRpcError("Deluge Web UI authentication failed")
|
||||
self._authenticated = True
|
||||
|
||||
def _select_daemon_host_id(self, hosts: list) -> str:
|
||||
# Hosts returned by web.get_hosts look like:
|
||||
# [[host_id, host, port, status], ...]
|
||||
preferred_hosts = {"127.0.0.1", "localhost"}
|
||||
|
||||
for entry in hosts:
|
||||
if isinstance(entry, list) and len(entry) >= 2 and entry[1] in preferred_hosts:
|
||||
return str(entry[0])
|
||||
|
||||
for entry in hosts:
|
||||
if isinstance(entry, list) and len(entry) >= 4 and str(entry[3]).lower() == "online":
|
||||
return str(entry[0])
|
||||
|
||||
return str(hosts[0][0])
|
||||
|
||||
def _ensure_connected(self) -> None:
|
||||
if not self._authenticated:
|
||||
self._login()
|
||||
|
||||
if self._connected:
|
||||
return
|
||||
|
||||
if self._rpc_call("web.connected") is True:
|
||||
self._connected = True
|
||||
return
|
||||
|
||||
hosts = self._rpc_call("web.get_hosts") or []
|
||||
if not hosts:
|
||||
raise DelugeRpcError(
|
||||
"Deluge Web UI isn't connected to Deluge core (no hosts configured). "
|
||||
"Add/connect a daemon in Deluge Web UI → Connection Manager."
|
||||
)
|
||||
|
||||
host_id = self._select_daemon_host_id(hosts)
|
||||
self._rpc_call("web.connect", host_id)
|
||||
|
||||
if self._rpc_call("web.connected") is not True:
|
||||
raise DelugeRpcError(
|
||||
"Deluge Web UI couldn't connect to Deluge core. "
|
||||
"Check daemon status in Deluge Web UI → Connection Manager."
|
||||
)
|
||||
|
||||
self._connected = True
|
||||
|
||||
def _get_daemon_version(self) -> Any:
|
||||
"""Fetch daemon version, preferring daemon.get_version when available."""
|
||||
try:
|
||||
methods = self._rpc_call("system.listMethods")
|
||||
if isinstance(methods, list) and "daemon.get_version" in methods:
|
||||
return self._rpc_call("daemon.get_version")
|
||||
except Exception:
|
||||
# Fall back to daemon.info to preserve existing behavior.
|
||||
pass
|
||||
|
||||
return self._rpc_call("daemon.info")
|
||||
|
||||
def _try_set_label(self, torrent_id: str, label: str) -> None:
|
||||
"""Best-effort label assignment (requires Deluge Label plugin)."""
|
||||
if not label:
|
||||
return
|
||||
|
||||
try:
|
||||
# label.add will error if the plugin is unavailable or the label exists.
|
||||
try:
|
||||
self._client.connect()
|
||||
self._connected = True
|
||||
logger.debug("Connected to Deluge daemon")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to Deluge daemon: {type(e).__name__}: {e}")
|
||||
raise
|
||||
self._rpc_call("label.add", label)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self._rpc_call("label.set_torrent", torrent_id, label)
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not set Deluge label '{label}' for {torrent_id}: {e}")
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
"""Check if Deluge is configured and selected as the torrent client."""
|
||||
client = config.get("PROWLARR_TORRENT_CLIENT", "")
|
||||
host = config.get("DELUGE_HOST", "")
|
||||
password = config.get("DELUGE_PASSWORD", "")
|
||||
return client == "deluge" and bool(host) and bool(password)
|
||||
|
||||
def test_connection(self) -> Tuple[bool, str]:
|
||||
"""Test connection to Deluge."""
|
||||
try:
|
||||
self._ensure_connected()
|
||||
# Get daemon info
|
||||
version = self._client.call('daemon.info')
|
||||
version = self._get_daemon_version()
|
||||
return True, f"Connected to Deluge {version}"
|
||||
except Exception as e:
|
||||
self._authenticated = False
|
||||
self._connected = False
|
||||
return False, f"Connection failed: {str(e)}"
|
||||
|
||||
def add_download(self, url: str, name: str, category: str = None) -> str:
|
||||
"""
|
||||
Add torrent by URL (magnet or .torrent).
|
||||
|
||||
Args:
|
||||
url: Magnet link or .torrent URL
|
||||
name: Display name for the torrent
|
||||
category: Category for organization (uses configured default if not specified)
|
||||
|
||||
Returns:
|
||||
Torrent hash (info_hash).
|
||||
|
||||
Raises:
|
||||
Exception: If adding fails.
|
||||
"""
|
||||
def add_download(
|
||||
self,
|
||||
url: str,
|
||||
name: str,
|
||||
category: Optional[str] = None,
|
||||
expected_hash: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
category = category or self._category
|
||||
category_value = str(category or self._category)
|
||||
|
||||
torrent_info = extract_torrent_info(url)
|
||||
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
if not torrent_info.is_magnet and not torrent_info.torrent_data:
|
||||
raise Exception("Failed to fetch torrent file")
|
||||
|
||||
options = {}
|
||||
options: dict[str, Any] = {}
|
||||
|
||||
if torrent_info.is_magnet:
|
||||
# Use magnet URL if available, otherwise original URL
|
||||
magnet_url = torrent_info.magnet_url or url
|
||||
torrent_id = self._client.call(
|
||||
'core.add_torrent_magnet',
|
||||
magnet_url,
|
||||
options,
|
||||
)
|
||||
torrent_id = self._rpc_call("core.add_torrent_magnet", magnet_url, options)
|
||||
else:
|
||||
filedump = base64.b64encode(torrent_info.torrent_data).decode('ascii')
|
||||
torrent_id = self._client.call(
|
||||
'core.add_torrent_file',
|
||||
torrent_data = torrent_info.torrent_data
|
||||
if torrent_data is None:
|
||||
raise Exception("Failed to fetch torrent file")
|
||||
|
||||
torrent_data_bytes: bytes = torrent_data
|
||||
filedump = base64.b64encode(torrent_data_bytes).decode("ascii")
|
||||
torrent_id = self._rpc_call(
|
||||
"core.add_torrent_file",
|
||||
f"{name}.torrent",
|
||||
filedump,
|
||||
options,
|
||||
)
|
||||
|
||||
if torrent_id:
|
||||
torrent_id = _decode(torrent_id)
|
||||
logger.info(f"Added torrent to Deluge: {torrent_id}")
|
||||
return torrent_id.lower()
|
||||
if not torrent_id:
|
||||
raise Exception("Deluge returned no torrent ID")
|
||||
|
||||
raise Exception("Deluge returned no torrent ID")
|
||||
torrent_id = str(torrent_id).lower()
|
||||
self._try_set_label(torrent_id, category_value)
|
||||
|
||||
logger.info(f"Added torrent to Deluge: {torrent_id}")
|
||||
return torrent_id
|
||||
|
||||
except Exception as e:
|
||||
self._authenticated = False
|
||||
self._connected = False
|
||||
logger.error(f"Deluge add failed: {e}")
|
||||
raise
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""
|
||||
Get torrent status by hash.
|
||||
|
||||
Args:
|
||||
download_id: Torrent info_hash
|
||||
|
||||
Returns:
|
||||
Current download status.
|
||||
"""
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
# Get torrent status
|
||||
status = self._client.call(
|
||||
'core.get_torrent_status',
|
||||
status = self._rpc_call(
|
||||
"core.get_torrent_status",
|
||||
download_id,
|
||||
['state', 'progress', 'download_payload_rate', 'eta', 'save_path', 'name'],
|
||||
["state", "progress", "download_payload_rate", "eta", "save_path", "name"],
|
||||
)
|
||||
|
||||
if not status:
|
||||
@@ -171,35 +280,43 @@ class DelugeClient(DownloadClient):
|
||||
|
||||
# Deluge states: Downloading, Seeding, Paused, Checking, Queued, Error, Moving
|
||||
state_map = {
|
||||
'Downloading': ('downloading', None),
|
||||
'Seeding': ('seeding', 'Seeding'),
|
||||
'Paused': ('paused', 'Paused'),
|
||||
'Checking': ('checking', 'Checking files'),
|
||||
'Queued': ('queued', 'Queued'),
|
||||
'Error': ('error', 'Error'),
|
||||
'Moving': ('processing', 'Moving files'),
|
||||
'Allocating': ('downloading', 'Allocating space'),
|
||||
"Downloading": ("downloading", None),
|
||||
"Seeding": ("seeding", "Seeding"),
|
||||
"Paused": ("paused", "Paused"),
|
||||
"Checking": ("checking", "Checking files"),
|
||||
"Queued": ("queued", "Queued"),
|
||||
"Error": ("error", "Error"),
|
||||
"Moving": ("processing", "Moving files"),
|
||||
"Allocating": ("downloading", "Allocating space"),
|
||||
}
|
||||
|
||||
deluge_state = _decode(status.get(b'state', b'Unknown'))
|
||||
state, message = state_map.get(deluge_state, ('unknown', deluge_state))
|
||||
progress = status.get(b'progress', 0)
|
||||
deluge_state = status.get("state", "Unknown")
|
||||
state, message = state_map.get(str(deluge_state), ("unknown", str(deluge_state)))
|
||||
|
||||
progress = float(status.get("progress", 0))
|
||||
# Don't mark complete while files are being moved
|
||||
complete = progress >= 100 and deluge_state != 'Moving'
|
||||
complete = progress >= 100 and deluge_state != "Moving"
|
||||
|
||||
if complete:
|
||||
message = "Complete"
|
||||
|
||||
eta = status.get(b'eta')
|
||||
if eta and eta > 604800:
|
||||
eta = status.get("eta")
|
||||
if eta is not None:
|
||||
try:
|
||||
eta = int(eta)
|
||||
except Exception:
|
||||
eta = None
|
||||
|
||||
if eta is not None and (eta < 0 or eta > 604800):
|
||||
eta = None
|
||||
|
||||
file_path = None
|
||||
if complete:
|
||||
save_path = _decode(status.get(b'save_path', b''))
|
||||
name = _decode(status.get(b'name', b''))
|
||||
if save_path and name:
|
||||
file_path = f"{save_path}/{name}"
|
||||
# Output path is save_path + torrent name
|
||||
file_path = self._build_path(
|
||||
str(status.get("save_path", "")),
|
||||
str(status.get("name", "")),
|
||||
)
|
||||
|
||||
return DownloadStatus(
|
||||
progress=progress,
|
||||
@@ -207,36 +324,18 @@ class DelugeClient(DownloadClient):
|
||||
message=message,
|
||||
complete=complete,
|
||||
file_path=file_path,
|
||||
download_speed=status.get(b'download_payload_rate'),
|
||||
download_speed=status.get("download_payload_rate"),
|
||||
eta=eta,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self._connected = False
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"Deluge get_status failed ({error_type}): {e}")
|
||||
return DownloadStatus.error(f"{error_type}: {e}")
|
||||
return DownloadStatus.error(self._log_error("get_status", e))
|
||||
|
||||
def remove(self, download_id: str, delete_files: bool = False) -> bool:
|
||||
"""
|
||||
Remove a torrent from Deluge.
|
||||
|
||||
Args:
|
||||
download_id: Torrent info_hash
|
||||
delete_files: Whether to also delete files
|
||||
|
||||
Returns:
|
||||
True if successful.
|
||||
"""
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
result = self._client.call(
|
||||
'core.remove_torrent',
|
||||
download_id,
|
||||
delete_files,
|
||||
)
|
||||
|
||||
result = self._rpc_call("core.remove_torrent", download_id, delete_files)
|
||||
if result:
|
||||
logger.info(
|
||||
f"Removed torrent from Deluge: {download_id}"
|
||||
@@ -246,45 +345,33 @@ class DelugeClient(DownloadClient):
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
self._connected = False
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"Deluge remove failed ({error_type}): {e}")
|
||||
self._log_error("remove", e)
|
||||
return False
|
||||
|
||||
def get_download_path(self, download_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get the path where torrent files are located.
|
||||
|
||||
Args:
|
||||
download_id: Torrent info_hash
|
||||
|
||||
Returns:
|
||||
Content path (file or directory), or None.
|
||||
"""
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
status = self._client.call(
|
||||
'core.get_torrent_status',
|
||||
status = self._rpc_call(
|
||||
"core.get_torrent_status",
|
||||
download_id,
|
||||
['save_path', 'name'],
|
||||
["save_path", "name"],
|
||||
)
|
||||
|
||||
if status:
|
||||
save_path = _decode(status.get(b'save_path', b''))
|
||||
name = _decode(status.get(b'name', b''))
|
||||
if save_path and name:
|
||||
return f"{save_path}/{name}"
|
||||
return self._build_path(
|
||||
str(status.get("save_path", "")),
|
||||
str(status.get("name", "")),
|
||||
)
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
self._connected = False
|
||||
error_type = type(e).__name__
|
||||
logger.debug(f"Deluge get_download_path failed ({error_type}): {e}")
|
||||
self._log_error("get_download_path", e, level="debug")
|
||||
return None
|
||||
|
||||
def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
"""Check if a torrent for this URL already exists in Deluge."""
|
||||
def find_existing(
|
||||
self, url: str, category: Optional[str] = None
|
||||
) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
@@ -292,10 +379,10 @@ class DelugeClient(DownloadClient):
|
||||
if not torrent_info.info_hash:
|
||||
return None
|
||||
|
||||
status = self._client.call(
|
||||
'core.get_torrent_status',
|
||||
status = self._rpc_call(
|
||||
"core.get_torrent_status",
|
||||
torrent_info.info_hash,
|
||||
['state'],
|
||||
["state"],
|
||||
)
|
||||
|
||||
if status:
|
||||
@@ -303,7 +390,9 @@ class DelugeClient(DownloadClient):
|
||||
return (torrent_info.info_hash, full_status)
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
self._authenticated = False
|
||||
self._connected = False
|
||||
logger.debug(f"Error checking for existing torrent: {e}")
|
||||
return None
|
||||
|
||||
@@ -11,10 +11,12 @@ import requests
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
register_client,
|
||||
with_retry,
|
||||
)
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -29,11 +31,13 @@ class NZBGetClient(DownloadClient):
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize NZBGet client with settings from config."""
|
||||
url = config.get("NZBGET_URL", "")
|
||||
if not url:
|
||||
raw_url = config.get("NZBGET_URL", "")
|
||||
if not raw_url:
|
||||
raise ValueError("NZBGET_URL is required")
|
||||
|
||||
self.url = url.rstrip("/")
|
||||
self.url = normalize_http_url(raw_url)
|
||||
if not self.url:
|
||||
raise ValueError("NZBGET_URL is invalid")
|
||||
self.username = config.get("NZBGET_USERNAME", "nzbget")
|
||||
self.password = config.get("NZBGET_PASSWORD", "")
|
||||
self._category = config.get("NZBGET_CATEGORY", "Books")
|
||||
@@ -42,10 +46,11 @@ class NZBGetClient(DownloadClient):
|
||||
def is_configured() -> bool:
|
||||
"""Check if NZBGet is configured and selected as the usenet client."""
|
||||
client = config.get("PROWLARR_USENET_CLIENT", "")
|
||||
url = config.get("NZBGET_URL", "")
|
||||
url = normalize_http_url(config.get("NZBGET_URL", ""))
|
||||
return client == "nzbget" and bool(url)
|
||||
|
||||
def _rpc_call(self, method: str, params: list = None) -> Any:
|
||||
@with_retry()
|
||||
def _rpc_call(self, method: str, params: Optional[list] = None) -> Any:
|
||||
"""
|
||||
Make a JSON-RPC call to NZBGet.
|
||||
|
||||
@@ -57,7 +62,7 @@ class NZBGetClient(DownloadClient):
|
||||
Result from NZBGet.
|
||||
|
||||
Raises:
|
||||
Exception: If RPC call fails.
|
||||
Exception: If RPC call fails after retries.
|
||||
"""
|
||||
rpc_url = f"{self.url}/jsonrpc"
|
||||
|
||||
@@ -96,7 +101,14 @@ class NZBGetClient(DownloadClient):
|
||||
except Exception as e:
|
||||
return False, f"Connection failed: {str(e)}"
|
||||
|
||||
def add_download(self, url: str, name: str, category: str = None) -> str:
|
||||
def add_download(
|
||||
self,
|
||||
url: str,
|
||||
name: str,
|
||||
category: Optional[str] = None,
|
||||
expected_hash: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""
|
||||
Add NZB by URL.
|
||||
|
||||
@@ -107,6 +119,7 @@ class NZBGetClient(DownloadClient):
|
||||
url: NZB URL (can be Prowlarr proxy URL)
|
||||
name: Display name for the download
|
||||
category: Category for organization (uses configured default if not specified)
|
||||
expected_hash: Optional info_hash hint (unused)
|
||||
|
||||
Returns:
|
||||
NZBGet download ID (NZBID).
|
||||
@@ -226,7 +239,19 @@ class NZBGetClient(DownloadClient):
|
||||
for item in history:
|
||||
if item.get("NZBID") == nzb_id:
|
||||
status = item.get("Status", "")
|
||||
dest_dir = item.get("DestDir", "")
|
||||
# Prefer FinalDir (post-processing result) over DestDir (original)
|
||||
final_dir = item.get("FinalDir", "") or None
|
||||
dest_dir = item.get("DestDir", "") or None
|
||||
file_path = final_dir or dest_dir # Use FinalDir if available
|
||||
|
||||
# Normalize for consistent downstream use.
|
||||
if isinstance(file_path, str) and file_path:
|
||||
import os
|
||||
|
||||
file_path = os.path.normpath(file_path)
|
||||
else:
|
||||
file_path = None
|
||||
|
||||
|
||||
if "SUCCESS" in status:
|
||||
return DownloadStatus(
|
||||
@@ -234,7 +259,7 @@ class NZBGetClient(DownloadClient):
|
||||
state="complete",
|
||||
message="Complete",
|
||||
complete=True,
|
||||
file_path=dest_dir,
|
||||
file_path=file_path,
|
||||
)
|
||||
else:
|
||||
return DownloadStatus(
|
||||
@@ -242,41 +267,54 @@ class NZBGetClient(DownloadClient):
|
||||
state="error",
|
||||
message=f"Download failed: {status}",
|
||||
complete=True,
|
||||
file_path=None,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
# Not found in queue or history
|
||||
return DownloadStatus.error("Download not found")
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"NZBGet get_status failed ({error_type}): {e}")
|
||||
return DownloadStatus.error(f"{error_type}: {e}")
|
||||
return DownloadStatus.error(self._log_error("get_status", e))
|
||||
|
||||
def remove(self, download_id: str, delete_files: bool = False) -> bool:
|
||||
"""
|
||||
Remove a download from NZBGet.
|
||||
"""Remove a download from NZBGet.
|
||||
|
||||
NZBGet can remove items from either the active queue (Group* commands) or from
|
||||
history (History* commands). Completed downloads are typically in history.
|
||||
|
||||
Args:
|
||||
download_id: NZBGet NZBID
|
||||
delete_files: Whether to permanently delete (vs move to history)
|
||||
delete_files: Whether to permanently delete downloaded files
|
||||
|
||||
Returns:
|
||||
True if successful.
|
||||
"""
|
||||
try:
|
||||
nzb_id = int(download_id)
|
||||
# editqueue params: Command (str), Param (str), IDs (int[])
|
||||
# GroupFinalDelete = permanent removal, GroupDelete = move to history
|
||||
command = "GroupFinalDelete" if delete_files else "GroupDelete"
|
||||
result = self._rpc_call("editqueue", [command, "", [nzb_id]])
|
||||
if result:
|
||||
logger.info(f"Removed NZB from NZBGet: {download_id}")
|
||||
return bool(result)
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"NZBGet remove failed ({error_type}): {e}")
|
||||
except (TypeError, ValueError) as e:
|
||||
self._log_error("remove", e)
|
||||
return False
|
||||
|
||||
if delete_files:
|
||||
# Keep HistoryDelete as a fallback for
|
||||
# older NZBGet versions where HistoryFinalDelete may not exist.
|
||||
commands = ["GroupFinalDelete", "HistoryFinalDelete", "HistoryDelete"]
|
||||
else:
|
||||
commands = ["GroupDelete", "HistoryDelete"]
|
||||
|
||||
last_error: Optional[Exception] = None
|
||||
for command in commands:
|
||||
try:
|
||||
result = self._rpc_call("editqueue", [command, 0, "", nzb_id])
|
||||
if result:
|
||||
logger.info(f"Removed NZB from NZBGet ({command}): {download_id}")
|
||||
return True
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
|
||||
if last_error is not None:
|
||||
self._log_error("remove", last_error)
|
||||
return False
|
||||
|
||||
def get_download_path(self, download_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get the path where NZB files are located.
|
||||
|
||||
@@ -2,10 +2,11 @@
|
||||
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
@@ -34,6 +35,55 @@ def _hashes_match(hash1: str, hash2: str) -> bool:
|
||||
class QBittorrentClient(DownloadClient):
|
||||
"""qBittorrent download client."""
|
||||
|
||||
def _is_torrent_loaded(self, torrent_hash: str) -> tuple[bool, Optional[str]]:
|
||||
"""Check whether qBittorrent has registered a torrent yet.
|
||||
|
||||
Uses `/api/v2/torrents/properties?hash=<hash>`.
|
||||
|
||||
Returns:
|
||||
(loaded, error_message)
|
||||
|
||||
Notes:
|
||||
A false result with no error means "not loaded yet".
|
||||
"""
|
||||
import requests
|
||||
|
||||
url = f"{self._base_url}/api/v2/torrents/properties"
|
||||
params = {"hash": torrent_hash}
|
||||
|
||||
try:
|
||||
self._client.auth_log_in()
|
||||
response = self._client._session.get(url, params=params, timeout=10)
|
||||
|
||||
# Re-authenticate and retry once on 403
|
||||
if response.status_code == 403:
|
||||
logger.debug("qBittorrent returned 403 for properties; re-authenticating and retrying")
|
||||
self._client.auth_log_in()
|
||||
response = self._client._session.get(url, params=params, timeout=10)
|
||||
|
||||
if response.status_code == 403:
|
||||
return False, "qBittorrent authentication failed (HTTP 403)"
|
||||
|
||||
# qBittorrent returns 404/409-ish responses depending on version when missing.
|
||||
if response.status_code == 404:
|
||||
return False, None
|
||||
|
||||
response.raise_for_status()
|
||||
return True, None
|
||||
except requests.exceptions.HTTPError as e:
|
||||
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||
if status == 404:
|
||||
return False, None
|
||||
if status:
|
||||
return False, f"qBittorrent API request failed (HTTP {status})"
|
||||
return False, "qBittorrent API request failed"
|
||||
except requests.exceptions.ConnectionError:
|
||||
return False, f"Cannot connect to qBittorrent at {self._base_url}"
|
||||
except requests.exceptions.Timeout:
|
||||
return False, f"qBittorrent request timed out at {self._base_url}"
|
||||
except Exception as e:
|
||||
return False, f"qBittorrent API error: {type(e).__name__}: {e}"
|
||||
|
||||
protocol = "torrent"
|
||||
name = "qbittorrent"
|
||||
|
||||
@@ -42,53 +92,132 @@ class QBittorrentClient(DownloadClient):
|
||||
# Lazy import to avoid dependency issues if not using torrents
|
||||
from qbittorrentapi import Client
|
||||
|
||||
url = config.get("QBITTORRENT_URL", "")
|
||||
if not url:
|
||||
raw_url = config.get("QBITTORRENT_URL", "")
|
||||
if not raw_url:
|
||||
raise ValueError("QBITTORRENT_URL is required")
|
||||
|
||||
self._base_url = url.rstrip("/")
|
||||
# We use `_base_url` for direct HTTP calls, so it must be a fully-qualified URL.
|
||||
self._base_url = normalize_http_url(raw_url)
|
||||
if not self._base_url:
|
||||
raise ValueError("QBITTORRENT_URL is invalid")
|
||||
|
||||
# qbittorrent-api accepts either a full URL or host:port; prefer the normalized URL
|
||||
# for consistency.
|
||||
self._client = Client(
|
||||
host=url,
|
||||
host=self._base_url,
|
||||
username=config.get("QBITTORRENT_USERNAME", ""),
|
||||
password=config.get("QBITTORRENT_PASSWORD", ""),
|
||||
)
|
||||
self._category = config.get("QBITTORRENT_CATEGORY", "cwabd")
|
||||
self._category = config.get("QBITTORRENT_CATEGORY", "books")
|
||||
|
||||
def _get_torrents_info(self, torrent_hash: Optional[str] = None) -> List:
|
||||
"""Get torrent info using GET (per API spec for read operations)."""
|
||||
|
||||
def _get_torrents_info(
|
||||
self, torrent_hash: Optional[str] = None
|
||||
) -> tuple[list[SimpleNamespace], Optional[str]]:
|
||||
"""Get torrent info using GET.
|
||||
|
||||
Behaviors:
|
||||
- Retry once on HTTP 403 by re-authenticating.
|
||||
- Keep "API/auth/connect" errors distinct from "torrent missing".
|
||||
- If a hash-specific query returns empty, fall back to listing by category
|
||||
and matching locally.
|
||||
|
||||
Returns:
|
||||
(torrents, error_message)
|
||||
"""
|
||||
import requests
|
||||
|
||||
try:
|
||||
url = f"{self._base_url}/api/v2/torrents/info"
|
||||
|
||||
def do_request(params: dict[str, str]) -> requests.Response:
|
||||
# Ensure session is authenticated before using it directly
|
||||
self._client.auth_log_in()
|
||||
return self._client._session.get(url, params=params, timeout=10)
|
||||
|
||||
def parse_response(
|
||||
response: requests.Response,
|
||||
*,
|
||||
request_params: dict[str, str],
|
||||
) -> tuple[list[SimpleNamespace], Optional[str]]:
|
||||
if response.status_code == 403:
|
||||
logger.debug("qBittorrent returned 403; re-authenticating and retrying")
|
||||
self._client.auth_log_in()
|
||||
response = self._client._session.get(url, params=request_params, timeout=10)
|
||||
|
||||
if response.status_code == 403:
|
||||
logger.warning("qBittorrent authentication failed (HTTP 403)")
|
||||
return [], "qBittorrent authentication failed (HTTP 403)"
|
||||
|
||||
params = {"hashes": torrent_hash} if torrent_hash else {}
|
||||
response = self._client._session.get(
|
||||
f"{self._base_url}/api/v2/torrents/info",
|
||||
params=params,
|
||||
timeout=10,
|
||||
)
|
||||
response.raise_for_status()
|
||||
torrents = response.json()
|
||||
return [SimpleNamespace(**t) for t in torrents]
|
||||
return [SimpleNamespace(**t) for t in torrents], None
|
||||
|
||||
try:
|
||||
primary_params: dict[str, str] = {}
|
||||
if torrent_hash:
|
||||
primary_params["hashes"] = torrent_hash
|
||||
|
||||
response = do_request(primary_params)
|
||||
torrents, error = parse_response(response, request_params=primary_params)
|
||||
if error:
|
||||
return [], error
|
||||
|
||||
if torrent_hash and not torrents:
|
||||
# Fallback 1: list by configured category
|
||||
category_params: dict[str, str] = {}
|
||||
if self._category:
|
||||
category_params["category"] = self._category
|
||||
|
||||
category_response = do_request(category_params)
|
||||
category_torrents, category_error = parse_response(
|
||||
category_response, request_params=category_params
|
||||
)
|
||||
if category_error:
|
||||
return [], category_error
|
||||
|
||||
if category_torrents:
|
||||
return category_torrents, None
|
||||
|
||||
# Fallback 2: list everything (handles per-task categories like audiobooks)
|
||||
all_response = do_request({})
|
||||
all_torrents, all_error = parse_response(all_response, request_params={})
|
||||
if all_error:
|
||||
return [], all_error
|
||||
|
||||
return all_torrents, None
|
||||
|
||||
return torrents, None
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response is not None and e.response.status_code == 403:
|
||||
logger.warning("qBittorrent auth failed - check credentials")
|
||||
else:
|
||||
logger.warning(f"qBittorrent API error: {e}")
|
||||
return []
|
||||
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||
if status:
|
||||
logger.warning(f"qBittorrent API error (HTTP {status}): {e}")
|
||||
return [], f"qBittorrent API request failed (HTTP {status})"
|
||||
|
||||
logger.warning(f"qBittorrent API error: {e}")
|
||||
return [], "qBittorrent API request failed"
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.warning(f"Cannot connect to qBittorrent at {self._base_url}")
|
||||
return []
|
||||
return [], f"Cannot connect to qBittorrent at {self._base_url}"
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning(f"qBittorrent request timed out at {self._base_url}")
|
||||
return [], f"qBittorrent request timed out at {self._base_url}"
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get torrents info: {e}")
|
||||
return []
|
||||
# requests raises InvalidSchema when the base URL doesn't include http(s)
|
||||
if type(e).__name__ == "InvalidSchema":
|
||||
return (
|
||||
[],
|
||||
"qBittorrent URL is invalid (missing http:// or https://). "
|
||||
f"Configured: {self._base_url}",
|
||||
)
|
||||
return [], f"qBittorrent API error: {type(e).__name__}: {e}"
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
"""Check if qBittorrent is configured and selected as the torrent client."""
|
||||
client = config.get("PROWLARR_TORRENT_CLIENT", "")
|
||||
url = config.get("QBITTORRENT_URL", "")
|
||||
url = normalize_http_url(config.get("QBITTORRENT_URL", ""))
|
||||
return client == "qbittorrent" and bool(url)
|
||||
|
||||
def test_connection(self) -> Tuple[bool, str]:
|
||||
@@ -100,7 +229,14 @@ class QBittorrentClient(DownloadClient):
|
||||
except Exception as e:
|
||||
return False, f"Connection failed: {str(e)}"
|
||||
|
||||
def add_download(self, url: str, name: str, category: str = None) -> str:
|
||||
def add_download(
|
||||
self,
|
||||
url: str,
|
||||
name: str,
|
||||
category: str | None = None,
|
||||
expected_hash: str | None = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""
|
||||
Add torrent by URL (magnet or .torrent).
|
||||
|
||||
@@ -108,6 +244,7 @@ class QBittorrentClient(DownloadClient):
|
||||
url: Magnet link or .torrent URL
|
||||
name: Display name for the torrent
|
||||
category: Category for organization (uses configured default if not specified)
|
||||
expected_hash: Optional info_hash hint (from Prowlarr)
|
||||
|
||||
Returns:
|
||||
Torrent hash (info_hash).
|
||||
@@ -128,7 +265,7 @@ class QBittorrentClient(DownloadClient):
|
||||
if "Conflict" not in type(e).__name__ and "409" not in str(e):
|
||||
logger.debug(f"Could not create category '{category}': {type(e).__name__}: {e}")
|
||||
|
||||
torrent_info = extract_torrent_info(url)
|
||||
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
expected_hash = torrent_info.info_hash
|
||||
torrent_data = torrent_info.torrent_data
|
||||
|
||||
@@ -154,13 +291,16 @@ class QBittorrentClient(DownloadClient):
|
||||
if not expected_hash:
|
||||
raise Exception("Could not determine torrent hash from URL")
|
||||
|
||||
# Wait for torrent to appear in client
|
||||
# Wait for torrent to appear in client.
|
||||
# Use `/torrents/properties?hash=` rather than relying on `torrents/info`
|
||||
# listing being immediately consistent.
|
||||
for _ in range(10):
|
||||
torrents = self._get_torrents_info(expected_hash)
|
||||
for t in torrents:
|
||||
if _hashes_match(t.hash, expected_hash):
|
||||
logger.info(f"Added torrent: {t.hash}")
|
||||
return t.hash.lower()
|
||||
loaded, error = self._is_torrent_loaded(expected_hash)
|
||||
if error:
|
||||
logger.debug(f"qBittorrent add_download: {error}")
|
||||
if loaded:
|
||||
logger.info(f"Added torrent: {expected_hash}")
|
||||
return expected_hash.lower()
|
||||
time.sleep(0.5)
|
||||
|
||||
# Client said Ok, trust it
|
||||
@@ -183,10 +323,21 @@ class QBittorrentClient(DownloadClient):
|
||||
Current download status.
|
||||
"""
|
||||
try:
|
||||
torrents = self._get_torrents_info(download_id)
|
||||
torrent = next((t for t in torrents if _hashes_match(t.hash, download_id)), None)
|
||||
torrents, error = self._get_torrents_info(download_id)
|
||||
if error:
|
||||
return DownloadStatus.error(error)
|
||||
|
||||
torrent = next(
|
||||
(
|
||||
t
|
||||
for t in torrents
|
||||
if isinstance(getattr(t, "hash", None), str)
|
||||
and _hashes_match(getattr(t, "hash"), download_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not torrent:
|
||||
return DownloadStatus.error("Torrent not found")
|
||||
return DownloadStatus.error("Torrent not found in qBittorrent")
|
||||
|
||||
# Map qBittorrent states to our states and user-friendly messages
|
||||
state_info = {
|
||||
@@ -211,43 +362,41 @@ class QBittorrentClient(DownloadClient):
|
||||
"unknown": ("unknown", "Unknown state"),
|
||||
}
|
||||
|
||||
state, message = state_info.get(torrent.state, ("unknown", torrent.state))
|
||||
torrent_state = getattr(torrent, "state", "unknown")
|
||||
state, message = state_info.get(torrent_state, ("unknown", str(torrent_state)))
|
||||
|
||||
torrent_progress = getattr(torrent, "progress", 0.0)
|
||||
# Don't mark complete while files are being moved to final location
|
||||
# (qBittorrent moves files from incomplete → complete folder)
|
||||
complete = torrent.progress >= 1.0 and torrent.state != "moving"
|
||||
complete = torrent_progress >= 1.0 and torrent_state != "moving"
|
||||
|
||||
# For active downloads without a special message, leave message as None
|
||||
# so the handler can build the progress message
|
||||
if complete:
|
||||
message = "Complete"
|
||||
|
||||
eta = torrent.eta if 0 < torrent.eta < 604800 else None
|
||||
torrent_eta = getattr(torrent, "eta", 0)
|
||||
eta = torrent_eta if isinstance(torrent_eta, int) and 0 < torrent_eta < 604800 else None
|
||||
|
||||
# Get file path for completed downloads
|
||||
file_path = None
|
||||
if complete:
|
||||
if getattr(torrent, 'content_path', ''):
|
||||
file_path = torrent.content_path
|
||||
else:
|
||||
# Fallback for Amarr which doesn't populate content_path
|
||||
save_path = getattr(torrent, 'save_path', '')
|
||||
name = getattr(torrent, 'name', '')
|
||||
if save_path and name:
|
||||
file_path = f"{save_path}/{name}"
|
||||
file_path = self._resolve_completed_download_path(torrent)
|
||||
|
||||
torrent_speed = getattr(torrent, "dlspeed", None)
|
||||
torrent_speed = torrent_speed if isinstance(torrent_speed, int) else None
|
||||
|
||||
return DownloadStatus(
|
||||
progress=torrent.progress * 100,
|
||||
progress=float(torrent_progress) * 100,
|
||||
state="complete" if complete else state,
|
||||
message=message,
|
||||
complete=complete,
|
||||
file_path=file_path,
|
||||
download_speed=torrent.dlspeed,
|
||||
download_speed=torrent_speed,
|
||||
eta=eta,
|
||||
)
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"qBittorrent get_status failed ({error_type}): {e}")
|
||||
return DownloadStatus.error(f"{error_type}: {e}")
|
||||
return DownloadStatus.error(self._log_error("get_status", e))
|
||||
|
||||
def remove(self, download_id: str, delete_files: bool = False) -> bool:
|
||||
"""
|
||||
@@ -270,39 +419,153 @@ class QBittorrentClient(DownloadClient):
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"qBittorrent remove failed ({error_type}): {e}")
|
||||
self._log_error("remove", e)
|
||||
return False
|
||||
|
||||
def get_download_path(self, download_id: str) -> Optional[str]:
|
||||
"""Get the path where torrent files are located."""
|
||||
"""Get the path where torrent files are located.
|
||||
|
||||
Prefer `content_path` when available.
|
||||
|
||||
When `content_path` is missing (commonly with qBittorrent-like emulators such
|
||||
as Amarr), derive the path using:
|
||||
- `/api/v2/torrents/properties?hash=<hash>` for `save_path`
|
||||
- `/api/v2/torrents/files?hash=<hash>` for the first file name
|
||||
- join `save_path` with the torrent's top-level directory
|
||||
"""
|
||||
import os
|
||||
|
||||
try:
|
||||
torrents = self._get_torrents_info(download_id)
|
||||
torrent = next((t for t in torrents if _hashes_match(t.hash, download_id)), None)
|
||||
torrents, error = self._get_torrents_info(download_id)
|
||||
if error:
|
||||
logger.debug(f"qBittorrent get_download_path: {error}")
|
||||
return None
|
||||
|
||||
torrent = next(
|
||||
(
|
||||
t
|
||||
for t in torrents
|
||||
if isinstance(getattr(t, "hash", None), str)
|
||||
and _hashes_match(getattr(t, "hash"), download_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not torrent:
|
||||
return None
|
||||
# Prefer content_path, fall back to save_path/name (for Amarr compatibility)
|
||||
if getattr(torrent, 'content_path', ''):
|
||||
return torrent.content_path
|
||||
save_path = getattr(torrent, 'save_path', '')
|
||||
name = getattr(torrent, 'name', '')
|
||||
return f"{save_path}/{name}" if save_path and name else None
|
||||
|
||||
return self._resolve_completed_download_path(torrent)
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.debug(f"qBittorrent get_download_path failed ({error_type}): {e}")
|
||||
self._log_error("get_download_path", e, level="debug")
|
||||
return None
|
||||
|
||||
def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
def _resolve_completed_download_path(self, torrent: SimpleNamespace) -> Optional[str]:
|
||||
"""Resolve the completed path for a torrent.
|
||||
|
||||
Centralizes the logic shared by `get_status()` and `get_download_path()`:
|
||||
- accept `content_path` only when it's not equal to `save_path`
|
||||
- otherwise derive via properties+files
|
||||
- finally fall back to `save_path + name`
|
||||
"""
|
||||
|
||||
# Prefer content_path, but treat content_path == save_path as invalid.
|
||||
content_path = getattr(torrent, "content_path", "")
|
||||
save_path = getattr(torrent, "save_path", "")
|
||||
if content_path and (not save_path or str(content_path) != str(save_path)):
|
||||
return str(content_path)
|
||||
|
||||
download_id = getattr(torrent, "hash", "")
|
||||
if isinstance(download_id, str) and download_id:
|
||||
derived = self._derive_download_path_from_files(download_id)
|
||||
if derived:
|
||||
return derived
|
||||
|
||||
# Legacy fallback: save_path + name (for older clients/emulators)
|
||||
return self._build_path(
|
||||
getattr(torrent, "save_path", ""),
|
||||
getattr(torrent, "name", ""),
|
||||
)
|
||||
|
||||
def _derive_download_path_from_files(self, download_id: str) -> Optional[str]:
|
||||
"""Derive completed download path using `/torrents/properties` + `/torrents/files`.
|
||||
|
||||
This mirrors how common automation apps derive the path when
|
||||
`content_path` isn't provided.
|
||||
"""
|
||||
import os
|
||||
import requests
|
||||
|
||||
def get_with_auth(url: str, params: dict[str, str]) -> requests.Response:
|
||||
self._client.auth_log_in()
|
||||
resp = self._client._session.get(url, params=params, timeout=10)
|
||||
if resp.status_code == 403:
|
||||
logger.debug("qBittorrent returned 403; re-authenticating and retrying")
|
||||
self._client.auth_log_in()
|
||||
resp = self._client._session.get(url, params=params, timeout=10)
|
||||
return resp
|
||||
|
||||
try:
|
||||
properties_url = f"{self._base_url}/api/v2/torrents/properties"
|
||||
files_url = f"{self._base_url}/api/v2/torrents/files"
|
||||
|
||||
props_resp = get_with_auth(properties_url, {"hash": download_id})
|
||||
if props_resp.status_code == 404:
|
||||
return None
|
||||
props_resp.raise_for_status()
|
||||
props = props_resp.json() if isinstance(props_resp.json(), dict) else {}
|
||||
|
||||
save_path = props.get("save_path") or props.get("savePath") or ""
|
||||
if not isinstance(save_path, str) or not save_path:
|
||||
return None
|
||||
|
||||
files_resp = get_with_auth(files_url, {"hash": download_id})
|
||||
if files_resp.status_code == 404:
|
||||
return None
|
||||
files_resp.raise_for_status()
|
||||
files = files_resp.json() if isinstance(files_resp.json(), list) else []
|
||||
if not files:
|
||||
return None
|
||||
|
||||
first_name = files[0].get("name") if isinstance(files[0], dict) else None
|
||||
if not isinstance(first_name, str) or not first_name:
|
||||
return None
|
||||
|
||||
# Get the first path segment (qBittorrent returns '/' even on Windows).
|
||||
first_name_norm = first_name.replace("\\", "/")
|
||||
top_level = first_name_norm.split("/", 1)[0]
|
||||
if not top_level:
|
||||
return None
|
||||
|
||||
return os.path.normpath(os.path.join(save_path, top_level))
|
||||
except Exception as e:
|
||||
logger.debug(f"qBittorrent could not derive path from files: {type(e).__name__}: {e}")
|
||||
return None
|
||||
|
||||
def find_existing(
|
||||
self, url: str, category: Optional[str] = None
|
||||
) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
"""Check if a torrent for this URL already exists in qBittorrent."""
|
||||
try:
|
||||
torrent_info = extract_torrent_info(url)
|
||||
if not torrent_info.info_hash:
|
||||
return None
|
||||
|
||||
torrents = self._get_torrents_info(torrent_info.info_hash)
|
||||
torrent = next((t for t in torrents if _hashes_match(t.hash, torrent_info.info_hash)), None)
|
||||
if torrent:
|
||||
return (torrent.hash.lower(), self.get_status(torrent.hash.lower()))
|
||||
torrents, error = self._get_torrents_info(torrent_info.info_hash)
|
||||
if error:
|
||||
logger.debug(f"qBittorrent find_existing: {error}")
|
||||
return None
|
||||
|
||||
torrent = next(
|
||||
(
|
||||
t
|
||||
for t in torrents
|
||||
if isinstance(getattr(t, "hash", None), str)
|
||||
and _hashes_match(getattr(t, "hash"), torrent_info.info_hash)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if torrent and isinstance(getattr(torrent, "hash", None), str):
|
||||
torrent_hash = getattr(torrent, "hash")
|
||||
return (torrent_hash.lower(), self.get_status(torrent_hash.lower()))
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
"""
|
||||
rTorrent download client for Prowlarr integration.
|
||||
|
||||
Uses xmlrpc to communicate with rTorrent's RPC interface.
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
register_client,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.clients.torrent_utils import (
|
||||
extract_torrent_info,
|
||||
)
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
@register_client("torrent")
|
||||
class RTorrentClient(DownloadClient):
|
||||
"""rTorrent download client using xmlrpc."""
|
||||
|
||||
protocol = "torrent"
|
||||
name = "rtorrent"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize rTorrent client with settings from config."""
|
||||
from xmlrpc.client import ServerProxy
|
||||
|
||||
raw_url = config.get("RTORRENT_URL", "")
|
||||
if not raw_url:
|
||||
raise ValueError("RTORRENT_URL is required")
|
||||
|
||||
self._base_url = normalize_http_url(raw_url)
|
||||
if not self._base_url:
|
||||
raise ValueError("RTORRENT_URL is invalid")
|
||||
|
||||
username = config.get("RTORRENT_USERNAME", "")
|
||||
password = config.get("RTORRENT_PASSWORD", "")
|
||||
|
||||
if username and password:
|
||||
parsed = urlparse(self._base_url)
|
||||
self._base_url = (
|
||||
f"{parsed.scheme}://{username}:{password}@{parsed.netloc}{parsed.path}"
|
||||
)
|
||||
|
||||
self._rpc = ServerProxy(self._base_url)
|
||||
self._download_dir = config.get("RTORRENT_DOWNLOAD_DIR", "")
|
||||
self._label = config.get("RTORRENT_LABEL", "")
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
"""Check if rTorrent is configured and selected as the torrent client."""
|
||||
client = config.get("PROWLARR_TORRENT_CLIENT", "")
|
||||
url = normalize_http_url(config.get("RTORRENT_URL", ""))
|
||||
return client == "rtorrent" and bool(url)
|
||||
|
||||
def test_connection(self) -> Tuple[bool, str]:
|
||||
"""Test connection to rTorrent."""
|
||||
try:
|
||||
version = self._rpc.system.client_version()
|
||||
return True, f"Connected to rTorrent {version}"
|
||||
except Exception as e:
|
||||
return False, f"Connection failed: {str(e)}"
|
||||
|
||||
def add_download(
|
||||
self,
|
||||
url: str,
|
||||
name: str,
|
||||
category: Optional[str] = None,
|
||||
expected_hash: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""
|
||||
Add torrent by URL (magnet or .torrent).
|
||||
|
||||
Args:
|
||||
url: Magnet link or .torrent URL
|
||||
name: Display name for the torrent
|
||||
category: Category for organization (uses configured label if not specified)
|
||||
expected_hash: Optional info_hash hint (from Prowlarr)
|
||||
|
||||
Returns:
|
||||
Torrent hash (info_hash).
|
||||
|
||||
Raises:
|
||||
Exception: If adding fails.
|
||||
"""
|
||||
try:
|
||||
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
|
||||
commands = []
|
||||
|
||||
label = category or self._label
|
||||
if label:
|
||||
logger.debug(f"Setting rTorrent label: {label}")
|
||||
commands.append(f"d.custom1.set={label}")
|
||||
|
||||
download_dir = self._download_dir or self._get_download_dir()
|
||||
if download_dir:
|
||||
logger.debug(f"Setting rTorrent download directory: {download_dir}")
|
||||
commands.append(f"d.directory_base.set={download_dir}")
|
||||
|
||||
if torrent_info.torrent_data:
|
||||
logger.debug(f"Adding torrent data directly to rTorrent for: {name} with commands: {commands} with data size: {len(torrent_info.torrent_data)}")
|
||||
self._rpc.load.raw_start(
|
||||
"", torrent_info.torrent_data, ";".join(commands)
|
||||
)
|
||||
else:
|
||||
logger.debug(f"Adding torrent URL to rTorrent for: {name} with commands: {commands} with URL: {url}")
|
||||
add_url = torrent_info.magnet_url or url
|
||||
self._rpc.load.start("", add_url, ";".join(commands))
|
||||
|
||||
torrent_hash = torrent_info.info_hash or expected_hash
|
||||
if not torrent_hash:
|
||||
raise Exception("Could not determine torrent hash from URL")
|
||||
|
||||
logger.debug(f"Added torrent to rTorrent: {torrent_hash}")
|
||||
return torrent_hash
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"rTorrent add failed: {e}")
|
||||
raise
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""
|
||||
Get torrent status by hash.
|
||||
|
||||
Args:
|
||||
download_id: Torrent info_hash
|
||||
|
||||
Returns:
|
||||
Current download status.
|
||||
"""
|
||||
try:
|
||||
# rtorrent is somehow case sensitive and requires uppercase hashes for look
|
||||
download_id = download_id.upper()
|
||||
torrent_list = self._rpc.d.multicall.filtered(
|
||||
"",
|
||||
"default",
|
||||
f"equal={{d.hash=,cat={download_id}}}",
|
||||
"d.hash=",
|
||||
"d.state=",
|
||||
"d.completed_bytes=",
|
||||
"d.size_bytes=",
|
||||
"d.down.rate=",
|
||||
"d.up.rate=",
|
||||
"d.custom1=",
|
||||
"d.complete=",
|
||||
)
|
||||
logger.debug(f"Fetched torrent status from rTorrent for: {download_id} - {torrent_list}")
|
||||
if not torrent_list:
|
||||
logger.warning(f"Torrent not found in rTorrent: {download_id}")
|
||||
return DownloadStatus.error("Torrent not found")
|
||||
|
||||
torrent = torrent_list[0]
|
||||
if not torrent:
|
||||
logger.warning(f"Torrent data is empty for: {download_id}")
|
||||
return DownloadStatus.error("Torrent not found")
|
||||
|
||||
logger.debug(f"Torrent data for {download_id}: {torrent}")
|
||||
(
|
||||
torrent_hash,
|
||||
state,
|
||||
bytes_downloaded,
|
||||
bytes_total,
|
||||
down_rate,
|
||||
up_rate,
|
||||
custom_category,
|
||||
complete,
|
||||
) = torrent
|
||||
|
||||
try:
|
||||
state = int(state)
|
||||
except Exception:
|
||||
state = 0
|
||||
|
||||
complete = bool(complete)
|
||||
|
||||
if bytes_total > 0:
|
||||
progress = (bytes_downloaded / bytes_total) * 100
|
||||
else:
|
||||
progress = 0
|
||||
|
||||
bytes_left = max(0, bytes_total - bytes_downloaded)
|
||||
|
||||
state_map = {
|
||||
0: ("paused", "Paused"),
|
||||
1: ("downloading", "Downloading"),
|
||||
2: ("downloading", "Downloading"),
|
||||
3: ("downloading", "Downloading"),
|
||||
4: ("seeding", "Seeding"),
|
||||
}
|
||||
|
||||
state_str, message = state_map.get(state, ("unknown", "Unknown state"))
|
||||
|
||||
if complete and not message:
|
||||
message = "Complete"
|
||||
|
||||
eta = None
|
||||
if down_rate > 0 and bytes_left > 0:
|
||||
eta_seconds = bytes_left / down_rate
|
||||
if eta_seconds < 604800:
|
||||
eta = int(eta_seconds)
|
||||
|
||||
file_path = None
|
||||
if complete:
|
||||
file_path = self._get_torrent_path(download_id)
|
||||
|
||||
return DownloadStatus(
|
||||
progress=min(100, progress),
|
||||
state="complete" if complete else state_str,
|
||||
message=message,
|
||||
complete=complete,
|
||||
file_path=file_path,
|
||||
download_speed=down_rate if down_rate > 0 else None,
|
||||
eta=eta,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"rTorrent get_status failed ({error_type}): {e}")
|
||||
return DownloadStatus.error(f"{error_type}: {e}")
|
||||
|
||||
def remove(self, download_id: str, delete_files: bool = False) -> bool:
|
||||
"""
|
||||
Remove a torrent from rTorrent.
|
||||
|
||||
Args:
|
||||
download_id: Torrent info_hash
|
||||
delete_files: Whether to also delete files
|
||||
|
||||
Returns:
|
||||
True if successful.
|
||||
"""
|
||||
try:
|
||||
if delete_files:
|
||||
self._rpc.d.delete_tied(download_id)
|
||||
self._rpc.d.erase(download_id)
|
||||
else:
|
||||
self._rpc.d.stop(download_id)
|
||||
self._rpc.d.erase(download_id)
|
||||
|
||||
logger.info(
|
||||
f"Removed torrent from rTorrent: {download_id}"
|
||||
+ (" (with files)" if delete_files else "")
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"rTorrent remove failed ({error_type}): {e}")
|
||||
return False
|
||||
|
||||
def get_download_path(self, download_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get the path where torrent files are located.
|
||||
|
||||
Args:
|
||||
download_id: Torrent info_hash
|
||||
|
||||
Returns:
|
||||
Content path (file or directory), or None.
|
||||
"""
|
||||
try:
|
||||
return self._get_torrent_path(download_id)
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.debug(f"rTorrent get_download_path failed ({error_type}): {e}")
|
||||
return None
|
||||
|
||||
def find_existing(
|
||||
self, url: str, category: Optional[str] = None
|
||||
) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
"""Check if a torrent for this URL already exists in rTorrent."""
|
||||
try:
|
||||
torrent_info = extract_torrent_info(url)
|
||||
if not torrent_info.info_hash:
|
||||
return None
|
||||
|
||||
try:
|
||||
status = self.get_status(torrent_info.info_hash)
|
||||
if status.state != DownloadStatus.error("").state:
|
||||
return (torrent_info.info_hash, status)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking for existing torrent: {e}")
|
||||
return None
|
||||
|
||||
def _get_download_dir(self) -> str:
|
||||
"""Get the download directory from rTorrent config."""
|
||||
try:
|
||||
download_dir = self._rpc.directory.default()
|
||||
return download_dir
|
||||
except Exception:
|
||||
return "/downloads"
|
||||
|
||||
def _get_torrent_path(self, download_id: str) -> Optional[str]:
|
||||
"""Get the file path of a torrent by hash.
|
||||
|
||||
Uses `d.base_path` for the item output path. In the xmlrpc interface
|
||||
this corresponds to `d.get_base_path()`.
|
||||
"""
|
||||
try:
|
||||
base_path = self._rpc.d.get_base_path(download_id)
|
||||
return base_path if base_path else None
|
||||
except Exception:
|
||||
return None
|
||||
@@ -5,15 +5,18 @@ Uses SABnzbd's REST API directly via requests (no external dependency).
|
||||
"""
|
||||
|
||||
from typing import Any, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
register_client,
|
||||
with_retry,
|
||||
)
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -66,32 +69,62 @@ def _parse_speed(slot: dict) -> Optional[int]:
|
||||
class SABnzbdClient(DownloadClient):
|
||||
"""SABnzbd download client using REST API."""
|
||||
|
||||
@staticmethod
|
||||
def _resolve_completed_storage_path(storage: str, title: str) -> str:
|
||||
"""Normalize SABnzbd's `storage` into a stable "job root" folder.
|
||||
|
||||
Walks up parent directories looking for a directory named exactly like the
|
||||
job `title`.
|
||||
|
||||
This helps when SABnzbd reports a nested path (e.g. sorting/post-processing)
|
||||
but we want the root folder for the completed job.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
storage = storage or ""
|
||||
title = (title or "").strip()
|
||||
if not storage or not title:
|
||||
return storage
|
||||
|
||||
# SAB returns absolute paths; don't require existence on disk.
|
||||
path = Path(storage)
|
||||
best_match: Path | None = None
|
||||
|
||||
for parent in [path, *path.parents]:
|
||||
if parent.name == title:
|
||||
best_match = parent
|
||||
|
||||
return str(best_match) if best_match is not None else storage
|
||||
|
||||
protocol = "usenet"
|
||||
name = "sabnzbd"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize SABnzbd client with settings from config."""
|
||||
url = config.get("SABNZBD_URL", "")
|
||||
if not url:
|
||||
raw_url = config.get("SABNZBD_URL", "")
|
||||
if not raw_url:
|
||||
raise ValueError("SABNZBD_URL is required")
|
||||
|
||||
api_key = config.get("SABNZBD_API_KEY", "")
|
||||
if not api_key:
|
||||
raise ValueError("SABNZBD_API_KEY is required")
|
||||
|
||||
self.url = url.rstrip("/")
|
||||
self.url = normalize_http_url(raw_url)
|
||||
if not self.url:
|
||||
raise ValueError("SABNZBD_URL is invalid")
|
||||
self.api_key = api_key
|
||||
self._category = config.get("SABNZBD_CATEGORY", "cwabd")
|
||||
self._category = config.get("SABNZBD_CATEGORY", "books")
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
"""Check if SABnzbd is configured and selected as the usenet client."""
|
||||
client = config.get("PROWLARR_USENET_CLIENT", "")
|
||||
url = config.get("SABNZBD_URL", "")
|
||||
url = normalize_http_url(config.get("SABNZBD_URL", ""))
|
||||
api_key = config.get("SABNZBD_API_KEY", "")
|
||||
return client == "sabnzbd" and bool(url) and bool(api_key)
|
||||
|
||||
def _api_call(self, mode: str, params: dict = None) -> Any:
|
||||
@with_retry()
|
||||
def _api_call(self, mode: str, params: Optional[dict] = None) -> Any:
|
||||
"""
|
||||
Make an API call to SABnzbd.
|
||||
|
||||
@@ -103,7 +136,7 @@ class SABnzbdClient(DownloadClient):
|
||||
JSON response from SABnzbd.
|
||||
|
||||
Raises:
|
||||
Exception: If API call fails.
|
||||
Exception: If API call fails after retries.
|
||||
"""
|
||||
api_url = f"{self.url}/api"
|
||||
|
||||
@@ -127,6 +160,100 @@ class SABnzbdClient(DownloadClient):
|
||||
|
||||
return result
|
||||
|
||||
def _api_post_file(self, nzb_content: bytes, filename: str, nzb_name: str, category: str) -> Any:
|
||||
"""
|
||||
Upload an NZB file to SABnzbd using addfile.
|
||||
|
||||
Returns:
|
||||
JSON response from SABnzbd.
|
||||
"""
|
||||
api_url = f"{self.url}/api"
|
||||
request_params = {
|
||||
"apikey": self.api_key,
|
||||
"mode": "addfile",
|
||||
"output": "json",
|
||||
"nzbname": nzb_name,
|
||||
"cat": category,
|
||||
}
|
||||
files = {"name": (filename, nzb_content, "application/x-nzb")}
|
||||
|
||||
response = requests.post(api_url, params=request_params, files=files, timeout=30)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
|
||||
if isinstance(result, dict) and result.get("status") is False:
|
||||
error = result.get("error", "Unknown error")
|
||||
raise Exception(f"SABnzbd error: {error}")
|
||||
|
||||
return result
|
||||
|
||||
def _fetch_nzb_content(self, url: str) -> bytes:
|
||||
"""Fetch NZB content, including Prowlarr auth headers when appropriate."""
|
||||
headers = self._get_prowlarr_headers(url)
|
||||
response = requests.get(url, timeout=30, headers=headers)
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
def _get_prowlarr_headers(self, url: str) -> dict:
|
||||
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
|
||||
if not api_key:
|
||||
return {}
|
||||
|
||||
prowlarr_url = normalize_http_url(config.get("PROWLARR_URL", ""))
|
||||
if not prowlarr_url:
|
||||
return {}
|
||||
|
||||
try:
|
||||
target = urlparse(url)
|
||||
base = urlparse(prowlarr_url)
|
||||
except ValueError:
|
||||
return {}
|
||||
|
||||
if target.hostname and base.hostname and target.hostname.lower() == base.hostname.lower():
|
||||
return {"X-Api-Key": api_key}
|
||||
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _build_nzb_filename(name: str, url: str) -> str:
|
||||
base_name = (name or "download").strip() or "download"
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path or ""
|
||||
lower_path = path.lower()
|
||||
|
||||
if lower_path.endswith(".nzb.gz"):
|
||||
suffix = ".nzb.gz"
|
||||
elif lower_path.endswith(".nzb"):
|
||||
suffix = ".nzb"
|
||||
else:
|
||||
suffix = ""
|
||||
|
||||
lower_base = base_name.lower()
|
||||
if suffix and lower_base.endswith(suffix):
|
||||
return base_name
|
||||
if suffix == ".nzb.gz" and lower_base.endswith(".nzb"):
|
||||
return f"{base_name}.gz"
|
||||
if suffix:
|
||||
return f"{base_name}{suffix}"
|
||||
if lower_base.endswith((".nzb", ".nzb.gz")):
|
||||
return base_name
|
||||
return f"{base_name}.nzb"
|
||||
|
||||
@staticmethod
|
||||
def _extract_nzo_id(result: Any) -> str:
|
||||
if not isinstance(result, dict):
|
||||
raise Exception("SABnzbd returned invalid response")
|
||||
|
||||
nzo_ids = result.get("nzo_ids") or result.get("nzo_id")
|
||||
if isinstance(nzo_ids, list) and nzo_ids:
|
||||
return str(nzo_ids[0])
|
||||
if isinstance(nzo_ids, str) and nzo_ids:
|
||||
return nzo_ids
|
||||
if isinstance(nzo_ids, int):
|
||||
return str(nzo_ids)
|
||||
|
||||
raise Exception("SABnzbd returned no nzo_id")
|
||||
|
||||
def test_connection(self) -> Tuple[bool, str]:
|
||||
"""Test connection to SABnzbd."""
|
||||
try:
|
||||
@@ -140,7 +267,14 @@ class SABnzbdClient(DownloadClient):
|
||||
except Exception as e:
|
||||
return False, f"Connection failed: {str(e)}"
|
||||
|
||||
def add_download(self, url: str, name: str, category: str = None) -> str:
|
||||
def add_download(
|
||||
self,
|
||||
url: str,
|
||||
name: str,
|
||||
category: Optional[str] = None,
|
||||
expected_hash: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""
|
||||
Add NZB by URL.
|
||||
|
||||
@@ -148,6 +282,7 @@ class SABnzbdClient(DownloadClient):
|
||||
url: NZB URL (can be Prowlarr proxy URL)
|
||||
name: Display name for the download
|
||||
category: Category for organization (uses configured default if not specified)
|
||||
expected_hash: Optional info_hash hint (unused)
|
||||
|
||||
Returns:
|
||||
SABnzbd nzo_id.
|
||||
@@ -160,7 +295,16 @@ class SABnzbdClient(DownloadClient):
|
||||
|
||||
try:
|
||||
logger.debug(f"Adding NZB to SABnzbd: {name}")
|
||||
nzb_filename = self._build_nzb_filename(name, url)
|
||||
nzb_content = self._fetch_nzb_content(url)
|
||||
result = self._api_post_file(nzb_content, nzb_filename, name, category)
|
||||
nzo_id = self._extract_nzo_id(result)
|
||||
logger.info(f"Added NZB to SABnzbd: {nzo_id}")
|
||||
return nzo_id
|
||||
except Exception as e:
|
||||
logger.warning(f"SABnzbd addfile failed, falling back to addurl: {e}")
|
||||
|
||||
try:
|
||||
result = self._api_call(
|
||||
"addurl",
|
||||
{
|
||||
@@ -169,15 +313,9 @@ class SABnzbdClient(DownloadClient):
|
||||
"cat": category,
|
||||
},
|
||||
)
|
||||
|
||||
# SABnzbd returns {"status": True, "nzo_ids": ["SABnzbd_nzo_xxx"]}
|
||||
nzo_ids = result.get("nzo_ids", [])
|
||||
if nzo_ids:
|
||||
nzo_id = nzo_ids[0]
|
||||
logger.info(f"Added NZB to SABnzbd: {nzo_id}")
|
||||
return nzo_id
|
||||
|
||||
raise Exception("SABnzbd returned no nzo_id")
|
||||
nzo_id = self._extract_nzo_id(result)
|
||||
logger.info(f"Added NZB to SABnzbd via addurl: {nzo_id}")
|
||||
return nzo_id
|
||||
except Exception as e:
|
||||
logger.error(f"SABnzbd add failed: {e}")
|
||||
raise
|
||||
@@ -241,24 +379,31 @@ class SABnzbdClient(DownloadClient):
|
||||
if slot.get("nzo_id") == download_id:
|
||||
status_text = slot.get("status", "").upper()
|
||||
storage = slot.get("storage", "")
|
||||
if storage is None:
|
||||
storage = ""
|
||||
logger.debug(f"SABnzbd history: {download_id} status={status_text} storage='{storage}'")
|
||||
|
||||
if status_text == "COMPLETED":
|
||||
title = slot.get("name") or slot.get("nzb_name") or ""
|
||||
resolved_storage = self._resolve_completed_storage_path(storage, title)
|
||||
|
||||
return DownloadStatus(
|
||||
progress=100,
|
||||
state="complete",
|
||||
message="Complete",
|
||||
complete=True,
|
||||
file_path=storage,
|
||||
file_path=resolved_storage,
|
||||
)
|
||||
elif status_text == "FAILED":
|
||||
fail_message = slot.get("fail_message", "Download failed")
|
||||
title = slot.get("name") or slot.get("nzb_name") or ""
|
||||
resolved_storage = self._resolve_completed_storage_path(storage, title)
|
||||
return DownloadStatus(
|
||||
progress=100,
|
||||
state="error",
|
||||
message=fail_message,
|
||||
complete=True,
|
||||
file_path=None,
|
||||
file_path=resolved_storage,
|
||||
)
|
||||
else:
|
||||
# Post-processing states: Queued, QuickCheck, Verifying,
|
||||
@@ -276,9 +421,7 @@ class SABnzbdClient(DownloadClient):
|
||||
logger.warning(f"SABnzbd: download {download_id} not found in queue or history")
|
||||
return DownloadStatus.error("Download not found")
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"SABnzbd get_status failed ({error_type}): {e}")
|
||||
return DownloadStatus.error(f"{error_type}: {e}")
|
||||
return DownloadStatus.error(self._log_error("get_status", e))
|
||||
|
||||
def remove(self, download_id: str, delete_files: bool = False, archive: bool = True) -> bool:
|
||||
"""
|
||||
@@ -292,8 +435,9 @@ class SABnzbdClient(DownloadClient):
|
||||
Returns:
|
||||
True if successful.
|
||||
"""
|
||||
# First try to remove from queue. If it isn't there (common for completed jobs),
|
||||
# fall back to history removal instead of failing fast on a SABnzbd error response.
|
||||
try:
|
||||
# First try to remove from queue
|
||||
result = self._api_call(
|
||||
"queue",
|
||||
{
|
||||
@@ -306,8 +450,11 @@ class SABnzbdClient(DownloadClient):
|
||||
if result.get("status"):
|
||||
logger.info(f"Removed NZB from SABnzbd queue: {download_id}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"SABnzbd queue delete skipped for {download_id}: {e}")
|
||||
|
||||
# If not in queue, try to remove from history
|
||||
# If not in queue (or queue delete failed), try to remove from history.
|
||||
try:
|
||||
result = self._api_call(
|
||||
"history",
|
||||
{
|
||||
@@ -322,13 +469,12 @@ class SABnzbdClient(DownloadClient):
|
||||
action = "archived" if archive else "removed"
|
||||
logger.info(f"NZB {action} from SABnzbd history: {download_id}")
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"SABnzbd remove failed ({error_type}): {e}")
|
||||
self._log_error("remove", e)
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def get_download_path(self, download_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get the path where NZB files are located.
|
||||
@@ -342,7 +488,9 @@ class SABnzbdClient(DownloadClient):
|
||||
status = self.get_status(download_id)
|
||||
return status.file_path
|
||||
|
||||
def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
def find_existing(
|
||||
self, url: str, category: Optional[str] = None
|
||||
) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
"""
|
||||
Check if an NZB for this URL already exists in SABnzbd.
|
||||
|
||||
@@ -353,6 +501,7 @@ class SABnzbdClient(DownloadClient):
|
||||
|
||||
Args:
|
||||
url: NZB URL
|
||||
category: Category to filter by (defaults to configured category)
|
||||
|
||||
Returns:
|
||||
Tuple of (nzo_id, status) if found, None if not found.
|
||||
@@ -378,10 +527,15 @@ class SABnzbdClient(DownloadClient):
|
||||
if not filename:
|
||||
return None
|
||||
|
||||
# Search queue
|
||||
# Use provided category or fall back to configured default
|
||||
search_category = category or self._category
|
||||
|
||||
# Search queue (SABnzbd uses "cat" field for category in queue)
|
||||
queue_result = self._api_call("queue")
|
||||
queue = queue_result.get("queue", {})
|
||||
for slot in queue.get("slots", []):
|
||||
if slot.get("cat", "") != search_category:
|
||||
continue
|
||||
slot_name = slot.get("filename", "")
|
||||
if filename.lower() in slot_name.lower():
|
||||
nzo_id = slot.get("nzo_id")
|
||||
@@ -390,10 +544,12 @@ class SABnzbdClient(DownloadClient):
|
||||
logger.debug(f"Found existing NZB in SABnzbd queue: {nzo_id}")
|
||||
return (nzo_id, status)
|
||||
|
||||
# Search history
|
||||
# Search history (SABnzbd uses "category" field in history)
|
||||
history_result = self._api_call("history", {"limit": 100})
|
||||
history = history_result.get("history", {})
|
||||
for slot in history.get("slots", []):
|
||||
if slot.get("category", "") != search_category:
|
||||
continue
|
||||
slot_name = slot.get("name", "")
|
||||
if filename.lower() in slot_name.lower():
|
||||
nzo_id = slot.get("nzo_id")
|
||||
|
||||
@@ -5,10 +5,11 @@ import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from urllib.parse import parse_qs, urljoin, urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -30,39 +31,78 @@ class TorrentInfo:
|
||||
magnet_url: Optional[str] = None
|
||||
"""The actual magnet URL, if available."""
|
||||
|
||||
def with_info_hash(self, info_hash: Optional[str]) -> "TorrentInfo":
|
||||
"""Return a copy with the info_hash replaced when provided."""
|
||||
if info_hash:
|
||||
return TorrentInfo(
|
||||
info_hash=info_hash,
|
||||
torrent_data=self.torrent_data,
|
||||
is_magnet=self.is_magnet,
|
||||
magnet_url=self.magnet_url,
|
||||
)
|
||||
return self
|
||||
|
||||
def extract_torrent_info(url: str, fetch_torrent: bool = True) -> TorrentInfo:
|
||||
"""Extract info_hash from magnet link or .torrent URL."""
|
||||
|
||||
def extract_torrent_info(
|
||||
url: str,
|
||||
fetch_torrent: bool = True,
|
||||
expected_hash: Optional[str] = None,
|
||||
) -> TorrentInfo:
|
||||
"""Extract info_hash from magnet link or .torrent URL.
|
||||
|
||||
Notes:
|
||||
When the URL points at Prowlarr's proxied download endpoint, it typically
|
||||
requires the `X-Api-Key` header. If `PROWLARR_API_KEY` is configured,
|
||||
include it for the torrent fetch request.
|
||||
|
||||
Redirects to magnet links are handled explicitly so we can extract a
|
||||
hash from the magnet when available.
|
||||
"""
|
||||
is_magnet = url.startswith("magnet:")
|
||||
|
||||
# Try to extract hash from magnet URL
|
||||
if is_magnet:
|
||||
info_hash = extract_hash_from_magnet(url)
|
||||
if not info_hash and expected_hash:
|
||||
info_hash = expected_hash
|
||||
return TorrentInfo(info_hash=info_hash, torrent_data=None, is_magnet=True, magnet_url=url)
|
||||
|
||||
# Not a magnet - try to fetch and parse the .torrent file
|
||||
if not fetch_torrent:
|
||||
return TorrentInfo(info_hash=None, torrent_data=None, is_magnet=False)
|
||||
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
|
||||
|
||||
headers: dict[str, str] = {"Accept": "application/x-bittorrent"}
|
||||
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
|
||||
if api_key:
|
||||
headers["X-Api-Key"] = api_key
|
||||
|
||||
def resolve_url(current: str, location: str) -> str:
|
||||
if not location:
|
||||
return current
|
||||
# Support relative redirect locations
|
||||
return urljoin(current, location)
|
||||
|
||||
try:
|
||||
logger.debug(f"Fetching torrent file from: {url[:80]}...")
|
||||
|
||||
# Use allow_redirects=False to handle magnet link redirects manually
|
||||
# Some indexers redirect download URLs to magnet links
|
||||
resp = requests.get(url, timeout=30, allow_redirects=False)
|
||||
resp = requests.get(url, timeout=30, allow_redirects=False, headers=headers)
|
||||
|
||||
# Check if this is a redirect to a magnet link
|
||||
if resp.status_code in (301, 302, 303, 307, 308):
|
||||
redirect_url = resp.headers.get("Location", "")
|
||||
redirect_url = resolve_url(url, resp.headers.get("Location", ""))
|
||||
if redirect_url.startswith("magnet:"):
|
||||
logger.debug(f"Download URL redirected to magnet link")
|
||||
logger.debug("Download URL redirected to magnet link")
|
||||
info_hash = extract_hash_from_magnet(redirect_url)
|
||||
if not info_hash and expected_hash:
|
||||
info_hash = expected_hash
|
||||
return TorrentInfo(
|
||||
info_hash=info_hash, torrent_data=None, is_magnet=True, magnet_url=redirect_url
|
||||
)
|
||||
# Not a magnet redirect, follow it manually
|
||||
logger.debug(f"Following redirect to: {redirect_url[:80]}...")
|
||||
resp = requests.get(redirect_url, timeout=30)
|
||||
resp = requests.get(redirect_url, timeout=30, headers=headers)
|
||||
|
||||
resp.raise_for_status()
|
||||
torrent_data = resp.content
|
||||
@@ -75,13 +115,15 @@ def extract_torrent_info(url: str, fetch_torrent: bool = True) -> TorrentInfo:
|
||||
if text_content.startswith("magnet:"):
|
||||
logger.debug("Download URL returned magnet link as response body")
|
||||
info_hash = extract_hash_from_magnet(text_content)
|
||||
if not info_hash and expected_hash:
|
||||
info_hash = expected_hash
|
||||
return TorrentInfo(
|
||||
info_hash=info_hash, torrent_data=None, is_magnet=True, magnet_url=text_content
|
||||
)
|
||||
except Exception:
|
||||
pass # Not text, continue with torrent parsing
|
||||
|
||||
info_hash = extract_info_hash_from_torrent(torrent_data)
|
||||
info_hash = extract_info_hash_from_torrent(torrent_data) or expected_hash
|
||||
if info_hash:
|
||||
logger.debug(f"Extracted hash from torrent file: {info_hash}")
|
||||
else:
|
||||
@@ -89,7 +131,7 @@ def extract_torrent_info(url: str, fetch_torrent: bool = True) -> TorrentInfo:
|
||||
return TorrentInfo(info_hash=info_hash, torrent_data=torrent_data, is_magnet=False)
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not fetch torrent file: {e}")
|
||||
return TorrentInfo(info_hash=None, torrent_data=None, is_magnet=False)
|
||||
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
|
||||
|
||||
|
||||
def parse_transmission_url(url: str) -> Tuple[str, int, str]:
|
||||
@@ -181,7 +223,10 @@ def extract_info_hash_from_torrent(torrent_data: bytes) -> Optional[str]:
|
||||
return None
|
||||
|
||||
info_bencoded = bencode_encode(decoded[b'info'])
|
||||
return hashlib.sha1(info_bencoded).hexdigest().lower()
|
||||
info_dict = decoded[b'info']
|
||||
if isinstance(info_dict, dict) and b'pieces' in info_dict:
|
||||
return hashlib.sha1(info_bencoded).hexdigest().lower()
|
||||
return hashlib.sha256(info_bencoded).hexdigest().lower()
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse torrent file: {e}")
|
||||
return None
|
||||
@@ -195,7 +240,42 @@ def extract_hash_from_magnet(magnet_url: str) -> Optional[str]:
|
||||
parsed = urlparse(magnet_url)
|
||||
params = parse_qs(parsed.query)
|
||||
|
||||
for xt in params.get("xt", []):
|
||||
def extract_btmh(value: str) -> Optional[str]:
|
||||
raw_value = value.strip()
|
||||
if not raw_value:
|
||||
return None
|
||||
|
||||
data: Optional[bytes] = None
|
||||
if re.fullmatch(r"[a-fA-F0-9]+", raw_value):
|
||||
if len(raw_value) % 2 != 0:
|
||||
return None
|
||||
try:
|
||||
data = bytes.fromhex(raw_value)
|
||||
except ValueError:
|
||||
return None
|
||||
else:
|
||||
padded = raw_value.upper() + "=" * (-len(raw_value) % 8)
|
||||
try:
|
||||
data = base64.b32decode(padded, casefold=True)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if not data:
|
||||
return None
|
||||
|
||||
if len(data) >= 34 and data[0] == 0x12 and data[1] == 0x20:
|
||||
digest = data[2:34]
|
||||
if len(digest) == 32:
|
||||
return digest.hex().lower()
|
||||
|
||||
if len(data) == 32:
|
||||
return data.hex().lower()
|
||||
|
||||
return None
|
||||
|
||||
xt_values = params.get("xt", [])
|
||||
|
||||
for xt in xt_values:
|
||||
# Format: urn:btih:<hash> (32 or 40 chars)
|
||||
match = re.match(r"urn:btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})", xt)
|
||||
if match:
|
||||
@@ -215,4 +295,11 @@ def extract_hash_from_magnet(magnet_url: str) -> Optional[str]:
|
||||
# Fallback: return as-is
|
||||
return hash_value.lower()
|
||||
|
||||
for xt in xt_values:
|
||||
if xt.startswith("urn:btmh:"):
|
||||
btmh_value = xt[len("urn:btmh:"):]
|
||||
btmh_hash = extract_btmh(btmh_value)
|
||||
if btmh_hash:
|
||||
return btmh_hash
|
||||
|
||||
return None
|
||||
|
||||
@@ -6,8 +6,10 @@ Uses the transmission-rpc library to communicate with Transmission's RPC API.
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
@@ -32,10 +34,14 @@ class TransmissionClient(DownloadClient):
|
||||
"""Initialize Transmission client with settings from config."""
|
||||
from transmission_rpc import Client
|
||||
|
||||
url = config.get("TRANSMISSION_URL", "")
|
||||
if not url:
|
||||
raw_url = config.get("TRANSMISSION_URL", "")
|
||||
if not raw_url:
|
||||
raise ValueError("TRANSMISSION_URL is required")
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
raise ValueError("TRANSMISSION_URL is invalid")
|
||||
|
||||
username = config.get("TRANSMISSION_USERNAME", "")
|
||||
password = config.get("TRANSMISSION_PASSWORD", "")
|
||||
|
||||
@@ -49,13 +55,13 @@ class TransmissionClient(DownloadClient):
|
||||
username=username if username else None,
|
||||
password=password if password else None,
|
||||
)
|
||||
self._category = config.get("TRANSMISSION_CATEGORY", "cwabd")
|
||||
self._category = config.get("TRANSMISSION_CATEGORY", "books")
|
||||
|
||||
@staticmethod
|
||||
def is_configured() -> bool:
|
||||
"""Check if Transmission is configured and selected as the torrent client."""
|
||||
client = config.get("PROWLARR_TORRENT_CLIENT", "")
|
||||
url = config.get("TRANSMISSION_URL", "")
|
||||
url = normalize_http_url(config.get("TRANSMISSION_URL", ""))
|
||||
return client == "transmission" and bool(url)
|
||||
|
||||
def test_connection(self) -> Tuple[bool, str]:
|
||||
@@ -67,7 +73,14 @@ class TransmissionClient(DownloadClient):
|
||||
except Exception as e:
|
||||
return False, f"Connection failed: {str(e)}"
|
||||
|
||||
def add_download(self, url: str, name: str, category: str = None) -> str:
|
||||
def add_download(
|
||||
self,
|
||||
url: str,
|
||||
name: str,
|
||||
category: Optional[str] = None,
|
||||
expected_hash: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""
|
||||
Add torrent by URL (magnet or .torrent).
|
||||
|
||||
@@ -75,6 +88,7 @@ class TransmissionClient(DownloadClient):
|
||||
url: Magnet link or .torrent URL
|
||||
name: Display name for the torrent
|
||||
category: Category for organization (uses configured default if not specified)
|
||||
expected_hash: Optional info_hash hint (from Prowlarr)
|
||||
|
||||
Returns:
|
||||
Torrent hash (info_hash).
|
||||
@@ -83,21 +97,21 @@ class TransmissionClient(DownloadClient):
|
||||
Exception: If adding fails.
|
||||
"""
|
||||
try:
|
||||
category = category or self._category
|
||||
resolved_category = category or self._category or ""
|
||||
|
||||
torrent_info = extract_torrent_info(url)
|
||||
torrent_info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
|
||||
if torrent_info.torrent_data:
|
||||
torrent = self._client.add_torrent(
|
||||
torrent=torrent_info.torrent_data,
|
||||
labels=[category],
|
||||
labels=[resolved_category] if resolved_category else None,
|
||||
)
|
||||
else:
|
||||
# Use magnet URL if available, otherwise original URL
|
||||
add_url = torrent_info.magnet_url or url
|
||||
torrent = self._client.add_torrent(
|
||||
torrent=add_url,
|
||||
labels=[category],
|
||||
labels=[resolved_category] if resolved_category else None,
|
||||
)
|
||||
|
||||
torrent_hash = torrent.hashString.lower()
|
||||
@@ -163,9 +177,14 @@ class TransmissionClient(DownloadClient):
|
||||
# Get file path for completed downloads
|
||||
file_path = None
|
||||
if complete:
|
||||
download_dir = torrent.download_dir
|
||||
name = torrent.name
|
||||
file_path = f"{download_dir}/{name}"
|
||||
# Output path is downloadDir + torrent name (with ':' replaced)
|
||||
torrent_name = getattr(torrent, 'name', '')
|
||||
if isinstance(torrent_name, str):
|
||||
torrent_name = torrent_name.replace(':', '_')
|
||||
file_path = self._build_path(
|
||||
getattr(torrent, 'download_dir', ''),
|
||||
torrent_name,
|
||||
)
|
||||
|
||||
return DownloadStatus(
|
||||
progress=progress,
|
||||
@@ -180,9 +199,7 @@ class TransmissionClient(DownloadClient):
|
||||
except KeyError:
|
||||
return DownloadStatus.error("Torrent not found")
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"Transmission get_status failed ({error_type}): {e}")
|
||||
return DownloadStatus.error(f"{error_type}: {e}")
|
||||
return DownloadStatus.error(self._log_error("get_status", e))
|
||||
|
||||
def remove(self, download_id: str, delete_files: bool = False) -> bool:
|
||||
"""
|
||||
@@ -206,8 +223,7 @@ class TransmissionClient(DownloadClient):
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.error(f"Transmission remove failed ({error_type}): {e}")
|
||||
self._log_error("remove", e)
|
||||
return False
|
||||
|
||||
def get_download_path(self, download_id: str) -> Optional[str]:
|
||||
@@ -221,16 +237,21 @@ class TransmissionClient(DownloadClient):
|
||||
Content path (file or directory), or None.
|
||||
"""
|
||||
try:
|
||||
torrent = self._client.get_torrent(download_id)
|
||||
download_dir = torrent.download_dir
|
||||
name = torrent.name
|
||||
return f"{download_dir}/{name}"
|
||||
torrent = self._client.get_torrent(download_id)
|
||||
torrent_name = getattr(torrent, 'name', '')
|
||||
if isinstance(torrent_name, str):
|
||||
torrent_name = torrent_name.replace(':', '_')
|
||||
return self._build_path(
|
||||
getattr(torrent, 'download_dir', ''),
|
||||
torrent_name,
|
||||
)
|
||||
except Exception as e:
|
||||
error_type = type(e).__name__
|
||||
logger.debug(f"Transmission get_download_path failed ({error_type}): {e}")
|
||||
self._log_error("get_download_path", e, level="debug")
|
||||
return None
|
||||
|
||||
def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
def find_existing(
|
||||
self, url: str, category: Optional[str] = None
|
||||
) -> Optional[Tuple[str, DownloadStatus]]:
|
||||
"""Check if a torrent for this URL already exists in Transmission."""
|
||||
try:
|
||||
torrent_info = extract_torrent_info(url)
|
||||
|
||||
@@ -12,11 +12,12 @@ from shelfmark.core.utils import is_audiobook
|
||||
from shelfmark.release_sources import DownloadHandler, register_handler
|
||||
from shelfmark.release_sources.prowlarr.cache import get_release, remove_release
|
||||
from shelfmark.release_sources.prowlarr.clients import (
|
||||
DownloadClient,
|
||||
DownloadState,
|
||||
get_client,
|
||||
list_configured_clients,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.utils import get_protocol, get_unique_path
|
||||
from shelfmark.release_sources.prowlarr.utils import get_preferred_download_url, get_protocol
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -24,10 +25,49 @@ logger = setup_logger(__name__)
|
||||
POLL_INTERVAL = 2
|
||||
|
||||
|
||||
def _diagnose_path_issue(path: str) -> str:
|
||||
"""
|
||||
Analyze a path and return diagnostic hints for common issues.
|
||||
|
||||
Args:
|
||||
path: The path that failed to be accessed
|
||||
|
||||
Returns:
|
||||
A hint string to help users diagnose the issue.
|
||||
"""
|
||||
# Detect Windows-style paths (won't work in Linux containers)
|
||||
if len(path) >= 2 and path[1] == ':':
|
||||
return (
|
||||
f"Path '{path}' appears to be a Windows path. "
|
||||
f"Shelfmark runs in Linux and cannot access Windows paths directly. "
|
||||
f"Ensure your download client uses Linux-style paths (/path/to/files)."
|
||||
)
|
||||
|
||||
# Detect backslashes (Windows path separators)
|
||||
if '\\' in path:
|
||||
return (
|
||||
f"Path '{path}' contains backslashes. "
|
||||
f"This may indicate a Windows path or incorrect path escaping. "
|
||||
f"Linux paths should use forward slashes (/)."
|
||||
)
|
||||
|
||||
# Generic hint for Linux paths
|
||||
return (
|
||||
f"Path '{path}' is not accessible from Shelfmark's container. "
|
||||
f"Ensure both containers have matching volume mounts for this directory, "
|
||||
f"or configure Remote Path Mappings in Settings > Advanced."
|
||||
)
|
||||
|
||||
|
||||
@register_handler("prowlarr")
|
||||
class ProwlarrHandler(DownloadHandler):
|
||||
"""Handler for Prowlarr downloads via configured torrent or usenet client."""
|
||||
|
||||
def __init__(self):
|
||||
# Track downloads that may need client-side cleanup after Shelfmark completes import.
|
||||
# task_id -> (client, download_id, protocol)
|
||||
self._cleanup_refs: dict[str, tuple[DownloadClient, str, str]] = {}
|
||||
|
||||
def _get_category_for_task(self, client, task: DownloadTask) -> Optional[str]:
|
||||
"""Get audiobook category if configured and applicable, else None for default."""
|
||||
if not is_audiobook(task.content_type):
|
||||
@@ -44,13 +84,116 @@ class ProwlarrHandler(DownloadHandler):
|
||||
audiobook_key = audiobook_keys.get(client.name)
|
||||
return config.get(audiobook_key, "") or None if audiobook_key else None
|
||||
|
||||
def _cleanup_client_history(self, client, download_id: str) -> None:
|
||||
"""Remove completed download from client history if configured."""
|
||||
if client.name == "sabnzbd" and config.get("SABNZBD_REMOVE_COMPLETED", True):
|
||||
try:
|
||||
client.remove(download_id, delete_files=True, archive=True)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to remove from SABnzbd history: {e}")
|
||||
def post_process_cleanup(self, task: DownloadTask, success: bool) -> None:
|
||||
if not success:
|
||||
self._cleanup_refs.pop(task.task_id, None)
|
||||
return
|
||||
|
||||
client_ref = self._cleanup_refs.pop(task.task_id, None)
|
||||
if client_ref is None:
|
||||
return
|
||||
|
||||
client, download_id, protocol = client_ref
|
||||
if protocol != "usenet":
|
||||
return
|
||||
|
||||
# "Move" means copy into ingest then let the usenet client delete its own files.
|
||||
if config.get("PROWLARR_USENET_ACTION", "move") != "move":
|
||||
return
|
||||
|
||||
try:
|
||||
self._delete_local_download_data(client, download_id)
|
||||
self._remove_usenet_download(client, download_id, delete_files=True, archive=True)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to cleanup usenet download {download_id} in {getattr(client, 'name', 'client')}: {e}")
|
||||
|
||||
def _remove_usenet_download(
|
||||
self,
|
||||
client: DownloadClient,
|
||||
download_id: str,
|
||||
*,
|
||||
delete_files: bool,
|
||||
archive: bool = True,
|
||||
) -> None:
|
||||
"""Remove a usenet download with SABnzbd-specific archive handling."""
|
||||
if getattr(client, "name", "") == "sabnzbd":
|
||||
client.remove(download_id, delete_files=delete_files, archive=archive)
|
||||
else:
|
||||
client.remove(download_id, delete_files=delete_files)
|
||||
|
||||
def _delete_local_download_data(self, client: DownloadClient, download_id: str) -> None:
|
||||
"""Best-effort local deletion of client download data."""
|
||||
try:
|
||||
raw_path = client.get_download_path(download_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to resolve download path for {client.name} {download_id}: {e}")
|
||||
return
|
||||
|
||||
if not raw_path:
|
||||
logger.debug(f"No download path available for {client.name} {download_id}")
|
||||
return
|
||||
|
||||
from shelfmark.core.path_mappings import (
|
||||
get_client_host_identifier,
|
||||
parse_remote_path_mappings,
|
||||
remap_remote_to_local_with_match,
|
||||
)
|
||||
|
||||
source_path_obj = Path(raw_path)
|
||||
host = get_client_host_identifier(client) or ""
|
||||
mapping_value = config.get("PROWLARR_REMOTE_PATH_MAPPINGS", [])
|
||||
mappings = parse_remote_path_mappings(mapping_value)
|
||||
remapped, matched_mapping = remap_remote_to_local_with_match(
|
||||
mappings=mappings,
|
||||
host=host,
|
||||
remote_path=source_path_obj,
|
||||
)
|
||||
|
||||
delete_path = remapped if matched_mapping else source_path_obj
|
||||
|
||||
if str(delete_path) in ("", "/"):
|
||||
logger.warning(f"Refusing to delete unsafe path for {client.name} {download_id}: {delete_path}")
|
||||
return
|
||||
|
||||
if not delete_path.exists():
|
||||
logger.debug(f"Local download path does not exist for cleanup: {delete_path}")
|
||||
return
|
||||
|
||||
try:
|
||||
if delete_path.is_dir():
|
||||
shutil.rmtree(delete_path)
|
||||
else:
|
||||
delete_path.unlink()
|
||||
logger.info(f"Deleted local download data for {client.name} {download_id}: {delete_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete local download data for {client.name} {download_id}: {e}")
|
||||
|
||||
def _safe_remove_download(self, client, download_id: str, protocol: str, reason: str) -> None:
|
||||
"""Best-effort removal of a failed/cancelled download from the client.
|
||||
|
||||
Safety policy:
|
||||
- torrents: never remove or delete client data (avoid breaking seeding)
|
||||
- usenet: keep legacy behavior (delete client files on removal)
|
||||
"""
|
||||
|
||||
if protocol != "usenet":
|
||||
logger.info(
|
||||
"Skipping download client cleanup for protocol=%s after %s (client=%s id=%s)",
|
||||
protocol,
|
||||
reason,
|
||||
getattr(client, "name", "client"),
|
||||
download_id,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
# Permanent delete for failed usenet downloads (SABnzbd archive=0).
|
||||
self._delete_local_download_data(client, download_id)
|
||||
self._remove_usenet_download(client, download_id, delete_files=True, archive=False)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to remove download {download_id} from {client.name} after {reason}: {e}"
|
||||
)
|
||||
|
||||
def _build_progress_message(self, status) -> str:
|
||||
"""Build a progress message from download status."""
|
||||
@@ -87,7 +230,7 @@ class ProwlarrHandler(DownloadHandler):
|
||||
return None
|
||||
|
||||
# Extract download URL
|
||||
download_url = prowlarr_result.get("downloadUrl") or prowlarr_result.get("magnetUrl")
|
||||
download_url = get_preferred_download_url(prowlarr_result)
|
||||
if not download_url:
|
||||
status_callback("error", "No download URL available")
|
||||
return None
|
||||
@@ -110,7 +253,8 @@ class ProwlarrHandler(DownloadHandler):
|
||||
|
||||
# Check if this download already exists in the client
|
||||
status_callback("resolving", f"Checking {client.name}")
|
||||
existing = client.find_existing(download_url)
|
||||
category = self._get_category_for_task(client, task)
|
||||
existing = client.find_existing(download_url, category=category)
|
||||
|
||||
if existing:
|
||||
download_id, existing_status = existing
|
||||
@@ -118,16 +262,91 @@ class ProwlarrHandler(DownloadHandler):
|
||||
|
||||
# If already complete, skip straight to file handling
|
||||
if existing_status.complete:
|
||||
logger.info(f"Existing download is complete, copying file directly")
|
||||
logger.info("Existing download is complete, copying file directly")
|
||||
status_callback("resolving", "Found existing download, copying to library")
|
||||
|
||||
source_path = client.get_download_path(download_id)
|
||||
if not source_path:
|
||||
status_callback("error", "Could not locate existing download file")
|
||||
logger.error(
|
||||
f"Could not get path for existing download. "
|
||||
f"Client: {client.name}, ID: {download_id}. "
|
||||
f"The download may have been moved or deleted."
|
||||
)
|
||||
status_callback(
|
||||
"error",
|
||||
f"Could not locate existing download in {client.name}. "
|
||||
f"Check that the file still exists."
|
||||
)
|
||||
return None
|
||||
|
||||
from shelfmark.core.path_mappings import (
|
||||
get_client_host_identifier,
|
||||
parse_remote_path_mappings,
|
||||
remap_remote_to_local_with_match,
|
||||
)
|
||||
|
||||
source_path_obj = Path(source_path)
|
||||
host = get_client_host_identifier(client) or ""
|
||||
mapping_value = config.get("PROWLARR_REMOTE_PATH_MAPPINGS", [])
|
||||
mappings = parse_remote_path_mappings(mapping_value)
|
||||
remapped, matched_mapping = remap_remote_to_local_with_match(
|
||||
mappings=mappings,
|
||||
host=host,
|
||||
remote_path=source_path_obj,
|
||||
)
|
||||
|
||||
if matched_mapping:
|
||||
if remapped.exists():
|
||||
logger.info(
|
||||
"Remapped existing download path for %s (%s): %s -> %s",
|
||||
client.name,
|
||||
download_id,
|
||||
source_path_obj,
|
||||
remapped,
|
||||
)
|
||||
source_path_obj = remapped
|
||||
else:
|
||||
logger.error(
|
||||
f"Download path does not exist after remapping: {source_path} -> {remapped}. "
|
||||
f"Client: {client.name}, ID: {download_id}. "
|
||||
f"Check that the local path in your mapping is mounted correctly."
|
||||
)
|
||||
status_callback(
|
||||
"error",
|
||||
f"Remapped path '{remapped}' does not exist. "
|
||||
f"Check your Docker volume mounts match the Local Path in Settings > Advanced > Remote Path Mappings.",
|
||||
)
|
||||
return None
|
||||
elif mappings:
|
||||
if source_path_obj.exists():
|
||||
logger.info(
|
||||
"No remote path mapping matched for %s (%s); using client path: %s",
|
||||
client.name,
|
||||
download_id,
|
||||
source_path_obj,
|
||||
)
|
||||
else:
|
||||
hint = _diagnose_path_issue(source_path)
|
||||
logger.error(
|
||||
f"Download path does not exist and no remote path mapping matched for {client.name} "
|
||||
f"({download_id}): {source_path}. {hint}"
|
||||
)
|
||||
status_callback(
|
||||
"error",
|
||||
f"{hint} No remote path mapping matched for client '{client.name}'.",
|
||||
)
|
||||
return None
|
||||
elif not source_path_obj.exists():
|
||||
hint = _diagnose_path_issue(source_path)
|
||||
logger.error(
|
||||
f"Download path does not exist: {source_path}. "
|
||||
f"Client: {client.name}, ID: {download_id}. {hint}"
|
||||
)
|
||||
status_callback("error", hint)
|
||||
return None
|
||||
|
||||
result = self._handle_completed_file(
|
||||
source_path=Path(source_path),
|
||||
source_path=source_path_obj,
|
||||
protocol=protocol,
|
||||
task=task,
|
||||
status_callback=status_callback,
|
||||
@@ -135,7 +354,7 @@ class ProwlarrHandler(DownloadHandler):
|
||||
|
||||
if result:
|
||||
remove_release(task.task_id)
|
||||
self._cleanup_client_history(client, download_id)
|
||||
self._cleanup_refs[task.task_id] = (client, download_id, protocol)
|
||||
return result
|
||||
|
||||
# Existing but still downloading - join the progress polling
|
||||
@@ -147,10 +366,12 @@ class ProwlarrHandler(DownloadHandler):
|
||||
try:
|
||||
release_name = prowlarr_result.get("title") or task.title or "Unknown"
|
||||
category = self._get_category_for_task(client, task)
|
||||
expected_hash = str(prowlarr_result.get("infoHash") or "").strip() or None
|
||||
download_id = client.add_download(
|
||||
url=download_url,
|
||||
name=release_name,
|
||||
category=category,
|
||||
expected_hash=expected_hash,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to add to {client.name}: {e}")
|
||||
@@ -186,6 +407,10 @@ class ProwlarrHandler(DownloadHandler):
|
||||
status_callback: Callable[[str, Optional[str]], None],
|
||||
) -> Optional[str]:
|
||||
"""Poll the download client for progress and handle completion."""
|
||||
# Track consecutive "not found" errors - torrents may take time to appear in client
|
||||
not_found_count = 0
|
||||
max_not_found_retries = 15 # 15 retries * 2s poll = 30s grace period
|
||||
|
||||
try:
|
||||
logger.debug(f"Starting poll for {download_id} (content_type={task.content_type})")
|
||||
while not cancel_flag.is_set():
|
||||
@@ -197,6 +422,7 @@ class ProwlarrHandler(DownloadHandler):
|
||||
if status.state == DownloadState.ERROR:
|
||||
logger.error(f"Download {download_id} completed with error: {status.message}")
|
||||
status_callback("error", status.message or "Download failed")
|
||||
self._safe_remove_download(client, download_id, protocol, "completion error")
|
||||
return None
|
||||
# Download complete - break to handle file
|
||||
logger.debug(f"Download {download_id} complete, file_path={status.file_path}")
|
||||
@@ -204,11 +430,57 @@ class ProwlarrHandler(DownloadHandler):
|
||||
|
||||
# Check for error state
|
||||
if status.state == DownloadState.ERROR:
|
||||
logger.error(f"Download {download_id} error state: {status.message}")
|
||||
message = (status.message or "").strip()
|
||||
message_lower = message.lower()
|
||||
|
||||
# Only treat *actual* "not found" as retryable.
|
||||
# qBittorrent auth/network/API failures should surface immediately (more actionable)
|
||||
# and must not be confused with "torrent missing".
|
||||
retryable_not_found = any(
|
||||
token in message_lower
|
||||
for token in (
|
||||
"torrent not found",
|
||||
"not found in qbittorrent",
|
||||
"download not found",
|
||||
)
|
||||
)
|
||||
|
||||
non_retryable = any(
|
||||
token in message_lower
|
||||
for token in (
|
||||
"authentication failed",
|
||||
"cannot connect",
|
||||
"timed out",
|
||||
"api request failed",
|
||||
)
|
||||
)
|
||||
|
||||
if retryable_not_found and not non_retryable:
|
||||
not_found_count += 1
|
||||
if not_found_count < max_not_found_retries:
|
||||
logger.debug(
|
||||
f"Download {download_id} not yet visible in client "
|
||||
f"(attempt {not_found_count}/{max_not_found_retries})"
|
||||
)
|
||||
status_callback("resolving", "Waiting for download client...")
|
||||
if cancel_flag.wait(timeout=POLL_INTERVAL):
|
||||
break
|
||||
continue
|
||||
|
||||
logger.error(
|
||||
f"Download {download_id} not found after {max_not_found_retries} attempts"
|
||||
)
|
||||
else:
|
||||
# Fail fast on actionable errors (auth, connectivity, API issues)
|
||||
logger.error(f"Download {download_id} error state: {status.message}")
|
||||
|
||||
status_callback("error", status.message or "Download failed")
|
||||
client.remove(download_id, delete_files=True)
|
||||
self._safe_remove_download(client, download_id, protocol, "download error")
|
||||
return None
|
||||
|
||||
# Reset not-found counter on successful status check
|
||||
not_found_count = 0
|
||||
|
||||
# Build status message - use client message if provided, else build progress
|
||||
msg = status.message or self._build_progress_message(status)
|
||||
if status.state == DownloadState.PROCESSING:
|
||||
@@ -223,8 +495,19 @@ class ProwlarrHandler(DownloadHandler):
|
||||
|
||||
# Handle cancellation
|
||||
if cancel_flag.is_set():
|
||||
logger.info(f"Download cancelled, removing from {client.name}: {download_id}")
|
||||
client.remove(download_id, delete_files=True)
|
||||
if protocol == "usenet":
|
||||
logger.info(f"Download cancelled, removing from {client.name}: {download_id}")
|
||||
try:
|
||||
self._delete_local_download_data(client, download_id)
|
||||
self._remove_usenet_download(client, download_id, delete_files=True, archive=True)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to remove download {download_id} from {client.name} after cancellation: {e}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Download cancelled for protocol={protocol}; leaving in {client.name}: {download_id}"
|
||||
)
|
||||
status_callback("cancelled", "Cancelled")
|
||||
return None
|
||||
|
||||
@@ -238,24 +521,94 @@ class ProwlarrHandler(DownloadHandler):
|
||||
)
|
||||
status_callback(
|
||||
"error",
|
||||
f"Download completed in {client.name} but path not returned. "
|
||||
f"Could not locate completed download in {client.name} (path not returned). "
|
||||
f"Check volume mappings and category settings."
|
||||
)
|
||||
return None
|
||||
|
||||
# Verify the path actually exists in our filesystem
|
||||
# Apply remote path mappings (client path -> shelfmark container path)
|
||||
from shelfmark.core.path_mappings import (
|
||||
get_client_host_identifier,
|
||||
parse_remote_path_mappings,
|
||||
remap_remote_to_local_with_match,
|
||||
)
|
||||
|
||||
source_path_obj = Path(source_path)
|
||||
if not source_path_obj.exists():
|
||||
host = get_client_host_identifier(client) or ""
|
||||
mapping_value = config.get("PROWLARR_REMOTE_PATH_MAPPINGS", [])
|
||||
mappings = parse_remote_path_mappings(mapping_value)
|
||||
|
||||
logger.debug(
|
||||
"Attempting path remap: client=%s, host=%s, path=%s, mappings=%s",
|
||||
client.name,
|
||||
host,
|
||||
source_path_obj,
|
||||
[(m.host, m.remote_path, m.local_path) for m in mappings],
|
||||
)
|
||||
|
||||
remapped, matched_mapping = remap_remote_to_local_with_match(
|
||||
mappings=mappings,
|
||||
host=host,
|
||||
remote_path=source_path_obj,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Remap result: %s -> %s (exists=%s, changed=%s, matched=%s)",
|
||||
source_path_obj,
|
||||
remapped,
|
||||
remapped.exists(),
|
||||
remapped != source_path_obj,
|
||||
matched_mapping,
|
||||
)
|
||||
|
||||
if matched_mapping:
|
||||
if remapped.exists():
|
||||
logger.info(
|
||||
"Remapped download path for %s (%s): %s -> %s",
|
||||
client.name,
|
||||
download_id,
|
||||
source_path_obj,
|
||||
remapped,
|
||||
)
|
||||
source_path_obj = remapped
|
||||
else:
|
||||
logger.error(
|
||||
f"Download path does not exist after remapping: {source_path} -> {remapped}. "
|
||||
f"Client: {client.name}, ID: {download_id}. "
|
||||
f"Check that the local path in your mapping is mounted correctly."
|
||||
)
|
||||
status_callback(
|
||||
"error",
|
||||
f"Remapped path '{remapped}' does not exist. "
|
||||
f"Check your Docker volume mounts match the Local Path in Settings > Advanced > Remote Path Mappings.",
|
||||
)
|
||||
return None
|
||||
elif mappings:
|
||||
if source_path_obj.exists():
|
||||
logger.info(
|
||||
"No remote path mapping matched for %s (%s); using client path: %s",
|
||||
client.name,
|
||||
download_id,
|
||||
source_path_obj,
|
||||
)
|
||||
else:
|
||||
hint = _diagnose_path_issue(source_path)
|
||||
logger.error(
|
||||
f"Download path does not exist and no remote path mapping matched for {client.name} "
|
||||
f"({download_id}): {source_path}. {hint}"
|
||||
)
|
||||
status_callback(
|
||||
"error",
|
||||
f"{hint} No remote path mapping matched for client '{client.name}'.",
|
||||
)
|
||||
return None
|
||||
elif not source_path_obj.exists():
|
||||
hint = _diagnose_path_issue(source_path)
|
||||
logger.error(
|
||||
f"Download path does not exist: {source_path}. "
|
||||
f"Client: {client.name}, ID: {download_id}. "
|
||||
f"The download client's path may not be mounted in Shelfmark's container. "
|
||||
f"Ensure both containers use identical volume mappings for the download folder."
|
||||
)
|
||||
status_callback(
|
||||
"error",
|
||||
f"Path not accessible: {source_path}. Check volume mappings between {client.name} and Shelfmark."
|
||||
f"Client: {client.name}, ID: {download_id}. {hint}"
|
||||
)
|
||||
status_callback("error", hint)
|
||||
return None
|
||||
|
||||
result = self._handle_completed_file(
|
||||
@@ -268,17 +621,14 @@ class ProwlarrHandler(DownloadHandler):
|
||||
# Clean up on success
|
||||
if result:
|
||||
remove_release(task.task_id)
|
||||
self._cleanup_client_history(client, download_id)
|
||||
self._cleanup_refs[task.task_id] = (client, download_id, protocol)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during download polling: {e}")
|
||||
status_callback("error", str(e))
|
||||
try:
|
||||
client.remove(download_id, delete_files=True)
|
||||
except Exception as cleanup_error:
|
||||
logger.error(f"Failed to cleanup download {download_id} after error: {cleanup_error}")
|
||||
self._safe_remove_download(client, download_id, protocol, "polling exception")
|
||||
return None
|
||||
|
||||
def _handle_completed_file(
|
||||
@@ -288,56 +638,26 @@ class ProwlarrHandler(DownloadHandler):
|
||||
task: DownloadTask,
|
||||
status_callback: Callable[[str, Optional[str]], None],
|
||||
) -> Optional[str]:
|
||||
"""Handle completed download. Torrents return original path; usenet stages to temp."""
|
||||
"""Handle a completed download and return its path.
|
||||
|
||||
For external download clients (torrents/usenet), staging large payloads into TMP_DIR
|
||||
is expensive (and can duplicate multi-GB files). Instead, return the client's
|
||||
completed path and let the orchestrator perform any required transfer (copy/move/
|
||||
hardlink) directly from that source.
|
||||
|
||||
Torrents also set ``task.original_download_path`` so the orchestrator can detect
|
||||
seeding data and enable hardlinking when configured.
|
||||
"""
|
||||
try:
|
||||
# For torrents, skip staging - return original path directly
|
||||
# Orchestrator will hardlink (library mode) or copy (ingest mode) as needed
|
||||
if protocol == "torrent":
|
||||
task.original_download_path = str(source_path)
|
||||
logger.debug(f"Torrent complete, returning original path: {source_path}")
|
||||
return str(source_path)
|
||||
|
||||
# Usenet: stage based on config
|
||||
status_callback("resolving", "Staging file")
|
||||
use_copy = config.get("PROWLARR_USENET_ACTION", "move") == "copy"
|
||||
logger.debug(f"Download complete, returning original path: {source_path}")
|
||||
return str(source_path)
|
||||
|
||||
from shelfmark.download.orchestrator import get_staging_dir
|
||||
staging_dir = get_staging_dir()
|
||||
|
||||
if source_path.is_dir():
|
||||
staged_path = get_unique_path(staging_dir, source_path.name)
|
||||
if use_copy:
|
||||
shutil.copytree(str(source_path), str(staged_path))
|
||||
else:
|
||||
shutil.move(str(source_path), str(staged_path))
|
||||
logger.debug(f"Staged directory: {staged_path.name}")
|
||||
else:
|
||||
staged_path = get_unique_path(staging_dir, source_path.stem, source_path.suffix)
|
||||
if use_copy:
|
||||
shutil.copy2(str(source_path), str(staged_path))
|
||||
else:
|
||||
shutil.move(str(source_path), str(staged_path))
|
||||
logger.debug(f"Staged: {staged_path.name}")
|
||||
|
||||
return str(staged_path)
|
||||
|
||||
except FileNotFoundError as e:
|
||||
logger.error(
|
||||
f"Source file not found during staging: {source_path}. "
|
||||
f"The file may have been moved or deleted by the download client. Error: {e}"
|
||||
)
|
||||
status_callback("error", f"File not found: {source_path}. It may have been moved or deleted.")
|
||||
return None
|
||||
except PermissionError as e:
|
||||
logger.error(
|
||||
f"Permission denied staging file from {source_path}. "
|
||||
f"Check that Shelfmark has read access to the download folder. Error: {e}"
|
||||
)
|
||||
status_callback("error", f"Permission denied accessing {source_path}. Check folder permissions.")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Staging failed for {source_path}: {e}")
|
||||
status_callback("error", f"Failed to stage file: {e}")
|
||||
logger.error(f"Failed to finalize completed download at {source_path}: {e}")
|
||||
status_callback("error", f"Failed to finalize completed download: {e}")
|
||||
return None
|
||||
|
||||
def cancel(self, task_id: str) -> bool:
|
||||
|
||||
@@ -6,7 +6,7 @@ Registers Prowlarr settings as a group with multiple tabs:
|
||||
- Download Clients: Torrent and usenet client settings
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from shelfmark.core.settings_registry import (
|
||||
register_group,
|
||||
@@ -19,6 +19,7 @@ from shelfmark.core.settings_registry import (
|
||||
SelectField,
|
||||
MultiSelectField,
|
||||
)
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
|
||||
|
||||
# ==================== Dynamic Options Loaders ====================
|
||||
@@ -34,10 +35,14 @@ def _get_indexer_options() -> List[Dict[str, str]]:
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
url = config.get("PROWLARR_URL", "")
|
||||
raw_url = config.get("PROWLARR_URL", "")
|
||||
api_key = config.get("PROWLARR_API_KEY", "")
|
||||
|
||||
if not url or not api_key:
|
||||
if not raw_url or not api_key:
|
||||
return []
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return []
|
||||
|
||||
try:
|
||||
@@ -72,7 +77,7 @@ def _get_indexer_options() -> List[Dict[str, str]]:
|
||||
|
||||
# ==================== Test Connection Callbacks ====================
|
||||
|
||||
def _test_prowlarr_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
|
||||
def _test_prowlarr_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Test the Prowlarr connection using current form values."""
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
@@ -81,11 +86,15 @@ def _test_prowlarr_connection(current_values: Dict[str, Any] = None) -> Dict[str
|
||||
logger = setup_logger(__name__)
|
||||
current_values = current_values or {}
|
||||
|
||||
url = current_values.get("PROWLARR_URL") or config.get("PROWLARR_URL", "")
|
||||
raw_url = current_values.get("PROWLARR_URL") or config.get("PROWLARR_URL", "")
|
||||
api_key = current_values.get("PROWLARR_API_KEY") or config.get("PROWLARR_API_KEY", "")
|
||||
|
||||
if not url:
|
||||
if not raw_url:
|
||||
return {"success": False, "message": "Prowlarr URL is required"}
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return {"success": False, "message": "Prowlarr URL is invalid"}
|
||||
if not api_key:
|
||||
return {"success": False, "message": "API key is required"}
|
||||
|
||||
@@ -97,22 +106,26 @@ def _test_prowlarr_connection(current_values: Dict[str, Any] = None) -> Dict[str
|
||||
return {"success": False, "message": f"Connection failed: {str(e)}"}
|
||||
|
||||
|
||||
def _test_qbittorrent_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
|
||||
def _test_qbittorrent_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Test the qBittorrent connection using current form values."""
|
||||
from shelfmark.core.config import config
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
url = current_values.get("QBITTORRENT_URL") or config.get("QBITTORRENT_URL", "")
|
||||
raw_url = current_values.get("QBITTORRENT_URL") or config.get("QBITTORRENT_URL", "")
|
||||
username = current_values.get("QBITTORRENT_USERNAME") or config.get("QBITTORRENT_USERNAME", "")
|
||||
password = current_values.get("QBITTORRENT_PASSWORD") or config.get("QBITTORRENT_PASSWORD", "")
|
||||
|
||||
if not url:
|
||||
if not raw_url:
|
||||
return {"success": False, "message": "qBittorrent URL is required"}
|
||||
|
||||
try:
|
||||
from qbittorrentapi import Client
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return {"success": False, "message": "qBittorrent URL is invalid"}
|
||||
|
||||
client = Client(host=url, username=username, password=password)
|
||||
client.auth_log_in()
|
||||
api_version = client.app.web_api_version
|
||||
@@ -123,7 +136,7 @@ def _test_qbittorrent_connection(current_values: Dict[str, Any] = None) -> Dict[
|
||||
return {"success": False, "message": f"Connection failed: {str(e)}"}
|
||||
|
||||
|
||||
def _test_transmission_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
|
||||
def _test_transmission_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Test the Transmission connection using current form values."""
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.release_sources.prowlarr.clients.torrent_utils import (
|
||||
@@ -132,13 +145,17 @@ def _test_transmission_connection(current_values: Dict[str, Any] = None) -> Dict
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
url = current_values.get("TRANSMISSION_URL") or config.get("TRANSMISSION_URL", "")
|
||||
raw_url = current_values.get("TRANSMISSION_URL") or config.get("TRANSMISSION_URL", "")
|
||||
username = current_values.get("TRANSMISSION_USERNAME") or config.get("TRANSMISSION_USERNAME", "")
|
||||
password = current_values.get("TRANSMISSION_PASSWORD") or config.get("TRANSMISSION_PASSWORD", "")
|
||||
|
||||
if not url:
|
||||
if not raw_url:
|
||||
return {"success": False, "message": "Transmission URL is required"}
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return {"success": False, "message": "Transmission URL is invalid"}
|
||||
|
||||
try:
|
||||
from transmission_rpc import Client
|
||||
|
||||
@@ -161,54 +178,166 @@ def _test_transmission_connection(current_values: Dict[str, Any] = None) -> Dict
|
||||
return {"success": False, "message": f"Connection failed: {str(e)}"}
|
||||
|
||||
|
||||
def _test_deluge_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
|
||||
"""Test the Deluge connection using current form values."""
|
||||
def _test_deluge_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Test Deluge Web UI JSON-RPC connection using current form values."""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from shelfmark.core.config import config
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
host = current_values.get("DELUGE_HOST") or config.get("DELUGE_HOST", "localhost")
|
||||
port = current_values.get("DELUGE_PORT") or config.get("DELUGE_PORT", "58846")
|
||||
username = current_values.get("DELUGE_USERNAME") or config.get("DELUGE_USERNAME", "")
|
||||
raw_host = current_values.get("DELUGE_HOST") or config.get("DELUGE_HOST", "localhost")
|
||||
raw_port = current_values.get("DELUGE_PORT") or config.get("DELUGE_PORT", "8112")
|
||||
password = current_values.get("DELUGE_PASSWORD") or config.get("DELUGE_PASSWORD", "")
|
||||
|
||||
if not host:
|
||||
if not raw_host:
|
||||
return {"success": False, "message": "Deluge host is required"}
|
||||
if not password:
|
||||
return {"success": False, "message": "Deluge password is required"}
|
||||
|
||||
try:
|
||||
from deluge_client import DelugeRPCClient
|
||||
raw_host = str(raw_host)
|
||||
raw_host = normalize_http_url(raw_host, strip_trailing_slash=False) if raw_host else ""
|
||||
if not raw_host:
|
||||
return {"success": False, "message": "Deluge host is invalid"}
|
||||
|
||||
client = DelugeRPCClient(
|
||||
host=host,
|
||||
port=int(port),
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
client.connect()
|
||||
version = client.call('daemon.info')
|
||||
raw_port = str(raw_port or "8112")
|
||||
|
||||
scheme = "http"
|
||||
base_path = ""
|
||||
host = raw_host
|
||||
port = int(raw_port) if raw_port.isdigit() else 8112
|
||||
|
||||
# Allow DELUGE_HOST to be a full URL (e.g. http://deluge:8112)
|
||||
if raw_host.startswith(("http://", "https://")):
|
||||
parsed = urlparse(raw_host)
|
||||
scheme = parsed.scheme or "http"
|
||||
host = parsed.hostname or "localhost"
|
||||
if parsed.port is not None:
|
||||
port = parsed.port
|
||||
base_path = (parsed.path or "").rstrip("/")
|
||||
else:
|
||||
# Allow "host:port" in DELUGE_HOST for convenience.
|
||||
if ":" in raw_host and raw_host.count(":") == 1:
|
||||
host_part, port_part = raw_host.split(":", 1)
|
||||
if host_part and port_part.isdigit():
|
||||
host = host_part
|
||||
port = int(port_part)
|
||||
|
||||
rpc_url = f"{scheme}://{host}:{port}{base_path}/json"
|
||||
|
||||
def rpc_call(session: requests.Session, rpc_id: int, method: str, *params: Any) -> Any:
|
||||
payload = {"id": rpc_id, "method": method, "params": list(params)}
|
||||
resp = session.post(rpc_url, json=payload, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("error"):
|
||||
error = data["error"]
|
||||
if isinstance(error, dict):
|
||||
raise Exception(error.get("message") or str(error))
|
||||
raise Exception(str(error))
|
||||
return data.get("result")
|
||||
|
||||
def get_daemon_version(session: requests.Session, rpc_id: int) -> Any:
|
||||
try:
|
||||
methods = rpc_call(session, rpc_id, "system.listMethods")
|
||||
if isinstance(methods, list) and "daemon.get_version" in methods:
|
||||
return rpc_call(session, rpc_id + 1, "daemon.get_version")
|
||||
except Exception:
|
||||
# Fall back to daemon.info to preserve existing behavior.
|
||||
pass
|
||||
|
||||
return rpc_call(session, rpc_id + 1, "daemon.info")
|
||||
|
||||
try:
|
||||
session = requests.Session()
|
||||
|
||||
if rpc_call(session, 1, "auth.login", password) is not True:
|
||||
return {"success": False, "message": "Deluge Web UI authentication failed"}
|
||||
|
||||
if rpc_call(session, 2, "web.connected") is not True:
|
||||
hosts = rpc_call(session, 3, "web.get_hosts") or []
|
||||
if not hosts:
|
||||
return {
|
||||
"success": False,
|
||||
"message": "Deluge Web UI isn't connected to Deluge core (no hosts configured). Add/connect a daemon in Deluge Web UI → Connection Manager.",
|
||||
}
|
||||
|
||||
host_id = hosts[0][0]
|
||||
for entry in hosts:
|
||||
if isinstance(entry, list) and len(entry) >= 2 and entry[1] in {"127.0.0.1", "localhost"}:
|
||||
host_id = entry[0]
|
||||
break
|
||||
|
||||
rpc_call(session, 4, "web.connect", host_id)
|
||||
|
||||
if rpc_call(session, 5, "web.connected") is not True:
|
||||
return {
|
||||
"success": False,
|
||||
"message": "Deluge Web UI couldn't connect to Deluge core. Check Deluge Web UI → Connection Manager.",
|
||||
}
|
||||
|
||||
version = get_daemon_version(session, 6)
|
||||
return {"success": True, "message": f"Connected to Deluge {version}"}
|
||||
except ImportError:
|
||||
return {"success": False, "message": "deluge-client package not installed"}
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
return {"success": False, "message": "Could not connect to Deluge Web UI"}
|
||||
except requests.exceptions.Timeout:
|
||||
return {"success": False, "message": "Connection timed out"}
|
||||
except Exception as e:
|
||||
return {"success": False, "message": f"Connection failed: {str(e)}"}
|
||||
|
||||
|
||||
def _test_nzbget_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
|
||||
def _test_rtorrent_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Test the rTorrent connection using current form values."""
|
||||
from shelfmark.core.config import config
|
||||
from urllib.parse import urlparse
|
||||
from xmlrpc.client import ServerProxy
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
raw_url = current_values.get("RTORRENT_URL") or config.get("RTORRENT_URL", "")
|
||||
username = current_values.get("RTORRENT_USERNAME") or config.get("RTORRENT_USERNAME", "")
|
||||
password = current_values.get("RTORRENT_PASSWORD") or config.get("RTORRENT_PASSWORD", "")
|
||||
|
||||
if not raw_url:
|
||||
return {"success": False, "message": "rTorrent URL is required"}
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return {"success": False, "message": "rTorrent URL is invalid"}
|
||||
|
||||
try:
|
||||
# Add HTTP auth to URL if credentials provided
|
||||
if username and password:
|
||||
parsed = urlparse(url)
|
||||
url = f"{parsed.scheme}://{username}:{password}@{parsed.netloc}{parsed.path}"
|
||||
|
||||
rpc = ServerProxy(url.rstrip("/"))
|
||||
version = rpc.system.client_version()
|
||||
return {"success": True, "message": f"Connected to rTorrent {version}"}
|
||||
except Exception as e:
|
||||
return {"success": False, "message": f"Connection failed: {str(e)}"}
|
||||
|
||||
|
||||
def _test_nzbget_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Test the NZBGet connection using current form values."""
|
||||
import requests
|
||||
from shelfmark.core.config import config
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
url = current_values.get("NZBGET_URL") or config.get("NZBGET_URL", "")
|
||||
raw_url = current_values.get("NZBGET_URL") or config.get("NZBGET_URL", "")
|
||||
username = current_values.get("NZBGET_USERNAME") or config.get("NZBGET_USERNAME", "nzbget")
|
||||
password = current_values.get("NZBGET_PASSWORD") or config.get("NZBGET_PASSWORD", "")
|
||||
|
||||
if not url:
|
||||
if not raw_url:
|
||||
return {"success": False, "message": "NZBGet URL is required"}
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return {"success": False, "message": "NZBGet URL is invalid"}
|
||||
|
||||
try:
|
||||
rpc_url = f"{url.rstrip('/')}/jsonrpc"
|
||||
payload = {"jsonrpc": "2.0", "method": "status", "params": [], "id": 1}
|
||||
@@ -227,18 +356,22 @@ def _test_nzbget_connection(current_values: Dict[str, Any] = None) -> Dict[str,
|
||||
return {"success": False, "message": f"Connection failed: {str(e)}"}
|
||||
|
||||
|
||||
def _test_sabnzbd_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
|
||||
def _test_sabnzbd_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Test the SABnzbd connection using current form values."""
|
||||
import requests
|
||||
from shelfmark.core.config import config
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
url = current_values.get("SABNZBD_URL") or config.get("SABNZBD_URL", "")
|
||||
raw_url = current_values.get("SABNZBD_URL") or config.get("SABNZBD_URL", "")
|
||||
api_key = current_values.get("SABNZBD_API_KEY") or config.get("SABNZBD_API_KEY", "")
|
||||
|
||||
if not url:
|
||||
if not raw_url:
|
||||
return {"success": False, "message": "SABnzbd URL is required"}
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return {"success": False, "message": "SABnzbd URL is invalid"}
|
||||
if not api_key:
|
||||
return {"success": False, "message": "API key is required"}
|
||||
|
||||
@@ -359,6 +492,7 @@ def prowlarr_clients_settings():
|
||||
{"value": "qbittorrent", "label": "qBittorrent"},
|
||||
{"value": "transmission", "label": "Transmission"},
|
||||
{"value": "deluge", "label": "Deluge"},
|
||||
{"value": "rtorrent", "label": "rTorrent"},
|
||||
],
|
||||
default="",
|
||||
),
|
||||
@@ -457,30 +591,24 @@ def prowlarr_clients_settings():
|
||||
# --- Deluge Settings ---
|
||||
TextField(
|
||||
key="DELUGE_HOST",
|
||||
label="Deluge Host",
|
||||
description="Hostname or IP of your Deluge daemon",
|
||||
placeholder="localhost",
|
||||
label="Deluge Web UI Host/URL",
|
||||
description="Hostname/IP or full URL of your Deluge Web UI (deluge-web)",
|
||||
placeholder="http://deluge:8112",
|
||||
default="localhost",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "deluge"},
|
||||
),
|
||||
TextField(
|
||||
key="DELUGE_PORT",
|
||||
label="Deluge Port",
|
||||
description="Deluge daemon RPC port (default: 58846). IMPORTANT: Ensure \"Allow Remote Connections\" is enabled in Deluge settings.",
|
||||
placeholder="58846",
|
||||
default="58846",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "deluge"},
|
||||
),
|
||||
TextField(
|
||||
key="DELUGE_USERNAME",
|
||||
label="Username",
|
||||
description="Deluge daemon username (from auth file)",
|
||||
label="Deluge Web UI Port",
|
||||
description="Deluge Web UI port (default: 8112)",
|
||||
placeholder="8112",
|
||||
default="8112",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "deluge"},
|
||||
),
|
||||
PasswordField(
|
||||
key="DELUGE_PASSWORD",
|
||||
label="Password",
|
||||
description="Deluge daemon password (from auth file)",
|
||||
description="Deluge Web UI password (default: deluge)",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "deluge"},
|
||||
),
|
||||
ActionButton(
|
||||
@@ -507,6 +635,50 @@ def prowlarr_clients_settings():
|
||||
default="",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "deluge"},
|
||||
),
|
||||
|
||||
# --- rTorrent Settings ---
|
||||
TextField(
|
||||
key="RTORRENT_URL",
|
||||
label="rTorrent URL",
|
||||
description="XML-RPC URL of your rTorrent instance",
|
||||
placeholder="http://rtorrent:6881/RPC2",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
TextField(
|
||||
key="RTORRENT_USERNAME",
|
||||
label="Username",
|
||||
description="HTTP Basic auth username (if authentication enabled)",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
PasswordField(
|
||||
key="RTORRENT_PASSWORD",
|
||||
label="Password",
|
||||
description="HTTP Basic auth password",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
ActionButton(
|
||||
key="test_rtorrent",
|
||||
label="Test Connection",
|
||||
description="Verify your rTorrent configuration",
|
||||
style="primary",
|
||||
callback=_test_rtorrent_connection,
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
TextField(
|
||||
key="RTORRENT_LABEL",
|
||||
label="Book Label",
|
||||
description="Label to assign to book downloads in rTorrent",
|
||||
placeholder="cwabd",
|
||||
default="cwabd",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
TextField(
|
||||
key="RTORRENT_DOWNLOAD_DIR",
|
||||
label="Download Directory",
|
||||
description="Server-side directory where torrents are downloaded (optional, uses rTorrent default if not specified)",
|
||||
placeholder="/downloads",
|
||||
show_when={"field": "PROWLARR_TORRENT_CLIENT", "value": "rtorrent"},
|
||||
),
|
||||
# Note: Torrent client download path must be mounted identically in both containers.
|
||||
# Torrents are always copied (not moved) to preserve seeding capability.
|
||||
|
||||
@@ -613,22 +785,14 @@ def prowlarr_clients_settings():
|
||||
default="",
|
||||
show_when={"field": "PROWLARR_USENET_CLIENT", "value": "sabnzbd"},
|
||||
),
|
||||
CheckboxField(
|
||||
key="SABNZBD_REMOVE_COMPLETED",
|
||||
label="Remove completed downloads from history",
|
||||
default=True,
|
||||
description="Remove downloads from SABnzbd history after successful import (archives them)",
|
||||
show_when={"field": "PROWLARR_USENET_CLIENT", "value": "sabnzbd"},
|
||||
),
|
||||
|
||||
# Note: Usenet client download path must be mounted identically in both containers.
|
||||
SelectField(
|
||||
key="PROWLARR_USENET_ACTION",
|
||||
label="NZB Completion Action",
|
||||
description="What to do with usenet files after download completes",
|
||||
description="Move deletes the job from your usenet client after import; Copy keeps it in the client",
|
||||
options=[
|
||||
{"value": "move", "label": "Move to ingest"},
|
||||
{"value": "copy", "label": "Copy to ingest"},
|
||||
{"value": "move", "label": "Move"},
|
||||
{"value": "copy", "label": "Copy"},
|
||||
],
|
||||
default="move",
|
||||
show_when={"field": "PROWLARR_USENET_CLIENT", "notEmpty": True},
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
"""Prowlarr release source - searches indexers for book releases (torrents/usenet)."""
|
||||
|
||||
import re
|
||||
from typing import List, Optional
|
||||
import time
|
||||
from typing import List, Optional, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.search_plan import ReleaseSearchPlan
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
from shelfmark.release_sources import (
|
||||
Release,
|
||||
ReleaseProtocol,
|
||||
ReleaseSource,
|
||||
register_source,
|
||||
ReleaseColumnConfig,
|
||||
@@ -19,8 +24,9 @@ from shelfmark.release_sources import (
|
||||
LeadingCellType,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.api import ProwlarrClient
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.cache import cache_release
|
||||
from shelfmark.release_sources.prowlarr.utils import get_protocol_display
|
||||
from shelfmark.release_sources.prowlarr.utils import get_preferred_download_url, get_protocol
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -53,6 +59,9 @@ AUDIOBOOK_FORMATS = ["m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "o
|
||||
# Combined list for format detection (audiobook formats first for priority)
|
||||
ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS
|
||||
|
||||
# Backend safeguard: cap total Prowlarr search time per request.
|
||||
PROWLARR_SEARCH_TIMEOUT_SECONDS = 120.0
|
||||
|
||||
|
||||
def _extract_format(title: str) -> Optional[str]:
|
||||
"""Extract ebook/audiobook format from release title (extension, bracketed, or standalone)."""
|
||||
@@ -104,11 +113,11 @@ def _extract_language(title: str) -> Optional[str]:
|
||||
# Prowlarr category IDs for content type detection
|
||||
# See: https://wiki.servarr.com/prowlarr/cardigann-yml-definition#categories
|
||||
AUDIOBOOK_CATEGORY_IDS = {3000, 3030} # 3000 = Audio, 3030 = Audio/Audiobook
|
||||
EBOOK_CATEGORY_IDS = {7000, 7020} # 7000 = Books, 7020 = Books/Ebook
|
||||
BOOK_CATEGORY_RANGE = range(7000, 8000) # 7000-7999 = Books (all subcategories)
|
||||
|
||||
|
||||
def _detect_content_type_from_categories(categories: list, fallback: str = "book") -> str:
|
||||
"""Detect content type from Prowlarr category IDs. Returns 'audiobook' or 'book'."""
|
||||
"""Detect content type from Prowlarr category IDs. Returns 'audiobook', 'book', or 'other'."""
|
||||
# Normalize fallback - convert "ebook" to "book" for display consistency
|
||||
normalized_fallback = "book" if fallback == "ebook" else fallback
|
||||
|
||||
@@ -122,13 +131,17 @@ def _detect_content_type_from_categories(categories: list, fallback: str = "book
|
||||
if (isinstance(cat, dict) and cat.get("id") is not None) or isinstance(cat, int)
|
||||
}
|
||||
|
||||
# Check for audiobook categories first (more specific), then ebook
|
||||
if not cat_ids:
|
||||
return normalized_fallback
|
||||
|
||||
# Check for audiobook categories first (more specific), then any book range
|
||||
if cat_ids & AUDIOBOOK_CATEGORY_IDS:
|
||||
return "audiobook"
|
||||
if cat_ids & EBOOK_CATEGORY_IDS:
|
||||
if any(cat_id in BOOK_CATEGORY_RANGE for cat_id in cat_ids):
|
||||
return "book"
|
||||
|
||||
return normalized_fallback
|
||||
# Categories are present but not book/audiobook
|
||||
return "other"
|
||||
|
||||
|
||||
def _prowlarr_result_to_release(result: dict, search_content_type: str = "ebook") -> Release:
|
||||
@@ -136,11 +149,11 @@ def _prowlarr_result_to_release(result: dict, search_content_type: str = "ebook"
|
||||
title = result.get("title", "Unknown")
|
||||
size_bytes = result.get("size")
|
||||
indexer = result.get("indexer", "Unknown")
|
||||
protocol = get_protocol_display(result)
|
||||
protocol = get_protocol(result)
|
||||
seeders = result.get("seeders")
|
||||
leechers = result.get("leechers")
|
||||
categories = result.get("categories", [])
|
||||
is_torrent = protocol == "torrent"
|
||||
is_torrent = protocol == ReleaseProtocol.TORRENT
|
||||
|
||||
# Format peers display string: "seeders / leechers"
|
||||
peers_display = (
|
||||
@@ -167,9 +180,15 @@ def _prowlarr_result_to_release(result: dict, search_content_type: str = "ebook"
|
||||
language=_extract_language(title),
|
||||
size=_parse_size(size_bytes),
|
||||
size_bytes=size_bytes,
|
||||
download_url=result.get("downloadUrl") or result.get("magnetUrl"),
|
||||
download_url=get_preferred_download_url(result),
|
||||
info_url=result.get("infoUrl") or result.get("guid"),
|
||||
protocol=protocol,
|
||||
protocol=(
|
||||
ReleaseProtocol.TORRENT
|
||||
if protocol == "torrent"
|
||||
else ReleaseProtocol.NZB
|
||||
if protocol == "usenet"
|
||||
else None
|
||||
),
|
||||
indexer=indexer,
|
||||
seeders=seeders if is_torrent else None,
|
||||
peers=peers_display,
|
||||
@@ -195,8 +214,7 @@ class ProwlarrSource(ReleaseSource):
|
||||
def __init__(self):
|
||||
self.last_search_type: Optional[str] = None
|
||||
|
||||
@classmethod
|
||||
def get_column_config(cls) -> ReleaseColumnConfig:
|
||||
def get_column_config(self) -> ReleaseColumnConfig:
|
||||
"""Column configuration for Prowlarr releases."""
|
||||
return ReleaseColumnConfig(
|
||||
columns=[
|
||||
@@ -254,15 +272,19 @@ class ProwlarrSource(ReleaseSource):
|
||||
],
|
||||
grid_template="minmax(0,2fr) minmax(80px,1fr) 60px 70px 90px 80px",
|
||||
leading_cell=LeadingCellConfig(type=LeadingCellType.NONE), # No leading cell for Prowlarr
|
||||
supported_filters=[], # Prowlarr has unreliable format/language metadata; content_type is auto-detected
|
||||
supported_filters=["language"], # Enables multi-language query expansion; Prowlarr language metadata is unreliable
|
||||
)
|
||||
|
||||
def _get_client(self) -> Optional[ProwlarrClient]:
|
||||
"""Get a configured Prowlarr client or None if not configured."""
|
||||
url = config.get("PROWLARR_URL", "")
|
||||
raw_url = config.get("PROWLARR_URL", "")
|
||||
api_key = config.get("PROWLARR_API_KEY", "")
|
||||
|
||||
if not url or not api_key:
|
||||
if not raw_url or not api_key:
|
||||
return None
|
||||
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
return None
|
||||
|
||||
return ProwlarrClient(url, api_key)
|
||||
@@ -294,8 +316,8 @@ class ProwlarrSource(ReleaseSource):
|
||||
def search(
|
||||
self,
|
||||
book: BookMetadata,
|
||||
plan: "ReleaseSearchPlan", # noqa: F821
|
||||
expand_search: bool = False,
|
||||
languages: Optional[List[str]] = None,
|
||||
content_type: str = "ebook"
|
||||
) -> List[Release]:
|
||||
"""Search Prowlarr indexers for releases matching the book."""
|
||||
@@ -304,27 +326,13 @@ class ProwlarrSource(ReleaseSource):
|
||||
logger.warning("Prowlarr not configured - skipping search")
|
||||
return []
|
||||
|
||||
# Build search query
|
||||
query_parts = []
|
||||
# Prefer search_title if available (cleaner title for searches)
|
||||
search_title = book.search_title or book.title
|
||||
if search_title:
|
||||
query_parts.append(search_title)
|
||||
if book.authors:
|
||||
# Use first author only - authors may be a list or a single string
|
||||
# that contains multiple comma-separated names (from frontend)
|
||||
first_author = book.authors[0]
|
||||
# If first author contains comma, split and use only the primary author
|
||||
if "," in first_author:
|
||||
first_author = first_author.split(",")[0].strip()
|
||||
query_parts.append(first_author)
|
||||
queries = [v.title for v in plan.title_variants if v.title]
|
||||
queries = [q for q in queries if q]
|
||||
|
||||
query = " ".join(query_parts)
|
||||
if not query:
|
||||
# Try ISBN as fallback
|
||||
query = book.isbn_13 or book.isbn_10 or ""
|
||||
if not queries and plan.isbn_candidates:
|
||||
queries = list(plan.isbn_candidates)
|
||||
|
||||
if not query:
|
||||
if not queries:
|
||||
logger.warning("No search query available for book")
|
||||
return []
|
||||
|
||||
@@ -334,17 +342,50 @@ class ProwlarrSource(ReleaseSource):
|
||||
# Get search categories based on content type
|
||||
# Audiobooks use 3030 (Audio/Audiobook), ebooks use 7000 (Books)
|
||||
search_categories = [3030] if content_type == "audiobook" else [7000]
|
||||
categories = None if expand_search else search_categories
|
||||
self.last_search_type = "expanded" if expand_search else "categories"
|
||||
|
||||
# Manual query override should behave like normal Prowlarr searches:
|
||||
# - default: search within the content-type categories
|
||||
# - expand: rerun without categories
|
||||
if plan.manual_query:
|
||||
categories = None if expand_search else search_categories
|
||||
self.last_search_type = "manual_expanded" if expand_search else "manual_query"
|
||||
else:
|
||||
categories = None if expand_search else search_categories
|
||||
self.last_search_type = "expanded" if expand_search else "categories"
|
||||
|
||||
if plan.manual_query:
|
||||
query_type = "manual"
|
||||
elif not plan.title_variants and plan.isbn_candidates:
|
||||
query_type = "isbn"
|
||||
else:
|
||||
query_type = "title"
|
||||
|
||||
indexer_desc = f"indexers={indexer_ids}" if indexer_ids else "all enabled indexers"
|
||||
logger.debug(f"Searching Prowlarr: query='{query}', {indexer_desc}, categories={categories}")
|
||||
if len(queries) == 1:
|
||||
logger.debug(
|
||||
f"Searching Prowlarr: {query_type}='{queries[0]}', {indexer_desc}, categories={categories}"
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
f"Searching Prowlarr: {query_type} ({len(queries)} variants), {indexer_desc}, categories={categories}"
|
||||
)
|
||||
|
||||
def search_indexers(cats: Optional[List[int]]) -> List[dict]:
|
||||
def search_indexers(query: str, cats: Optional[List[int]]) -> List[dict]:
|
||||
"""Search indexers with given categories, collecting results."""
|
||||
results = []
|
||||
if indexer_ids:
|
||||
# Search specific indexers one at a time
|
||||
# Prefer a single request for all selected indexers to reduce latency.
|
||||
try:
|
||||
raw = client.search(query=query, indexer_ids=indexer_ids, categories=cats)
|
||||
if raw:
|
||||
results.extend(raw)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Search failed for selected indexers {indexer_ids}: {e}. Falling back to per-indexer search."
|
||||
)
|
||||
|
||||
# Fallback: search specific indexers one at a time
|
||||
for indexer_id in indexer_ids:
|
||||
try:
|
||||
raw = client.search(query=query, indexer_ids=[indexer_id], categories=cats)
|
||||
@@ -362,22 +403,50 @@ class ProwlarrSource(ReleaseSource):
|
||||
logger.warning(f"Search failed for all indexers: {e}")
|
||||
return results
|
||||
|
||||
all_results = []
|
||||
try:
|
||||
all_results = search_indexers(categories)
|
||||
|
||||
# Auto-expand: if no results with categories and auto-expand enabled, retry without
|
||||
auto_expand_enabled = config.get("PROWLARR_AUTO_EXPAND", False)
|
||||
if not all_results and categories and auto_expand_enabled:
|
||||
logger.info("Prowlarr: no results with category filter, auto-expanding search")
|
||||
all_results = search_indexers(None)
|
||||
self.last_search_type = "expanded"
|
||||
deadline = time.monotonic() + PROWLARR_SEARCH_TIMEOUT_SECONDS
|
||||
|
||||
def _check_timeout() -> None:
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(
|
||||
f"Prowlarr search timed out after {int(PROWLARR_SEARCH_TIMEOUT_SECONDS)}s"
|
||||
)
|
||||
seen_keys: set[str] = set()
|
||||
all_results: List[dict] = []
|
||||
|
||||
for idx, query in enumerate(queries, start=1):
|
||||
_check_timeout()
|
||||
if len(queries) > 1:
|
||||
logger.debug(f"Prowlarr query {idx}/{len(queries)}: '{query}'")
|
||||
|
||||
raw_results = search_indexers(query=query, cats=categories)
|
||||
|
||||
# Auto-expand: if no results with categories and auto-expand enabled, retry without
|
||||
if not raw_results and categories and auto_expand_enabled:
|
||||
_check_timeout()
|
||||
logger.info(f"Prowlarr: no results for query '{query}' with category filter, auto-expanding search")
|
||||
raw_results = search_indexers(query=query, cats=None)
|
||||
self.last_search_type = "expanded"
|
||||
|
||||
for r in raw_results:
|
||||
key = (
|
||||
r.get("guid")
|
||||
or r.get("downloadUrl")
|
||||
or r.get("magnetUrl")
|
||||
or r.get("infoUrl")
|
||||
or f"{r.get('indexerId')}:{r.get('title')}"
|
||||
)
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
|
||||
results = [_prowlarr_result_to_release(r, content_type) for r in all_results]
|
||||
|
||||
if results:
|
||||
torrent_count = sum(1 for r in results if r.protocol == "torrent")
|
||||
nzb_count = sum(1 for r in results if r.protocol == "nzb")
|
||||
torrent_count = sum(1 for r in results if r.protocol == ReleaseProtocol.TORRENT)
|
||||
nzb_count = sum(1 for r in results if r.protocol == ReleaseProtocol.NZB)
|
||||
indexers = sorted(set(r.indexer for r in results if r.indexer))
|
||||
indexer_str = ", ".join(indexers) if indexers else "unknown"
|
||||
logger.info(f"Prowlarr: {len(results)} results ({torrent_count} torrent, {nzb_count} nzb) from {indexer_str}")
|
||||
@@ -386,6 +455,9 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
return results
|
||||
|
||||
except TimeoutError as e:
|
||||
logger.warning(f"Prowlarr search timed out: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Prowlarr search failed: {e}")
|
||||
return []
|
||||
@@ -394,6 +466,6 @@ class ProwlarrSource(ReleaseSource):
|
||||
"""Check if Prowlarr is enabled and configured."""
|
||||
if not config.get("PROWLARR_ENABLED", False):
|
||||
return False
|
||||
url = config.get("PROWLARR_URL", "")
|
||||
url = normalize_http_url(config.get("PROWLARR_URL", ""))
|
||||
api_key = config.get("PROWLARR_API_KEY", "")
|
||||
return bool(url and api_key)
|
||||
|
||||
@@ -6,36 +6,91 @@ Provides common helper functions used across the Prowlarr plugin.
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
|
||||
|
||||
|
||||
def get_protocol(result: dict) -> str:
|
||||
"""Get the download protocol from a Prowlarr result.
|
||||
|
||||
Uses the protocol field directly if available, otherwise infers from URLs.
|
||||
"""
|
||||
Get the download protocol from a Prowlarr result.
|
||||
|
||||
Uses the protocol field directly if available, otherwise infers from URL.
|
||||
|
||||
Args:
|
||||
result: Prowlarr search result dictionary
|
||||
|
||||
Returns:
|
||||
Protocol string: "torrent", "usenet", or "unknown"
|
||||
"""
|
||||
# Prowlarr provides protocol directly - use it
|
||||
protocol = result.get("protocol", "").lower()
|
||||
protocol = str(result.get("protocol", "")).lower()
|
||||
if protocol in ("torrent", "usenet"):
|
||||
return protocol
|
||||
|
||||
# Fallback: infer from download URL
|
||||
download_url = result.get("downloadUrl") or result.get("magnetUrl") or ""
|
||||
url_lower = download_url.lower()
|
||||
if url_lower.startswith("magnet:") or ".torrent" in url_lower:
|
||||
magnet_url = str(result.get("magnetUrl") or "").lower()
|
||||
download_url = str(result.get("downloadUrl") or "").lower()
|
||||
|
||||
# Prefer magnetUrl for inference if present.
|
||||
if magnet_url.startswith("magnet:"):
|
||||
return "torrent"
|
||||
if ".nzb" in url_lower:
|
||||
|
||||
if download_url.startswith("magnet:") or ".torrent" in download_url:
|
||||
return "torrent"
|
||||
if ".nzb" in download_url:
|
||||
return "usenet"
|
||||
|
||||
return "unknown"
|
||||
|
||||
|
||||
def get_preferred_download_url(result: dict) -> str:
|
||||
"""Pick the best URL to hand to a download client.
|
||||
|
||||
For torrent results, prefer magnetUrl when available (downloadUrl may be a
|
||||
Prowlarr proxy URL that needs auth/headers).
|
||||
"""
|
||||
protocol = str(result.get("protocol", "")).lower()
|
||||
magnet_url = str(result.get("magnetUrl") or "").strip()
|
||||
download_url = sanitize_download_url(str(result.get("downloadUrl") or "").strip())
|
||||
|
||||
if protocol == "torrent":
|
||||
return magnet_url or download_url
|
||||
if protocol == "usenet":
|
||||
return download_url or magnet_url
|
||||
|
||||
# Unknown protocol: if it looks like a magnet, still prefer it.
|
||||
if magnet_url.lower().startswith("magnet:"):
|
||||
return magnet_url
|
||||
|
||||
return download_url or magnet_url
|
||||
|
||||
|
||||
def sanitize_download_url(download_url: str) -> str:
|
||||
"""Normalize Prowlarr download URLs to avoid malformed query strings."""
|
||||
if not download_url:
|
||||
return download_url
|
||||
|
||||
normalized = download_url.strip()
|
||||
if not normalized:
|
||||
return normalized
|
||||
|
||||
lower = normalized.lower()
|
||||
if not (lower.startswith("http://") or lower.startswith("https://")):
|
||||
return normalized
|
||||
|
||||
if " " not in normalized:
|
||||
return normalized
|
||||
|
||||
parsed = urlparse(normalized)
|
||||
if not parsed.query:
|
||||
return normalized
|
||||
|
||||
cleaned_pairs = []
|
||||
changed = False
|
||||
for key, value in parse_qsl(parsed.query, keep_blank_values=True):
|
||||
cleaned_key = key.strip()
|
||||
cleaned_value = value.strip()
|
||||
if cleaned_key != key or cleaned_value != value:
|
||||
changed = True
|
||||
cleaned_pairs.append((cleaned_key, cleaned_value))
|
||||
|
||||
if not changed:
|
||||
return normalized
|
||||
|
||||
cleaned_query = urlencode(cleaned_pairs, doseq=True)
|
||||
return urlunparse(parsed._replace(query=cleaned_query))
|
||||
|
||||
|
||||
def get_protocol_display(result: dict) -> str:
|
||||
"""
|
||||
Get a user-friendly display label for the protocol.
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
||||
<meta name="description" content="Shelfmark - Book search and download" />
|
||||
<base href="/" data-shelfmark-base />
|
||||
|
||||
<!-- Theme color with media queries for light/dark mode -->
|
||||
<meta name="theme-color" content="#f8f8f8" media="(prefers-color-scheme: light)" />
|
||||
@@ -15,8 +16,8 @@
|
||||
<meta name="apple-mobile-web-app-title" content="Shelfmark" />
|
||||
|
||||
<!-- App Icons -->
|
||||
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
|
||||
<link rel="apple-touch-icon" href="/logo.png" />
|
||||
<link rel="icon" type="image/x-icon" href="favicon.ico" />
|
||||
<link rel="apple-touch-icon" href="logo.png" />
|
||||
<title>Shelfmark</title>
|
||||
<script>
|
||||
// Apply theme immediately before first paint to prevent flash
|
||||
|
||||
@@ -29,6 +29,7 @@ import { ConfigSetupBanner } from './components/ConfigSetupBanner';
|
||||
import { OnboardingModal } from './components/OnboardingModal';
|
||||
import { DEFAULT_LANGUAGES, DEFAULT_SUPPORTED_FORMATS } from './data/languages';
|
||||
import { buildSearchQuery } from './utils/buildSearchQuery';
|
||||
import { withBasePath } from './utils/basePath';
|
||||
import { SearchModeProvider } from './contexts/SearchModeContext';
|
||||
import './styles.css';
|
||||
|
||||
@@ -128,15 +129,6 @@ function App() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
const [featureNoticeDismissed, setFeatureNoticeDismissed] = useState(() => {
|
||||
return localStorage.getItem('cwa-bd-prowlarr-irc-notice-dismissed') === 'true';
|
||||
});
|
||||
|
||||
const handleDismissFeatureNotice = useCallback(() => {
|
||||
localStorage.setItem('cwa-bd-prowlarr-irc-notice-dismissed', 'true');
|
||||
setFeatureNoticeDismissed(true);
|
||||
}, []);
|
||||
|
||||
// URL-based search: parse URL params for automatic search on page load
|
||||
const urlSearchEnabled = isAuthenticated && config !== null;
|
||||
const { parsedParams, wasProcessed } = useUrlSearch({ enabled: urlSearchEnabled });
|
||||
@@ -210,7 +202,7 @@ function App() {
|
||||
// Auto-download to browser if enabled
|
||||
if (config?.download_to_browser && book.download_path) {
|
||||
const link = document.createElement('a');
|
||||
link.href = `/api/localdownload?id=${encodeURIComponent(bookId)}`;
|
||||
link.href = withBasePath(`/api/localdownload?id=${encodeURIComponent(bookId)}`);
|
||||
link.download = '';
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
@@ -437,6 +429,7 @@ function App() {
|
||||
await fetchStatus();
|
||||
} catch (error) {
|
||||
console.error('Cancel failed:', error);
|
||||
showToast('Failed to cancel/clear download', 'error');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -447,6 +440,7 @@ function App() {
|
||||
await fetchStatus();
|
||||
} catch (error) {
|
||||
console.error('Clear completed failed:', error);
|
||||
showToast('Failed to clear finished downloads', 'error');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -624,18 +618,6 @@ function App() {
|
||||
onContentTypeChange={setContentType}
|
||||
/>
|
||||
|
||||
{isInitialState && !featureNoticeDismissed && (
|
||||
<div className="absolute bottom-4 left-0 right-0 px-4 text-center text-sm opacity-40">
|
||||
<span>We've renamed to Shelfmark. New: Torrent, Usenet, IRC and Audiobook support.</span>
|
||||
<button
|
||||
onClick={handleDismissFeatureNotice}
|
||||
className="ml-2 text-blue-500 hover:text-blue-600 dark:text-blue-400 dark:hover:text-blue-300 underline"
|
||||
>
|
||||
Dismiss
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ResultsSection
|
||||
books={books}
|
||||
visible={hasResults}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useEffect } from 'react';
|
||||
import { StatusData, Book } from '../types';
|
||||
import { withBasePath } from '../utils/basePath';
|
||||
|
||||
interface DownloadsSidebarProps {
|
||||
isOpen: boolean;
|
||||
@@ -142,6 +143,8 @@ export const DownloadsSidebar = ({
|
||||
};
|
||||
|
||||
const isInProgress = ['queued', 'resolving', 'downloading'].includes(statusName);
|
||||
const isQueued = statusName === 'queued';
|
||||
const isActive = statusName === 'resolving' || statusName === 'downloading';
|
||||
const isCompleted = statusName === 'complete';
|
||||
const hasError = statusName === 'error';
|
||||
|
||||
@@ -169,25 +172,27 @@ export const DownloadsSidebar = ({
|
||||
className="relative rounded-lg border hover:shadow-md transition-shadow overflow-hidden"
|
||||
style={{ borderColor: 'var(--border-muted)', background: 'var(--bg-soft)' }}
|
||||
>
|
||||
{/* Cancel/Clear Button - top right corner */}
|
||||
{/* Action Button - top right corner */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onCancel(book.id);
|
||||
}}
|
||||
className={`absolute top-1 right-1 z-10 flex items-center justify-center rounded-full transition-colors ${
|
||||
isInProgress
|
||||
? 'px-2 py-0.5 text-xs font-medium text-red-600 dark:text-red-400 hover:bg-red-100 dark:hover:bg-red-900/30'
|
||||
: 'w-6 h-6 text-gray-500 hover:text-red-600 hover:bg-red-100 dark:hover:bg-red-900/30'
|
||||
className={`absolute top-1 right-1 z-10 flex h-8 w-8 items-center justify-center rounded-full transition-colors ${
|
||||
isActive || isQueued
|
||||
? 'text-red-600 dark:text-red-400 hover:bg-red-100 dark:hover:bg-red-900/30'
|
||||
: 'text-gray-500 hover:text-red-600 hover:bg-red-100 dark:hover:bg-red-900/30'
|
||||
}`}
|
||||
title={isInProgress ? "Stop download" : "Clear from list"}
|
||||
aria-label={isInProgress ? "Stop download" : "Clear from list"}
|
||||
title={isActive ? 'Stop download' : isQueued ? 'Remove from queue' : 'Clear from list'}
|
||||
aria-label={isActive ? 'Stop download' : isQueued ? 'Remove from queue' : 'Clear from list'}
|
||||
>
|
||||
{isInProgress ? (
|
||||
'Stop'
|
||||
{isActive ? (
|
||||
<svg className="w-4 h-4" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">
|
||||
<rect x="6" y="6" width="12" height="12" rx="2" />
|
||||
</svg>
|
||||
) : (
|
||||
<svg className="w-3.5 h-3.5" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.5">
|
||||
<svg className="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.5" aria-hidden="true">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
)}
|
||||
@@ -207,7 +212,7 @@ export const DownloadsSidebar = ({
|
||||
<h3 className="font-semibold text-sm truncate" title={book.title}>
|
||||
{isCompleted && book.download_path ? (
|
||||
<a
|
||||
href={`/api/localdownload?id=${encodeURIComponent(book.id)}`}
|
||||
href={withBasePath(`/api/localdownload?id=${encodeURIComponent(book.id)}`)}
|
||||
className="text-sky-600 hover:underline"
|
||||
>
|
||||
{book.title || 'Unknown Title'}
|
||||
|
||||
@@ -189,7 +189,7 @@ export const Dropdown = ({
|
||||
panelDirection === 'down'
|
||||
? renderTrigger ? 'mt-2' : ''
|
||||
: renderTrigger ? 'bottom-full mb-2' : 'bottom-full'
|
||||
} border shadow-lg z-20 ${panelClassName || widthClassName}`}
|
||||
} border z-20 ${panelDirection === 'down' ? 'shadow-lg' : ''} ${panelClassName || widthClassName}`}
|
||||
style={{
|
||||
background: 'var(--bg)',
|
||||
borderColor: 'var(--border-muted)',
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useState, useEffect, useRef, forwardRef, useImperativeHandle } from 'react';
|
||||
import { SearchBar, SearchBarHandle } from './SearchBar';
|
||||
import { ContentType } from '../types';
|
||||
import { withBasePath } from '../utils/basePath';
|
||||
|
||||
export interface HeaderHandle {
|
||||
submitSearch: () => void;
|
||||
@@ -345,7 +346,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(({
|
||||
// Show persistent toast while gathering logs
|
||||
const loadingToastId = onShowToast?.('Gathering debug logs... This may take a minute.', 'info', true);
|
||||
try {
|
||||
const response = await fetch('/api/debug', {
|
||||
const response = await fetch(withBasePath('/api/debug'), {
|
||||
method: 'GET',
|
||||
credentials: 'include',
|
||||
});
|
||||
@@ -394,7 +395,7 @@ export const Header = forwardRef<HeaderHandle, HeaderProps>(({
|
||||
</svg>
|
||||
<span>Debug</span>
|
||||
</button>
|
||||
<form action="/api/restart" method="get" className="w-full">
|
||||
<form action={withBasePath('/api/restart')} method="get" className="w-full">
|
||||
<button
|
||||
className="w-full text-left px-4 py-2 hover-surface transition-colors flex items-center gap-3 text-orange-600 dark:text-orange-400"
|
||||
type="submit"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { FormEvent, KeyboardEvent, useEffect, useRef, useState } from 'react';
|
||||
import { LoginCredentials } from '../types';
|
||||
import { withBasePath } from '../utils/basePath';
|
||||
|
||||
interface LoginFormProps {
|
||||
onSubmit: (credentials: LoginCredentials) => void;
|
||||
@@ -97,7 +98,7 @@ export const LoginForm = ({
|
||||
)}
|
||||
<form
|
||||
method="post"
|
||||
action="/api/login"
|
||||
action={withBasePath('/api/login')}
|
||||
autoComplete="on"
|
||||
id="login-form"
|
||||
name="login"
|
||||
@@ -227,5 +228,3 @@ export const LoginForm = ({
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
HeadingFieldConfig,
|
||||
ActionButtonConfig,
|
||||
ActionResult,
|
||||
ShowWhenCondition,
|
||||
} from '../types/settings';
|
||||
import { FieldWrapper } from './settings/shared';
|
||||
import {
|
||||
@@ -35,6 +36,24 @@ interface OnboardingModalProps {
|
||||
onShowToast?: (message: string, type: 'success' | 'error' | 'info') => void;
|
||||
}
|
||||
|
||||
function evaluateShowWhenCondition(
|
||||
showWhen: ShowWhenCondition,
|
||||
values: Record<string, unknown>
|
||||
): boolean {
|
||||
const currentValue = values[showWhen.field];
|
||||
|
||||
if (showWhen.notEmpty) {
|
||||
if (Array.isArray(currentValue)) {
|
||||
return currentValue.length > 0;
|
||||
}
|
||||
return currentValue !== undefined && currentValue !== null && currentValue !== '';
|
||||
}
|
||||
|
||||
return Array.isArray(showWhen.value)
|
||||
? showWhen.value.includes(currentValue as string)
|
||||
: currentValue === showWhen.value;
|
||||
}
|
||||
|
||||
// Check if a field should be visible based on showWhen condition
|
||||
function isFieldVisible(
|
||||
field: SettingsField,
|
||||
@@ -43,20 +62,11 @@ function isFieldVisible(
|
||||
const showWhen = field.showWhen;
|
||||
if (!showWhen) return true;
|
||||
|
||||
const currentValue = values[showWhen.field];
|
||||
|
||||
// Handle notEmpty condition
|
||||
if (showWhen.notEmpty) {
|
||||
if (Array.isArray(currentValue)) {
|
||||
return currentValue.length > 0;
|
||||
}
|
||||
return currentValue !== undefined && currentValue !== null && currentValue !== '';
|
||||
if (Array.isArray(showWhen)) {
|
||||
return showWhen.every((condition) => evaluateShowWhenCondition(condition, values));
|
||||
}
|
||||
|
||||
// Handle array of allowed values or single value
|
||||
return Array.isArray(showWhen.value)
|
||||
? showWhen.value.includes(currentValue as string)
|
||||
: currentValue === showWhen.value;
|
||||
return evaluateShowWhenCondition(showWhen, values);
|
||||
}
|
||||
|
||||
// Check if a step should be visible based on its showWhen conditions (all must be true)
|
||||
|
||||
@@ -622,6 +622,8 @@ export const ReleaseModal = ({
|
||||
// A specific value means "show only that format"
|
||||
const [formatFilter, setFormatFilter] = useState<string>('');
|
||||
const [languageFilter, setLanguageFilter] = useState<string[]>([LANGUAGE_OPTION_DEFAULT]);
|
||||
const [manualQuery, setManualQuery] = useState<string>('');
|
||||
const [showManualQuery, setShowManualQuery] = useState<boolean>(false);
|
||||
|
||||
// Sort state - keyed by source name, persisted to localStorage
|
||||
// null means "Default" (backend order), undefined means "not set yet"
|
||||
@@ -671,6 +673,8 @@ export const ReleaseModal = ({
|
||||
setExpandedBySource({});
|
||||
setFormatFilter('');
|
||||
setLanguageFilter([LANGUAGE_OPTION_DEFAULT]);
|
||||
setManualQuery('');
|
||||
setShowManualQuery(false);
|
||||
setSearchStatus(null);
|
||||
lastStatusTimeRef.current = 0;
|
||||
pendingStatusRef.current = null;
|
||||
@@ -844,7 +848,7 @@ export const ReleaseModal = ({
|
||||
setErrorBySource((prev) => ({ ...prev, [activeTab]: null }));
|
||||
|
||||
try {
|
||||
const response = await getReleases(provider, bookId, activeTab, book.title, book.author, undefined, undefined, contentType);
|
||||
const response = await getReleases(provider, bookId, activeTab, book.title, book.author, undefined, undefined, contentType, manualQuery.trim() || undefined);
|
||||
setCachedReleases(provider, bookId, activeTab, contentType, response);
|
||||
setReleasesBySource((prev) => ({ ...prev, [activeTab]: response }));
|
||||
} catch (err) {
|
||||
@@ -856,7 +860,7 @@ export const ReleaseModal = ({
|
||||
};
|
||||
|
||||
fetchReleases();
|
||||
}, [book, activeTab, releasesBySource, loadingBySource, errorBySource, contentType]);
|
||||
}, [book, activeTab, releasesBySource, loadingBySource, errorBySource, contentType, manualQuery]);
|
||||
|
||||
// Handler for expanding search (title+author instead of ISBN)
|
||||
// Fetches additional results and merges with existing ISBN results
|
||||
@@ -879,7 +883,7 @@ export const ReleaseModal = ({
|
||||
|
||||
// Fetch with expand_search=true (title+author search)
|
||||
const expandedResponse = await getReleases(
|
||||
provider, bookId, activeTab, book.title, book.author, true, languagesParam, contentType
|
||||
provider, bookId, activeTab, book.title, book.author, true, languagesParam, contentType, manualQuery.trim() || undefined
|
||||
);
|
||||
|
||||
// Merge with existing results, deduplicating by source_id
|
||||
@@ -906,7 +910,7 @@ export const ReleaseModal = ({
|
||||
} finally {
|
||||
setLoadingBySource((prev) => ({ ...prev, [activeTab]: false }));
|
||||
}
|
||||
}, [activeTab, book, languageFilter, bookLanguages, defaultLanguages, contentType]);
|
||||
}, [activeTab, book, languageFilter, bookLanguages, defaultLanguages, contentType, manualQuery]);
|
||||
|
||||
// Build list of tabs to show
|
||||
// Only show enabled sources that support the current content type
|
||||
@@ -1230,20 +1234,22 @@ export const ReleaseModal = ({
|
||||
{book.author || 'Unknown author'}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClose}
|
||||
className="rounded-full p-2 text-gray-500 transition-colors hover-action hover:text-gray-900 dark:hover:text-gray-100 flex-shrink-0"
|
||||
aria-label="Close"
|
||||
>
|
||||
<svg className="h-5 w-5" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={1.5}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
<div className="flex items-center gap-2 flex-shrink-0">
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClose}
|
||||
className="rounded-full p-2 text-gray-500 transition-colors hover-action hover:text-gray-900 dark:hover:text-gray-100"
|
||||
aria-label="Close"
|
||||
>
|
||||
<svg className="h-5 w-5" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={1.5}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* Scrollable content */}
|
||||
<div ref={scrollContainerRef} className="flex-1 min-h-0 overflow-y-auto">
|
||||
{/* Scrollable content */}
|
||||
<div ref={scrollContainerRef} className="flex-1 min-h-0 overflow-y-auto">
|
||||
{/* Book summary - scrolls with content */}
|
||||
<div ref={bookSummaryRef} className="flex gap-4 px-5 py-4 border-b border-[var(--border-muted)]">
|
||||
{book.preview ? (
|
||||
@@ -1401,123 +1407,148 @@ export const ReleaseModal = ({
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Sort dropdown - only show if source has sortable columns */}
|
||||
{sortableColumns.length > 0 && (
|
||||
<Dropdown
|
||||
align="right"
|
||||
widthClassName="w-auto flex-shrink-0"
|
||||
panelClassName="w-48"
|
||||
renderTrigger={({ isOpen, toggle }) => (
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggle}
|
||||
className={`relative p-2 rounded-full transition-colors hover-surface text-gray-500 dark:text-gray-400 ${
|
||||
isOpen ? 'bg-[var(--hover-surface)]' : ''
|
||||
}`}
|
||||
aria-label="Sort releases"
|
||||
>
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={1.5}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M3 7.5 7.5 3m0 0L12 7.5M7.5 3v13.5m13.5 0L16.5 21m0 0L12 16.5m4.5 4.5V7.5" />
|
||||
</svg>
|
||||
{currentSort && (
|
||||
<span className="absolute top-1 right-1 w-2 h-2 bg-emerald-500 rounded-full" />
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
<div className="flex items-center gap-3 pl-2 pr-1">
|
||||
{/* Manual query button */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setShowManualQuery((prev) => {
|
||||
const next = !prev;
|
||||
if (next && !manualQuery.trim()) {
|
||||
const baseTitle = book?.search_title || book?.title || '';
|
||||
const baseAuthor = book?.search_author || book?.author || '';
|
||||
const defaultQuery = `${baseTitle} ${baseAuthor}`.trim();
|
||||
setManualQuery(defaultQuery);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}}
|
||||
className={`p-2.5 rounded-full transition-colors hover-surface text-gray-500 dark:text-gray-400 ${
|
||||
manualQuery.trim() ? 'text-emerald-600 dark:text-emerald-400' : ''
|
||||
}`}
|
||||
aria-label="Manual search query"
|
||||
title="Manual query"
|
||||
>
|
||||
{({ close }) => (
|
||||
<div className="py-1">
|
||||
{/* Default option - no client-side sorting */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
handleSortChange(null, null);
|
||||
close();
|
||||
}}
|
||||
className={`w-full px-3 py-2 text-left text-sm flex items-center justify-between hover-surface rounded ${
|
||||
!currentSort
|
||||
? 'text-emerald-600 dark:text-emerald-400 font-medium'
|
||||
: 'text-gray-700 dark:text-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span>Default</span>
|
||||
{!currentSort && (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={2}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="m4.5 12.75 6 6 9-13.5" />
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
{sortableColumns.map((col) => {
|
||||
const sortKey = col.sort_key || col.key;
|
||||
const isSelected = currentSort?.key === sortKey;
|
||||
const direction = isSelected ? currentSort?.direction : null;
|
||||
return (
|
||||
<button
|
||||
key={sortKey}
|
||||
type="button"
|
||||
onClick={() => {
|
||||
handleSortChange(sortKey, col);
|
||||
// Don't close - allow toggling direction
|
||||
if (!isSelected) close();
|
||||
}}
|
||||
className={`w-full px-3 py-2 text-left text-sm flex items-center justify-between hover-surface rounded ${
|
||||
isSelected
|
||||
? 'text-emerald-600 dark:text-emerald-400 font-medium'
|
||||
: 'text-gray-700 dark:text-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span>{col.label}</span>
|
||||
{isSelected && direction && (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={2}>
|
||||
{direction === 'asc' ? (
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M4.5 15.75l7.5-7.5 7.5 7.5" />
|
||||
) : (
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M19.5 8.25l-7.5 7.5-7.5-7.5" />
|
||||
)}
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</Dropdown>
|
||||
)}
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={1.5}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="m16.862 4.487 1.687-1.688a1.875 1.875 0 0 1 2.652 2.652L10.582 16.07a4.5 4.5 0 0 1-1.897 1.13L6 18l.8-2.685a4.5 4.5 0 0 1 1.13-1.897l8.932-8.931Zm0 0L19.5 7.125M18 14v4.75A2.25 2.25 0 0 1 15.75 21H5.25A2.25 2.25 0 0 1 3 18.75V8.25A2.25 2.25 0 0 1 5.25 6H10" />
|
||||
</svg>
|
||||
</button>
|
||||
|
||||
{/* Filter funnel button - stays fixed */}
|
||||
{/* Only show filter button if source supports at least one filter type */}
|
||||
{((columnConfig.supported_filters?.includes('format') && availableFormats.length > 0) ||
|
||||
(columnConfig.supported_filters?.includes('language') && bookLanguages.length > 0)) && (
|
||||
<Dropdown
|
||||
align="right"
|
||||
widthClassName="w-auto flex-shrink-0"
|
||||
panelClassName="w-56"
|
||||
noScrollLimit
|
||||
renderTrigger={({ isOpen, toggle }) => {
|
||||
// Active filter: format is set, or language is not just default
|
||||
const hasLanguageFilter = !(languageFilter.length === 1 && languageFilter[0] === LANGUAGE_OPTION_DEFAULT);
|
||||
const hasActiveFilter = formatFilter !== '' || hasLanguageFilter;
|
||||
return (
|
||||
{/* Sort dropdown - only show if source has sortable columns */}
|
||||
{sortableColumns.length > 0 && (
|
||||
<Dropdown
|
||||
align="right"
|
||||
widthClassName="w-auto flex-shrink-0"
|
||||
panelClassName="w-48"
|
||||
renderTrigger={({ isOpen, toggle }) => (
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggle}
|
||||
className={`relative p-2 rounded-full transition-colors ${
|
||||
isOpen
|
||||
? 'bg-gray-200 dark:bg-gray-700 text-gray-900 dark:text-gray-100'
|
||||
: 'text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-200 hover:bg-gray-100 dark:hover:bg-gray-800'
|
||||
className={`relative p-2.5 rounded-full transition-colors hover-surface text-gray-500 dark:text-gray-400 ${
|
||||
isOpen ? 'bg-[var(--hover-surface)]' : ''
|
||||
}`}
|
||||
aria-label="Filter releases"
|
||||
aria-label="Sort releases"
|
||||
>
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={1.5}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M12 3c2.755 0 5.455.232 8.083.678.533.09.917.556.917 1.096v1.044a2.25 2.25 0 0 1-.659 1.591l-5.432 5.432a2.25 2.25 0 0 0-.659 1.591v2.927a2.25 2.25 0 0 1-1.244 2.013L9.75 21v-6.568a2.25 2.25 0 0 0-.659-1.591L3.659 7.409A2.25 2.25 0 0 1 3 5.818V4.774c0-.54.384-1.006.917-1.096A48.32 48.32 0 0 1 12 3Z" />
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M3 7.5 7.5 3m0 0L12 7.5M7.5 3v13.5m13.5 0L16.5 21m0 0L12 16.5m4.5 4.5V7.5" />
|
||||
</svg>
|
||||
{hasActiveFilter && (
|
||||
{currentSort && (
|
||||
<span className="absolute top-1 right-1 w-2 h-2 bg-emerald-500 rounded-full" />
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
}}
|
||||
>
|
||||
)}
|
||||
>
|
||||
{({ close }) => (
|
||||
<div className="py-1">
|
||||
{/* Default option - no client-side sorting */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
handleSortChange(null, null);
|
||||
close();
|
||||
}}
|
||||
className={`w-full px-3 py-2 text-left text-sm flex items-center justify-between hover-surface rounded ${
|
||||
!currentSort
|
||||
? 'text-emerald-600 dark:text-emerald-400 font-medium'
|
||||
: 'text-gray-700 dark:text-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span>Default</span>
|
||||
{!currentSort && (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={2}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="m4.5 12.75 6 6 9-13.5" />
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
{sortableColumns.map((col) => {
|
||||
const sortKey = col.sort_key || col.key;
|
||||
const isSelected = currentSort?.key === sortKey;
|
||||
const direction = isSelected ? currentSort?.direction : null;
|
||||
return (
|
||||
<button
|
||||
key={sortKey}
|
||||
type="button"
|
||||
onClick={() => {
|
||||
handleSortChange(sortKey, col);
|
||||
// Don't close - allow toggling direction
|
||||
if (!isSelected) close();
|
||||
}}
|
||||
className={`w-full px-3 py-2 text-left text-sm flex items-center justify-between hover-surface rounded ${
|
||||
isSelected
|
||||
? 'text-emerald-600 dark:text-emerald-400 font-medium'
|
||||
: 'text-gray-700 dark:text-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span>{col.label}</span>
|
||||
{isSelected && direction && (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={2}>
|
||||
{direction === 'asc' ? (
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M4.5 15.75l7.5-7.5 7.5 7.5" />
|
||||
) : (
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M19.5 8.25l-7.5 7.5-7.5-7.5" />
|
||||
)}
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</Dropdown>
|
||||
)}
|
||||
|
||||
{/* Filter funnel button - stays fixed */}
|
||||
{/* Only show filter button if source supports at least one filter type */}
|
||||
{((columnConfig.supported_filters?.includes('format') && availableFormats.length > 0) ||
|
||||
(columnConfig.supported_filters?.includes('language') && bookLanguages.length > 0)) && (
|
||||
<Dropdown
|
||||
align="right"
|
||||
widthClassName="w-auto flex-shrink-0"
|
||||
panelClassName="w-56"
|
||||
noScrollLimit
|
||||
renderTrigger={({ isOpen, toggle }) => {
|
||||
// Active filter: format is set, or language is not just default
|
||||
const hasLanguageFilter = !(languageFilter.length === 1 && languageFilter[0] === LANGUAGE_OPTION_DEFAULT);
|
||||
const hasActiveFilter = formatFilter !== '' || hasLanguageFilter;
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggle}
|
||||
className={`relative p-2.5 rounded-full transition-colors hover-surface text-gray-500 dark:text-gray-400 ${
|
||||
isOpen ? 'bg-[var(--hover-surface)]' : ''
|
||||
}`}
|
||||
aria-label="Filter releases"
|
||||
>
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" strokeWidth={1.5}>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M12 3c2.755 0 5.455.232 8.083.678.533.09.917.556.917 1.096v1.044a2.25 2.25 0 0 1-.659 1.591l-5.432 5.432a2.25 2.25 0 0 0-.659 1.591v2.927a2.25 2.25 0 0 1-1.244 2.013L9.75 21v-6.568a2.25 2.25 0 0 0-.659-1.591L3.659 7.409A2.25 2.25 0 0 1 3 5.818V4.774c0-.54.384-1.006.917-1.096A48.32 48.32 0 0 1 12 3Z" />
|
||||
</svg>
|
||||
{hasActiveFilter && (
|
||||
<span className="absolute top-1 right-1 w-2 h-2 bg-emerald-500 rounded-full" />
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
}}
|
||||
>
|
||||
{({ close }) => (
|
||||
<div className="p-4 space-y-4">
|
||||
{columnConfig.supported_filters?.includes('format') && availableFormats.length > 0 && (
|
||||
@@ -1538,8 +1569,8 @@ export const ReleaseModal = ({
|
||||
defaultLanguageCodes={defaultLanguages}
|
||||
/>
|
||||
)}
|
||||
{/* Apply button - for AA, re-fetches with language filter; for others, just closes */}
|
||||
{activeTab === 'direct_download' && (
|
||||
{/* Apply button - re-fetch with server-side filters/expansion (e.g. language-aware searches) */}
|
||||
{(activeTab === 'direct_download' || activeTab === 'prowlarr') && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={async () => {
|
||||
@@ -1575,7 +1606,7 @@ export const ReleaseModal = ({
|
||||
: langCodes;
|
||||
|
||||
const response = await getReleases(
|
||||
provider, bookId, activeTab, book.title, book.author, false, languagesParam, contentType
|
||||
provider, bookId, activeTab, book.title, book.author, false, languagesParam, contentType, manualQuery.trim() || undefined
|
||||
);
|
||||
setCachedReleases(provider, bookId, activeTab, contentType, response);
|
||||
setReleasesBySource((prev) => ({ ...prev, [activeTab]: response }));
|
||||
@@ -1595,10 +1626,86 @@ export const ReleaseModal = ({
|
||||
)}
|
||||
</Dropdown>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Manual query panel (below source tabs) */}
|
||||
{showManualQuery && (
|
||||
<div className="px-5 py-3 border-b border-[var(--border-muted)] bg-[var(--bg)] sm:bg-[var(--bg-soft)]">
|
||||
<form
|
||||
className="flex items-center gap-2"
|
||||
onSubmit={async (e) => {
|
||||
e.preventDefault();
|
||||
if (!book?.provider || !book?.provider_id) return;
|
||||
|
||||
const q = manualQuery.trim();
|
||||
if (!q) return;
|
||||
|
||||
const provider = book.provider;
|
||||
const bookId = book.provider_id;
|
||||
|
||||
// Clear cache + clear visible results so user gets feedback.
|
||||
const key = getCacheKey(provider, bookId, activeTab, contentType);
|
||||
releaseCache.delete(key);
|
||||
cacheTimestamps.delete(key);
|
||||
setExpandedBySource((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[activeTab];
|
||||
return next;
|
||||
});
|
||||
setErrorBySource((prev) => ({ ...prev, [activeTab]: null }));
|
||||
setReleasesBySource((prev) => ({ ...prev, [activeTab]: null }));
|
||||
|
||||
setLoadingBySource((prev) => ({ ...prev, [activeTab]: true }));
|
||||
try {
|
||||
const response = await getReleases(
|
||||
provider,
|
||||
bookId,
|
||||
activeTab,
|
||||
book.title,
|
||||
book.author,
|
||||
false,
|
||||
undefined,
|
||||
contentType,
|
||||
q
|
||||
);
|
||||
setCachedReleases(provider, bookId, activeTab, contentType, response);
|
||||
setReleasesBySource((prev) => ({ ...prev, [activeTab]: response }));
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Failed to fetch releases';
|
||||
setErrorBySource((prev) => ({ ...prev, [activeTab]: message }));
|
||||
} finally {
|
||||
setLoadingBySource((prev) => ({ ...prev, [activeTab]: false }));
|
||||
}
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="text"
|
||||
value={manualQuery}
|
||||
onChange={(e) => setManualQuery(e.target.value)}
|
||||
placeholder="Type a custom search query (overrides all sources)"
|
||||
className="w-full px-3 py-2 text-sm rounded-lg border border-[var(--border-muted)] bg-[var(--bg)] text-[var(--text)]"
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={currentTabLoading || !manualQuery.trim()}
|
||||
className={`px-3 py-2 text-sm font-medium text-white rounded-lg transition-colors ${
|
||||
currentTabLoading || !manualQuery.trim()
|
||||
? 'bg-emerald-600/60 cursor-not-allowed'
|
||||
: 'bg-emerald-600 hover:bg-emerald-700'
|
||||
}`}
|
||||
>
|
||||
{currentTabLoading ? 'Searching…' : 'Search'}
|
||||
</button>
|
||||
</form>
|
||||
<p className="mt-2 text-xs text-gray-500 dark:text-gray-400">
|
||||
Manual query overrides ISBN/title/author/language expansion.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Release list content */}
|
||||
<div className="min-h-[200px]">
|
||||
{sourcesLoading ? (
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
OrderableListItem,
|
||||
ActionButtonConfig,
|
||||
HeadingFieldConfig,
|
||||
ShowWhenCondition,
|
||||
TableFieldConfig,
|
||||
} from '../../types/settings';
|
||||
import { FieldWrapper } from './shared';
|
||||
import {
|
||||
@@ -25,6 +27,7 @@ import {
|
||||
OrderableListField,
|
||||
ActionButton,
|
||||
HeadingField,
|
||||
TableField,
|
||||
} from './fields';
|
||||
|
||||
interface SettingsContentProps {
|
||||
@@ -38,6 +41,24 @@ interface SettingsContentProps {
|
||||
isUniversalMode?: boolean; // Whether app is in Universal search mode
|
||||
}
|
||||
|
||||
function evaluateShowWhenCondition(
|
||||
showWhen: ShowWhenCondition,
|
||||
values: Record<string, unknown>
|
||||
): boolean {
|
||||
const currentValue = values[showWhen.field];
|
||||
|
||||
if (showWhen.notEmpty) {
|
||||
if (Array.isArray(currentValue)) {
|
||||
return currentValue.length > 0;
|
||||
}
|
||||
return currentValue !== undefined && currentValue !== null && currentValue !== '';
|
||||
}
|
||||
|
||||
return Array.isArray(showWhen.value)
|
||||
? showWhen.value.includes(currentValue as string)
|
||||
: currentValue === showWhen.value;
|
||||
}
|
||||
|
||||
// Check if a field should be visible based on showWhen condition and search mode
|
||||
function isFieldVisible(
|
||||
field: SettingsField,
|
||||
@@ -52,20 +73,11 @@ function isFieldVisible(
|
||||
const showWhen = field.showWhen;
|
||||
if (!showWhen) return true;
|
||||
|
||||
const currentValue = values[showWhen.field];
|
||||
|
||||
// Handle notEmpty condition - show when field has any non-empty value
|
||||
if (showWhen.notEmpty) {
|
||||
if (Array.isArray(currentValue)) {
|
||||
return currentValue.length > 0;
|
||||
}
|
||||
return currentValue !== undefined && currentValue !== null && currentValue !== '';
|
||||
if (Array.isArray(showWhen)) {
|
||||
return showWhen.every((condition) => evaluateShowWhenCondition(condition, values));
|
||||
}
|
||||
|
||||
// Handle array of allowed values or single value
|
||||
return Array.isArray(showWhen.value)
|
||||
? showWhen.value.includes(currentValue as string)
|
||||
: currentValue === showWhen.value;
|
||||
return evaluateShowWhenCondition(showWhen, values);
|
||||
}
|
||||
|
||||
// Check if a field should be disabled based on disabledWhen condition
|
||||
@@ -117,7 +129,8 @@ const renderField = (
|
||||
value: unknown,
|
||||
onChange: (value: unknown) => void,
|
||||
onAction: () => Promise<ActionResult>,
|
||||
isDisabled: boolean
|
||||
isDisabled: boolean,
|
||||
allValues: Record<string, unknown> // All form values for cascading dropdown support
|
||||
) => {
|
||||
switch (field.type) {
|
||||
case 'TextField':
|
||||
@@ -156,15 +169,26 @@ const renderField = (
|
||||
disabled={isDisabled}
|
||||
/>
|
||||
);
|
||||
case 'SelectField':
|
||||
case 'SelectField': {
|
||||
const selectConfig = field as SelectFieldConfig;
|
||||
// Get filter value for cascading dropdowns
|
||||
const rawFilterValue = selectConfig.filterByField
|
||||
? allValues[selectConfig.filterByField]
|
||||
: undefined;
|
||||
const filterValue =
|
||||
rawFilterValue === undefined || rawFilterValue === null || rawFilterValue === ''
|
||||
? undefined
|
||||
: String(rawFilterValue);
|
||||
return (
|
||||
<SelectField
|
||||
field={field as SelectFieldConfig}
|
||||
field={selectConfig}
|
||||
value={(value as string) ?? ''}
|
||||
onChange={onChange}
|
||||
disabled={isDisabled}
|
||||
filterValue={filterValue}
|
||||
/>
|
||||
);
|
||||
}
|
||||
case 'MultiSelectField':
|
||||
return (
|
||||
<MultiSelectField
|
||||
@@ -185,6 +209,15 @@ const renderField = (
|
||||
);
|
||||
case 'ActionButton':
|
||||
return <ActionButton field={field as ActionButtonConfig} onAction={onAction} disabled={isDisabled} />;
|
||||
case 'TableField':
|
||||
return (
|
||||
<TableField
|
||||
field={field as TableFieldConfig}
|
||||
value={(value as Record<string, unknown>[]) ?? []}
|
||||
onChange={onChange}
|
||||
disabled={isDisabled}
|
||||
/>
|
||||
);
|
||||
case 'HeadingField':
|
||||
return <HeadingField field={field as HeadingFieldConfig} />;
|
||||
default:
|
||||
@@ -227,24 +260,25 @@ export const SettingsContent = ({
|
||||
>
|
||||
<div className="space-y-5">
|
||||
{visibleFields.map((field) => {
|
||||
const disabledState = getDisabledState(field, values);
|
||||
return (
|
||||
<FieldWrapper
|
||||
key={`${tab.name}-${field.key}`}
|
||||
field={field}
|
||||
disabledOverride={disabledState.disabled}
|
||||
disabledReasonOverride={disabledState.reason}
|
||||
>
|
||||
{renderField(
|
||||
field,
|
||||
values[field.key],
|
||||
(v) => onChange(field.key, v),
|
||||
() => onAction(field.key),
|
||||
disabledState.disabled
|
||||
)}
|
||||
</FieldWrapper>
|
||||
);
|
||||
})}
|
||||
const disabledState = getDisabledState(field, values);
|
||||
return (
|
||||
<FieldWrapper
|
||||
key={`${tab.name}-${field.key}`}
|
||||
field={field}
|
||||
disabledOverride={disabledState.disabled}
|
||||
disabledReasonOverride={disabledState.reason}
|
||||
>
|
||||
{renderField(
|
||||
field,
|
||||
values[field.key],
|
||||
(v) => onChange(field.key, v),
|
||||
() => onAction(field.key),
|
||||
disabledState.disabled,
|
||||
values
|
||||
)}
|
||||
</FieldWrapper>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ interface HeadingFieldProps {
|
||||
}
|
||||
|
||||
export const HeadingField = ({ field }: HeadingFieldProps) => (
|
||||
<div className="pb-2">
|
||||
<div className="pb-1 [&:not(:first-child)]:pt-5 [&:not(:first-child)]:mt-1 [&:not(:first-child)]:border-t [&:not(:first-child)]:border-black/10 [&:not(:first-child)]:dark:border-white/10">
|
||||
<h3 className="text-base font-semibold mb-1">{field.title}</h3>
|
||||
{field.description && (
|
||||
<p className="text-sm opacity-70">
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { useEffect, useMemo, useRef } from 'react';
|
||||
import { SelectFieldConfig } from '../../../types/settings';
|
||||
import { DropdownList } from '../../DropdownList';
|
||||
|
||||
@@ -6,17 +7,53 @@ interface SelectFieldProps {
|
||||
value: string;
|
||||
onChange: (value: string) => void;
|
||||
disabled?: boolean;
|
||||
filterValue?: string;
|
||||
}
|
||||
|
||||
export const SelectField = ({ field, value, onChange, disabled }: SelectFieldProps) => {
|
||||
// disabled prop is already computed by SettingsContent.getDisabledState()
|
||||
export const SelectField = ({ field, value, onChange, disabled, filterValue }: SelectFieldProps) => {
|
||||
const isDisabled = disabled ?? false;
|
||||
|
||||
const prevFilterValue = useRef(filterValue);
|
||||
|
||||
const normalizedOptions = useMemo(
|
||||
() =>
|
||||
field.options.map((opt) => ({
|
||||
...opt,
|
||||
value: String(opt.value),
|
||||
childOf:
|
||||
opt.childOf === undefined || opt.childOf === null
|
||||
? undefined
|
||||
: String(opt.childOf),
|
||||
label: opt.label ?? String(opt.value),
|
||||
})),
|
||||
[field.options]
|
||||
);
|
||||
|
||||
// Filter options based on filterValue (cascading dropdown support)
|
||||
const filteredOptions = useMemo(() => {
|
||||
if (!filterValue) {
|
||||
return normalizedOptions.filter((opt) => !opt.childOf);
|
||||
}
|
||||
// Filter to options that belong to the selected parent or have no parent
|
||||
return normalizedOptions.filter((opt) => !opt.childOf || opt.childOf === filterValue);
|
||||
}, [normalizedOptions, filterValue]);
|
||||
|
||||
// Clear selection when filter value changes and current value is not in filtered options
|
||||
useEffect(() => {
|
||||
if (prevFilterValue.current !== filterValue && filterValue !== undefined) {
|
||||
const currentValueInOptions = filteredOptions.some((opt) => opt.value === value);
|
||||
if (!currentValueInOptions && value) {
|
||||
onChange('');
|
||||
}
|
||||
}
|
||||
prevFilterValue.current = filterValue;
|
||||
}, [filterValue, filteredOptions, value, onChange]);
|
||||
|
||||
// Use field's default value as fallback when value is empty
|
||||
const effectiveValue = value || field.default || '';
|
||||
|
||||
// Convert options to DropdownList format
|
||||
const dropdownOptions = field.options.map((opt) => ({
|
||||
const dropdownOptions = filteredOptions.map((opt) => ({
|
||||
value: opt.value,
|
||||
label: opt.label,
|
||||
description: opt.description,
|
||||
@@ -30,7 +67,7 @@ export const SelectField = ({ field, value, onChange, disabled }: SelectFieldPro
|
||||
|
||||
if (isDisabled) {
|
||||
// When disabled, show a static display instead of the dropdown
|
||||
const selectedOption = field.options.find((opt) => opt.value === effectiveValue);
|
||||
const selectedOption = filteredOptions.find((opt) => opt.value === effectiveValue);
|
||||
return (
|
||||
<div className="w-full px-3 py-2 rounded-lg border border-[var(--border-muted)] bg-[var(--bg-soft)] text-sm opacity-60 cursor-not-allowed">
|
||||
{selectedOption?.label || 'Select...'}
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { useMemo } from 'react';
|
||||
import { TableFieldConfig, TableFieldColumn } from '../../../types/settings';
|
||||
import { DropdownList } from '../../DropdownList';
|
||||
|
||||
interface TableFieldProps {
|
||||
field: TableFieldConfig;
|
||||
value: Record<string, unknown>[];
|
||||
onChange: (value: Record<string, unknown>[]) => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
function defaultCellValue(column: TableFieldColumn): unknown {
|
||||
if (column.defaultValue !== undefined) {
|
||||
return column.defaultValue;
|
||||
}
|
||||
if (column.type === 'checkbox') {
|
||||
return false;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function normalizeRows(rows: Record<string, unknown>[], columns: TableFieldColumn[]): Record<string, unknown>[] {
|
||||
return (rows ?? []).map((row) => {
|
||||
const normalized: Record<string, unknown> = { ...row };
|
||||
for (const col of columns) {
|
||||
if (!(col.key in normalized)) {
|
||||
normalized[col.key] = defaultCellValue(col);
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
});
|
||||
}
|
||||
|
||||
export const TableField = ({ field, value, onChange, disabled }: TableFieldProps) => {
|
||||
const isDisabled = disabled ?? false;
|
||||
|
||||
const columns = useMemo(() => field.columns ?? [], [field.columns]);
|
||||
const rows = useMemo(() => normalizeRows(value ?? [], columns), [value, columns]);
|
||||
|
||||
const updateCell = (rowIndex: number, key: string, cellValue: unknown) => {
|
||||
const next = rows.map((row, idx) => (idx === rowIndex ? { ...row, [key]: cellValue } : row));
|
||||
onChange(next);
|
||||
};
|
||||
|
||||
const addRow = () => {
|
||||
const newRow: Record<string, unknown> = {};
|
||||
columns.forEach((col) => {
|
||||
newRow[col.key] = defaultCellValue(col);
|
||||
});
|
||||
onChange([...(rows ?? []), newRow]);
|
||||
};
|
||||
|
||||
const removeRow = (rowIndex: number) => {
|
||||
const next = rows.filter((_, idx) => idx !== rowIndex);
|
||||
onChange(next);
|
||||
};
|
||||
|
||||
if (rows.length === 0) {
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
{field.emptyMessage && <p className="text-sm opacity-70">{field.emptyMessage}</p>}
|
||||
<button
|
||||
type="button"
|
||||
onClick={addRow}
|
||||
disabled={isDisabled}
|
||||
className="px-3 py-2 rounded-lg text-sm font-medium
|
||||
bg-[var(--bg-soft)] border border-[var(--border-muted)]
|
||||
hover:bg-[var(--hover-surface)] transition-colors
|
||||
disabled:opacity-60 disabled:cursor-not-allowed"
|
||||
>
|
||||
{field.addLabel || 'Add'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Use minmax(0, ...) so the grid can shrink inside the settings modal.
|
||||
// Use fixed width for delete button column to ensure header/data alignment.
|
||||
const gridTemplate = 'sm:grid-cols-[minmax(0,180px)_minmax(0,1fr)_minmax(0,1fr)_2rem]';
|
||||
|
||||
return (
|
||||
<div className="space-y-3 min-w-0">
|
||||
<div className={`hidden sm:grid ${gridTemplate} gap-3 items-start min-w-0 text-xs font-medium opacity-70`}>
|
||||
{columns.map((col) => (
|
||||
<div key={col.key} className="min-w-0 truncate">
|
||||
{col.label}
|
||||
</div>
|
||||
))}
|
||||
<div />
|
||||
</div>
|
||||
|
||||
<div className="space-y-3 min-w-0">
|
||||
{rows.map((row, rowIndex) => (
|
||||
<div
|
||||
key={rowIndex}
|
||||
className={`grid grid-cols-1 ${gridTemplate} gap-3 items-start min-w-0`}
|
||||
style={{ overflow: 'visible' }}
|
||||
>
|
||||
{columns.map((col) => {
|
||||
const cellValue = row[col.key];
|
||||
|
||||
const mobileLabel = <div className="sm:hidden text-xs font-medium opacity-70">{col.label}</div>;
|
||||
|
||||
if (col.type === 'checkbox') {
|
||||
return (
|
||||
<div key={col.key} className="flex flex-col gap-1 min-w-0">
|
||||
{mobileLabel}
|
||||
<div className="pt-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={Boolean(cellValue)}
|
||||
onChange={(e) => updateCell(rowIndex, col.key, e.target.checked)}
|
||||
disabled={isDisabled}
|
||||
className="h-4 w-4 rounded border-gray-300 text-sky-600 focus:ring-sky-500
|
||||
disabled:opacity-60 disabled:cursor-not-allowed"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (col.type === 'select') {
|
||||
const options = (col.options ?? []).map((opt) => ({
|
||||
value: String(opt.value),
|
||||
label: opt.label,
|
||||
description: opt.description,
|
||||
}));
|
||||
|
||||
return (
|
||||
<div key={col.key} className="flex flex-col gap-1 min-w-0">
|
||||
{mobileLabel}
|
||||
{isDisabled ? (
|
||||
<div className="w-full px-3 py-2 rounded-lg border border-[var(--border-muted)] bg-[var(--bg-soft)] text-sm opacity-60 cursor-not-allowed">
|
||||
{options.find((o) => o.value === String(cellValue ?? ''))?.label || 'Select...'}
|
||||
</div>
|
||||
) : (
|
||||
<DropdownList
|
||||
options={options}
|
||||
value={String(cellValue ?? '')}
|
||||
onChange={(val) => updateCell(rowIndex, col.key, Array.isArray(val) ? val[0] : val)}
|
||||
placeholder={col.placeholder || 'Select...'}
|
||||
widthClassName="w-full"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// text/path
|
||||
return (
|
||||
<div key={col.key} className="flex flex-col gap-1 min-w-0">
|
||||
{mobileLabel}
|
||||
<input
|
||||
type="text"
|
||||
value={String(cellValue ?? '')}
|
||||
onChange={(e) => updateCell(rowIndex, col.key, e.target.value)}
|
||||
placeholder={col.placeholder}
|
||||
disabled={isDisabled}
|
||||
className="w-full px-3 py-2 rounded-lg border border-[var(--border-muted)]
|
||||
bg-[var(--bg-soft)] text-sm
|
||||
focus:outline-none focus:ring-2 focus:ring-sky-500/50 focus:border-sky-500
|
||||
disabled:opacity-60 disabled:cursor-not-allowed
|
||||
transition-colors"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div className="flex items-start pt-1.5">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeRow(rowIndex)}
|
||||
disabled={isDisabled}
|
||||
className="p-1.5 rounded-full hover:bg-[var(--hover-surface)]
|
||||
disabled:opacity-60 disabled:cursor-not-allowed"
|
||||
aria-label="Remove row"
|
||||
>
|
||||
<svg
|
||||
className="w-4 h-4"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth={1.5}
|
||||
stroke="currentColor"
|
||||
>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="col-span-full border-t border-[var(--border-muted)] opacity-60" />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={addRow}
|
||||
disabled={isDisabled}
|
||||
className="px-3 py-2 rounded-lg text-sm font-medium
|
||||
bg-[var(--bg-soft)] border border-[var(--border-muted)]
|
||||
hover:bg-[var(--hover-surface)] transition-colors
|
||||
disabled:opacity-60 disabled:cursor-not-allowed"
|
||||
>
|
||||
{field.addLabel || 'Add'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -6,4 +6,5 @@ export { SelectField } from './SelectField';
|
||||
export { MultiSelectField } from './MultiSelectField';
|
||||
export { OrderableListField } from './OrderableListField';
|
||||
export { ActionButton } from './ActionButton';
|
||||
export { TableField } from './TableField';
|
||||
export { HeadingField } from './HeadingField';
|
||||
|
||||
@@ -66,8 +66,9 @@ export const FieldWrapper = ({
|
||||
disabledOverride,
|
||||
disabledReasonOverride,
|
||||
}: FieldWrapperProps) => {
|
||||
// Action buttons and headings handle their own layout
|
||||
if (field.type === 'ActionButton' || field.type === 'HeadingField') {
|
||||
// Action buttons, headings, and table fields handle their own layout
|
||||
// Table fields have column headers, so they don't need a separate label
|
||||
if (field.type === 'ActionButton' || field.type === 'HeadingField' || field.type === 'TableField') {
|
||||
return <>{children}</>;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { createContext, useContext, useEffect, useRef, useState, ReactNode } from 'react';
|
||||
import { io, Socket } from 'socket.io-client';
|
||||
import { withBasePath } from '../utils/basePath';
|
||||
|
||||
interface SocketContextValue {
|
||||
socket: Socket | null;
|
||||
@@ -23,11 +24,12 @@ export const SocketProvider = ({ children }: SocketProviderProps) => {
|
||||
const wsUrl = window.location.port === '5173'
|
||||
? 'http://localhost:8084'
|
||||
: window.location.origin;
|
||||
const socketPath = withBasePath('/socket.io');
|
||||
|
||||
console.log('SocketProvider: Connecting to', wsUrl);
|
||||
|
||||
const socket = io(wsUrl, {
|
||||
path: '/socket.io',
|
||||
path: socketPath,
|
||||
transports: ['polling', 'websocket'],
|
||||
withCredentials: false,
|
||||
});
|
||||
|
||||
@@ -83,7 +83,11 @@ export function useAuth(options: UseAuthOptions = {}): UseAuthReturn {
|
||||
|
||||
const handleLogout = useCallback(async () => {
|
||||
try {
|
||||
await logout();
|
||||
const { logout_url } = await logout();
|
||||
if (logout_url?.startsWith('https://') || logout_url?.startsWith('http://')) {
|
||||
window.location.href = logout_url;
|
||||
return;
|
||||
}
|
||||
setIsAuthenticated(false);
|
||||
onLogoutSuccess?.();
|
||||
navigate('/login', { replace: true });
|
||||
|
||||
@@ -37,6 +37,11 @@ function getFieldValue(field: SettingsField): unknown {
|
||||
if (field.type === 'ActionButton' || field.type === 'HeadingField') {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (field.type === 'TableField') {
|
||||
return (field as unknown as { value?: unknown }).value ?? [];
|
||||
}
|
||||
|
||||
// All other fields have a value property
|
||||
return field.value ?? '';
|
||||
}
|
||||
|
||||
@@ -3,13 +3,17 @@ import { createRoot } from 'react-dom/client';
|
||||
import { BrowserRouter } from 'react-router-dom';
|
||||
import { SocketProvider } from './contexts/SocketContext';
|
||||
import App from './App';
|
||||
import { getBasePath } from './utils/basePath';
|
||||
|
||||
const root = document.getElementById('root');
|
||||
if (!root) throw new Error('Root element not found');
|
||||
|
||||
const basePath = getBasePath();
|
||||
const routerBase = basePath === '/' ? undefined : basePath;
|
||||
|
||||
createRoot(root).render(
|
||||
<StrictMode>
|
||||
<BrowserRouter>
|
||||
<BrowserRouter basename={routerBase}>
|
||||
<SocketProvider>
|
||||
<App />
|
||||
</SocketProvider>
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { Book, StatusData, AppConfig, LoginCredentials, AuthResponse, ReleaseSource, ReleasesResponse } from '../types';
|
||||
import { SettingsResponse, ActionResult, UpdateResult } from '../types/settings';
|
||||
import { MetadataBookData, transformMetadataToBook } from '../utils/bookTransformers';
|
||||
import { getApiBase } from '../utils/basePath';
|
||||
|
||||
const API_BASE = '/api';
|
||||
const API_BASE = getApiBase();
|
||||
|
||||
// API endpoints
|
||||
const API = {
|
||||
@@ -41,9 +42,15 @@ export class TimeoutError extends Error {
|
||||
const DEFAULT_TIMEOUT_MS = 30000;
|
||||
|
||||
// Utility function for JSON fetch with credentials and timeout
|
||||
async function fetchJSON<T>(url: string, opts: RequestInit = {}, timeoutMs: number = DEFAULT_TIMEOUT_MS): Promise<T> {
|
||||
async function fetchJSON<T>(
|
||||
url: string,
|
||||
opts: RequestInit = {},
|
||||
timeoutMs: number | null = DEFAULT_TIMEOUT_MS
|
||||
): Promise<T> {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const timeoutId = timeoutMs && timeoutMs > 0
|
||||
? setTimeout(() => controller.abort(), timeoutMs)
|
||||
: null;
|
||||
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
@@ -59,13 +66,16 @@ async function fetchJSON<T>(url: string, opts: RequestInit = {}, timeoutMs: numb
|
||||
if (!res.ok) {
|
||||
// Try to parse error message from response body
|
||||
let errorMessage = `${res.status} ${res.statusText}`;
|
||||
let hasServerMessage = false;
|
||||
try {
|
||||
const errorData = await res.json();
|
||||
// Prefer user-friendly 'message' field, fall back to 'error'
|
||||
if (errorData.message) {
|
||||
errorMessage = errorData.message;
|
||||
hasServerMessage = true;
|
||||
} else if (errorData.error) {
|
||||
errorMessage = errorData.error;
|
||||
hasServerMessage = true;
|
||||
}
|
||||
} catch (e) {
|
||||
// Log parse failure for debugging - server may have returned non-JSON (e.g., HTML error page)
|
||||
@@ -74,7 +84,9 @@ async function fetchJSON<T>(url: string, opts: RequestInit = {}, timeoutMs: numb
|
||||
|
||||
// Provide helpful message for gateway/proxy errors
|
||||
if (res.status === 502 || res.status === 503 || res.status === 504) {
|
||||
errorMessage = `Server unavailable (${res.status}). If using a reverse proxy, check its configuration.`;
|
||||
if (!hasServerMessage) {
|
||||
errorMessage = `Server unavailable (${res.status}). If using a reverse proxy, check its configuration.`;
|
||||
}
|
||||
}
|
||||
|
||||
// Throw appropriate error based on status code
|
||||
@@ -93,7 +105,9 @@ async function fetchJSON<T>(url: string, opts: RequestInit = {}, timeoutMs: numb
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,6 +213,7 @@ export const downloadRelease = async (release: {
|
||||
series_name?: string;
|
||||
series_position?: number;
|
||||
subtitle?: string;
|
||||
search_author?: string;
|
||||
}): Promise<void> => {
|
||||
await fetchJSON(`${API_BASE}/releases/download`, {
|
||||
method: 'POST',
|
||||
@@ -323,7 +338,8 @@ export const getReleases = async (
|
||||
author?: string,
|
||||
expandSearch?: boolean,
|
||||
languages?: string[],
|
||||
contentType?: string
|
||||
contentType?: string,
|
||||
manualQuery?: string
|
||||
): Promise<ReleasesResponse> => {
|
||||
const params = new URLSearchParams({
|
||||
provider,
|
||||
@@ -347,5 +363,9 @@ export const getReleases = async (
|
||||
if (contentType) {
|
||||
params.set('content_type', contentType);
|
||||
}
|
||||
return fetchJSON<ReleasesResponse>(`${API_BASE}/releases?${params.toString()}`);
|
||||
if (manualQuery) {
|
||||
params.set('manual_query', manualQuery);
|
||||
}
|
||||
// Let the backend control timeouts for release searches (can be long-running).
|
||||
return fetchJSON<ReleasesResponse>(`${API_BASE}/releases?${params.toString()}`, {}, null);
|
||||
};
|
||||
|
||||
@@ -46,6 +46,8 @@ export interface Book {
|
||||
series_position?: number; // This book's position (e.g., 3, 1.5 for novellas)
|
||||
series_count?: number; // Total books in the series
|
||||
subtitle?: string;
|
||||
search_title?: string;
|
||||
search_author?: string;
|
||||
}
|
||||
|
||||
// Status response types
|
||||
@@ -181,6 +183,7 @@ export interface AuthResponse {
|
||||
auth_required?: boolean;
|
||||
is_admin?: boolean;
|
||||
error?: string;
|
||||
logout_url?: string;
|
||||
}
|
||||
|
||||
// Type guard to check if a book is from a metadata provider
|
||||
@@ -279,6 +282,8 @@ export interface ReleasesResponse {
|
||||
provider_id: string;
|
||||
title: string;
|
||||
subtitle?: string;
|
||||
search_author?: string;
|
||||
search_title?: string;
|
||||
authors?: string[];
|
||||
isbn_10?: string;
|
||||
isbn_13?: string;
|
||||
|
||||
@@ -8,6 +8,7 @@ export type FieldType =
|
||||
| 'SelectField'
|
||||
| 'MultiSelectField'
|
||||
| 'OrderableListField'
|
||||
| 'TableField'
|
||||
| 'ActionButton'
|
||||
| 'HeadingField';
|
||||
|
||||
@@ -25,6 +26,8 @@ export interface ShowWhenCondition {
|
||||
notEmpty?: boolean; // If true, show when field has any non-empty value
|
||||
}
|
||||
|
||||
export type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
|
||||
|
||||
// Conditional disable configuration
|
||||
export interface DisabledWhenCondition {
|
||||
field: string; // The field key to check
|
||||
@@ -42,7 +45,7 @@ export interface BaseField {
|
||||
fromEnv?: boolean; // True if value is set via environment variable
|
||||
disabled?: boolean; // True if field is disabled/greyed out
|
||||
disabledReason?: string; // Explanation shown when field is disabled
|
||||
showWhen?: ShowWhenCondition; // Conditional visibility based on another field's value
|
||||
showWhen?: ShowWhen; // Conditional visibility based on another field's value
|
||||
disabledWhen?: DisabledWhenCondition; // Conditional disable based on another field's value
|
||||
requiresRestart?: boolean; // True if changing this setting requires a container restart
|
||||
universalOnly?: boolean; // Only show in Universal search mode (hide in Direct mode)
|
||||
@@ -80,6 +83,7 @@ export interface SelectFieldConfig extends BaseField {
|
||||
value: string;
|
||||
options: SelectOption[];
|
||||
default?: string;
|
||||
filterByField?: string; // Field key whose value filters options via childOf property
|
||||
}
|
||||
|
||||
export interface MultiSelectFieldConfig extends BaseField {
|
||||
@@ -115,6 +119,31 @@ export interface ActionButtonConfig extends BaseField {
|
||||
style: 'default' | 'primary' | 'danger';
|
||||
}
|
||||
|
||||
export interface TableFieldColumnOption {
|
||||
value: string;
|
||||
label: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export type TableFieldColumnType = 'text' | 'select' | 'checkbox' | 'path';
|
||||
|
||||
export interface TableFieldColumn {
|
||||
key: string;
|
||||
label: string;
|
||||
type: TableFieldColumnType;
|
||||
placeholder?: string;
|
||||
options?: TableFieldColumnOption[];
|
||||
defaultValue?: string | boolean;
|
||||
}
|
||||
|
||||
export interface TableFieldConfig extends BaseField {
|
||||
type: 'TableField';
|
||||
value: Record<string, unknown>[];
|
||||
columns: TableFieldColumn[];
|
||||
addLabel?: string;
|
||||
emptyMessage?: string;
|
||||
}
|
||||
|
||||
export interface HeadingFieldConfig {
|
||||
key: string;
|
||||
type: 'HeadingField';
|
||||
@@ -122,7 +151,7 @@ export interface HeadingFieldConfig {
|
||||
description?: string;
|
||||
linkUrl?: string;
|
||||
linkText?: string;
|
||||
showWhen?: ShowWhenCondition; // Conditional visibility based on another field's value
|
||||
showWhen?: ShowWhen; // Conditional visibility based on another field's value
|
||||
universalOnly?: boolean; // Only show in Universal search mode (hide in Direct mode)
|
||||
}
|
||||
|
||||
@@ -135,6 +164,7 @@ export type SettingsField =
|
||||
| SelectFieldConfig
|
||||
| MultiSelectFieldConfig
|
||||
| OrderableListFieldConfig
|
||||
| TableFieldConfig
|
||||
| ActionButtonConfig
|
||||
| HeadingFieldConfig;
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user