starred/shelfmark
1
0
Fork 0
mirror of https://github.com/calibrain/shelfmark.git synced 2026-09-24 13:40:21 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Linter followup (#868)

Browse Source 
Expanded Ruff rules and completed fixes
This commit is contained in:
Alex
2026-04-11 16:18:52 +01:00
committed by GitHub
parent 28eef75de0
commit 8d98e122ec
117 changed files with 2628 additions and 1037 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+27 -4
View File
@@ -66,17 +66,40 @@ extend-exclude = [".local"]
[tool.ruff.lint]
select = [
"F", "I", "UP", "B", "C4", "SIM", "PTH", "RET", "PIE", "FURB", "PERF", "TRY",
"ANN001", "ANN201", "ANN202", "ANN204",
"A", "DTZ", "N",
"BLE001",
"ANN001", "ANN002", "ANN003", "ANN201", "ANN202", "ANN204",
"E402",
"ERA001",
"E731",
"FBT002", "FBT003",
"S101",
"S110",
"S105", "S108",
"S311", "S324",
"S607", "S608",
"G003", "G004",
"PGH003",
"PLC0414",
"PLR1714",
"PLW1510",
"PLW2901",
"PLW0108",
"PT028",
"PYI034",
"Q000",
"RUF005", "RUF012", "RUF013", "RUF059", "RUF100",
"TC001", "TC003",
"TC001", "TC002", "TC003",
]
ignore = ["D", "EM", "FBT", "PLR2004", "UP035", "TRY003", "E501", "TD002", "S104", "S603"]
[tool.ruff.lint.per-file-ignores]
"tests/**/*.py" = [
"ANN",
"S101",
"S105",
"S108",
"S311",
]
ignore = ["UP035", "TRY003", "E501"]
[tool.basedpyright]
include = ["shelfmark"]
shelfmark/api/websocket.py
+5 -1
View File
@@ -1,15 +1,18 @@
"""WebSocket manager for real-time status updates."""
from __future__ import annotations
import logging
import threading
from typing import TYPE_CHECKING, Any
from flask import Flask
from flask_socketio import SocketIO, join_room, leave_room
if TYPE_CHECKING:
from collections.abc import Callable
from flask import Flask
logger = logging.getLogger(__name__)
@@ -17,6 +20,7 @@ class WebSocketManager:
"""Manages WebSocket connections and broadcasts."""
def __init__(self) -> None:
"""Initialize in-memory connection and room tracking."""
self.socketio: SocketIO | None = None
self._enabled = False
self._connection_count = 0
shelfmark/bypass/__init__.py
+1 -1
View File
@@ -1,5 +1,5 @@
"""Cloudflare bypass utilities."""
class BypassCancelledException(Exception):
class BypassCancelledError(Exception):
"""Raised when a bypass operation is cancelled."""
shelfmark/bypass/external_bypasser.py
+4 -3
View File
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
import requests
from shelfmark.bypass import BypassCancelledException
from shelfmark.bypass import BypassCancelledError
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url
@@ -18,6 +18,7 @@ if TYPE_CHECKING:
from shelfmark.download import network
logger = setup_logger(__name__)
_RNG = random.SystemRandom()
# Timeout constants (seconds)
CONNECT_TIMEOUT = 10
@@ -102,7 +103,7 @@ def _check_cancelled(cancel_flag: Event | None, context: str) -> None:
if cancel_flag and cancel_flag.is_set():
logger.info("External bypasser cancelled %s", context)
msg = "Bypass cancelled"
raise BypassCancelledException(msg)
raise BypassCancelledError(msg)
def _sleep_with_cancellation(seconds: float, cancel_flag: Event | None) -> None:
@@ -136,7 +137,7 @@ def get_bypassed_page(
if attempt == MAX_RETRY:
break
delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + _RNG.random()
logger.info(
"External bypasser attempt %s/%s failed, retrying in %.1fs",
attempt,
shelfmark/bypass/fingerprint.py
+5 -1
View File
@@ -21,9 +21,11 @@ COMMON_RESOLUTIONS = [
# Current screen size (module-level singleton)
_current_screen_size: tuple[int, int] | None = None
_RNG = random.SystemRandom()
def get_screen_size() -> tuple[int, int]:
"""Return the current synthetic screen size, generating one if needed."""
global _current_screen_size
if _current_screen_size is None:
_current_screen_size = _generate_screen_size()
@@ -36,6 +38,7 @@ def get_screen_size() -> tuple[int, int]:
def rotate_screen_size() -> tuple[int, int]:
"""Rotate to a new synthetic screen size and return it."""
global _current_screen_size
old_size = _current_screen_size
_current_screen_size = _generate_screen_size()
@@ -56,6 +59,7 @@ def rotate_screen_size() -> tuple[int, int]:
def clear_screen_size() -> None:
"""Clear the cached synthetic screen size."""
global _current_screen_size
_current_screen_size = None
@@ -63,4 +67,4 @@ def clear_screen_size() -> None:
def _generate_screen_size() -> tuple[int, int]:
resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS]
weights = [weight for _, _, weight in COMMON_RESOLUTIONS]
return random.choices(resolutions, weights=weights)[0]
return _RNG.choices(resolutions, weights=weights)[0]
shelfmark/bypass/internal_bypasser.py
+162 -118
View File
@@ -1,15 +1,20 @@
"""Internal Cloudflare bypass implementation using SeleniumBase and CDP helpers."""
import asyncio
import os
import random
import shutil
import signal
import socket
import stat
import subprocess
import tempfile
import threading
import time
import traceback
from contextlib import suppress
from datetime import datetime
from datetime import UTC, datetime
from http import HTTPStatus
from pathlib import Path
from threading import Event
from typing import Any
@@ -17,8 +22,9 @@ from urllib.parse import urlparse
import requests
from seleniumbase import cdp_driver
from seleniumbase.undetected.cdp_driver.connection import ProtocolException
from shelfmark.bypass import BypassCancelledException
from shelfmark.bypass import BypassCancelledError
from shelfmark.bypass.fingerprint import get_screen_size
from shelfmark.config import env
from shelfmark.config.env import LOG_DIR
@@ -30,8 +36,12 @@ from shelfmark.download.network import get_proxies, get_ssl_verify
logger = setup_logger(__name__)
SELENIUMBASE_RUNTIME_ROOT = Path("/tmp/shelfmark/seleniumbase")
SELENIUMBASE_RUNTIME_ROOT = Path(tempfile.gettempdir()) / "shelfmark" / "seleniumbase"
SELENIUMBASE_DOWNLOADS_DIR = SELENIUMBASE_RUNTIME_ROOT / "downloaded_files"
_BYPASSED_BODY_LENGTH_MIN = 100_000
_BYPASS_EMOJI_MATCH_MIN = 3
_LOADING_BODY_LENGTH_MAX = 50
_PAGE_BODY_PREVIEW_CHARS = 500
# Challenge detection indicators
CLOUDFLARE_INDICATORS = [
@@ -54,6 +64,35 @@ DISPLAY = {
"ffmpeg_output": None,
}
LOCKED = threading.Lock()
_PGREP_PATH = shutil.which("pgrep")
_PKILL_PATH = shutil.which("pkill")
_RNG = random.SystemRandom()
_CDP_OPERATION_ERRORS = (
asyncio.TimeoutError,
AttributeError,
NameError,
OSError,
ProtocolException,
RuntimeError,
TypeError,
ValueError,
)
_PATH_INSPECTION_ERRORS = (OSError, RuntimeError, TypeError, ValueError)
_REQUEST_OPERATION_ERRORS = (
OSError,
RuntimeError,
TypeError,
ValueError,
requests.RequestException,
)
_SUBPROCESS_OPERATION_ERRORS = (
OSError,
RuntimeError,
TypeError,
ValueError,
subprocess.SubprocessError,
)
def _describe_runtime_path(path: str | Path) -> str:
@@ -68,7 +107,7 @@ def _describe_runtime_path(path: str | Path) -> str:
return f"{path}{link_target} exists uid={st.st_uid} gid={st.st_gid} mode={oct(mode)}"
except FileNotFoundError:
return f"{path} missing"
except Exception as e:
except _PATH_INSPECTION_ERRORS as e:
return f"{path} error={type(e).__name__}: {e}"
@@ -85,16 +124,13 @@ class _CdpWorker:
self._loop = loop
self._ready.set()
loop.run_forever()
try:
with suppress(Exception):
pending = asyncio.all_tasks(loop)
for task in pending:
task.cancel()
if pending:
loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
except Exception:
pass
finally:
loop.close()
loop.close()
def start(self) -> None:
with self._lock:
@@ -108,12 +144,14 @@ class _CdpWorker:
)
self._thread.start()
if not self._ready.wait(timeout=10):
raise RuntimeError("CDP worker loop failed to start")
msg = "CDP worker loop failed to start"
raise RuntimeError(msg)
def run(self, coro: Any, timeout: float | None = None) -> Any:
self.start()
if not self._loop or self._loop.is_closed():
raise RuntimeError("CDP worker loop not available")
msg = "CDP worker loop not available"
raise RuntimeError(msg)
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
return future.result(timeout=timeout)
@@ -121,7 +159,7 @@ class _CdpWorker:
_CDP_WORKER = _CdpWorker()
# Cookie storage - shared with requests library for Cloudflare bypass
# Structure: {domain: {cookie_name: {value, expiry, ...}}}
# Nested mapping of domain to cookie name to cookie metadata.
_cf_cookies: dict[str, dict] = {}
_cf_cookies_lock = threading.Lock()
@@ -212,18 +250,18 @@ async def _extract_cookies_from_cdp(driver: Any, page: Any, url: str) -> None:
try:
try:
all_cookies = await driver.cookies.get_all(requests_cookie_format=True)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Failed to get cookies via CDP: %s", e)
return
try:
user_agent = await page.evaluate("navigator.userAgent")
except Exception:
except _CDP_OPERATION_ERRORS:
user_agent = None
_store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Failed to extract cookies: %s", e)
@@ -290,10 +328,18 @@ def _cleanup_orphan_processes() -> int:
logger.debug("Checking for orphan processes...")
logger.log_resource_usage()
if _PGREP_PATH is None or _PKILL_PATH is None:
logger.warning("Skipping orphan-process cleanup because pgrep/pkill are unavailable")
return 0
for proc_name in processes_to_kill:
try:
result = subprocess.run(
["pgrep", "-f", proc_name], capture_output=True, text=True, timeout=5
[_PGREP_PATH, "-f", proc_name],
capture_output=True,
check=False,
text=True,
timeout=5,
)
if result.returncode != 0 or not result.stdout.strip():
continue
@@ -303,7 +349,10 @@ def _cleanup_orphan_processes() -> int:
logger.info("Found %s orphan %s process(es), killing...", count, proc_name)
kill_result = subprocess.run(
["pkill", "-9", "-f", proc_name], capture_output=True, timeout=5
[_PKILL_PATH, "-9", "-f", proc_name],
capture_output=True,
check=False,
timeout=5,
)
if kill_result.returncode == 0:
total_killed += count
@@ -312,7 +361,7 @@ def _cleanup_orphan_processes() -> int:
except subprocess.TimeoutExpired:
logger.warning("Timeout while checking for %s processes", proc_name)
except Exception as e:
except _SUBPROCESS_OPERATION_ERRORS as e:
logger.debug("Error checking for %s processes: %s", proc_name, e)
if total_killed > 0:
@@ -329,16 +378,16 @@ async def _get_page_info(page: Any) -> tuple[str, str, str]:
"""Extract page title, body text, and current URL safely."""
try:
title = (await page.get_title() or "").lower()
except Exception:
except _CDP_OPERATION_ERRORS:
title = ""
try:
body = await page.evaluate("document.body ? document.body.innerText : ''")
body = (body or "").lower()
except Exception:
except _CDP_OPERATION_ERRORS:
body = ""
try:
current_url = await page.get_current_url() or ""
except Exception:
except _CDP_OPERATION_ERRORS:
current_url = ""
return title, body, current_url
@@ -358,85 +407,75 @@ def _has_cloudflare_patterns(body: str, url: str) -> bool:
async def _detect_challenge_type(page: Any) -> str:
"""Detect challenge type: 'cloudflare', 'ddos_guard', or 'none'."""
try:
title, body, current_url = await _get_page_info(page)
except Exception as e:
logger.warning("Error detecting challenge type: %s", e)
return "none"
else:
# DDOS-Guard indicators
if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS):
logger.debug("DDOS-Guard indicator found: '%s'", found)
return "ddos_guard"
title, body, current_url = await _get_page_info(page)
# Cloudflare indicators
if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS):
logger.debug("Cloudflare indicator found: '%s'", found)
return "cloudflare"
# DDOS-Guard indicators
if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS):
logger.debug("DDOS-Guard indicator found: '%s'", found)
return "ddos_guard"
# Check URL patterns
if _has_cloudflare_patterns(body, current_url):
return "cloudflare"
# Cloudflare indicators
if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS):
logger.debug("Cloudflare indicator found: '%s'", found)
return "cloudflare"
return "none"
# Check URL patterns
if _has_cloudflare_patterns(body, current_url):
return "cloudflare"
return "none"
async def _is_bypassed(page: Any, *, escape_emojis: bool = True) -> bool:
"""Check if the protection has been bypassed."""
try:
title, body, current_url = await _get_page_info(page)
except Exception as e:
logger.warning("Error checking bypass status: %s", e)
return False
title, body, current_url = await _get_page_info(page)
body_len = len(body.strip())
else:
body_len = len(body.strip())
# Long page content = probably bypassed
if body_len > _BYPASSED_BODY_LENGTH_MIN:
logger.debug("Page content too long, probably bypassed (len: %s)", body_len)
return True
# Long page content = probably bypassed
if body_len > 100000:
logger.debug("Page content too long, probably bypassed (len: %s)", body_len)
# Multiple emojis = probably real content
if escape_emojis:
import emoji
if len(emoji.emoji_list(body)) >= _BYPASS_EMOJI_MATCH_MIN:
logger.debug("Detected emojis in page, probably bypassed")
return True
# Multiple emojis = probably real content
if escape_emojis:
import emoji
# Check for protection indicators (means NOT bypassed)
if _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS):
return False
if len(emoji.emoji_list(body)) >= 3:
logger.debug("Detected emojis in page, probably bypassed")
return True
# Cloudflare URL patterns
if _has_cloudflare_patterns(body, current_url):
logger.debug("Cloudflare patterns detected in page")
return False
# Check for protection indicators (means NOT bypassed)
if _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS):
return False
# Page too short = still loading
if body_len < _LOADING_BODY_LENGTH_MAX:
logger.debug("Page content too short, might still be loading")
return False
# Cloudflare URL patterns
if _has_cloudflare_patterns(body, current_url):
logger.debug("Cloudflare patterns detected in page")
return False
# Page too short = still loading
if body_len < 50:
logger.debug("Page content too short, might still be loading")
return False
logger.debug("Bypass check passed - Title: '%s', Body length: %s", title[:100], body_len)
return True
logger.debug("Bypass check passed - Title: '%s', Body length: %s", title[:100], body_len)
return True
async def _bypass_method_humanlike(page: Any) -> bool:
"""Human-like behavior with scroll, wait, and reload."""
try:
logger.debug("Attempting bypass: human-like interaction")
await asyncio.sleep(random.uniform(6, 10))
await asyncio.sleep(_RNG.uniform(6, 10))
try:
await page.evaluate("window.scrollTo(0, 10000);")
await page.wait()
await asyncio.sleep(random.uniform(1, 2))
await asyncio.sleep(_RNG.uniform(1, 2))
await page.evaluate("window.scrollTo(0, 0);")
await page.wait()
await asyncio.sleep(random.uniform(2, 3))
except Exception as e:
await asyncio.sleep(_RNG.uniform(2, 3))
except _CDP_OPERATION_ERRORS as e:
logger.debug("Scroll behavior failed: %s", e)
if await _is_bypassed(page):
@@ -444,19 +483,19 @@ async def _bypass_method_humanlike(page: Any) -> bool:
logger.debug("Trying page refresh...")
await page.reload(ignore_cache=True)
await asyncio.sleep(random.uniform(5, 8))
await asyncio.sleep(_RNG.uniform(5, 8))
if await _is_bypassed(page):
return True
try:
await page.solve_captcha()
await asyncio.sleep(random.uniform(3, 5))
except Exception as e:
await asyncio.sleep(_RNG.uniform(3, 5))
except _CDP_OPERATION_ERRORS as e:
logger.debug("Final captcha click failed: %s", e)
return await _is_bypassed(page)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Human-like method failed: %s", e)
return False
@@ -466,9 +505,9 @@ async def _bypass_method_cdp_solve(page: Any) -> bool:
try:
logger.debug("Attempting bypass: CDP solve_captcha")
await page.solve_captcha()
await asyncio.sleep(random.uniform(3, 5))
await asyncio.sleep(_RNG.uniform(3, 5))
return await _is_bypassed(page)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("CDP solve_captcha failed: %s", e)
return False
@@ -495,15 +534,15 @@ async def _bypass_method_cdp_click(page: Any) -> bool:
logger.debug("CDP clicking: %s", selector)
await page.click(selector)
await asyncio.sleep(random.uniform(2, 4))
await asyncio.sleep(_RNG.uniform(2, 4))
if await _is_bypassed(page):
return True
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("CDP click on '%s' failed: %s", selector, e)
return await _is_bypassed(page)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("CDP Mode click failed: %s", e)
return False
@@ -525,11 +564,11 @@ async def _bypass_method_cdp_gui_click(page: Any) -> bool:
try:
logger.debug("Trying solve_captcha()")
await page.solve_captcha()
await asyncio.sleep(random.uniform(3, 5))
await asyncio.sleep(_RNG.uniform(3, 5))
if await _is_bypassed(page):
return True
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("solve_captcha() failed: %s", e)
for selector in CDP_GUI_CLICK_SELECTORS:
@@ -539,15 +578,15 @@ async def _bypass_method_cdp_gui_click(page: Any) -> bool:
logger.debug("CDP click_with_offset: %s", selector)
await page.click_with_offset(selector, 0, 0, center=True)
await asyncio.sleep(random.uniform(3, 5))
await asyncio.sleep(_RNG.uniform(3, 5))
if await _is_bypassed(page):
return True
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("CDP gui_click on '%s' failed: %s", selector, e)
return await _is_bypassed(page)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("CDP Mode gui_click failed: %s", e)
return False
@@ -566,7 +605,8 @@ def _check_cancellation(cancel_flag: Event | None, message: str) -> None:
"""Check if cancellation was requested and raise if so."""
if cancel_flag and cancel_flag.is_set():
logger.info(message)
raise BypassCancelledException("Bypass cancelled")
msg = "Bypass cancelled"
raise BypassCancelledError(msg)
async def _bypass(
@@ -594,17 +634,17 @@ async def _bypass(
# No challenge detected but page doesn't look bypassed - wait and retry
if challenge_type == "none":
logger.info("No challenge detected, waiting for page to settle...")
await asyncio.sleep(random.uniform(2, 3))
await asyncio.sleep(_RNG.uniform(2, 3))
if await _is_bypassed(page):
return True
# Try a simple refresh instead of captcha methods
try:
await page.reload(ignore_cache=True)
await asyncio.sleep(random.uniform(1, 2))
await asyncio.sleep(_RNG.uniform(1, 2))
if await _is_bypassed(page):
logger.info("Bypass successful after refresh")
return True
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Refresh during no-challenge wait failed: %s", e)
continue
@@ -625,7 +665,7 @@ async def _bypass(
logger.info("Bypass attempt %s/%s using %s", try_count + 1, max_retries, method.__name__)
if try_count > 0:
wait_time = min(random.uniform(2, 4) * try_count, 12)
wait_time = min(_RNG.uniform(2, 4) * try_count, 12)
logger.info("Waiting %0.1fs before trying...", wait_time)
for _ in range(int(wait_time)):
_check_cancellation(cancel_flag, "Bypass cancelled during wait")
@@ -636,9 +676,9 @@ async def _bypass(
if await method(page):
logger.info("Bypass successful using %s", method.__name__)
return True
except BypassCancelledException:
except BypassCancelledError:
raise
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.warning("Exception in %s: %s", method.__name__, e)
logger.info("Bypass method %s failed.", method.__name__)
@@ -700,7 +740,7 @@ def _build_host_resolver_rules() -> list[str]:
logger.warning("Chrome: No addresses returned for %s", hostname)
except socket.gaierror as e:
logger.warning("Chrome: Could not pre-resolve %s: %s", hostname, e)
except Exception as e:
except (OSError, RuntimeError, TypeError, ValueError) as e:
logger.error_trace(f"Error pre-resolving hostnames for Chrome: {e}")
return host_rules
@@ -726,7 +766,7 @@ async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str:
current_url = await page.get_current_url()
title = await page.get_title()
logger.debug("Page loaded - URL: %s, Title: %s", current_url, title)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Could not get page info: %s", e)
logger.debug("Starting bypass process...")
@@ -738,9 +778,12 @@ async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str:
try:
body = await page.evaluate("document.body ? document.body.innerText : ''")
if body:
logger.debug(f"Page content: {body[:500]}..." if len(body) > 500 else body)
except Exception:
pass
preview = body
if len(body) > _PAGE_BODY_PREVIEW_CHARS:
preview = body[:_PAGE_BODY_PREVIEW_CHARS] + "..."
logger.debug("Page content: %s", preview)
except _CDP_OPERATION_ERRORS as exc:
logger.debug("Could not inspect protected page body: %s", exc)
return ""
@@ -767,9 +810,9 @@ def get(url: str, retry: int | None = None, cancel_flag: Event | None = None) ->
result = await _get(url, driver, cancel_flag)
if result:
return result
except BypassCancelledException:
except BypassCancelledError:
raise
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
error_details = f"{type(e).__name__}: {e}"
logger.warning(
"Bypass failed (attempt %s/%s): %s", attempt + 1, retry, error_details
@@ -824,7 +867,7 @@ async def _create_cdp_browser(url: str) -> Any:
proxy=proxy,
browser_args=browser_args,
)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.warning("Pure CDP browser startup failed: %s: %s", type(e).__name__, e)
logger.warning(
"SeleniumBase runtime paths: cwd=%s; %s; %s; %s; %s",
@@ -832,13 +875,13 @@ async def _create_cdp_browser(url: str) -> Any:
_describe_runtime_path(SELENIUMBASE_DOWNLOADS_DIR),
_describe_runtime_path("/app/downloaded_files"),
_describe_runtime_path("downloaded_files"),
_describe_runtime_path("/tmp"),
_describe_runtime_path(tempfile.gettempdir()),
)
raise
try:
await driver.page.set_window_rect(0, 0, screen_width, screen_height)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Failed to set window size: %s", e)
# Start FFmpeg recording if debug mode (record each bypass session)
@@ -868,13 +911,13 @@ async def _close_cdp_driver(driver: Any) -> None:
connections.extend(driver.targets)
for conn in connections:
await _close_websocket_connection(conn)
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Error during connection cleanup: %s", e)
try:
driver.stop()
logger.debug("Stopped CDP browser")
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("CDP stop: %s", e)
if env.DOCKERMODE:
@@ -898,9 +941,9 @@ async def _close_cdp_driver(driver: Any) -> None:
if _pid_alive(pid):
os.kill(pid, signal.SIGKILL)
logger.debug("Killed Chrome pid %s", pid)
except Exception as e:
except (OSError, RuntimeError, TypeError, ValueError) as e:
logger.debug("Failed to kill Chrome pid %s: %s", pid, e)
except Exception as e:
except (OSError, RuntimeError, TypeError, ValueError) as e:
logger.debug("Process cleanup failed: %s", e)
logger.log_resource_usage()
@@ -910,7 +953,7 @@ async def _close_websocket_connection(conn: Any) -> None:
"""Close one websocket-like connection, ignoring best-effort failures."""
try:
await conn.aclose()
except Exception as e:
except _CDP_OPERATION_ERRORS as e:
logger.debug("Failed to close websocket connection: %s", e)
@@ -918,7 +961,7 @@ def _start_ffmpeg_recording(display: str) -> None:
"""Start FFmpeg screen recording for debug mode."""
global DISPLAY
RECORDING_DIR.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now().strftime("%y%m%d-%H%M%S")
timestamp = datetime.now(UTC).strftime("%y%m%d-%H%M%S")
output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4"
screen_width, screen_height = get_screen_size()
@@ -981,7 +1024,7 @@ def _stop_ffmpeg_recording() -> None:
proc.send_signal(signal.SIGINT)
proc.wait(timeout=5)
logger.debug("Stopped ffmpeg recording")
except Exception as e:
except _SUBPROCESS_OPERATION_ERRORS as e:
logger.debug("ffmpeg stop: %s", e)
with suppress(Exception):
proc.terminate()
@@ -1013,11 +1056,11 @@ def _try_with_cached_cookies(url: str, hostname: str) -> str | None:
timeout=(5, 10),
verify=get_ssl_verify(url),
)
if response.status_code == 200:
if response.status_code == HTTPStatus.OK:
logger.debug("Cached cookies worked, skipped Chrome bypass")
return response.text
except Exception:
pass
except _REQUEST_OPERATION_ERRORS as exc:
logger.debug("Cached cookie retry failed for %s: %s", url, exc)
return None
@@ -1036,9 +1079,9 @@ def get_bypassed_page(
try:
response_html = get(attempt_url, cancel_flag=cancel_flag)
except BypassCancelledException:
except BypassCancelledError:
raise
except Exception:
except _CDP_OPERATION_ERRORS + _REQUEST_OPERATION_ERRORS:
_check_cancellation(cancel_flag, "Bypass cancelled")
new_base, action = sel.next_mirror_or_rotate_dns()
if action in ("mirror", "dns") and new_base:
@@ -1048,6 +1091,7 @@ def get_bypassed_page(
raise
if not response_html.strip():
raise requests.exceptions.RequestException("Failed to bypass Cloudflare")
msg = "Failed to bypass Cloudflare"
raise requests.exceptions.RequestException(msg)
return response_html
shelfmark/config/booklore_settings.py
+3 -1
View File
@@ -1,3 +1,5 @@
"""Helpers for Booklore settings validation, option loading, and connection tests."""
from __future__ import annotations
from typing import Any
@@ -164,7 +166,7 @@ def get_booklore_path_options() -> list[dict[str, Any]]:
return path_options
def test_booklore_connection(
def check_booklore_connection(
current_values: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Test the Booklore connection using current form values."""
shelfmark/config/email_settings.py
+5 -2
View File
@@ -1,5 +1,8 @@
"""Helpers for email settings validation and SMTP connection tests."""
from __future__ import annotations
import smtplib
from typing import Any
from shelfmark.core.config import config
@@ -10,7 +13,7 @@ from shelfmark.download.outputs.email import (
)
def test_email_connection(
def check_email_connection(
current_values: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Test SMTP connectivity using current form values (including unsaved changes)."""
@@ -41,7 +44,7 @@ def test_email_connection(
test_smtp_connection(smtp_config)
except EmailOutputError as exc:
return {"success": False, "message": str(exc)}
except Exception as exc:
except (OSError, smtplib.SMTPException) as exc:
return {"success": False, "message": f"SMTP test failed: {exc}"}
else:
return {"success": True, "message": "Connected to SMTP server"}
shelfmark/config/env.py
+2 -1
View File
@@ -3,6 +3,7 @@
import json
import os
import shutil
import tempfile
from pathlib import Path
@@ -89,7 +90,7 @@ CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
LOG_DIR = LOG_ROOT / "shelfmark"
LOG_FILE = LOG_DIR / "shelfmark.log"
TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark"))
TMP_DIR = Path(os.getenv("TMP_DIR", (Path(tempfile.gettempdir()) / "shelfmark").as_posix()))
INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
shelfmark/config/notifications_settings.py
+1
View File
@@ -143,6 +143,7 @@ def _extract_unique_route_urls(routes: list[dict[str, Any]]) -> list[str]:
def build_notification_test_result(routes_input: Any, *, scope_label: str) -> dict[str, Any]:
"""Validate routes and return a test-notification result payload."""
invalid_event_count = _count_invalid_route_events(routes_input)
if invalid_event_count:
return {
shelfmark/config/security.py
+2 -2
View File
@@ -4,8 +4,8 @@ from typing import TYPE_CHECKING, Any
from shelfmark.config.migrations import migrate_security_settings
from shelfmark.config.security_handlers import (
check_oidc_connection,
on_save_security,
test_oidc_connection,
)
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
@@ -61,7 +61,7 @@ def _on_save_security(values: dict[str, Any]) -> dict[str, Any]:
def _test_oidc_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
return test_oidc_connection(
return check_oidc_connection(
load_security_config=lambda: {
"OIDC_DISCOVERY_URL": app_config.get("OIDC_DISCOVERY_URL", ""),
},
shelfmark/config/security_handlers.py
+1 -1
View File
@@ -50,7 +50,7 @@ def on_save_security(
return {"error": False, "values": normalized_values}
def test_oidc_connection(
def check_oidc_connection(
*,
load_security_config: Callable[[], dict[str, Any]],
current_values: dict[str, Any] | None = None,
shelfmark/config/settings.py
+44 -40
View File
@@ -4,6 +4,33 @@ import json
from pathlib import Path
from typing import Any
from shelfmark.config import env
from shelfmark.config.booklore_settings import (
check_booklore_connection,
get_booklore_library_options,
get_booklore_path_options,
)
from shelfmark.config.email_settings import check_email_connection
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import (
ActionButton,
CheckboxField,
HeadingField,
MultiSelectField,
NumberField,
OrderableListField,
PasswordField,
SelectField,
SettingsField,
TableField,
TagListField,
TextField,
load_config_file,
register_group,
register_on_save,
register_settings,
)
def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
"""Validate advanced settings before persisting."""
@@ -64,16 +91,9 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
return {"error": False, "values": values}
from shelfmark.config import env
from shelfmark.config.booklore_settings import (
get_booklore_library_options,
get_booklore_path_options,
test_booklore_connection,
)
from shelfmark.config.email_settings import test_email_connection
from shelfmark.core.logger import setup_logger
logger = setup_logger(__name__)
_SMTP_PORT_MAX = 65535
_EMAIL_ATTACHMENT_LIMIT_MB_MAX = 600
# Log bootstrap configuration values at DEBUG level
logger.debug("Bootstrap configuration:")
@@ -117,25 +137,6 @@ def _log_external_bypasser_warning() -> None:
)
from shelfmark.core.settings_registry import (
ActionButton,
CheckboxField,
HeadingField,
MultiSelectField,
NumberField,
OrderableListField,
PasswordField,
SelectField,
SettingsField,
TableField,
TagListField,
TextField,
load_config_file,
register_group,
register_on_save,
register_settings,
)
register_group("direct_download", "Direct Download", icon="download", order=20)
register_group(
@@ -301,8 +302,8 @@ def _get_zlib_mirror_options() -> list[dict[str, str]]:
# Add custom mirrors
additional = config.get("ZLIB_ADDITIONAL_URLS", "")
if additional:
for url in additional.split(","):
url = url.strip()
for raw_url in additional.split(","):
url = raw_url.strip()
if url and url not in DEFAULT_ZLIB_MIRRORS:
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
options.append({"value": url, "label": f"{domain} (custom)"})
@@ -325,8 +326,8 @@ def _get_welib_mirror_options() -> list[dict[str, str]]:
# Add custom mirrors
additional = config.get("WELIB_ADDITIONAL_URLS", "")
if additional:
for url in additional.split(","):
url = url.strip()
for raw_url in additional.split(","):
url = raw_url.strip()
if url and url not in DEFAULT_WELIB_MIRRORS:
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
options.append({"value": url, "label": f"{domain} (custom)"})
@@ -777,10 +778,10 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
except TypeError, ValueError:
return {"error": True, "message": "SMTP port must be a number", "values": values}
if port < 1 or port > 65535:
if port < 1 or port > _SMTP_PORT_MAX:
return {
"error": True,
"message": "SMTP port must be between 1 and 65535",
"message": f"SMTP port must be between 1 and {_SMTP_PORT_MAX}",
"values": values,
}
@@ -818,10 +819,13 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
"values": values,
}
if attachment_limit_mb < 1 or attachment_limit_mb > 600:
if attachment_limit_mb < 1 or attachment_limit_mb > _EMAIL_ATTACHMENT_LIMIT_MB_MAX:
return {
"error": True,
"message": "Attachment size limit (MB) must be between 1 and 600",
"message": (
"Attachment size limit (MB) must be between 1 and "
f"{_EMAIL_ATTACHMENT_LIMIT_MB_MAX}"
),
"values": values,
}
@@ -1049,7 +1053,7 @@ def download_settings() -> list[SettingsField]:
label="Test Connection",
description="Verify your Grimmory configuration",
style="primary",
callback=test_booklore_connection,
callback=check_booklore_connection,
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"},
),
HeadingField(
@@ -1157,7 +1161,7 @@ def download_settings() -> list[SettingsField]:
label="Test SMTP Connection",
description="Verify your SMTP configuration (connect + optional login).",
style="primary",
callback=test_email_connection,
callback=check_email_connection,
show_when={"field": "BOOKS_OUTPUT_MODE", "value": "email"},
),
# === AUDIOBOOKS SECTION ===
@@ -1358,7 +1362,7 @@ def _get_slow_source_defaults() -> list[dict[str, str | bool]]:
"download_sources", "Download Sources", icon="download", order=21, group="direct_download"
)
def download_source_settings() -> list[SettingsField]:
"""Settings for download source behavior."""
"""Return settings for download source behavior."""
return [
PasswordField(
key="AA_DONATOR_KEY",
@@ -1466,7 +1470,7 @@ def download_source_settings() -> list[SettingsField]:
"cloudflare_bypass", "Cloudflare Bypass", icon="shield", order=22, group="direct_download"
)
def cloudflare_bypass_settings() -> list[SettingsField]:
"""Settings for Cloudflare bypass behavior."""
"""Return settings for Cloudflare bypass behavior."""
return [
CheckboxField(
key="USE_CF_BYPASS",
shelfmark/core/activity_routes.py
+3 -1
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import sqlite3
from typing import TYPE_CHECKING, Any, NamedTuple
from flask import Flask, Response, jsonify, request, session
@@ -37,6 +38,7 @@ if TYPE_CHECKING:
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
_USER_DB_IDENTITY_ERRORS = (sqlite3.Error, OSError)
def _normalize_log_field(value: object) -> str:
@@ -192,7 +194,7 @@ def _resolve_db_user_id(
if user_db is not None:
try:
db_user = user_db.get_user(user_id=parsed_db_user_id)
except Exception as exc:
except _USER_DB_IDENTITY_ERRORS as exc:
logger.warning("Failed to validate activity db identity %s: %s", parsed_db_user_id, exc)
db_user = None
if db_user is None:
shelfmark/core/activity_view_state_service.py
+14 -7
View File
@@ -15,6 +15,7 @@ USER_VIEWER_SCOPE_PREFIX = "user:"
def user_viewer_scope(user_id: int) -> str:
"""Build the persisted viewer scope string for a specific user."""
if not isinstance(user_id, int) or user_id < 1:
msg = "user_id must be a positive integer"
raise ValueError(msg)
@@ -22,6 +23,7 @@ def user_viewer_scope(user_id: int) -> str:
def normalize_viewer_scope(viewer_scope: object) -> str:
"""Validate and normalize a persisted viewer scope string."""
if not isinstance(viewer_scope, str) or not viewer_scope.strip():
msg = "viewer_scope must be a non-empty string"
raise ValueError(msg)
@@ -75,6 +77,7 @@ class ActivityViewStateService:
"""Service for per-viewer activity dismissal and history visibility."""
def __init__(self, db_path: str) -> None:
"""Initialize the service with the SQLite state database path."""
self._db_path = db_path
self._lock = threading.Lock()
@@ -90,6 +93,7 @@ class ActivityViewStateService:
viewer_scope: str,
limit: int | None = None,
) -> list[dict[str, Any]]:
"""Return dismissed rows for a viewer, including cleared history entries."""
normalized_scope = normalize_viewer_scope(viewer_scope)
normalized_limit = None if limit is None else max(1, int(limit))
query = """
@@ -118,6 +122,7 @@ class ActivityViewStateService:
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
"""Return active dismissal history rows for a viewer."""
normalized_scope = normalize_viewer_scope(viewer_scope)
normalized_limit = max(1, min(int(limit), 5000))
normalized_offset = max(0, int(offset))
@@ -147,6 +152,7 @@ class ActivityViewStateService:
item_type: str,
item_key: str,
) -> int:
"""Mark a single activity item as dismissed for a viewer."""
normalized_scope = normalize_viewer_scope(viewer_scope)
normalized_type = _normalize_item_type(item_type)
normalized_key = _normalize_item_key(item_key, item_type=normalized_type)
@@ -183,6 +189,7 @@ class ActivityViewStateService:
viewer_scope: str,
items: list[dict[str, str]],
) -> int:
"""Mark multiple activity items as dismissed for a viewer."""
normalized_scope = normalize_viewer_scope(viewer_scope)
if not items:
return 0
@@ -236,6 +243,7 @@ class ActivityViewStateService:
conn.close()
def clear_history(self, *, viewer_scope: str) -> int:
"""Mark all dismissed items as cleared for a viewer."""
normalized_scope = normalize_viewer_scope(viewer_scope)
cleared_at = now_utc_iso()
@@ -259,6 +267,7 @@ class ActivityViewStateService:
conn.close()
def clear_item_for_all_viewers(self, *, item_type: str, item_key: str) -> int:
"""Delete a dismissed item record for every viewer."""
normalized_type = _normalize_item_type(item_type)
normalized_key = _normalize_item_key(item_key, item_type=normalized_type)
@@ -279,6 +288,7 @@ class ActivityViewStateService:
conn.close()
def delete_viewer_scope(self, *, viewer_scope: str) -> int:
"""Delete all activity-view state rows for a viewer scope."""
normalized_scope = normalize_viewer_scope(viewer_scope)
with self._lock:
@@ -295,6 +305,7 @@ class ActivityViewStateService:
conn.close()
def delete_items(self, *, item_type: str, item_keys: list[str]) -> int:
"""Delete multiple dismissed item records for a given item type."""
normalized_type = _normalize_item_type(item_type)
normalized_keys = [
_normalize_item_key(item_key, item_type=normalized_type) for item_key in item_keys
@@ -302,16 +313,12 @@ class ActivityViewStateService:
if not normalized_keys:
return 0
placeholders = ",".join("?" for _ in normalized_keys)
with self._lock:
conn = self._connect()
try:
cursor = conn.execute(
f"""
DELETE FROM activity_view_state
WHERE item_type = ? AND item_key IN ({placeholders})
""",
(normalized_type, *normalized_keys),
cursor = conn.executemany(
"DELETE FROM activity_view_state WHERE item_type = ? AND item_key = ?",
[(normalized_type, normalized_key) for normalized_key in normalized_keys],
)
conn.commit()
rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0
shelfmark/core/admin_routes.py
+20 -11
View File
@@ -40,6 +40,8 @@ if TYPE_CHECKING:
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
MIN_PASSWORD_LENGTH = 4
_CONFIG_REFRESH_ERRORS = (ImportError, OSError, RuntimeError, TypeError, ValueError)
__all__ = [
"get_booklore_library_options",
@@ -122,7 +124,8 @@ def _serialize_user(
def _sync_all_cwa_users(user_db: UserDB) -> dict[str, int]:
"""Sync all users from the Calibre-Web database into users.db."""
if not CWA_DB_PATH or not CWA_DB_PATH.exists():
raise FileNotFoundError("Calibre-Web database is not available")
msg = "Calibre-Web database is not available"
raise FileNotFoundError(msg)
db_path = os.fspath(CWA_DB_PATH)
db_uri = f"file:{db_path}?mode=ro&immutable=1"
@@ -143,7 +146,7 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
def _require_admin(
f: Callable[..., Response | tuple[Response, int]],
) -> Callable[..., Response | tuple[Response, int]]:
"""Decorator to require admin session for admin routes.
"""Require an admin session for admin routes.
In no-auth mode, everyone has access (is_admin defaults True).
In auth-required modes, requires an authenticated session with admin role.
@@ -151,7 +154,7 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
"""
@wraps(f)
def decorated(*args, **kwargs) -> Response | tuple[Response, int]:
def decorated(*args: object, **kwargs: object) -> Response | tuple[Response, int]:
auth_mode = load_active_auth_mode(CWA_DB_PATH, user_db=user_db)
g.auth_mode = auth_mode
if auth_mode != "none":
@@ -197,8 +200,10 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
if not username:
return jsonify({"error": "Username is required"}), 400
if not password or len(password) < 4:
return jsonify({"error": "Password must be at least 4 characters"}), 400
if not password or len(password) < MIN_PASSWORD_LENGTH:
return jsonify(
{"error": f"Password must be at least {MIN_PASSWORD_LENGTH} characters"}
), 400
if role not in ("admin", "user"):
return jsonify({"error": "Role must be 'admin' or 'user'"}), 400
@@ -277,8 +282,10 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
"message": "Password authentication is only available for local users.",
}
), 400
if len(password) < 4:
return jsonify({"error": "Password must be at least 4 characters"}), 400
if len(password) < MIN_PASSWORD_LENGTH:
return jsonify(
{"error": f"Password must be at least {MIN_PASSWORD_LENGTH} characters"}
), 400
user_db.update_user(user_id, password_hash=generate_password_hash(password))
# Update user fields
@@ -365,11 +372,13 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None:
user_db.set_user_settings(user_id, validated_settings)
# Ensure runtime reads see updated per-user overrides immediately.
try:
from shelfmark.core.config import config as app_config
app_config.refresh(force=True)
except Exception:
pass
except _CONFIG_REFRESH_ERRORS as exc:
logger.warning(
"Updated settings for user %s but failed to refresh runtime config: %s",
user_id,
exc,
)
updated = user_db.get_user(user_id=user_id)
result = _serialize_user(
shelfmark/core/admin_settings_routes.py
+4
View File
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
def validate_user_settings(
settings: dict[str, Any],
) -> tuple[dict[str, Any], list[str]]:
"""Validate and normalize per-user settings overrides."""
settings_registry = _get_settings_registry()
field_map = settings_registry.get_settings_field_map()
overridable_map = settings_registry.get_user_overridable_fields()
@@ -132,6 +133,7 @@ def build_user_notification_test_response(
user_id: int,
payload: object,
) -> tuple[dict[str, Any], int]:
"""Build a notification test response using effective per-user routes."""
from shelfmark.core.config import config as app_config
routes_input = app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=user_id)
@@ -151,6 +153,8 @@ def register_admin_settings_routes(
user_db: UserDB,
require_admin: Callable[[Callable[..., object]], Callable[..., object]],
) -> None:
"""Register admin endpoints for user-specific settings and defaults."""
@app.route("/api/admin/download-defaults", methods=["GET"])
@require_admin
def admin_download_defaults() -> Response | tuple[Response, int]:
shelfmark/core/auth_modes.py
+3 -2
View File
@@ -1,6 +1,7 @@
"""Authentication mode, auth-source normalization, and admin access policy helpers."""
import os
import sqlite3
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -33,7 +34,7 @@ def has_local_password_admin(user_db: object | None = None) -> bool:
db.initialize()
return db.has_admin_with_password()
except Exception:
except AttributeError, ImportError, OSError, RuntimeError, TypeError, ValueError, sqlite3.Error:
return False
@@ -99,7 +100,7 @@ def load_active_auth_mode(
cwa_db_path,
has_local_admin=has_local_password_admin(user_db),
)
except Exception:
except ImportError, OSError, RuntimeError, TypeError, ValueError, sqlite3.Error:
return "none"
shelfmark/core/cache.py
+2 -2
View File
@@ -113,7 +113,7 @@ def get_metadata_cache() -> CacheService:
return _metadata_cache
def cache_key(*args, **kwargs) -> str:
def cache_key(*args: object, **kwargs: object) -> str:
"""Generate cache key from arguments."""
parts = [str(arg) for arg in args]
parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
@@ -126,7 +126,7 @@ def cacheable(
ttl_default: int = 300,
key_prefix: str = "",
) -> Callable[[Callable[P, R]], Callable[P, R]]:
"""Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
"""Cache function results with a static or config-backed TTL."""
def decorator(func: Callable[P, R]) -> Callable[P, R]:
@wraps(func)
shelfmark/core/config.py
+5 -3
View File
@@ -6,7 +6,7 @@ import time
from importlib import import_module
from pathlib import Path
from threading import Lock
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Self
if TYPE_CHECKING:
from types import ModuleType
@@ -61,7 +61,8 @@ class Config:
_instance: Config | None = None
_lock = Lock()
def __new__(cls) -> Config:
def __new__(cls) -> Self:
"""Return the shared configuration singleton instance."""
if cls._instance is None:
with cls._lock:
if cls._instance is None:
@@ -70,6 +71,7 @@ class Config:
return cls._instance
def __init__(self) -> None:
"""Initialize caches and backing stores for the singleton."""
if self._initialized:
return
self._cache: dict[str, Any] = {}
@@ -161,7 +163,7 @@ class Config:
db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db")
user_db = user_db_cls(db_path)
user_db.initialize()
except Exception:
except ImportError, OSError, sqlite3.Error:
# Multi-user support is optional; fall back to global config when unavailable.
return None
else:
shelfmark/core/download_history_service.py
+8 -2
View File
@@ -75,6 +75,7 @@ class DownloadHistoryService:
"""Service for persisted canonical download activity rows."""
def __init__(self, db_path: str) -> None:
"""Initialize the service with the SQLite history database path."""
self._db_path = db_path
self._lock = threading.Lock()
@@ -146,6 +147,7 @@ class DownloadHistoryService:
@staticmethod
def is_retry_available(row: dict[str, Any]) -> bool:
"""Return whether a persisted download row can be retried."""
final_status = (
str(row.get("retry_final_status") or row.get("final_status") or "").strip().lower()
)
@@ -175,6 +177,7 @@ class DownloadHistoryService:
@staticmethod
def to_download_payload(row: dict[str, Any]) -> dict[str, Any]:
"""Build the sidebar/history download payload for a persisted row."""
return {
"id": row.get("task_id"),
"title": row.get("title"),
@@ -211,6 +214,7 @@ class DownloadHistoryService:
@classmethod
def to_history_row(cls, row: dict[str, Any], *, dismissed_at: str) -> dict[str, Any]:
"""Build the activity-history payload for a persisted download row."""
task_id = str(row.get("task_id") or "").strip()
item_key = cls._to_item_key(task_id)
download_payload = cls.to_download_payload(row)
@@ -245,7 +249,7 @@ class DownloadHistoryService:
source_display_name: str | None,
title: str,
author: str | None,
format: str | None,
file_format: str | None,
size: str | None,
preview: str | None,
content_type: str | None,
@@ -303,7 +307,7 @@ class DownloadHistoryService:
normalize_optional_text(source_display_name),
normalized_title,
normalize_optional_text(author),
normalize_optional_text(format),
normalize_optional_text(file_format),
normalize_optional_text(size),
normalize_optional_text(preview),
normalize_optional_text(content_type),
@@ -368,6 +372,7 @@ class DownloadHistoryService:
conn.close()
def get_by_task_id(self, task_id: str) -> dict[str, Any] | None:
"""Return a persisted download row for the given task id."""
normalized_task_id = _normalize_task_id(task_id)
conn = self._connect()
try:
@@ -385,6 +390,7 @@ class DownloadHistoryService:
user_id: int | None,
limit: int = 200,
) -> list[dict[str, Any]]:
"""Return recent persisted download rows, optionally scoped to one user."""
normalized_user_id = normalize_optional_positive_int(user_id, "user_id")
normalized_limit = _normalize_limit(limit, default=200, minimum=1, maximum=1000)
query = "SELECT * FROM download_history"
shelfmark/core/external_user_linking.py
+1
View File
@@ -60,6 +60,7 @@ def _get_by_subject(
def find_unique_user_by_email(user_db: UserDB, email: str | None) -> dict[str, Any] | None:
"""Return the unique local user matching an email address, if any."""
key = _email_key(_normalize_email(email))
if not key:
return None
shelfmark/core/image_cache.py
+3 -4
View File
@@ -485,13 +485,12 @@ class ImageCacheService:
"""Check that a URL is safe to fetch (no SSRF to internal resources)."""
try:
parsed = urlparse(url)
except Exception:
hostname = parsed.hostname
except ValueError:
return False
if parsed.scheme not in ("http", "https"):
return False
hostname = parsed.hostname
if not hostname:
return False
@@ -570,7 +569,7 @@ class ImageCacheService:
is_404 = e.response is not None and e.response.status_code == HTTP_NOT_FOUND
self.put_negative(cache_id, transient=not is_404)
return None
except Exception:
except requests.exceptions.RequestException:
return None
else:
return cached_data
shelfmark/core/logger.py
+9 -4
View File
@@ -28,9 +28,14 @@ class CustomLogger(logging.Logger):
self.debug(msg, *args, exc_info=has_exception, **kwargs)
def log_resource_usage(self) -> None:
# Best-effort only; this should never raise during exception logging.
"""Log best-effort CPU and memory usage for the current container."""
try:
import psutil
except ImportError:
return
# Best-effort only; this should never raise during exception logging.
try:
def _get_process_rss_mb(proc: object) -> float | None:
try:
@@ -57,7 +62,7 @@ class CustomLogger(logging.Logger):
except PermissionError, psutil.AccessDenied, OSError:
try:
app_memory_mb = psutil.Process().memory_info().rss / (1024 * 1024)
except Exception:
except AttributeError, OSError, psutil.Error:
app_memory_mb = 0.0
memory = psutil.virtual_memory()
@@ -68,7 +73,7 @@ class CustomLogger(logging.Logger):
f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, "
f"Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%"
)
except Exception:
except AttributeError, OSError, psutil.Error:
# Avoid breaking the original log call if psutil is missing or restricted.
return
@@ -124,7 +129,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
except Exception as e:
except (OSError, TypeError, ValueError) as e:
logger.error_trace(f"Failed to create log file: {e}", exc_info=True)
return logger
shelfmark/core/models.py
+5
View File
@@ -14,6 +14,7 @@ def build_filename(
year: str | None = None,
fmt: str | None = None,
) -> str:
"""Build a filesystem-safe filename from book metadata."""
parts = []
if author:
parts.append(author)
@@ -62,6 +63,8 @@ ACTIVE_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset(
class SearchMode(StrEnum):
"""Search modes supported by the Shelfmark UI and API."""
DIRECT = "direct"
UNIVERSAL = "universal"
@@ -83,6 +86,8 @@ class QueueItem:
@dataclass
class DownloadTask:
"""Mutable download task state tracked throughout the pipeline."""
task_id: str # Unique ID (e.g., AA MD5 hash, Prowlarr GUID)
source: str # Handler name ("direct_download", "prowlarr")
title: str # Display title for queue sidebar
shelfmark/core/naming.py
+17 -14
View File
@@ -54,6 +54,7 @@ sanitize_path_component = sanitize_filename
def format_series_position(position: str | float | None) -> str:
"""Format a series position for naming templates."""
if position is None:
return ""
@@ -95,36 +96,37 @@ def parse_naming_template(
*,
allow_path_separators: bool = True,
) -> str:
"""Render a naming template with Shelfmark metadata placeholders."""
if not template:
return ""
# Normalize metadata keys to lowercase for case-insensitive matching
normalized = {k.lower(): v for k, v in metadata.items()}
def find_token(content: str) -> tuple[str | None, int]:
def find_placeholder(content: str) -> tuple[str | None, int]:
content_lower = content.lower()
for token in KNOWN_TOKENS:
idx = content_lower.find(token)
for placeholder_name in KNOWN_TOKENS:
idx = content_lower.find(placeholder_name)
if idx != -1:
return token, idx
return placeholder_name, idx
return None, -1
def token_value(token: str) -> str:
value = normalized.get(token)
if token == "seriesposition":
def placeholder_value(placeholder_name: str) -> str:
value = normalized.get(placeholder_name)
if placeholder_name == "seriesposition":
value = format_series_position(value)
if value is None:
return ""
return str(value).strip()
def render_block(content: str) -> str | None:
token, idx = find_token(content)
if token is None:
placeholder_name, idx = find_placeholder(content)
if placeholder_name is None:
return None
prefix = content[:idx]
suffix = content[idx + len(token) :]
value = token_value(token)
suffix = content[idx + len(placeholder_name) :]
value = placeholder_value(placeholder_name)
if not value:
return ""
@@ -153,10 +155,10 @@ def parse_naming_template(
include_literal = False
if idx + 1 < len(matches) and match.end() == matches[idx + 1].start():
next_content = matches[idx + 1].group(1)
next_token, _next_idx = find_token(next_content)
if next_token is not None:
next_placeholder_name, _next_idx = find_placeholder(next_content)
if next_placeholder_name is not None:
conditional_literal = True
include_literal = bool(token_value(next_token))
include_literal = bool(placeholder_value(next_placeholder_name))
if include_literal:
parts.append(content)
elif not conditional_literal and re.search(r"\s", content):
@@ -194,6 +196,7 @@ def build_library_path(
metadata: Mapping[str, str | int | float | None],
extension: str | None = None,
) -> Path:
"""Build a final library path from a template and metadata."""
relative = parse_naming_template(template, metadata, allow_path_separators=True)
if not relative:
shelfmark/core/notifications.py
+13 -11
View File
@@ -33,6 +33,7 @@ _APPRISE_LOGO_URL = (
"https://raw.githubusercontent.com/calibrain/shelfmark/main/src/frontend/public/logo.png"
)
_APPRISE_LOGGER_NAME = "apprise"
_APPRISE_DISPATCH_ERRORS = (RuntimeError, TypeError, ValueError)
class NotificationEvent(StrEnum):
@@ -401,7 +402,7 @@ def _dispatch_to_apprise(
with _capture_apprise_logs(min_level=logging.INFO) as apprise_records:
try:
plugin = apprise.Apprise.instantiate(url, asset=getattr(apobj, "asset", None))
except Exception as exc:
except _APPRISE_DISPATCH_ERRORS as exc:
logger.warning(
"Failed to register notification route URL for scheme '%s': %s",
scheme,
@@ -435,7 +436,7 @@ def _dispatch_to_apprise(
try:
delivered = bool(apobj.notify(title=title, body=body, notify_type=notify_type))
except Exception as exc:
except _APPRISE_DISPATCH_ERRORS as exc:
_log_apprise_records(apprise_records)
failed_delivery_urls += 1
logger.warning(
@@ -526,16 +527,17 @@ def _create_apprise_client() -> object:
)
except TypeError:
# Support older Apprise versions that do not expose image_url_logo.
asset = apprise_asset_cls(
app_id=_APPRISE_APP_ID,
app_desc=_APPRISE_APP_DESC,
)
except Exception:
return apprise_cls()
try:
asset = apprise_asset_cls(
app_id=_APPRISE_APP_ID,
app_desc=_APPRISE_APP_DESC,
)
except TypeError:
return apprise_cls()
try:
return apprise_cls(asset=asset)
except Exception:
except TypeError:
return apprise_cls()
@@ -556,7 +558,7 @@ def notify_admin(event: NotificationEvent, context: NotificationContext) -> None
try:
_executor.submit(_dispatch_admin_async, event, context, urls)
except Exception as exc:
except RuntimeError as exc:
logger.warning("Failed to queue admin notification '%s': %s", event.value, exc)
@@ -575,7 +577,7 @@ def notify_user(
try:
_executor.submit(_dispatch_user_async, normalized_user_id, event, context, urls)
except Exception as exc:
except RuntimeError as exc:
logger.warning(
"Failed to queue user notification '%s' for user_id=%s: %s",
event.value,
shelfmark/core/oidc_routes.py
+4 -2
View File
@@ -7,6 +7,7 @@ Business logic remains in oidc_auth.py.
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode, urlsplit, urlunsplit
from authlib.integrations.base_client.errors import OAuthError
from authlib.integrations.flask_client import OAuth
from authlib.jose.errors import InvalidClaimError
from flask import Flask, Response, jsonify, redirect, request, session
@@ -26,6 +27,7 @@ if TYPE_CHECKING:
logger = setup_logger(__name__)
oauth = OAuth()
_RETURN_TO_SESSION_KEY = "oidc_return_to"
_OIDC_CLIENT_ERRORS = (OAuthError, OSError, RuntimeError, TypeError, ValueError)
def _normalize_claims(raw_claims: object) -> dict[str, Any]:
@@ -38,7 +40,7 @@ def _normalize_claims(raw_claims: object) -> dict[str, Any]:
return raw_claims.to_dict() # type: ignore[no-any-return]
try:
return dict(raw_claims)
except Exception:
except TypeError, ValueError:
return {}
@@ -214,7 +216,7 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None:
metadata = client.load_server_metadata()
if isinstance(metadata, dict):
provider_issuer = str(metadata.get("issuer", ""))
except Exception as metadata_error:
except _OIDC_CLIENT_ERRORS as metadata_error:
logger.debug(
"OIDC metadata lookup failed during claim diagnostics: %s",
metadata_error,
shelfmark/core/onboarding.py
+1 -1
View File
@@ -86,7 +86,7 @@ def _get_field_from_tab(tab_name: str, field_key: str) -> SettingsField | None:
return None
def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField:
def _clone_field_with_overrides(field: SettingsField, **overrides: object) -> SettingsField:
"""Clone a field with optional attribute overrides.
Useful for customizing labels, descriptions, or defaults for onboarding context.
shelfmark/core/path_mappings.py
+5
View File
@@ -21,6 +21,8 @@ _WINDOWS_DRIVE_PREFIX_LENGTH = 2
@dataclass(frozen=True)
class RemotePathMapping:
"""Mapping from a remote path prefix to a local path prefix."""
host: str
remote_path: str
local_path: str
@@ -49,6 +51,7 @@ def _normalize_host(host: str) -> str:
def parse_remote_path_mappings(value: object) -> list[RemotePathMapping]:
"""Parse configured remote-path mapping rows into normalized mappings."""
if not value or not isinstance(value, list):
return []
@@ -79,6 +82,7 @@ def remap_remote_to_local_with_match(
host: str,
remote_path: str | Path,
) -> tuple[Path, bool]:
"""Remap a remote path and report whether a configured mapping matched."""
host_normalized = _normalize_host(host)
remote_normalized = _normalize_prefix(str(remote_path))
@@ -124,6 +128,7 @@ def remap_remote_to_local_with_match(
def remap_remote_to_local(
*, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path
) -> Path:
"""Remap a remote path to a local path using the configured mappings."""
remapped, _ = remap_remote_to_local_with_match(
mappings=mappings,
host=host,
shelfmark/core/prefix_middleware.py
+2
View File
@@ -17,11 +17,13 @@ class PrefixMiddleware:
prefix: str,
bypass_paths: Iterable[str] | None = None,
) -> None:
"""Initialize the middleware with a prefix and optional bypass paths."""
self.app = app
self.prefix = prefix.rstrip("/")
self.bypass_paths = set(bypass_paths or [])
def __call__(self, environ: dict[str, object], start_response: Callable[..., object]) -> object:
"""Rewrite prefixed requests before handing them to the wrapped app."""
path = environ.get("PATH_INFO", "") or ""
if path in self.bypass_paths:
shelfmark/core/queue.py
+8 -6
View File
@@ -2,7 +2,7 @@
import queue
import time
from datetime import datetime, timedelta
from datetime import UTC, datetime, timedelta
from pathlib import Path
from threading import Event, Lock
from typing import TYPE_CHECKING, Any
@@ -20,12 +20,14 @@ if TYPE_CHECKING:
from collections.abc import Callable
logger = setup_logger(__name__)
_QUEUE_HOOK_ERRORS = (OSError, RuntimeError, TypeError, ValueError)
class BookQueue:
"""Thread-safe download queue manager with priority support and cancellation."""
def __init__(self) -> None:
"""Initialize queue state, locks, and lifecycle hooks."""
self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
self._lock = Lock()
self._status: dict[str, QueueStatus] = {}
@@ -67,7 +69,7 @@ class BookQueue:
if hook is not None:
try:
hook(task_id, task)
except Exception as exc:
except _QUEUE_HOOK_ERRORS as exc:
logger.warning("Queue hook failed while adding task %s: %s", task_id, exc)
return True
@@ -104,9 +106,9 @@ class BookQueue:
return self._status.get(task_id)
def _update_status(self, book_id: str, status: QueueStatus) -> None:
"""Internal method to update status and timestamp."""
"""Update the status and timestamp for a task."""
self._status[book_id] = status
self._status_timestamps[book_id] = datetime.now()
self._status_timestamps[book_id] = datetime.now(UTC)
def set_terminal_status_hook(
self,
@@ -306,7 +308,7 @@ class BookQueue:
if hook is not None and hook_task is not None:
try:
hook(task_id, hook_task)
except Exception as exc:
except _QUEUE_HOOK_ERRORS as exc:
logger.warning("Queue hook failed while requeueing task %s: %s", task_id, exc)
return True
@@ -349,7 +351,7 @@ class BookQueue:
"""Remove any tasks that are done downloading or have stale status."""
terminal_statuses = TERMINAL_QUEUE_STATUSES
with self._lock:
current_time = datetime.now()
current_time = datetime.now(UTC)
to_remove = []
for task_id, status in self._status.items():
shelfmark/core/request_helpers.py
+1 -1
View File
@@ -32,7 +32,7 @@ def emit_ws_event(
if socketio is None or not callable(is_enabled) or not is_enabled():
return
socketio.emit(event_name, payload, to=room)
except Exception as exc:
except (AttributeError, RuntimeError, TypeError, ValueError) as exc:
_logger.warning(
"Failed to emit WebSocket event '%s' to room '%s': %s",
event_name,
shelfmark/core/request_routes.py
+3 -2
View File
@@ -48,6 +48,7 @@ if TYPE_CHECKING:
from shelfmark.core.user_db import UserDB
logger = setup_logger(__name__)
_NOTIFICATION_TRIGGER_ERRORS = (RuntimeError, TypeError, ValueError)
def _error_response(
@@ -508,7 +509,7 @@ def _notify_admin_for_request_event(
owner_user_id = normalize_positive_int(request_row.get("user_id"))
try:
notify_admin(event, context)
except Exception as exc:
except _NOTIFICATION_TRIGGER_ERRORS as exc:
logger.warning(
"Failed to trigger admin notification for request event '%s': %s",
event.value,
@@ -518,7 +519,7 @@ def _notify_admin_for_request_event(
return
try:
notify_user(owner_user_id, event, context)
except Exception as exc:
except _NOTIFICATION_TRIGGER_ERRORS as exc:
logger.warning(
"Failed to trigger user notification for request event '%s' (user_id=%s): %s",
event.value,
shelfmark/core/requests_service.py
+1
View File
@@ -40,6 +40,7 @@ class RequestServiceError(ValueError):
code: str | None = None,
required_mode: str | None = None,
) -> None:
"""Initialize the error with HTTP metadata for API callers."""
super().__init__(message)
self.status_code = status_code
self.code = code
shelfmark/core/search_plan.py
+7 -2
View File
@@ -1,10 +1,10 @@
"""Helpers for building release search plans from metadata and user input."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
MANUAL_QUERY_MAX_LEN = 256
from shelfmark.core.config import config
from shelfmark.metadata_providers import (
BookMetadata,
@@ -15,6 +15,8 @@ from shelfmark.metadata_providers import (
if TYPE_CHECKING:
from shelfmark.core.models import SearchFilters
MANUAL_QUERY_MAX_LEN = 256
@dataclass(frozen=True)
class ReleaseSearchVariant:
@@ -26,6 +28,7 @@ class ReleaseSearchVariant:
@property
def query(self) -> str:
"""Return the combined title-and-author query for this variant."""
return " ".join(part for part in [self.title, self.author] if part).strip()
@@ -44,6 +47,7 @@ class ReleaseSearchPlan:
@property
def primary_query(self) -> str:
"""Return the first expanded title query, if one exists."""
return self.title_variants[0].query if self.title_variants else ""
@@ -94,6 +98,7 @@ def build_release_search_plan(
indexers: list[str] | None = None,
source_filters: SearchFilters | None = None,
) -> ReleaseSearchPlan:
"""Build normalized search variants shared across release sources."""
resolved_languages = _normalize_languages(languages)
resolved_manual_query = None
shelfmark/core/self_user_routes.py
+69 -45
View File
@@ -1,5 +1,6 @@
"""Self-service user account routes."""
import sqlite3
from functools import wraps
from typing import TYPE_CHECKING, Any
@@ -20,6 +21,7 @@ from shelfmark.core.auth_modes import (
load_active_auth_mode,
normalize_auth_source,
)
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import load_config_file
from shelfmark.core.user_settings_overrides import (
@@ -47,6 +49,8 @@ _VALID_SELF_SETTINGS_SECTIONS = (
_SELF_SETTINGS_SECTION_NOTIFICATIONS,
)
_DEFAULT_VISIBLE_SELF_SETTINGS_SECTIONS = list(_VALID_SELF_SETTINGS_SECTIONS)
_USER_PREFERENCES_FALLBACK_ERRORS = (ImportError, OSError, RuntimeError, TypeError, sqlite3.Error)
_CONFIG_REFRESH_ERRORS = (ImportError, OSError, RuntimeError, TypeError, ValueError)
def _get_current_user(
@@ -91,6 +95,36 @@ def _serialize_self_user(user: Mapping[str, Any], auth_mode: str) -> dict[str, A
return payload
def _build_optional_user_preferences(
user_db: UserDB,
*,
user_id: int,
tab_name: str,
missing_tab_error: str,
preference_label: str,
) -> tuple[dict[str, Any] | None, tuple[Response, int] | None]:
try:
return _build_user_preferences_payload(user_db, user_id, tab_name), None
except ValueError as exc:
if str(exc) == missing_tab_error:
return None, (jsonify({"error": missing_tab_error}), 500)
logger.warning(
"Failed to build user %s preferences for user_id=%s: %s",
preference_label,
user_id,
exc,
)
return None, None
except _USER_PREFERENCES_FALLBACK_ERRORS as exc:
logger.warning(
"Failed to build user %s preferences for user_id=%s: %s",
preference_label,
user_id,
exc,
)
return None, None
def _normalize_visible_self_settings_sections(raw_sections: object) -> list[str]:
"""Normalize users.VISIBLE_SELF_SETTINGS_SECTIONS to a safe ordered list."""
if raw_sections is None:
@@ -147,13 +181,13 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
def _require_authenticated_user(
f: Callable[..., Response | tuple[Response, int]],
) -> Callable[..., Response | tuple[Response, int]]:
"""Decorator requiring an authenticated session linked to a local user row.
"""Require an authenticated session linked to a local user row.
Caches the resolved auth_mode in ``g.auth_mode`` for the request.
"""
@wraps(f)
def decorated(*args, **kwargs) -> Response | tuple[Response, int]:
def decorated(*args: object, **kwargs: object) -> Response | tuple[Response, int]:
auth_mode = load_active_auth_mode(CWA_DB_PATH, user_db=user_db)
g.auth_mode = auth_mode
if auth_mode != "none" and "user_id" not in session:
@@ -179,51 +213,39 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
delivery_preferences = None
if _SELF_SETTINGS_SECTION_DELIVERY in visible_self_settings_sections:
try:
delivery_preferences = _build_user_preferences_payload(
user_db, user_id, "downloads"
)
except ValueError:
return jsonify({"error": "Downloads settings tab not found"}), 500
except Exception as exc:
logger.warning(
"Failed to build user delivery preferences for user_id=%s: %s",
user_id,
exc,
)
delivery_preferences = None
delivery_preferences, error_response = _build_optional_user_preferences(
user_db,
user_id=user_id,
tab_name="downloads",
missing_tab_error="Downloads settings tab not found",
preference_label="delivery",
)
if error_response:
return error_response
search_preferences = None
if _SELF_SETTINGS_SECTION_SEARCH in visible_self_settings_sections:
try:
search_preferences = _build_user_preferences_payload(
user_db, user_id, "search_mode"
)
except ValueError:
return jsonify({"error": "Search mode settings tab not found"}), 500
except Exception as exc:
logger.warning(
"Failed to build user search preferences for user_id=%s: %s",
user_id,
exc,
)
search_preferences = None
search_preferences, error_response = _build_optional_user_preferences(
user_db,
user_id=user_id,
tab_name="search_mode",
missing_tab_error="Search mode settings tab not found",
preference_label="search",
)
if error_response:
return error_response
notification_preferences = None
if _SELF_SETTINGS_SECTION_NOTIFICATIONS in visible_self_settings_sections:
try:
notification_preferences = _build_user_preferences_payload(
user_db, user_id, "notifications"
)
except ValueError:
return jsonify({"error": "Notifications settings tab not found"}), 500
except Exception as exc:
logger.warning(
"Failed to build user notification preferences for user_id=%s: %s",
user_id,
exc,
)
notification_preferences = None
notification_preferences, error_response = _build_optional_user_preferences(
user_db,
user_id=user_id,
tab_name="notifications",
missing_tab_error="Notifications settings tab not found",
preference_label="notification",
)
if error_response:
return error_response
user_overridable_keys = sorted(
set(delivery_preferences.get("keys", []) if delivery_preferences else [])
@@ -370,11 +392,13 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None:
user_db.set_user_settings(user_id, validated_settings)
try:
from shelfmark.core.config import config as app_config
app_config.refresh(force=True)
except Exception:
pass
except _CONFIG_REFRESH_ERRORS as exc:
logger.warning(
"Updated settings for user %s but failed to refresh runtime config: %s",
user_id,
exc,
)
updated = user_db.get_user(user_id=user_id)
if not updated:
shelfmark/core/settings_registry.py
+21 -3
View File
@@ -12,6 +12,7 @@ from werkzeug.utils import secure_filename
from shelfmark.core.logger import setup_logger
logger = setup_logger(__name__)
_SETTINGS_LIVE_APPLY_ERRORS = (OSError, RuntimeError, TypeError, ValueError)
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
@@ -114,6 +115,8 @@ class TagListField(FieldBase):
@dataclass
class OrderableListField(FieldBase):
"""Settings field for ordered, toggleable option lists."""
# Options can be a list or a callable that returns a list (for lazy evaluation)
# Each option: {id, label, description?, disabledReason?, isLocked?, section?, isPinned?}
# - isLocked: toggle is disabled (can't enable/disable)
@@ -154,9 +157,11 @@ class CustomComponentField:
universal_only: bool = False
def get_field_type(self) -> str:
"""Return the serialized field type for this custom component."""
return "CustomComponentField"
def get_bind_keys(self) -> list[str]:
"""Return the config keys this custom component reads or writes."""
if self.bind_keys:
return self.bind_keys
return [f.key for f in self.value_fields if getattr(f, "key", None)]
@@ -164,6 +169,8 @@ class CustomComponentField:
@dataclass
class ActionButton:
"""Definition for a custom action button in the settings UI."""
key: str # Action identifier
label: str # Button text
description: str = "" # Help text
@@ -181,6 +188,7 @@ class ActionButton:
)
def get_field_type(self) -> str:
"""Return the serialized field type for this action button."""
return "ActionButton"
@@ -206,6 +214,7 @@ class HeadingField:
universal_only: bool = False # Only show in Universal search mode (hide in Direct mode)
def get_field_type(self) -> str:
"""Return the serialized field type for this heading field."""
return "HeadingField"
@@ -255,6 +264,7 @@ _REGISTRY_LOCK = Lock()
def register_group(name: str, display_name: str, icon: str | None = None, order: int = 100) -> None:
"""Register a settings group used to organize tabs in the UI."""
with _REGISTRY_LOCK:
group = SettingsGroup(
name=name,
@@ -273,6 +283,8 @@ def register_settings(
order: int = 100,
group: str | None = None,
) -> Callable[[Callable[[], list[SettingsField]]], Callable[[], list[SettingsField]]]:
"""Register a settings tab and its field factory."""
def decorator(func: Callable[[], list[SettingsField]]) -> Callable[[], list[SettingsField]]:
with _REGISTRY_LOCK:
fields = func()
@@ -297,6 +309,7 @@ def register_settings(
def register_on_save(tab_name: str, handler: Callable[[dict[str, Any]], dict[str, Any]]) -> None:
"""Register an on-save hook for a settings tab."""
with _REGISTRY_LOCK:
_ON_SAVE_HANDLERS[tab_name] = handler
logger.debug("Registered on_save handler for tab: %s", tab_name)
@@ -407,6 +420,7 @@ def _ensure_config_dir(tab_name: str) -> None:
def load_config_file(tab_name: str) -> dict[str, Any]:
"""Load a settings tab config file, returning an empty dict on failure."""
config_path = _get_config_file_path(tab_name)
if not config_path.exists():
@@ -421,6 +435,7 @@ def load_config_file(tab_name: str) -> dict[str, Any]:
def save_config_file(tab_name: str, values: dict[str, Any]) -> bool:
"""Merge and save persisted settings values for a tab."""
try:
_ensure_config_dir(tab_name)
config_path = _get_config_file_path(tab_name)
@@ -506,6 +521,7 @@ def initialize_default_configs() -> bool:
def sync_env_to_config() -> None:
"""Sync supported environment-backed settings into config files."""
# Initialize default configs first (for fresh installs)
initialize_default_configs()
@@ -821,6 +837,7 @@ def migrate_download_to_browser_settings() -> None:
def get_setting_value(field: SettingsField, tab_name: str) -> object:
"""Resolve the effective value for a settings field."""
if isinstance(field, (ActionButton, HeadingField, CustomComponentField)):
return None # Actions and headings don't have values
@@ -1162,7 +1179,7 @@ def _apply_dns_settings(config: Config) -> None:
network.set_dns_provider(provider, manual_servers, use_doh=use_doh)
except ImportError:
pass # Network module not available
except Exception as e:
except _SETTINGS_LIVE_APPLY_ERRORS as e:
logger.warning("Failed to apply DNS settings: %s", e)
@@ -1179,11 +1196,12 @@ def _apply_aa_mirror_settings(config: Config) -> None:
network.init_aa(force=True)
except ImportError:
pass # Network module not available
except Exception as e:
except _SETTINGS_LIVE_APPLY_ERRORS as e:
logger.warning("Failed to apply AA mirror settings: %s", e)
def update_settings(tab_name: str, values: dict[str, Any]) -> dict[str, Any]:
"""Validate, persist, and post-process updates for a settings tab."""
tab = get_settings_tab(tab_name)
if not tab:
return {
@@ -1290,7 +1308,7 @@ def update_settings(tab_name: str, values: dict[str, Any]) -> dict[str, Any]:
)
_apply_ssl_warning_suppression()
except Exception as e:
except _SETTINGS_LIVE_APPLY_ERRORS as e:
logger.warning("Failed to apply certificate validation setting: %s", e)
# Apply AA mirror settings changes live (mirrors tab)
shelfmark/core/user_db.py
+78 -41
View File
@@ -175,6 +175,7 @@ class UserDB:
_VALID_AUTH_SOURCES: ClassVar[frozenset[str]] = frozenset(AUTH_SOURCE_SET)
def __init__(self, db_path: str) -> None:
"""Initialize the user database wrapper for the given SQLite path."""
self._db_path = db_path
self._lock = threading.Lock()
@@ -273,7 +274,8 @@ class UserDB:
) -> dict[str, Any]:
"""Create a new user. Raises ValueError if username or oidc_subject already exists."""
if auth_source not in self._VALID_AUTH_SOURCES:
raise ValueError(f"Invalid auth_source: {auth_source}")
msg = f"Invalid auth_source: {auth_source}"
raise ValueError(msg)
with self._lock:
conn = self._connect()
try:
@@ -296,7 +298,8 @@ class UserDB:
user_id = cursor.lastrowid
return self._get_user_by_id(conn, user_id)
except sqlite3.IntegrityError as e:
raise ValueError(f"User already exists: {e}") from e
msg = f"User already exists: {e}"
raise ValueError(msg) from e
finally:
conn.close()
@@ -337,25 +340,35 @@ class UserDB:
"role",
}
)
_USER_UPDATE_STATEMENTS: ClassVar[dict[str, str]] = {
"email": "UPDATE users SET email = ? WHERE id = ?",
"display_name": "UPDATE users SET display_name = ? WHERE id = ?",
"password_hash": "UPDATE users SET password_hash = ? WHERE id = ?",
"oidc_subject": "UPDATE users SET oidc_subject = ? WHERE id = ?",
"auth_source": "UPDATE users SET auth_source = ? WHERE id = ?",
"role": "UPDATE users SET role = ? WHERE id = ?",
}
def update_user(self, user_id: int, **kwargs) -> None:
def update_user(self, user_id: int, **kwargs: object) -> None:
"""Update user fields. Raises ValueError if user not found or invalid column."""
if not kwargs:
return
for k in kwargs:
if k not in self._ALLOWED_UPDATE_COLUMNS:
raise ValueError(f"Invalid column: {k}")
msg = f"Invalid column: {k}"
raise ValueError(msg)
if "auth_source" in kwargs and kwargs["auth_source"] not in self._VALID_AUTH_SOURCES:
raise ValueError(f"Invalid auth_source: {kwargs['auth_source']}")
msg = f"Invalid auth_source: {kwargs['auth_source']}"
raise ValueError(msg)
with self._lock:
conn = self._connect()
try:
# Verify user exists
if not self._get_user_by_id(conn, user_id):
raise ValueError(f"User {user_id} not found")
sets = ", ".join(f"{k} = ?" for k in kwargs)
values = [*list(kwargs.values()), user_id]
conn.execute(f"UPDATE users SET {sets} WHERE id = ?", values)
msg = f"User {user_id} not found"
raise ValueError(msg)
for column, value in kwargs.items():
conn.execute(self._USER_UPDATE_STATEMENTS[column], (value, user_id))
conn.commit()
finally:
conn.close()
@@ -371,14 +384,9 @@ class UserDB:
).fetchall()
request_item_keys = [f"request:{row['id']}" for row in request_rows]
if request_item_keys:
placeholders = ",".join("?" for _ in request_item_keys)
conn.execute(
f"""
DELETE FROM activity_view_state
WHERE item_type = 'request'
AND item_key IN ({placeholders})
""",
request_item_keys,
conn.executemany(
"DELETE FROM activity_view_state WHERE item_type = 'request' AND item_key = ?",
[(item_key,) for item_key in request_item_keys],
)
conn.execute(
"DELETE FROM activity_view_state WHERE viewer_scope = ?",
@@ -461,7 +469,8 @@ class UserDB:
try:
return json.dumps(value)
except TypeError as exc:
raise ValueError(f"{field} must be JSON-serializable") from exc
msg = f"{field} must be JSON-serializable"
raise ValueError(msg) from exc
@staticmethod
def _parse_request_row(row: sqlite3.Row | None) -> dict[str, Any] | None:
@@ -543,7 +552,8 @@ class UserDB:
).fetchone()
parsed = self._parse_request_row(row)
if parsed is None:
raise ValueError(f"Request {request_id} not found after creation")
msg = f"Request {request_id} not found after creation"
raise ValueError(msg)
return parsed
def create_request(
@@ -566,11 +576,14 @@ class UserDB:
) -> dict[str, Any]:
"""Create a download request row and return the created record."""
if not isinstance(book_data, dict):
raise TypeError("book_data must be an object")
msg = "book_data must be an object"
raise TypeError(msg)
if release_data is not None and not isinstance(release_data, dict):
raise TypeError("release_data must be an object when provided")
msg = "release_data must be an object when provided"
raise TypeError(msg)
if not content_type:
raise ValueError("content_type is required")
msg = "content_type is required"
raise ValueError(msg)
normalized_status = normalize_request_status(status)
normalized_delivery_state = normalize_delivery_state(delivery_state)
@@ -690,27 +703,46 @@ class UserDB:
"last_failure_reason",
}
)
_REQUEST_UPDATE_STATEMENTS: ClassVar[dict[str, str]] = {
"status": "UPDATE download_requests SET status = ? WHERE id = ?",
"source_hint": "UPDATE download_requests SET source_hint = ? WHERE id = ?",
"content_type": "UPDATE download_requests SET content_type = ? WHERE id = ?",
"request_level": "UPDATE download_requests SET request_level = ? WHERE id = ?",
"policy_mode": "UPDATE download_requests SET policy_mode = ? WHERE id = ?",
"book_data": "UPDATE download_requests SET book_data = ? WHERE id = ?",
"release_data": "UPDATE download_requests SET release_data = ? WHERE id = ?",
"note": "UPDATE download_requests SET note = ? WHERE id = ?",
"admin_note": "UPDATE download_requests SET admin_note = ? WHERE id = ?",
"reviewed_by": "UPDATE download_requests SET reviewed_by = ? WHERE id = ?",
"reviewed_at": "UPDATE download_requests SET reviewed_at = ? WHERE id = ?",
"delivery_state": "UPDATE download_requests SET delivery_state = ? WHERE id = ?",
"delivery_updated_at": "UPDATE download_requests SET delivery_updated_at = ? WHERE id = ?",
"last_failure_reason": "UPDATE download_requests SET last_failure_reason = ? WHERE id = ?",
}
def update_request(
self,
request_id: int,
expected_current_status: str | None = None,
**kwargs,
**kwargs: object,
) -> dict[str, Any]:
"""Update request fields and return the updated record."""
if not kwargs:
request = self.get_request(request_id)
if request is None:
raise ValueError(f"Request {request_id} not found")
msg = f"Request {request_id} not found"
raise ValueError(msg)
if expected_current_status is not None:
normalized_expected_status = normalize_request_status(expected_current_status)
if request["status"] != normalized_expected_status:
raise ValueError("Request state changed before update")
msg = "Request state changed before update"
raise ValueError(msg)
return request
for key in kwargs:
if key not in self._ALLOWED_REQUEST_UPDATE_COLUMNS:
raise ValueError(f"Invalid request column: {key}")
msg = f"Invalid request column: {key}"
raise ValueError(msg)
with self._lock:
conn = self._connect()
@@ -721,12 +753,14 @@ class UserDB:
).fetchone()
current = self._parse_request_row(row)
if current is None:
raise ValueError(f"Request {request_id} not found")
msg = f"Request {request_id} not found"
raise ValueError(msg)
if expected_current_status is not None:
normalized_expected_status = normalize_request_status(expected_current_status)
if current["status"] != normalized_expected_status:
raise ValueError("Request state changed before update")
msg = "Request state changed before update"
raise ValueError(msg)
updates = dict(kwargs)
@@ -746,35 +780,35 @@ class UserDB:
if "delivery_updated_at" in updates:
delivery_updated_at = updates["delivery_updated_at"]
if delivery_updated_at is not None and not isinstance(delivery_updated_at, str):
raise TypeError("delivery_updated_at must be a string when provided")
msg = "delivery_updated_at must be a string when provided"
raise TypeError(msg)
if "content_type" in updates and not updates["content_type"]:
raise ValueError("content_type is required")
msg = "content_type is required"
raise ValueError(msg)
if "request_level" in updates:
updates["request_level"] = normalize_request_level(updates["request_level"])
if "book_data" in updates:
if not isinstance(updates["book_data"], dict):
raise TypeError("book_data must be an object")
msg = "book_data must be an object"
raise TypeError(msg)
updates["book_data"] = self._serialize_json(updates["book_data"], "book_data")
if "release_data" in updates:
if updates["release_data"] is not None and not isinstance(
updates["release_data"], dict
):
raise TypeError("release_data must be an object when provided")
msg = "release_data must be an object when provided"
raise TypeError(msg)
updates["release_data"] = self._serialize_json(
updates["release_data"],
"release_data",
)
set_clause = ", ".join(f"{column} = ?" for column in updates)
values = [*list(updates.values()), request_id]
conn.execute(
f"UPDATE download_requests SET {set_clause} WHERE id = ?",
values,
)
for column, value in updates.items():
conn.execute(self._REQUEST_UPDATE_STATEMENTS[column], (value, request_id))
conn.commit()
updated_row = conn.execute(
@@ -783,7 +817,8 @@ class UserDB:
).fetchone()
parsed = self._parse_request_row(updated_row)
if parsed is None:
raise ValueError(f"Request {request_id} not found after update")
msg = f"Request {request_id} not found after update"
raise ValueError(msg)
return parsed
finally:
conn.close()
@@ -865,7 +900,8 @@ class UserDB:
).fetchone()
current = self._parse_request_row(row)
if current is None:
raise ValueError(f"Request {request_id} not found")
msg = f"Request {request_id} not found"
raise ValueError(msg)
conn.execute(
"""
@@ -893,7 +929,8 @@ class UserDB:
conn.commit()
parsed = self._parse_request_row(updated_row)
if parsed is None:
raise ValueError(f"Request {request_id} not found after rollback")
msg = f"Request {request_id} not found after rollback"
raise ValueError(msg)
return parsed
finally:
conn.close()
shelfmark/core/user_settings_overrides.py
+3
View File
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
def get_settings_registry() -> ModuleType:
"""Load settings modules and return the shared settings registry module."""
# Ensure settings modules are loaded before reading registry metadata.
import_module("shelfmark.config.notifications_settings")
import_module("shelfmark.config.security")
@@ -23,6 +24,7 @@ def get_settings_registry() -> ModuleType:
def get_ordered_user_overridable_fields(tab_name: str) -> list[tuple[str, Any]]:
"""Return user-overridable fields for a tab in UI display order."""
settings_registry = get_settings_registry()
tab = settings_registry.get_settings_tab(tab_name)
if not tab:
@@ -32,6 +34,7 @@ def get_ordered_user_overridable_fields(tab_name: str) -> list[tuple[str, Any]]:
def build_user_preferences_payload(user_db: UserDB, user_id: int, tab_name: str) -> dict[str, Any]:
"""Build the effective user-preferences payload for a settings tab."""
from shelfmark.core.config import config as app_config
settings_registry = get_settings_registry()
shelfmark/core/utils.py
+5 -3
View File
@@ -4,6 +4,7 @@ import base64
import importlib
import os
import re
import sqlite3
from pathlib import Path
from threading import Lock
from typing import TYPE_CHECKING
@@ -57,6 +58,7 @@ def normalize_http_url(
_xmlrpc_patch_lock = Lock()
_xmlrpc_patch_applied = False
_XMLRPC_PATCH_ERRORS = (ImportError, AttributeError, OSError, RuntimeError)
def get_hardened_xmlrpc_client() -> ModuleType:
@@ -70,7 +72,7 @@ def get_hardened_xmlrpc_client() -> ModuleType:
monkey_patch()
_xmlrpc_patch_applied = True
except Exception:
except _XMLRPC_PATCH_ERRORS:
# Keep runtime behavior unchanged if defusedxml is unavailable.
_xmlrpc_patch_applied = False
@@ -173,7 +175,7 @@ def _resolve_destination_username(
if not user:
return ""
return str(user.get("username") or "").strip()
except Exception:
except ImportError, OSError, sqlite3.Error:
return ""
@@ -257,7 +259,7 @@ def get_aa_content_type_dir(content_type: str | None = None) -> Path | None:
def get_ingest_dir(content_type: str | None = None) -> Path:
"""DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead."""
"""Return the legacy ingest directory for a content type."""
from shelfmark.core.config import config
# Check new DESTINATION setting first, then legacy INGEST_DIR
shelfmark/download/archive.py
+22 -11
View File
@@ -134,7 +134,8 @@ def extract_archive(
elif suffix == "rar":
extracted_files, warnings = _extract_rar(archive_path, output_dir)
else:
raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
msg = f"Unsupported archive format: {suffix}"
raise ArchiveExtractionError(msg)
is_audiobook = check_audiobook(content_type)
file_type_label = "audiobook" if is_audiobook else "book"
@@ -174,7 +175,8 @@ def extract_archive_raw(
if suffix == "rar":
return _extract_rar(archive_path, output_dir)
raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
msg = f"Unsupported archive format: {suffix}"
raise ArchiveExtractionError(msg)
def _extract_files_from_archive(archive: ArchiveType, output_dir: Path) -> list[Path]:
@@ -222,31 +224,37 @@ def _extract_zip(archive_path: Path, output_dir: Path) -> tuple[list[Path], list
# Check for password protection
for info in zf.infolist():
if info.flag_bits & 0x1: # Encrypted flag
raise PasswordProtectedError("ZIP archive is password protected")
msg = "ZIP archive is password protected"
raise PasswordProtectedError(msg)
# Test archive integrity
bad_file = zf.testzip()
if bad_file:
raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}")
msg = f"Corrupted file in archive: {bad_file}"
raise CorruptedArchiveError(msg)
return _extract_files_from_archive(zf, output_dir), []
except zipfile.BadZipFile as e:
raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}") from e
msg = f"Invalid or corrupted ZIP: {e}"
raise CorruptedArchiveError(msg) from e
except PermissionError as e:
raise ArchiveExtractionError(f"Permission denied: {e}") from e
msg = f"Permission denied: {e}"
raise ArchiveExtractionError(msg) from e
def _extract_rar(archive_path: Path, output_dir: Path) -> tuple[list[Path], list[str]]:
"""Extract files from a RAR archive."""
if not RAR_AVAILABLE:
raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
msg = "RAR extraction not available - rarfile library not installed"
raise ArchiveExtractionError(msg)
try:
with rarfile.RarFile(archive_path, "r") as rf:
# Check for password protection
if rf.needs_password():
raise PasswordProtectedError("RAR archive is password protected")
msg = "RAR archive is password protected"
raise PasswordProtectedError(msg)
# Test archive integrity
rf.testrar()
@@ -254,8 +262,11 @@ def _extract_rar(archive_path: Path, output_dir: Path) -> tuple[list[Path], list
return _extract_files_from_archive(rf, output_dir), []
except rarfile.BadRarFile as e:
raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}") from e
msg = f"Invalid or corrupted RAR: {e}"
raise CorruptedArchiveError(msg) from e
except rarfile.RarCannotExec as e:
raise ArchiveExtractionError("unrar binary not found - install unrar package") from e
msg = "unrar binary not found - install unrar package"
raise ArchiveExtractionError(msg) from e
except PermissionError as e:
raise ArchiveExtractionError(f"Permission denied: {e}") from e
msg = f"Permission denied: {e}"
raise ArchiveExtractionError(msg) from e
shelfmark/download/clients/__init__.py
+34 -13
View File
@@ -17,6 +17,7 @@ from abc import ABC, abstractmethod
from dataclasses import dataclass
from enum import Enum
from functools import wraps
from importlib import import_module
from pathlib import Path
from typing import TYPE_CHECKING, TypeVar, cast
@@ -39,6 +40,7 @@ RETRYABLE_EXCEPTIONS = (
_MIN_RETRYABLE_STATUS = 500
_MIN_PROGRESS_PERCENT = 0
_MAX_PROGRESS_PERCENT = 100
_RNG = random.SystemRandom()
def with_retry(
@@ -47,7 +49,7 @@ def with_retry(
max_delay: float = 10.0,
jitter: float = 0.5,
) -> Callable[[Callable[..., T]], Callable[..., T]]:
"""Decorator for retrying API calls with exponential backoff.
"""Retry API calls with exponential backoff.
Args:
max_attempts: Maximum number of attempts (default 3)
@@ -68,7 +70,7 @@ def with_retry(
def decorator(func: Callable[..., T]) -> Callable[..., T]:
@wraps(func)
def wrapper(*args, **kwargs) -> T:
def wrapper(*args: object, **kwargs: object) -> T:
last_exception = None
for attempt in range(1, max_attempts + 1):
@@ -86,7 +88,7 @@ def with_retry(
# Calculate delay with exponential backoff
delay = min(base_delay * (2 ** (attempt - 1)), max_delay)
# Add jitter to prevent thundering herd
delay += random.uniform(0, delay * jitter)
delay += _RNG.uniform(0, delay * jitter)
_logger.debug(
"Retry %s/%s for %s after %.1fs (error: %s)",
attempt,
@@ -232,7 +234,7 @@ class DownloadClient(ABC):
# Join and normalize
return os.path.normpath(str(Path(valid[0]).joinpath(*valid[1:])))
def __init_subclass__(cls, **kwargs) -> None:
def __init_subclass__(cls, **kwargs: object) -> None:
"""Validate that subclasses define required class attributes."""
super().__init_subclass__(**kwargs)
@@ -288,6 +290,7 @@ class DownloadClient(ABC):
name: Display name for the download
category: Category/label for organization (None = client default)
expected_hash: Optional info_hash hint (torrents only)
**kwargs: Client-specific options passed through to the implementation.
Returns:
Client-specific download ID (hash for torrents, ID for NZBGet).
@@ -356,12 +359,32 @@ class DownloadClient(ABC):
# Client registry: protocol -> list of client classes
_CLIENTS: dict[str, list[type[DownloadClient]]] = {}
_BUILTIN_CLIENT_MODULES = (
"shelfmark.download.clients.deluge",
"shelfmark.download.clients.nzbget",
"shelfmark.download.clients.qbittorrent",
"shelfmark.download.clients.rtorrent",
"shelfmark.download.clients.sabnzbd",
"shelfmark.download.clients.transmission",
)
_builtin_client_state = {"loaded": False}
def _ensure_builtin_clients_registered() -> None:
"""Import built-in client modules once to populate the registry."""
if _builtin_client_state["loaded"]:
return
for module_name in _BUILTIN_CLIENT_MODULES:
import_module(module_name)
_builtin_client_state["loaded"] = True
def register_client(
protocol: str,
) -> Callable[[type[DownloadClient]], type[DownloadClient]]:
"""Decorator to register a download client for a protocol.
"""Register a download client for a protocol.
Multiple clients can be registered for the same protocol.
The `is_configured()` method determines which one is active.
@@ -398,6 +421,8 @@ def get_client(protocol: str) -> DownloadClient | None:
Configured client instance, or None if not available/configured.
"""
_ensure_builtin_clients_registered()
if protocol not in _CLIENTS:
return None
@@ -415,6 +440,8 @@ def list_configured_clients() -> list[str]:
List of protocol names (e.g., ["torrent", "usenet"]).
"""
_ensure_builtin_clients_registered()
result = []
for protocol, client_classes in _CLIENTS.items():
for cls in client_classes:
@@ -431,14 +458,8 @@ def get_all_clients() -> dict[str, list[type[DownloadClient]]]:
Dict of protocol -> list of client classes.
"""
_ensure_builtin_clients_registered()
return dict(_CLIENTS)
# Import client implementations to trigger registration
# These imports are at the bottom to avoid circular imports
from shelfmark.download.clients import deluge as deluge
from shelfmark.download.clients import nzbget as nzbget
from shelfmark.download.clients import qbittorrent as qbittorrent
from shelfmark.download.clients import rtorrent as rtorrent
from shelfmark.download.clients import sabnzbd as sabnzbd
from shelfmark.download.clients import transmission as transmission
_ensure_builtin_clients_registered()
shelfmark/download/clients/base_handler.py
+22 -13
View File
@@ -29,9 +29,13 @@ if TYPE_CHECKING:
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
_CLIENT_CLEANUP_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError)
# How often to poll the download client for status (seconds)
POLL_INTERVAL = 2
WINDOWS_DRIVE_PREFIX_LENGTH = 2
SECONDS_PER_MINUTE = 60
SECONDS_PER_HOUR = 3600
# How long to wait for completed files to appear (seconds)
COMPLETED_PATH_RETRY_INTERVAL = 5
COMPLETED_PATH_MAX_ATTEMPTS = 12 # 12 attempts * 5s = 60s grace period
@@ -60,7 +64,7 @@ def _diagnose_path_issue(path: str) -> str:
"""
# Detect Windows-style paths (won't work in Linux containers)
if len(path) >= 2 and path[1] == ":":
if len(path) >= WINDOWS_DRIVE_PREFIX_LENGTH and path[1] == ":":
return (
f"Path '{path}' appears to be a Windows path. "
f"Shelfmark runs in Linux and cannot access Windows paths directly. "
@@ -110,6 +114,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
"""Shared lifecycle handler for sources that hand off to torrent/usenet clients."""
def __init__(self) -> None:
"""Initialize cleanup tracking for client-managed downloads."""
# Track downloads that may need client-side cleanup after Shelfmark completes import.
# task_id -> (client, download_id, protocol)
self._cleanup_refs: dict[str, tuple[DownloadClient, str, str]] = {}
@@ -123,7 +128,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
"""Resolve source-specific task metadata into a client download request."""
def _on_download_complete(self, task: DownloadTask) -> None:
"""Hook called after successful completion; override for source cleanup."""
"""Run post-completion source cleanup hooks."""
return
def _get_client(self, protocol: str) -> DownloadClient | None:
@@ -135,7 +140,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
return list_configured_clients()
def _poll_interval(self) -> float:
"""Polling interval for status checks (seconds)."""
"""Return the polling interval for status checks."""
return POLL_INTERVAL
def _completed_path_retry_interval(self) -> float:
@@ -163,6 +168,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
return config.get(audiobook_key, "") or None if audiobook_key else None
def post_process_cleanup(self, task: DownloadTask, *, success: bool) -> None:
"""Clean up external-client state after post-processing finishes."""
if not success:
self._cleanup_refs.pop(task.task_id, None)
return
@@ -180,7 +186,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
try:
self._delete_local_download_data(client, download_id)
self._remove_usenet_download(client, download_id, delete_files=True, archive=True)
except Exception as e:
except _CLIENT_CLEANUP_ERRORS as e:
logger.warning(
"Failed to cleanup usenet download %s in %s: %s",
download_id,
@@ -193,7 +199,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
return
try:
client.remove(download_id, delete_files=False)
except Exception as e:
except _CLIENT_CLEANUP_ERRORS as e:
logger.warning(
"Failed to remove torrent %s from %s: %s",
download_id,
@@ -219,7 +225,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
"""Best-effort local deletion of client download data."""
try:
raw_path = client.get_download_path(download_id)
except Exception as e:
except _CLIENT_CLEANUP_ERRORS as e:
logger.debug(
"Failed to resolve download path for %s %s: %s", client.name, download_id, e
)
@@ -268,7 +274,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
logger.info(
"Deleted local download data for %s %s: %s", client.name, download_id, delete_path
)
except Exception as e:
except _CLIENT_CLEANUP_ERRORS as e:
logger.warning(
"Failed to delete local download data for %s %s: %s", client.name, download_id, e
)
@@ -300,7 +306,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
# Permanent delete for failed usenet downloads (SABnzbd archive=0).
self._delete_local_download_data(client, download_id)
self._remove_usenet_download(client, download_id, delete_files=True, archive=False)
except Exception as e:
except _CLIENT_CLEANUP_ERRORS as e:
logger.warning(
"Failed to remove download %s from %s after %s: %s",
download_id,
@@ -321,7 +327,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
try:
self._delete_local_download_data(client, download_id)
self._remove_usenet_download(client, download_id, delete_files=True, archive=True)
except Exception as e:
except _CLIENT_CLEANUP_ERRORS as e:
logger.warning(
"Failed to remove download %s from %s after cancellation: %s",
download_id,
@@ -557,12 +563,15 @@ class ExternalClientHandler(DownloadHandler, ABC):
msg += f" ({speed_mb:.1f} MB/s)"
if status.eta and status.eta > 0:
if status.eta < 60:
if status.eta < SECONDS_PER_MINUTE:
msg += f" - {status.eta}s left"
elif status.eta < 3600:
msg += f" - {status.eta // 60}m left"
elif status.eta < SECONDS_PER_HOUR:
msg += f" - {status.eta // SECONDS_PER_MINUTE}m left"
else:
msg += f" - {status.eta // 3600}h {(status.eta % 3600) // 60}m left"
msg += (
f" - {status.eta // SECONDS_PER_HOUR}h "
f"{(status.eta % SECONDS_PER_HOUR) // SECONDS_PER_MINUTE}m left"
)
return msg
shelfmark/download/clients/deluge.py
+34 -16
View File
@@ -40,11 +40,25 @@ ONE_WEEK_IN_SECONDS = 604800
class DelugeRpcError(RuntimeError):
"""Raised when Deluge returns a JSON-RPC error response."""
def __init__(self, message: str, code: int | None = None) -> None:
"""Initialize the RPC error with an optional Deluge error code."""
super().__init__(message)
self.code = code
_DELUGE_CLIENT_ERRORS = (
AttributeError,
DelugeRpcError,
OSError,
requests.exceptions.RequestException,
RuntimeError,
TypeError,
ValueError,
)
def _get_error_message(error: object) -> tuple[str, int | None]:
if isinstance(error, dict):
return str(error.get("message") or error), error.get("code")
@@ -63,6 +77,7 @@ class DelugeClient(DownloadClient):
name = "deluge"
def __init__(self) -> None:
"""Initialize the client from the configured Deluge connection settings."""
raw_host = str(config.get("DELUGE_HOST", "localhost") or "")
raw_port = str(config.get("DELUGE_PORT", "8112") or "8112")
password = str(config.get("DELUGE_PASSWORD", "") or "")
@@ -146,8 +161,7 @@ class DelugeClient(DownloadClient):
self._authenticated = True
def _select_daemon_host_id(self, hosts: list) -> str:
# Hosts returned by web.get_hosts look like:
# [[host_id, host, port, status], ...]
# Deluge returns entries containing host id, host, port, and status.
preferred_hosts = {"127.0.0.1", "localhost"}
for entry in hosts:
@@ -201,13 +215,10 @@ class DelugeClient(DownloadClient):
def _get_daemon_version(self) -> object:
"""Fetch daemon version, preferring daemon.get_version when available."""
try:
with suppress(*_DELUGE_CLIENT_ERRORS):
methods = self._rpc_call("system.listMethods")
if isinstance(methods, list) and "daemon.get_version" in methods:
return self._rpc_call("daemon.get_version")
except Exception:
# Fall back to daemon.info to preserve existing behavior.
pass
return self._rpc_call("daemon.info")
@@ -218,25 +229,27 @@ class DelugeClient(DownloadClient):
try:
# label.add will error if the plugin is unavailable or the label exists.
with suppress(Exception):
with suppress(*_DELUGE_CLIENT_ERRORS):
self._rpc_call("label.add", label)
self._rpc_call("label.set_torrent", torrent_id, label)
except Exception as e:
except _DELUGE_CLIENT_ERRORS as e:
logger.debug("Could not set Deluge label '%s' for %s: %s", label, torrent_id, e)
@staticmethod
def is_configured() -> bool:
"""Return whether Deluge is the active configured torrent client."""
client = config.get("PROWLARR_TORRENT_CLIENT", "")
host = config.get("DELUGE_HOST", "")
password = config.get("DELUGE_PASSWORD", "")
return client == "deluge" and bool(host) and bool(password)
def test_connection(self) -> tuple[bool, str]:
"""Test connectivity and authentication against the Deluge server."""
try:
self._ensure_connected()
version = self._get_daemon_version()
except Exception as e:
except _DELUGE_CLIENT_ERRORS as e:
self._authenticated = False
self._connected = False
return False, f"Connection failed: {e!s}"
@@ -249,8 +262,9 @@ class DelugeClient(DownloadClient):
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
**kwargs: object,
) -> str:
"""Add a torrent to Deluge and return the torrent id."""
try:
self._ensure_connected()
@@ -298,7 +312,7 @@ class DelugeClient(DownloadClient):
logger.info("Added torrent to Deluge: %s", torrent_id)
except Exception:
except _DELUGE_CLIENT_ERRORS:
self._authenticated = False
self._connected = False
logger.exception("Deluge add failed")
@@ -307,6 +321,7 @@ class DelugeClient(DownloadClient):
return torrent_id
def get_status(self, download_id: str) -> DownloadStatus:
"""Return the current Deluge status for a torrent."""
try:
self._ensure_connected()
@@ -352,7 +367,7 @@ class DelugeClient(DownloadClient):
if eta is not None:
try:
eta = int(eta)
except Exception:
except TypeError, ValueError:
eta = None
if eta is not None and (eta < 0 or eta > ONE_WEEK_IN_SECONDS):
@@ -376,10 +391,11 @@ class DelugeClient(DownloadClient):
eta=eta,
)
except Exception as e:
except _DELUGE_CLIENT_ERRORS as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
"""Remove a torrent from Deluge, optionally deleting its files."""
try:
self._ensure_connected()
@@ -392,13 +408,14 @@ class DelugeClient(DownloadClient):
)
return True
except Exception as e:
except _DELUGE_CLIENT_ERRORS as e:
self._log_error("remove", e)
return False
else:
return False
def get_download_path(self, download_id: str) -> str | None:
"""Return the resolved download path for a Deluge torrent."""
try:
self._ensure_connected()
@@ -414,7 +431,7 @@ class DelugeClient(DownloadClient):
str(status.get("name", "")),
)
except Exception as e:
except _DELUGE_CLIENT_ERRORS as e:
self._log_error("get_download_path", e, level="debug")
return None
else:
@@ -423,6 +440,7 @@ class DelugeClient(DownloadClient):
def find_existing(
self, url: str, category: str | None = None
) -> tuple[str, DownloadStatus] | None:
"""Find an existing Deluge torrent matching a release URL."""
try:
self._ensure_connected()
@@ -440,7 +458,7 @@ class DelugeClient(DownloadClient):
full_status = self.get_status(torrent_info.info_hash)
return (torrent_info.info_hash, full_status)
except Exception as e:
except _DELUGE_CLIENT_ERRORS as e:
self._authenticated = False
self._connected = False
logger.debug("Error checking for existing torrent: %s", e)
shelfmark/download/clients/nzbget.py
+7 -5
View File
@@ -19,6 +19,7 @@ from shelfmark.download.clients import (
from shelfmark.download.network import get_ssl_verify
logger = setup_logger(__name__)
_NZBGET_CLIENT_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, ValueError)
@register_client("usenet")
@@ -59,7 +60,7 @@ class NZBGetClient(DownloadClient):
if result:
logger.info("Removed NZB from NZBGet (%s): %s", command, download_id)
return True, None
except Exception as e:
except _NZBGET_CLIENT_ERRORS as e:
return False, e
return False, None
@@ -115,7 +116,7 @@ class NZBGetClient(DownloadClient):
return False, "Could not connect to NZBGet"
except requests.exceptions.Timeout:
return False, "Connection timed out"
except Exception as e:
except _NZBGET_CLIENT_ERRORS as e:
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to NZBGet {version}"
@@ -126,7 +127,7 @@ class NZBGetClient(DownloadClient):
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
**kwargs: object,
) -> str:
"""Add NZB by URL.
@@ -138,6 +139,7 @@ class NZBGetClient(DownloadClient):
name: Display name for the download
category: Category for organization (uses configured default if not specified)
expected_hash: Optional info_hash hint (unused)
**kwargs: Client-specific options passed through to the implementation.
Returns:
NZBGet download ID (NZBID).
@@ -193,7 +195,7 @@ class NZBGetClient(DownloadClient):
logger.exception("Failed to fetch NZB from URL")
msg = f"Failed to fetch NZB: {e}"
raise RuntimeError(msg) from e
except Exception:
except _NZBGET_CLIENT_ERRORS:
logger.exception("NZBGet add failed")
raise
@@ -286,7 +288,7 @@ class NZBGetClient(DownloadClient):
# Not found in queue or history
return DownloadStatus.error("Download not found")
except Exception as e:
except _NZBGET_CLIENT_ERRORS as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
shelfmark/download/clients/qbittorrent.py
+57 -24
View File
@@ -1,11 +1,15 @@
"""qBittorrent download client for Prowlarr integration."""
from __future__ import annotations
import time
from http import HTTPStatus
from pathlib import Path
from types import SimpleNamespace
from typing import NoReturn
import requests
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import normalize_http_url
@@ -19,6 +23,15 @@ from shelfmark.download.clients.torrent_utils import (
)
from shelfmark.download.network import get_ssl_verify
try:
import qbittorrentapi as _qbittorrentapi
except ImportError:
_ImportedQBittorrentApiError = RuntimeError
_ImportedQBittorrentLoginFailed = RuntimeError
else:
_ImportedQBittorrentApiError = getattr(_qbittorrentapi, "APIError", RuntimeError)
_ImportedQBittorrentLoginFailed = getattr(_qbittorrentapi, "LoginFailed", RuntimeError)
logger = setup_logger(__name__)
_HASH_LENGTH_40 = 40
@@ -28,6 +41,25 @@ _HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
_ONE_WEEK_IN_SECONDS = 604800
def _resolve_qbittorrent_exception_type(candidate: object) -> type[Exception]:
if isinstance(candidate, type) and issubclass(candidate, Exception):
return candidate
return RuntimeError
_QBittorrentApiError = _resolve_qbittorrent_exception_type(_ImportedQBittorrentApiError)
_QBittorrentLoginFailed = _resolve_qbittorrent_exception_type(_ImportedQBittorrentLoginFailed)
_QBITTORRENT_CLIENT_ERRORS = (
_QBittorrentLoginFailed,
_QBittorrentApiError,
AttributeError,
OSError,
RuntimeError,
TypeError,
ValueError,
)
def _hashes_match(hash1: str, hash2: str) -> bool:
"""Compare hashes, handling Amarr's 40-char zero-padded hashes vs 32-char ed2k hashes."""
h1, h2 = hash1.lower(), hash2.lower()
@@ -106,8 +138,6 @@ class QBittorrentClient(DownloadClient):
A false result with no error means "not loaded yet".
"""
import requests
url = f"{self._base_url}/api/v2/torrents/properties"
params = {"hash": torrent_hash}
@@ -142,7 +172,13 @@ class QBittorrentClient(DownloadClient):
return False, f"Cannot connect to qBittorrent at {self._base_url}"
except requests.exceptions.Timeout:
return False, f"qBittorrent request timed out at {self._base_url}"
except Exception as e:
except requests.exceptions.InvalidSchema:
return (
False,
"qBittorrent URL is invalid (missing http:// or https://). "
f"Configured: {self._base_url}",
)
except _QBITTORRENT_CLIENT_ERRORS as e:
return False, f"qBittorrent API error: {type(e).__name__}: {e}"
else:
return True, None
@@ -193,8 +229,6 @@ class QBittorrentClient(DownloadClient):
(torrents, error_message)
"""
import requests
url = f"{self._base_url}/api/v2/torrents/info"
def do_request(params: dict[str, str]) -> requests.Response:
@@ -268,15 +302,15 @@ class QBittorrentClient(DownloadClient):
except requests.exceptions.Timeout:
logger.warning("qBittorrent request timed out at %s", self._base_url)
return [], f"qBittorrent request timed out at {self._base_url}"
except Exception as e:
except requests.exceptions.InvalidSchema:
logger.debug("Failed to get torrents info: invalid qBittorrent URL: %s", self._base_url)
return (
[],
"qBittorrent URL is invalid (missing http:// or https://). "
f"Configured: {self._base_url}",
)
except _QBITTORRENT_CLIENT_ERRORS as e:
logger.debug("Failed to get torrents info: %s", e)
# requests raises InvalidSchema when the base URL doesn't include http(s)
if type(e).__name__ == "InvalidSchema":
return (
[],
"qBittorrent URL is invalid (missing http:// or https://). "
f"Configured: {self._base_url}",
)
return [], f"qBittorrent API error: {type(e).__name__}: {e}"
else:
return torrents, None
@@ -293,7 +327,7 @@ class QBittorrentClient(DownloadClient):
try:
self._client.auth_log_in()
api_version = self._client.app.web_api_version
except Exception as e:
except _QBITTORRENT_CLIENT_ERRORS as e:
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to qBittorrent (API v{api_version})"
@@ -304,7 +338,7 @@ class QBittorrentClient(DownloadClient):
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
**kwargs: object,
) -> str:
"""Add torrent by URL (magnet or .torrent).
@@ -313,6 +347,7 @@ class QBittorrentClient(DownloadClient):
name: Display name for the torrent
category: Category for organization (uses configured default if not specified)
expected_hash: Optional info_hash hint (from Prowlarr)
**kwargs: Client-specific options passed through to the implementation.
Returns:
Torrent hash (info_hash).
@@ -330,7 +365,7 @@ class QBittorrentClient(DownloadClient):
if category:
try:
self._client.torrents_create_category(name=category)
except Exception as e:
except _QBITTORRENT_CLIENT_ERRORS as e:
# Conflict409Error means category exists - that's expected
# Log other errors but continue since download may still work
if "Conflict" not in type(e).__name__ and "409" not in str(e):
@@ -402,7 +437,7 @@ class QBittorrentClient(DownloadClient):
"Torrent add was not confirmed within the visibility grace period (response=%s), returning expected hash",
result_text,
)
except Exception:
except _QBITTORRENT_CLIENT_ERRORS:
logger.exception("qBittorrent add failed")
raise
else:
@@ -498,7 +533,7 @@ class QBittorrentClient(DownloadClient):
download_speed=torrent_speed,
eta=eta,
)
except Exception as e:
except _QBITTORRENT_CLIENT_ERRORS as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
@@ -519,7 +554,7 @@ class QBittorrentClient(DownloadClient):
download_id,
" (with files)" if delete_files else "",
)
except Exception as e:
except _QBITTORRENT_CLIENT_ERRORS as e:
self._log_error("remove", e)
return False
else:
@@ -555,7 +590,7 @@ class QBittorrentClient(DownloadClient):
return None
return self._resolve_completed_download_path(torrent)
except Exception as e:
except _QBITTORRENT_CLIENT_ERRORS as e:
self._log_error("get_download_path", e, level="debug")
return None
@@ -593,8 +628,6 @@ class QBittorrentClient(DownloadClient):
"""
import os
import requests
def get_with_auth(url: str, params: dict[str, str]) -> requests.Response:
self._client.auth_log_in()
resp = self._client._session.get(url, params=params, timeout=10)
@@ -637,7 +670,7 @@ class QBittorrentClient(DownloadClient):
return None
return os.path.normpath(str(Path(save_path) / top_level))
except Exception as e:
except _QBITTORRENT_CLIENT_ERRORS as e:
logger.debug(
"qBittorrent could not derive path from files: %s: %s",
type(e).__name__,
@@ -671,7 +704,7 @@ class QBittorrentClient(DownloadClient):
if torrent and isinstance(getattr(torrent, "hash", None), str):
torrent_hash = torrent.hash
return (torrent_hash.lower(), self.get_status(torrent_hash.lower()))
except Exception as e:
except _QBITTORRENT_CLIENT_ERRORS as e:
logger.debug("Error checking for existing torrent: %s", e)
return None
else:
shelfmark/download/clients/rtorrent.py
+26 -12
View File
@@ -4,6 +4,7 @@ Uses xmlrpc to communicate with rTorrent's RPC interface.
"""
import ssl
import xmlrpc.client as stdlib_xmlrpc_client
from typing import NoReturn
from urllib.parse import urlparse
@@ -24,6 +25,14 @@ logger = setup_logger(__name__)
_ETA_MAX_SECONDS = 604800
_RTORRENT_CLIENT_ERRORS = (
AttributeError,
OSError,
RuntimeError,
TypeError,
ValueError,
stdlib_xmlrpc_client.Error,
)
def _create_rtorrent_server_proxy(url: str) -> object:
@@ -86,7 +95,7 @@ class RTorrentClient(DownloadClient):
"""Test connection to rTorrent."""
try:
version = self._rpc.system.client_version()
except Exception as e:
except _RTORRENT_CLIENT_ERRORS as e:
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to rTorrent {version}"
@@ -97,7 +106,7 @@ class RTorrentClient(DownloadClient):
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
**kwargs: object,
) -> str:
"""Add torrent by URL (magnet or .torrent).
@@ -106,6 +115,7 @@ class RTorrentClient(DownloadClient):
name: Display name for the torrent
category: Category for organization (uses configured label if not specified)
expected_hash: Optional info_hash hint (from Prowlarr)
**kwargs: Client-specific options passed through to the implementation.
Returns:
Torrent hash (info_hash).
@@ -153,7 +163,7 @@ class RTorrentClient(DownloadClient):
logger.debug("Added torrent to rTorrent: %s", torrent_hash)
except Exception:
except _RTORRENT_CLIENT_ERRORS:
logger.exception("rTorrent add failed")
raise
else:
@@ -213,7 +223,7 @@ class RTorrentClient(DownloadClient):
try:
state = int(state)
except Exception:
except TypeError, ValueError:
state = 0
complete = bool(complete)
@@ -255,7 +265,7 @@ class RTorrentClient(DownloadClient):
eta=eta,
)
except Exception as e:
except _RTORRENT_CLIENT_ERRORS as e:
error_type = type(e).__name__
logger.exception("rTorrent get_status failed (%s)", error_type)
return DownloadStatus.error(f"{error_type}: {e}")
@@ -284,7 +294,7 @@ class RTorrentClient(DownloadClient):
download_id,
" (with files)" if delete_files else "",
)
except Exception as e:
except _RTORRENT_CLIENT_ERRORS as e:
error_type = type(e).__name__
logger.exception("rTorrent remove failed (%s)", error_type)
return False
@@ -303,7 +313,7 @@ class RTorrentClient(DownloadClient):
"""
try:
return self._get_torrent_path(download_id)
except Exception as e:
except _RTORRENT_CLIENT_ERRORS as e:
error_type = type(e).__name__
logger.debug("rTorrent get_download_path failed (%s): %s", error_type, e)
return None
@@ -321,9 +331,13 @@ class RTorrentClient(DownloadClient):
status = self.get_status(torrent_info.info_hash)
if status.state != DownloadStatus.error("").state:
return (torrent_info.info_hash, status)
except Exception:
pass
except Exception as e:
except _RTORRENT_CLIENT_ERRORS as exc:
logger.debug(
"Could not fetch existing rTorrent status for %s: %s",
torrent_info.info_hash,
exc,
)
except _RTORRENT_CLIENT_ERRORS as e:
logger.debug("Error checking for existing torrent: %s", e)
return None
else:
@@ -333,7 +347,7 @@ class RTorrentClient(DownloadClient):
"""Get the download directory from rTorrent config."""
try:
return self._rpc.directory.default()
except Exception:
except _RTORRENT_CLIENT_ERRORS:
return "/downloads"
def _get_torrent_path(self, download_id: str) -> str | None:
@@ -355,7 +369,7 @@ class RTorrentClient(DownloadClient):
if not details:
return None
path = details[0][0]
except Exception:
except _RTORRENT_CLIENT_ERRORS:
return None
else:
return path or None
shelfmark/download/clients/sabnzbd.py
+17 -9
View File
@@ -22,6 +22,13 @@ logger = setup_logger(__name__)
_ETA_PART_COUNT = 3
_SPEED_PARTS_MIN = 2
_SABNZBD_CLIENT_ERRORS = (
requests.exceptions.RequestException,
AttributeError,
RuntimeError,
TypeError,
ValueError,
)
def _parse_eta(eta_str: str) -> int | None:
@@ -212,7 +219,7 @@ class SABnzbdClient(DownloadClient):
return response.content
def _get_prowlarr_headers(self, url: str) -> dict:
# TODO: Move this source-specific Prowlarr auth handling into a source hook.
# TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook.
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
if not api_key:
return {}
@@ -283,7 +290,7 @@ class SABnzbdClient(DownloadClient):
return False, "Could not connect to SABnzbd"
except requests.exceptions.Timeout:
return False, "Connection timed out"
except Exception as e:
except _SABNZBD_CLIENT_ERRORS as e:
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to SABnzbd {version}"
@@ -294,7 +301,7 @@ class SABnzbdClient(DownloadClient):
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
**kwargs: object,
) -> str:
"""Add NZB by URL.
@@ -303,6 +310,7 @@ class SABnzbdClient(DownloadClient):
name: Display name for the download
category: Category for organization (uses configured default if not specified)
expected_hash: Optional info_hash hint (unused)
**kwargs: Client-specific options passed through to the implementation.
Returns:
SABnzbd nzo_id.
@@ -321,7 +329,7 @@ class SABnzbdClient(DownloadClient):
result = self._api_post_file(nzb_content, nzb_filename, name, category)
nzo_id = self._extract_nzo_id(result)
logger.info("Added NZB to SABnzbd: %s", nzo_id)
except Exception as e:
except _SABNZBD_CLIENT_ERRORS as e:
logger.warning("SABnzbd addfile failed, falling back to addurl: %s", e)
else:
return nzo_id
@@ -337,7 +345,7 @@ class SABnzbdClient(DownloadClient):
)
nzo_id = self._extract_nzo_id(result)
logger.info("Added NZB to SABnzbd via addurl: %s", nzo_id)
except Exception:
except _SABNZBD_CLIENT_ERRORS:
logger.exception("SABnzbd add failed")
raise
else:
@@ -447,7 +455,7 @@ class SABnzbdClient(DownloadClient):
# Not found
logger.warning("SABnzbd: download %s not found in queue or history", download_id)
return DownloadStatus.error("Download not found")
except Exception as e:
except _SABNZBD_CLIENT_ERRORS as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, *, delete_files: bool = False, archive: bool = True) -> bool:
@@ -477,7 +485,7 @@ class SABnzbdClient(DownloadClient):
if result.get("status"):
logger.info("Removed NZB from SABnzbd queue: %s", download_id)
return True
except Exception as e:
except _SABNZBD_CLIENT_ERRORS as e:
logger.debug("SABnzbd queue delete skipped for %s: %s", download_id, e)
# If not in queue (or queue delete failed), try to remove from history.
@@ -496,7 +504,7 @@ class SABnzbdClient(DownloadClient):
action = "archived" if archive else "removed"
logger.info("NZB %s from SABnzbd history: %s", action, download_id)
return True
except Exception as e:
except _SABNZBD_CLIENT_ERRORS as e:
self._log_error("remove", e)
return False
@@ -583,7 +591,7 @@ class SABnzbdClient(DownloadClient):
logger.debug("Found existing NZB in SABnzbd history: %s", nzo_id)
return (nzo_id, status)
except Exception as e:
except _SABNZBD_CLIENT_ERRORS as e:
logger.debug("Error checking for existing NZB: %s", e)
return None
else:
shelfmark/download/clients/settings.py
+46 -3
View File
@@ -16,10 +16,53 @@ from shelfmark.core.settings_registry import (
from shelfmark.core.utils import get_hardened_xmlrpc_client, normalize_http_url
from shelfmark.download.network import get_ssl_verify
try:
import qbittorrentapi as _qbittorrentapi
except ImportError:
_ImportedQBittorrentApiError = RuntimeError
_ImportedQBittorrentLoginFailed = RuntimeError
else:
_ImportedQBittorrentApiError = getattr(_qbittorrentapi, "APIError", RuntimeError)
_ImportedQBittorrentLoginFailed = getattr(_qbittorrentapi, "LoginFailed", RuntimeError)
try:
from transmission_rpc import TransmissionError as _ImportedTransmissionError
except ImportError:
_ImportedTransmissionError = RuntimeError
if TYPE_CHECKING:
from collections.abc import Iterator
# ==================== Test Connection Callbacks ====================
_DELUGE_HOST_ENTRY_MIN_LENGTH = 2
def _resolve_exception_type(candidate: object) -> type[Exception]:
if isinstance(candidate, type) and issubclass(candidate, Exception):
return candidate
return RuntimeError
_QBittorrentApiError = _resolve_exception_type(_ImportedQBittorrentApiError)
_QBittorrentLoginFailed = _resolve_exception_type(_ImportedQBittorrentLoginFailed)
_TransmissionError = _resolve_exception_type(_ImportedTransmissionError)
_QBITTORRENT_SETTINGS_ERRORS = (
_QBittorrentLoginFailed,
_QBittorrentApiError,
AttributeError,
OSError,
RuntimeError,
TypeError,
ValueError,
)
_TRANSMISSION_SETTINGS_ERRORS = (
_TransmissionError,
AttributeError,
OSError,
RuntimeError,
TypeError,
ValueError,
)
def _raise_runtime_error(message: str) -> NoReturn:
@@ -84,7 +127,7 @@ def _test_qbittorrent_connection(current_values: dict[str, Any] | None = None) -
api_version = client.app.web_api_version
except ImportError:
return {"success": False, "message": "qbittorrent-api package not installed"}
except Exception as e:
except _QBITTORRENT_SETTINGS_ERRORS as e:
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to qBittorrent (API v{api_version})"}
@@ -150,7 +193,7 @@ def _test_transmission_connection(current_values: dict[str, Any] | None = None)
version = session.version
except ImportError:
return {"success": False, "message": "transmission-rpc package not installed"}
except Exception as e:
except _TRANSMISSION_SETTINGS_ERRORS as e:
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": True, "message": f"Connected to Transmission {version}"}
@@ -245,7 +288,7 @@ def _test_deluge_connection(current_values: dict[str, Any] | None = None) -> dic
for entry in hosts:
if (
isinstance(entry, list)
and len(entry) >= 2
and len(entry) >= _DELUGE_HOST_ENTRY_MIN_LENGTH
and entry[1] in {"127.0.0.1", "localhost"}
):
host_id = entry[0]
shelfmark/download/clients/torrent_utils.py
+31 -22
View File
@@ -3,6 +3,7 @@
import base64
import hashlib
import re
from binascii import Error as BinasciiError
from dataclasses import dataclass
from urllib.parse import parse_qs, urljoin, urlparse
@@ -21,6 +22,14 @@ _BTIH_PREFIX_BYTE = 0x12
_BTIH_DIGEST_LENGTH = 32
_BTIH_HASH_LENGTH_40 = 40
_BTIH_HASH_LENGTH_32 = 32
_TORRENT_FETCH_ERRORS = (
requests.exceptions.RequestException,
OSError,
RuntimeError,
TypeError,
ValueError,
)
_TORRENT_PARSE_ERRORS = (IndexError, KeyError, TypeError, ValueError)
@dataclass
@@ -82,7 +91,7 @@ def extract_torrent_info(
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
headers: dict[str, str] = {"Accept": "application/x-bittorrent"}
# TODO: Move this source-specific Prowlarr auth handling into a source hook.
# TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook.
api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip()
if api_key:
headers["X-Api-Key"] = api_key
@@ -135,21 +144,18 @@ def extract_torrent_info(
# Check if response is actually a magnet link (text response)
# Some indexers return magnet links as plain text instead of redirecting
if len(torrent_data) < _MAGNET_RESPONSE_MAX_BYTES: # Magnet links are typically short
try:
text_content = torrent_data.decode("utf-8", errors="ignore").strip()
if text_content.startswith("magnet:"):
logger.debug("Download URL returned magnet link as response body")
info_hash = extract_hash_from_magnet(text_content)
if not info_hash and expected_hash:
info_hash = expected_hash
return TorrentInfo(
info_hash=info_hash,
torrent_data=None,
is_magnet=True,
magnet_url=text_content,
)
except Exception:
pass # Not text, continue with torrent parsing
text_content = torrent_data.decode("utf-8", errors="ignore").strip()
if text_content.startswith("magnet:"):
logger.debug("Download URL returned magnet link as response body")
info_hash = extract_hash_from_magnet(text_content)
if not info_hash and expected_hash:
info_hash = expected_hash
return TorrentInfo(
info_hash=info_hash,
torrent_data=None,
is_magnet=True,
magnet_url=text_content,
)
info_hash = extract_info_hash_from_torrent(torrent_data) or expected_hash
if info_hash:
@@ -157,7 +163,7 @@ def extract_torrent_info(
else:
logger.warning("Could not extract hash from torrent file")
return TorrentInfo(info_hash=info_hash, torrent_data=torrent_data, is_magnet=False)
except Exception as e:
except _TORRENT_FETCH_ERRORS as e:
logger.debug("Could not fetch torrent file: %s", e)
return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False)
@@ -256,9 +262,10 @@ def extract_info_hash_from_torrent(torrent_data: bytes) -> str | None:
info_bencoded = bencode_encode(decoded[b"info"])
info_dict = decoded[b"info"]
if isinstance(info_dict, dict) and b"pieces" in info_dict:
return hashlib.sha1(info_bencoded).hexdigest().lower()
# BitTorrent v1 info hashes are defined as SHA-1.
return hashlib.sha1(info_bencoded).hexdigest().lower() # noqa: S324
return hashlib.sha256(info_bencoded).hexdigest().lower()
except Exception as e:
except _TORRENT_PARSE_ERRORS as e:
logger.debug("Failed to parse torrent file: %s", e)
return None
@@ -288,7 +295,7 @@ def extract_hash_from_magnet(magnet_url: str) -> str | None:
padded = raw_value.upper() + "=" * (-len(raw_value) % 8)
try:
data = base64.b32decode(padded, casefold=True)
except Exception:
except BinasciiError, ValueError:
return None
if not data:
@@ -326,8 +333,10 @@ def extract_hash_from_magnet(magnet_url: str) -> str | None:
if re.match(r"^[A-Z2-7]{32}$", hash_value.upper()):
try:
return base64.b32decode(hash_value.upper()).hex().lower()
except Exception:
pass
except BinasciiError, ValueError:
logger.debug(
"Could not decode base32 BTIH hash from magnet URI: %s", hash_value
)
# Fallback: return as-is
return hash_value.lower()
shelfmark/download/clients/transmission.py
+32 -12
View File
@@ -20,6 +20,11 @@ from shelfmark.download.clients.torrent_utils import (
)
from shelfmark.download.network import get_ssl_verify
try:
from transmission_rpc import TransmissionError as _ImportedTransmissionError
except ImportError:
_ImportedTransmissionError = RuntimeError
if TYPE_CHECKING:
from collections.abc import Iterator
@@ -27,6 +32,20 @@ logger = setup_logger(__name__)
_SEEDING_PROGRESS_PERCENT = 100
_ETA_MAX_SECONDS = 604800
_TransmissionError = (
_ImportedTransmissionError
if isinstance(_ImportedTransmissionError, type)
and issubclass(_ImportedTransmissionError, Exception)
else RuntimeError
)
_TRANSMISSION_CLIENT_ERRORS = (
_TransmissionError,
AttributeError,
OSError,
RuntimeError,
TypeError,
ValueError,
)
@contextmanager
@@ -43,13 +62,13 @@ def _transmission_session_verify_override(url: str) -> Iterator[None]:
try:
import transmission_rpc.client as transmission_rpc_client
except Exception:
original_session_factory = transmission_rpc_client.requests.Session
except AttributeError, ImportError:
# If internals differ, gracefully fall back to default behavior.
yield
return
original_session_factory = transmission_rpc_client.requests.Session
def _session_factory(*args: object, **kwargs: object) -> object:
session = original_session_factory(*args, **kwargs)
session.verify = False
@@ -69,7 +88,7 @@ def _apply_transmission_ssl_verify(client: object, url: str) -> None:
return
try:
session.verify = get_ssl_verify(url)
except Exception as e:
except (AttributeError, OSError, TypeError, ValueError) as e:
logger.debug("Unable to apply Transmission TLS verify setting: %s", e)
@@ -138,7 +157,7 @@ class TransmissionClient(DownloadClient):
try:
session = self._client.get_session()
version = session.version
except Exception as e:
except _TRANSMISSION_CLIENT_ERRORS as e:
return False, f"Connection failed: {e!s}"
else:
return True, f"Connected to Transmission {version}"
@@ -149,7 +168,7 @@ class TransmissionClient(DownloadClient):
name: str,
category: str | None = None,
expected_hash: str | None = None,
**kwargs,
**kwargs: object,
) -> str:
"""Add torrent by URL (magnet or .torrent).
@@ -158,6 +177,7 @@ class TransmissionClient(DownloadClient):
name: Display name for the torrent
category: Category for organization (uses configured default if not specified)
expected_hash: Optional info_hash hint (from Prowlarr)
**kwargs: Client-specific options passed through to the implementation.
Returns:
Torrent hash (info_hash).
@@ -206,10 +226,10 @@ class TransmissionClient(DownloadClient):
if seed_kwargs:
try:
self._client.change_torrent(ids=torrent_hash, **seed_kwargs)
except Exception as e:
except _TRANSMISSION_CLIENT_ERRORS as e:
logger.warning("Failed to set seeding limits for %s: %s", torrent_hash, e)
except Exception:
except _TRANSMISSION_CLIENT_ERRORS:
logger.exception("Transmission add failed")
raise
else:
@@ -292,7 +312,7 @@ class TransmissionClient(DownloadClient):
except KeyError:
return DownloadStatus.error("Torrent not found")
except Exception as e:
except _TRANSMISSION_CLIENT_ERRORS as e:
return DownloadStatus.error(self._log_error("get_status", e))
def remove(self, download_id: str, *, delete_files: bool = False) -> bool:
@@ -316,7 +336,7 @@ class TransmissionClient(DownloadClient):
download_id,
" (with files)" if delete_files else "",
)
except Exception as e:
except _TRANSMISSION_CLIENT_ERRORS as e:
self._log_error("remove", e)
return False
else:
@@ -341,7 +361,7 @@ class TransmissionClient(DownloadClient):
getattr(torrent, "download_dir", ""),
torrent_name,
)
except Exception as e:
except _TRANSMISSION_CLIENT_ERRORS as e:
self._log_error("get_download_path", e, level="debug")
return None
@@ -361,6 +381,6 @@ class TransmissionClient(DownloadClient):
return None
else:
return (torrent_info.info_hash, status)
except Exception as e:
except _TRANSMISSION_CLIENT_ERRORS as e:
logger.debug("Error checking for existing torrent: %s", e)
return None
shelfmark/download/fs.py
+18 -6
View File
@@ -53,7 +53,14 @@ def _call_and_capture[T](
) -> tuple[bool, T | Exception]:
try:
return True, func(*args, **kwargs)
except Exception as exc:
except (
AttributeError,
OSError,
RuntimeError,
TypeError,
ValueError,
subprocess.SubprocessError,
) as exc:
return False, exc
@@ -118,10 +125,11 @@ def _verify_transfer_size(
actual_size = run_blocking_io(dest.stat).st_size
if actual_size != expected_size:
raise OSError(
msg = (
f"File {action} incomplete, data loss may have occurred. "
f"'{dest}' was {actual_size} bytes instead of expected {expected_size}."
)
raise OSError(msg)
def _is_stale_handle_error(error: Exception) -> bool:
@@ -206,7 +214,8 @@ def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
else:
return try_path
raise RuntimeError(f"Could not write file after {max_attempts} attempts: {dest_path}")
msg = f"Could not write file after {max_attempts} attempts: {dest_path}"
raise RuntimeError(msg)
def _is_permission_error(e: Exception) -> bool:
@@ -521,7 +530,8 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
else:
return try_path
raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}")
msg = f"Could not move file after {max_attempts} attempts: {dest_path}"
raise RuntimeError(msg)
def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
@@ -572,7 +582,8 @@ def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100)
else:
return try_path
raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
msg = f"Could not create hardlink after {max_attempts} attempts: {dest_path}"
raise RuntimeError(msg)
def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
@@ -665,4 +676,5 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) ->
else:
return try_path
raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}")
msg = f"Could not copy file after {max_attempts} attempts: {dest_path}"
raise RuntimeError(msg)
shelfmark/download/http.py
+25 -5
View File
@@ -11,6 +11,7 @@ from urllib.parse import urljoin, urlparse
import requests
from tqdm import tqdm
from shelfmark.bypass import BypassCancelledError
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
from shelfmark.download import network
@@ -21,6 +22,7 @@ if TYPE_CHECKING:
from types import ModuleType
logger = setup_logger(__name__)
_RNG = random.SystemRandom()
_MAX_REDIRECTS = 5
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
@@ -30,6 +32,17 @@ _HTTP_STATUS_OK = HTTPStatus.OK
_HTTP_STATUS_RANGE_NOT_SATISFIABLE = HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE
_HTTP_STATUS_PARTIAL_CONTENT = HTTPStatus.PARTIAL_CONTENT
_HTTP_STATUS_NON_RETRYABLE = (_HTTP_STATUS_FORBIDDEN, _HTTP_STATUS_NOT_FOUND)
_STATUS_CALLBACK_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError)
_BYPASSER_ERRORS = (
AttributeError,
BypassCancelledError,
KeyError,
OSError,
RuntimeError,
TypeError,
ValueError,
requests.exceptions.RequestException,
)
# Bypasser modules are imported lazily to support dynamic selection based on config
_internal_bypasser = None
@@ -90,7 +103,7 @@ def get_bypassed_page(
selector: network.AAMirrorSelector | None = None,
cancel_flag: Event | None = None,
) -> str | None:
"""Wrapper that delegates to the appropriate bypasser based on config."""
"""Fetch a bypassed page using the active bypasser implementation."""
if _is_using_external_bypasser():
return _get_external_bypasser().get_bypassed_page(url, selector, cancel_flag)
return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag)
@@ -168,7 +181,7 @@ def parse_size_string(size: str) -> float | None:
def _backoff_delay(attempt: int, base: float = 0.25, cap: float = 3.0) -> float:
"""Exponential backoff with jitter."""
return min(cap, base * (2 ** (attempt - 1))) + random.random() * base
return min(cap, base * (2 ** (attempt - 1))) + _RNG.random() * base
def _get_status_code(e: Exception) -> int | None:
@@ -218,11 +231,18 @@ def html_get_page(
"""Fetch HTML content from a URL with retry mechanism.
Args:
url: URL to fetch.
retry: Maximum number of attempts before giving up.
selector: Mirror selector used for AA mirror and DNS rotation.
cancel_flag: Optional event used to abort retries early.
status_callback: Optional callback for UI status updates.
allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
instead of switching to the bypasser. Use for search operations.
use_bypasser: Whether to start with the bypasser instead of direct HTTP.
include_response_url: If True, return `(html, final_url)` to expose the
resolved response URL after redirects.
success_delay: Optional delay (seconds) after successful fetch.
session: Optional requests session to reuse across attempts.
"""
@@ -258,7 +278,7 @@ def html_get_page(
return
try:
status_callback("resolving", "Bypassing protection...")
except Exception:
except _STATUS_CALLBACK_ERRORS:
return
heartbeat_thread = Thread(
@@ -268,7 +288,7 @@ def html_get_page(
try:
result = get_bypassed_page(current_url, selector, cancel_flag)
return _result(result or "", current_url)
except Exception as e:
except _BYPASSER_ERRORS as e:
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
return _result("", current_url)
finally:
@@ -521,7 +541,7 @@ def download_url(
time.sleep(0.5)
# Retry with fresh cookies (don't increment attempt)
continue
except Exception as cookie_err:
except _BYPASSER_ERRORS as cookie_err:
logger.warning("Z-Library cookie refresh failed: %s", cookie_err)
# Non-retryable errors
shelfmark/download/network.py
+74 -42
View File
@@ -5,12 +5,14 @@ import ipaddress
import socket
import urllib.parse
import urllib.request
from datetime import datetime, timedelta
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
from socket import AddressFamily, SocketKind
from typing import TYPE_CHECKING, Any, cast
import dns.resolver
import requests
from dns.exception import DNSException
from shelfmark.core.config import config as app_config
from shelfmark.core.logger import setup_logger
@@ -92,8 +94,7 @@ def get_proxies(url: str = "") -> dict:
def get_ssl_verify(url: str = "") -> bool:
"""Return the ``verify`` value for outbound requests based on the
CERTIFICATE_VALIDATION setting.
"""Return the ``verify`` value for outbound requests.
- ``enabled`` → always ``True``
- ``disabled_local`` → ``False`` for local/private addresses, ``True`` otherwise
@@ -120,8 +121,7 @@ _ssl_warnings_suppressed = False
def _apply_ssl_warning_suppression() -> None:
"""Suppress or restore urllib3 InsecureRequestWarning based on the
CERTIFICATE_VALIDATION setting.
"""Suppress or restore urllib3 InsecureRequestWarning.
Called once at init and again whenever the setting changes via the UI.
Only modifies warning filters when the mode is not 'enabled', so the
@@ -161,6 +161,7 @@ except ImportError:
_using_gevent_locks = False
logger = setup_logger(__name__)
_GETADDRINFO_SOCKADDR_INDEX = 4
def _call_dns_rotation_callback(
@@ -173,7 +174,7 @@ def _call_dns_rotation_callback(
try:
logger.debug("Calling DNS rotation callback: %s", callback.__name__)
callback(provider_name, servers, doh_url)
except Exception as e:
except (OSError, RuntimeError, TypeError, ValueError) as e:
logger.warning("DNS rotation callback %s failed: %s", callback.__name__, e)
@@ -228,7 +229,9 @@ def _load_state() -> dict[str, Any]:
"""Return current in-memory network state (no disk persistence)."""
if state.get("chosen_at"):
chosen = datetime.fromisoformat(state["chosen_at"])
if datetime.now() - chosen > timedelta(days=STATE_TTL_DAYS):
if chosen.tzinfo is None:
chosen = chosen.replace(tzinfo=UTC)
if datetime.now(UTC) - chosen > timedelta(days=STATE_TTL_DAYS):
state.clear()
return state
@@ -239,7 +242,7 @@ def _save_state(aa_url: str | None = None, dns_provider: str | None = None) -> N
state["aa_base_url"] = aa_url
if dns_provider:
state["dns_provider"] = dns_provider
state["chosen_at"] = datetime.now().isoformat()
state["chosen_at"] = datetime.now(UTC).isoformat()
# AA URL failover state
@@ -451,7 +454,7 @@ class DoHResolver:
key = (hostname, record_type)
if key in self._cache:
ips, timestamp = self._cache[key]
if datetime.now() - timestamp < timedelta(seconds=self.CACHE_TTL):
if datetime.now(UTC) - timestamp < timedelta(seconds=self.CACHE_TTL):
logger.debug("DoH cache hit for %s: %s", hostname, ips)
return ips
# Cache expired, remove it
@@ -461,7 +464,7 @@ class DoHResolver:
def _set_cached(self, hostname: str, record_type: str, ips: list[str]) -> None:
"""Cache DNS result."""
if ips: # Only cache non-empty results
self._cache[(hostname, record_type)] = (ips, datetime.now())
self._cache[(hostname, record_type)] = (ips, datetime.now(UTC))
def resolve(self, hostname: str, record_type: str) -> list[str]:
"""Resolve a hostname using DoH.
@@ -522,7 +525,7 @@ class DoHResolver:
self._set_cached(hostname, record_type, answers)
# Don't log here - the caller (custom_getaddrinfo) will log the final result
except Exception as e:
except (OSError, ValueError, requests.RequestException) as e:
logger.warning("DoH resolution failed for %s: %s", hostname, e)
return []
else:
@@ -545,7 +548,7 @@ def resolve_with_custom_dns(
try:
answers = resolver.resolve(hostname, record_type)
return [str(answer) for answer in answers]
except Exception:
except DNSException:
# Don't log here - let the caller handle it to prevent spam
# Don't trigger DNS switch here either - caller handles it
return []
@@ -575,7 +578,7 @@ def create_custom_getaddrinfo(
host: str | bytes | None,
port: str | bytes | int | None,
family: int = 0,
type: int = 0,
socket_type: int = 0,
proto: int = 0,
flags: int = 0,
) -> Sequence[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]]:
@@ -601,15 +604,21 @@ def create_custom_getaddrinfo(
# Skip logging entirely for localhost to reduce noise
if host_str in ("localhost", "127.0.0.1", "::1"):
return
try:
ips = [entry[4][0] for entry in res if len(entry) >= 5 and entry[4]]
msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}"
if is_bypass:
logger.debug(msg)
else:
logger.info(msg)
except Exception:
pass # Silently ignore logging failures
ips = []
for entry in res:
if not isinstance(entry, tuple) or len(entry) <= _GETADDRINFO_SOCKADDR_INDEX:
continue
sockaddr = entry[_GETADDRINFO_SOCKADDR_INDEX]
if not isinstance(sockaddr, tuple) or not sockaddr:
continue
ip = sockaddr[0]
if isinstance(ip, str):
ips.append(ip)
msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}"
if is_bypass:
logger.debug(msg)
else:
logger.info(msg)
# Skip custom resolution for IP addresses, local addresses, or if skip check passes
if (
@@ -618,7 +627,7 @@ def create_custom_getaddrinfo(
or (skip_check and skip_check(host_str))
):
# Quietly bypass custom resolution for IP/local targets
res = original_getaddrinfo(host, port, family, type, proto, flags)
res = original_getaddrinfo(host, port, family, socket_type, proto, flags)
_log_results("system resolver (bypass)", "system", res, is_bypass=True)
return res
@@ -630,7 +639,13 @@ def create_custom_getaddrinfo(
ipv4_answers = resolve_ipv4(host_str)
results.extend(
[
(socket.AF_INET, cast("SocketKind", type), proto, "", (answer, port_int))
(
socket.AF_INET,
cast("SocketKind", socket_type),
proto,
"",
(answer, port_int),
)
for answer in ipv4_answers
]
)
@@ -639,7 +654,14 @@ def create_custom_getaddrinfo(
_log_results("custom resolver", _current_dns_label(), results)
return results
except Exception as e:
except (
DNSException,
OSError,
RuntimeError,
TypeError,
ValueError,
requests.RequestException,
) as e:
logger.warning(
"Custom DNS resolution failed for %s: %s, falling back to system DNS", host_str, e
)
@@ -660,14 +682,22 @@ def create_custom_getaddrinfo(
"Custom DNS returned no addresses for %s; falling back to system resolver", host_str
)
try:
res = original_getaddrinfo(host, port, family, type, proto, flags)
res = original_getaddrinfo(host, port, family, socket_type, proto, flags)
_log_results("system resolver (fallback)", "system", res)
except Exception:
except OSError:
logger.exception("System DNS resolution also failed for %s", host_str)
# Last resort: Try to connect to the hostname directly
if family in {0, socket.AF_INET}:
logger.warning("Using direct hostname as last resort for %s", host_str)
return [(socket.AF_INET, cast("SocketKind", type), proto, "", (host_str, port_int))]
return [
(
socket.AF_INET,
cast("SocketKind", socket_type),
proto,
"",
(host_str, port_int),
)
]
raise # Re-raise the exception if we can't provide a last resort
else:
return res
@@ -686,14 +716,14 @@ def create_system_failover_getaddrinfo() -> Callable[
host: str | bytes | None,
port: str | bytes | int | None,
family: int = 0,
type: int = 0,
socket_type: int = 0,
proto: int = 0,
flags: int = 0,
) -> Sequence[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]]:
host_str = _decode_host(host)
try:
return original_getaddrinfo(host, port, family, type, proto, flags)
except Exception as e:
return original_getaddrinfo(host, port, family, socket_type, proto, flags)
except OSError as e:
if host_str not in _switch_logged:
logger.warning("System DNS resolution failed for %s: %s", host_str, e)
@@ -708,14 +738,14 @@ def create_system_failover_getaddrinfo() -> Callable[
logger.info("Switching DNS provider after system DNS failure for %s", host_str)
_switch_logged.add(host_str)
if switch_dns_provider():
return socket.getaddrinfo(host, port, family, type, proto, flags)
return socket.getaddrinfo(host, port, family, socket_type, proto, flags)
raise
return system_failover_getaddrinfo
def _init_doh_resolver_internal(doh_server: str) -> DoHResolver:
"""Internal: Initialize DNS over HTTPS resolver with specified server.
"""Initialize a DNS-over-HTTPS resolver for the given server.
Args:
doh_server: The DoH server URL
@@ -739,7 +769,7 @@ def _init_doh_resolver_internal(doh_server: str) -> DoHResolver:
# Restore custom getaddrinfo if it was previously set
socket.getaddrinfo = temp_getaddrinfo
except Exception:
except OSError:
logger.exception("Failed to resolve DoH server %s", server_hostname)
# Fall back to a known public DNS if resolution fails
server_ip = "1.1.1.1"
@@ -773,7 +803,7 @@ def _init_doh_resolver_internal(doh_server: str) -> DoHResolver:
def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver:
"""Internal: Initialize custom DNS resolver with specified servers.
"""Initialize a custom DNS resolver for the given servers.
Args:
servers: List of DNS server IPs to use
@@ -856,8 +886,7 @@ def rotate_dns_provider() -> bool:
def rotate_dns_and_reset_aa() -> bool:
"""Switch DNS provider (auto mode) and reset AA URL list to the first entry.
Returns True if DNS switched; False if no providers left or not in auto mode.
"""Switch DNS provider and reset the AA URL list.
Note: This function can be called during initialization, so we must NOT call
_ensure_initialized() here to avoid recursive init loops.
@@ -1090,13 +1119,13 @@ def _initialize_aa_state() -> None:
response = requests.get(
url, proxies=get_proxies(url), timeout=3, verify=get_ssl_verify(url)
)
if response.status_code == 200:
if response.status_code == HTTPStatus.OK:
_current_aa_url_index = i
_aa_base_url = url
_save_state(aa_url=_aa_base_url)
break
except Exception:
pass
except (OSError, requests.RequestException) as exc:
logger.debug("Could not reach AA mirror candidate %s: %s", url, exc)
if not _aa_base_url or _aa_base_url == "auto":
_aa_base_url = _aa_urls[0]
_current_aa_url_index = 0
@@ -1223,11 +1252,13 @@ def set_aa_url_index(new_index: int) -> bool:
class AAMirrorSelector:
"""Small helper to keep AA mirror switching consistent across call sites.
"""Keep AA mirror switching consistent across call sites.
Tracks attempts per DNS cycle and rewrites URLs safely.
"""
def __init__(self) -> None:
"""Initialize mirror state from the current AA configuration."""
self._ensure_fresh_state(reset_attempts=True)
def _ensure_fresh_state(self, *, reset_attempts: bool = False) -> None:
@@ -1251,7 +1282,8 @@ class AAMirrorSelector:
return url
def next_mirror_or_rotate_dns(self, *, allow_dns: bool = True) -> tuple[str | None, str]:
"""Advance to next mirror; if exhausted and allowed, rotate DNS and reset to first.
"""Advance to the next mirror or rotate DNS if needed.
Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'.
"""
self.attempts_this_dns += 1
shelfmark/download/orchestrator.py
+180 -186
View File
@@ -33,6 +33,7 @@ from shelfmark.release_sources import (
)
logger = setup_logger(__name__)
_RNG = random.SystemRandom()
# =============================================================================
@@ -263,7 +264,7 @@ def queue_release(
error_msg = f"Missing required field in release data: {e}"
logger.warning(error_msg)
return False, error_msg
except Exception as e:
except (AttributeError, OSError, RuntimeError, TypeError) as e:
error_msg = f"Error queueing release: {e}"
logger.error_trace(error_msg)
return False, error_msg
@@ -303,7 +304,7 @@ def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]:
with Path(path).open("rb") as f:
return f.read(), task
except Exception as e:
except OSError as e:
logger.error_trace(f"Error getting book data: {e}")
if task:
task.download_path = None
@@ -545,7 +546,7 @@ def _capture_task_error(
task.last_error_type = normalized_type
def _format_download_exception_message(exc: Exception) -> str:
def _format_download_exception_message(exc: BaseException) -> str:
if isinstance(exc, PermissionError) and "/cwa-book-ingest" in str(exc):
return "Destination misconfigured. Go to Settings → Downloads to update."
if isinstance(exc, PermissionError):
@@ -555,141 +556,122 @@ def _format_download_exception_message(exc: Exception) -> str:
def _download_task(task_id: str, cancel_flag: Event) -> str | None:
"""Download a task via appropriate handler, then post-process to ingest."""
try:
# Check for cancellation before starting
if cancel_flag.is_set():
logger.info("Task %s: cancelled before starting", task_id)
return None
task = book_queue.get_task(task_id)
if not task:
logger.error("Task not found in queue: %s", task_id)
return None
title_label = task.title or "Unknown title"
logger.info(
"Task %s: starting download (%s) - %s",
task_id,
get_source_display_name(task.source),
title_label,
)
def progress_callback(progress: float) -> None:
update_download_progress(task_id, progress)
def status_callback(status: str, message: str | None = None) -> None:
status_key = status.lower()
if status_key == "error":
_capture_task_error(
task,
message=message or "Download failed",
exc_type="StatusCallbackError",
)
return
# Don't propagate terminal statuses to the queue here. Output modules
# call status_callback("complete") before returning the download path,
# but _process_single_download needs to set download_path on the task
# first so the terminal hook captures it for history persistence.
if status_key in ("complete", "cancelled"):
if message is not None:
book_queue.update_status_message(task_id, message)
return
update_download_status(task_id, status, message)
# Get the download handler based on the task's source
handler = get_handler(task.source)
temp_file: Path | None = None
if task.staged_path:
staged_file = Path(task.staged_path)
if run_blocking_io(staged_file.exists):
temp_file = staged_file
logger.info("Task %s: reusing staged file for retry: %s", task_id, staged_file)
else:
task.staged_path = None
if temp_file is None:
temp_path = handler.download(
task,
cancel_flag,
progress_callback,
status_callback,
)
# Handler returns temp path - orchestrator handles post-processing
if not temp_path:
return None
temp_file = Path(temp_path)
if not run_blocking_io(temp_file.exists):
logger.error("Handler returned non-existent path: %s", temp_path)
_capture_task_error(
task,
message=f"Download file missing: {temp_path}",
exc_type="MissingDownloadPath",
)
return None
# Check cancellation before post-processing
if cancel_flag.is_set():
logger.info("Task %s: cancelled before post-processing", task_id)
if not is_torrent_source(temp_file, task):
safe_cleanup_path(temp_file, task)
return None
logger.info("Task %s: download finished; starting post-processing", task_id)
logger.debug("Task %s: post-processing input path: %s", task_id, temp_file)
task.staged_path = str(temp_file)
preserve_source_on_failure = True
# Post-processing: output routing + file processing pipeline
result = post_process_download(
temp_file,
task,
cancel_flag,
status_callback,
preserve_source_on_failure=preserve_source_on_failure,
)
if cancel_flag.is_set():
logger.info("Task %s: post-processing cancelled", task_id)
elif result:
logger.info("Task %s: post-processing complete", task_id)
logger.debug("Task %s: post-processing result: %s", task_id, result)
else:
logger.warning("Task %s: post-processing failed", task_id)
if not task.last_error_message:
_capture_task_error(
task,
message="Download failed",
exc_type="UnknownFailure",
)
try:
handler.post_process_cleanup(task, success=bool(result))
except Exception as e:
logger.warning("Post-processing cleanup hook failed for %s: %s", task_id, e)
if result:
task.staged_path = None
_clear_task_error_state(task)
except Exception as e:
if cancel_flag.is_set():
logger.info("Task %s: cancelled during error handling", task_id)
else:
logger.error_trace("Task %s: error downloading: %s", task_id, e)
task = book_queue.get_task(task_id)
if task:
_capture_task_error(
task,
message=_format_download_exception_message(e),
exc_type=type(e).__name__,
)
# Check for cancellation before starting
if cancel_flag.is_set():
logger.info("Task %s: cancelled before starting", task_id)
return None
task = book_queue.get_task(task_id)
if not task:
logger.error("Task not found in queue: %s", task_id)
return None
title_label = task.title or "Unknown title"
logger.info(
"Task %s: starting download (%s) - %s",
task_id,
get_source_display_name(task.source),
title_label,
)
def progress_callback(progress: float) -> None:
update_download_progress(task_id, progress)
def status_callback(status: str, message: str | None = None) -> None:
status_key = status.lower()
if status_key == "error":
_capture_task_error(
task,
message=message or "Download failed",
exc_type="StatusCallbackError",
)
return
# Don't propagate terminal statuses to the queue here. Output modules
# call status_callback("complete") before returning the download path,
# but _process_single_download needs to set download_path on the task
# first so the terminal hook captures it for history persistence.
if status_key in ("complete", "cancelled"):
if message is not None:
book_queue.update_status_message(task_id, message)
return
update_download_status(task_id, status, message)
# Get the download handler based on the task's source
handler = get_handler(task.source)
temp_file: Path | None = None
if task.staged_path:
staged_file = Path(task.staged_path)
if run_blocking_io(staged_file.exists):
temp_file = staged_file
logger.info("Task %s: reusing staged file for retry: %s", task_id, staged_file)
else:
task.staged_path = None
if temp_file is None:
temp_path = handler.download(
task,
cancel_flag,
progress_callback,
status_callback,
)
# Handler returns temp path - orchestrator handles post-processing
if not temp_path:
return None
temp_file = Path(temp_path)
if not run_blocking_io(temp_file.exists):
logger.error("Handler returned non-existent path: %s", temp_path)
_capture_task_error(
task,
message=f"Download file missing: {temp_path}",
exc_type="MissingDownloadPath",
)
return None
# Check cancellation before post-processing
if cancel_flag.is_set():
logger.info("Task %s: cancelled before post-processing", task_id)
if not is_torrent_source(temp_file, task):
safe_cleanup_path(temp_file, task)
return None
logger.info("Task %s: download finished; starting post-processing", task_id)
logger.debug("Task %s: post-processing input path: %s", task_id, temp_file)
task.staged_path = str(temp_file)
preserve_source_on_failure = True
# Post-processing: output routing + file processing pipeline
result = post_process_download(
temp_file,
task,
cancel_flag,
status_callback,
preserve_source_on_failure=preserve_source_on_failure,
)
if cancel_flag.is_set():
logger.info("Task %s: post-processing cancelled", task_id)
elif result:
logger.info("Task %s: post-processing complete", task_id)
logger.debug("Task %s: post-processing result: %s", task_id, result)
else:
return result
logger.warning("Task %s: post-processing failed", task_id)
if not task.last_error_message:
_capture_task_error(
task,
message="Download failed",
exc_type="UnknownFailure",
)
handler.post_process_cleanup(task, success=bool(result))
if result:
task.staged_path = None
_clear_task_error_state(task)
return result
def update_download_progress(book_id: str, progress: float) -> None:
@@ -854,61 +836,38 @@ def _finalize_download_failure(task_id: str) -> None:
def _process_single_download(task_id: str, cancel_flag: Event) -> None:
"""Process a single download job."""
try:
# Status will be updated through callbacks during download process
# (resolving -> downloading -> complete)
download_path = _download_task(task_id, cancel_flag)
# Status will be updated through callbacks during download process
# (resolving -> downloading -> complete)
download_path = _download_task(task_id, cancel_flag)
# Clean up progress tracking
_cleanup_progress_tracking(task_id)
# Clean up progress tracking
_cleanup_progress_tracking(task_id)
if cancel_flag.is_set():
book_queue.update_status(task_id, QueueStatus.CANCELLED)
# Broadcast cancellation
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
return
if download_path:
book_queue.update_download_path(task_id, download_path)
book_queue.update_status(task_id, QueueStatus.COMPLETE)
else:
_finalize_download_failure(task_id)
# Broadcast final status (completed or error)
if cancel_flag.is_set():
book_queue.update_status(task_id, QueueStatus.CANCELLED)
# Broadcast cancellation
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
return
except Exception as e:
# Clean up progress tracking even on error
_cleanup_progress_tracking(task_id)
if download_path:
book_queue.update_download_path(task_id, download_path)
book_queue.update_status(task_id, QueueStatus.COMPLETE)
else:
_finalize_download_failure(task_id)
if not cancel_flag.is_set():
logger.error_trace(f"Error in download processing: {e}")
task = book_queue.get_task(task_id)
if task:
_capture_task_error(
task,
message=f"Download failed: {type(e).__name__}: {e!s}",
exc_type=type(e).__name__,
)
_finalize_download_failure(task_id)
else:
logger.info("Download cancelled: %s", task_id)
book_queue.update_status(task_id, QueueStatus.CANCELLED)
# Broadcast error/cancelled status
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
# Broadcast final status (completed or error)
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
def concurrent_download_loop() -> None:
"""Main download coordinator using ThreadPoolExecutor for concurrent downloads."""
"""Run the main concurrent download coordinator."""
max_workers = config.MAX_CONCURRENT_DOWNLOADS
logger.info("Starting concurrent download loop with %s workers", max_workers)
with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="Download") as executor:
active_futures: dict[Future, str] = {} # Track active download futures
active_futures: dict[Future, tuple[str, Event]] = {} # Track active download futures
stalled_tasks: set[str] = set() # Track tasks already cancelled due to stall
while True:
@@ -916,17 +875,52 @@ def concurrent_download_loop() -> None:
# Clean up completed futures
completed_futures = [f for f in active_futures if f.done()]
for future in completed_futures:
task_id = active_futures.pop(future)
task_id, cancel_flag = active_futures.pop(future)
stalled_tasks.discard(task_id)
try:
future.result() # This will raise any exceptions from the worker
except Exception as e:
logger.error_trace(f"Future exception for {task_id}: {e}")
if future.cancelled():
_cleanup_progress_tracking(task_id)
if cancel_flag.is_set():
logger.info("Download cancelled: %s", task_id)
book_queue.update_status(task_id, QueueStatus.CANCELLED)
else:
logger.warning("Future cancelled unexpectedly for %s", task_id)
task = book_queue.get_task(task_id)
if task:
_capture_task_error(
task,
message="Download failed: CancelledError",
exc_type="CancelledError",
)
_finalize_download_failure(task_id)
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
continue
worker_error = future.exception()
if worker_error is None:
continue
_cleanup_progress_tracking(task_id)
if cancel_flag.is_set():
logger.info("Download cancelled: %s", task_id)
book_queue.update_status(task_id, QueueStatus.CANCELLED)
else:
logger.error_trace("Future exception for %s: %s", task_id, worker_error)
task = book_queue.get_task(task_id)
if task:
_capture_task_error(
task,
message=_format_download_exception_message(worker_error),
exc_type=type(worker_error).__name__,
)
_finalize_download_failure(task_id)
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
# Check for stalled downloads (no activity in STALL_TIMEOUT seconds)
current_time = time.time()
with _progress_lock:
for _future, task_id in list(active_futures.items()):
for _future, (task_id, _cancel_flag) in list(active_futures.items()):
if task_id in stalled_tasks:
continue
last_active = _last_activity.get(task_id, current_time)
@@ -948,7 +942,7 @@ def concurrent_download_loop() -> None:
# Stagger concurrent downloads to avoid rate limiting on shared download servers
# Only delay if other downloads are already active
if active_futures:
stagger_delay = random.uniform(2, 5)
stagger_delay = _RNG.uniform(2, 5)
logger.debug("Staggering download start by %.1fs", stagger_delay)
time.sleep(stagger_delay)
@@ -956,11 +950,11 @@ def concurrent_download_loop() -> None:
# Submit download job to thread pool
future = executor.submit(_process_single_download, task_id, cancel_flag)
active_futures[future] = task_id
active_futures[future] = (task_id, cancel_flag)
# Brief sleep to prevent busy waiting
time.sleep(config.MAIN_LOOP_SLEEP_TIME)
except Exception as e:
except (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError) as e:
logger.error_trace("Download coordinator loop error: %s", e)
time.sleep(COORDINATOR_LOOP_ERROR_RETRY_DELAY)
shelfmark/download/outputs/__init__.py
+8
View File
@@ -1,3 +1,5 @@
"""Output registry and shared types for post-download delivery handlers."""
from __future__ import annotations
from collections.abc import Callable
@@ -13,6 +15,8 @@ OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback, bool], str
@dataclass(frozen=True)
class OutputRegistration:
"""Registered output handler with support checks and priority metadata."""
mode: str
supports_task: Callable[[DownloadTask], bool]
handler: OutputHandler
@@ -28,6 +32,8 @@ def register_output(
supports_task: Callable[[DownloadTask], bool],
priority: int = 0,
) -> Callable[[OutputHandler], OutputHandler]:
"""Register an output handler for a named delivery mode."""
def decorator(handler: OutputHandler) -> OutputHandler:
_OUTPUT_REGISTRY.append(
OutputRegistration(
@@ -44,6 +50,7 @@ def register_output(
def load_output_handlers() -> None:
"""Load built-in output handlers exactly once."""
global _OUTPUTS_LOADED
if _OUTPUTS_LOADED:
return
@@ -80,6 +87,7 @@ def _derive_output_mode(task: DownloadTask) -> str:
def resolve_output_handler(task: DownloadTask) -> OutputRegistration | None:
"""Resolve the best output handler for a download task."""
load_output_handlers()
desired_mode = _derive_output_mode(task)
shelfmark/download/outputs/booklore.py
+11 -1
View File
@@ -1,3 +1,5 @@
"""Booklore output integration for uploading completed downloads."""
from __future__ import annotations
import os
@@ -53,6 +55,8 @@ class BookloreError(Exception):
@dataclass(frozen=True)
class BookloreConfig:
"""Configuration required to upload files into Booklore."""
base_url: str
username: str
password: str
@@ -85,6 +89,7 @@ def build_booklore_config(
values: Mapping[str, Any],
user_id: int | None = None,
) -> BookloreConfig:
"""Build and validate the effective Booklore configuration."""
base_url = str(values.get("BOOKLORE_HOST", "")).strip()
username = str(values.get("BOOKLORE_USERNAME", "")).strip()
password = values.get("BOOKLORE_PASSWORD", "") or ""
@@ -139,6 +144,7 @@ def build_booklore_config(
def booklore_login(booklore_config: BookloreConfig) -> str:
"""Authenticate with Booklore and return an API token."""
url = f"{booklore_config.base_url}/api/v1/auth/login"
payload = {
"username": booklore_config.username,
@@ -182,6 +188,7 @@ def booklore_login(booklore_config: BookloreConfig) -> str:
def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list[dict[str, Any]]:
"""Fetch the available Booklore libraries for the current user."""
url = f"{booklore_config.base_url}/api/v1/libraries"
headers = {"Authorization": f"Bearer {token}"}
@@ -200,6 +207,7 @@ def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list
def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: Path) -> None:
"""Upload a completed file into Booklore."""
if booklore_config.upload_to_bookdrop:
url = f"{booklore_config.base_url}/api/v1/files/upload/bookdrop"
params = None
@@ -244,6 +252,7 @@ def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path:
def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> None:
"""Trigger a Booklore library refresh after upload."""
url = f"{booklore_config.base_url}/api/v1/libraries/{booklore_config.library_id}/refresh"
headers = {"Authorization": f"Bearer {token}"}
@@ -438,7 +447,7 @@ def _post_process_booklore(
logger.warning("Task %s: Booklore upload failed: %s", task.task_id, e)
status_callback("error", str(e))
return None
except Exception as e:
except (OSError, TypeError, ValueError) as e:
logger.error_trace("Task %s: unexpected error uploading to Booklore: %s", task.task_id, e)
status_callback("error", f"{BOOKLORE_DISPLAY_NAME} upload failed: {e}")
return None
@@ -464,6 +473,7 @@ def process_booklore_output(
*,
preserve_source_on_failure: bool = False,
) -> str | None:
"""Process a completed download through the Booklore output."""
return _post_process_booklore(
temp_file,
task,
shelfmark/download/outputs/email.py
+31 -16
View File
@@ -1,3 +1,5 @@
"""Email output integration for delivering completed downloads as attachments."""
from __future__ import annotations
import mimetypes
@@ -44,6 +46,8 @@ class EmailOutputError(Exception):
@dataclass(frozen=True)
class EmailSmtpConfig:
"""SMTP connection settings for the email output."""
host: str
port: int
security: str
@@ -57,25 +61,28 @@ class EmailSmtpConfig:
def _parse_int(value: Any, label: str, *, minimum: int = 1) -> int:
if value is None or value == "":
raise EmailOutputError(f"{label} is required")
msg = f"{label} is required"
raise EmailOutputError(msg)
try:
parsed = int(value)
except (TypeError, ValueError) as exc:
raise EmailOutputError(f"{label} must be a number") from exc
msg = f"{label} must be a number"
raise EmailOutputError(msg) from exc
if parsed < minimum:
raise EmailOutputError(f"{label} must be >= {minimum}")
msg = f"{label} must be >= {minimum}"
raise EmailOutputError(msg)
return parsed
def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig:
"""Build and validate SMTP settings for the email output."""
host = str(values.get("EMAIL_SMTP_HOST", "") or "").strip()
port = _parse_int(values.get("EMAIL_SMTP_PORT", 587), "SMTP port", minimum=1)
security = str(values.get("EMAIL_SMTP_SECURITY", SECURITY_STARTTLS) or "").strip().lower()
if security not in ALLOWED_SECURITY:
raise EmailOutputError(
f"SMTP security must be one of: {', '.join(sorted(ALLOWED_SECURITY))}"
)
msg = f"SMTP security must be one of: {', '.join(sorted(ALLOWED_SECURITY))}"
raise EmailOutputError(msg)
username = str(values.get("EMAIL_SMTP_USERNAME", "") or "").strip()
password = values.get("EMAIL_SMTP_PASSWORD", "") or ""
@@ -88,9 +95,11 @@ def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig:
allow_unverified_tls = bool(values.get("EMAIL_ALLOW_UNVERIFIED_TLS", False))
if not host:
raise EmailOutputError("SMTP host is required")
msg = "SMTP host is required"
raise EmailOutputError(msg)
if username and not password:
raise EmailOutputError("SMTP password is required when username is set")
msg = "SMTP password is required when username is set"
raise EmailOutputError(msg)
if not from_addr:
# If From is not configured, fall back to the SMTP username if it is an email address.
@@ -98,9 +107,8 @@ def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig:
if username_email and "@" in username_email:
from_addr = f"Shelfmark <{username_email}>"
else:
raise EmailOutputError(
"From address is required (or set SMTP username to an email address)."
)
msg = "From address is required (or set SMTP username to an email address)."
raise EmailOutputError(msg)
return EmailSmtpConfig(
host=host,
@@ -161,6 +169,7 @@ def compose_email_message(
recipient: str,
files: list[Path],
) -> EmailMessage:
"""Compose the outbound email message for a completed download."""
message = EmailMessage()
message["From"] = smtp_config.from_addr
message["To"] = recipient
@@ -220,9 +229,11 @@ def test_smtp_connection(smtp_config: EmailSmtpConfig) -> None:
if smtp_config.username:
smtp.login(smtp_config.username, smtp_config.password)
except smtplib.SMTPAuthenticationError as exc:
raise EmailOutputError("SMTP authentication failed") from exc
msg = "SMTP authentication failed"
raise EmailOutputError(msg) from exc
except (smtplib.SMTPConnectError, smtplib.SMTPServerDisconnected, TimeoutError, OSError) as exc:
raise EmailOutputError(f"Could not connect to SMTP server: {exc}") from exc
msg = f"Could not connect to SMTP server: {exc}"
raise EmailOutputError(msg) from exc
finally:
if smtp is not None:
with suppress(Exception):
@@ -232,6 +243,7 @@ def test_smtp_connection(smtp_config: EmailSmtpConfig) -> None:
def send_email_message(smtp_config: EmailSmtpConfig, message: EmailMessage) -> None:
"""Send a prepared email message using the configured SMTP transport."""
smtp: smtplib.SMTP | None = None
try:
if smtp_config.security == SECURITY_SSL:
@@ -259,9 +271,11 @@ def send_email_message(smtp_config: EmailSmtpConfig, message: EmailMessage) -> N
smtp.send_message(message)
except smtplib.SMTPAuthenticationError as exc:
raise EmailOutputError("SMTP authentication failed") from exc
msg = "SMTP authentication failed"
raise EmailOutputError(msg) from exc
except (smtplib.SMTPException, TimeoutError, OSError) as exc:
raise EmailOutputError(f"Failed to send email: {exc}") from exc
msg = f"Failed to send email: {exc}"
raise EmailOutputError(msg) from exc
finally:
if smtp is not None:
with suppress(Exception):
@@ -428,7 +442,7 @@ def _post_process_email(
logger.warning("Task %s: email send failed: %s", task.task_id, exc)
status_callback("error", str(exc))
return None
except Exception as exc:
except (OSError, TypeError, ValueError) as exc:
logger.error_trace("Task %s: unexpected error sending email: %s", task.task_id, exc)
status_callback("error", f"Email send failed: {exc}")
return None
@@ -454,6 +468,7 @@ def process_email_output(
*,
preserve_source_on_failure: bool = False,
) -> str | None:
"""Process a completed download through the email output."""
return _post_process_email(
temp_file,
task,
shelfmark/download/outputs/folder.py
+2
View File
@@ -1,3 +1,5 @@
"""Folder output handler for copying or linking files into a target directory."""
from __future__ import annotations
from dataclasses import dataclass
shelfmark/download/permissions_debug.py
+11 -9
View File
@@ -21,6 +21,8 @@ if TYPE_CHECKING:
logger = setup_logger(__name__)
_T = TypeVar("_T")
_PERMISSION_DEBUG_ERRORS = (LookupError, OSError, RuntimeError, TypeError, ValueError)
_IO_OFFLOAD_FALLBACK_ERRORS = (RuntimeError, TypeError)
def _log_path_permissions(probe: Path, label: str) -> None:
@@ -39,7 +41,7 @@ def _log_path_permissions(probe: Path, label: str) -> None:
_run_io(probe.exists),
_run_io(probe.is_dir),
)
except Exception as stat_error:
except _PERMISSION_DEBUG_ERRORS as stat_error:
logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error)
@@ -51,12 +53,12 @@ def _run_io[T](func: Callable[..., _T], *args: Any, **kwargs: Any) -> _T:
"""
try:
from shelfmark.download.fs import run_blocking_io as _run_blocking_io
except Exception:
except ImportError:
return func(*args, **kwargs)
try:
return _run_blocking_io(func, *args, **kwargs)
except Exception:
except _IO_OFFLOAD_FALLBACK_ERRORS:
# Fall back to direct call if threadpool offload is unavailable.
return func(*args, **kwargs)
@@ -66,7 +68,7 @@ def _format_uid(uid: int) -> str:
import pwd
return pwd.getpwuid(uid).pw_name
except Exception:
except ImportError, KeyError:
return str(uid)
@@ -75,7 +77,7 @@ def _format_gid(gid: int) -> str:
import grp
return grp.getgrgid(gid).gr_name
except Exception:
except ImportError, KeyError:
return str(gid)
@@ -103,7 +105,7 @@ def log_path_permission_context(label: str, path: Path) -> None:
for probe in [path, path.parent]:
try:
resolved = _run_io(probe.resolve)
except Exception:
except OSError, RuntimeError:
resolved = probe
try:
@@ -121,14 +123,14 @@ def log_path_permission_context(label: str, path: Path) -> None:
_run_io(probe.is_dir),
_run_io(probe.is_symlink),
)
except Exception as stat_error:
except _PERMISSION_DEBUG_ERRORS as stat_error:
logger.debug(
"Path permissions (%s): stat failed for %s: %s",
label,
probe,
stat_error,
)
except Exception as context_error:
except _PERMISSION_DEBUG_ERRORS as context_error:
logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
@@ -153,5 +155,5 @@ def log_transfer_permission_context(label: str, source: Path, dest: Path, error:
for probe in [source, dest, dest.parent]:
_log_path_permissions(probe, label)
except Exception as context_error:
except _PERMISSION_DEBUG_ERRORS as context_error:
logger.debug("Permission context (%s): failed to collect: %s", label, context_error)
shelfmark/download/postprocess/custom_script.py
+10
View File
@@ -1,3 +1,5 @@
"""Custom script execution helpers for post-processing hooks."""
from __future__ import annotations
import json
@@ -48,6 +50,8 @@ def resolve_custom_script_target(target_path: Path, destination: Path, path_mode
@dataclass(frozen=True)
class CustomScriptExecution:
"""Resolved command inputs for a single custom script run."""
script_path: str
target_arg: Path
target_abs: Path
@@ -59,6 +63,8 @@ class CustomScriptExecution:
@dataclass(frozen=True)
class CustomScriptTransferSummary:
"""Transfer metadata exposed to custom post-process scripts."""
op_counts: dict[str, int]
use_hardlink: bool
is_torrent: bool
@@ -67,6 +73,8 @@ class CustomScriptTransferSummary:
@dataclass(frozen=True)
class CustomScriptContext:
"""Runtime context exposed to custom post-process scripts."""
task: DownloadTask
phase: str
output_mode: str
@@ -87,6 +95,7 @@ def prepare_custom_script_execution(
phase: str,
payload: dict[str, Any] | None = None,
) -> CustomScriptExecution:
"""Resolve script arguments and payload for a custom hook invocation."""
mode = (path_mode or "absolute").strip().lower()
if mode != "relative":
mode = "absolute"
@@ -110,6 +119,7 @@ def run_custom_script(
status_callback: Callable[[str, str | None], None],
timeout_seconds: int = DEFAULT_CUSTOM_SCRIPT_TIMEOUT_SECONDS,
) -> bool:
"""Run a prepared custom script and report success."""
cwd: str | None = None
if execution.mode == "relative":
# Make relative paths unambiguous by running the script from the destination folder.
shelfmark/download/postprocess/destination.py
+3 -1
View File
@@ -1,3 +1,5 @@
"""Destination planning helpers for post-processing outputs."""
from __future__ import annotations
import uuid
@@ -56,7 +58,7 @@ def validate_destination(
)
run_blocking_io(test_path.write_text, test_content)
run_blocking_io(test_path.unlink, missing_ok=True)
except Exception as exc:
except OSError as exc:
logger.debug("Destination write probe path: %s", test_path)
log_path_permission_context("destination_write_probe", destination)
logger.warning("Destination not writable: %s (%s)", destination, exc)
shelfmark/download/postprocess/policy.py
+2 -2
View File
@@ -44,7 +44,7 @@ def get_supported_audiobook_formats() -> list[str]:
return [fmt.lower() for fmt in formats]
def get_file_organization(is_audiobook: bool) -> str:
def get_file_organization(*, is_audiobook: bool) -> str:
"""Get the file organization mode for the content type."""
key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
mode = core_config.config.get(key, "rename")
@@ -62,7 +62,7 @@ def get_file_organization(is_audiobook: bool) -> str:
return mode
def get_template(is_audiobook: bool, organization_mode: str) -> str:
def get_template(*, is_audiobook: bool, organization_mode: str) -> str:
"""Get the template for the content type and organization mode."""
# Determine the correct key based on content type and organization mode
if is_audiobook:
shelfmark/download/postprocess/prepare.py
+3
View File
@@ -1,3 +1,5 @@
"""Preparation helpers for staging files before final output handling."""
from __future__ import annotations
from typing import TYPE_CHECKING
@@ -54,6 +56,7 @@ def prepare_output_files(
*,
preserve_source_on_failure: bool = False,
) -> PreparedFiles | None:
"""Prepare staged files and output metadata for final processing."""
if output_plan is None:
output_plan = build_output_plan(
temp_file,
shelfmark/download/postprocess/scan.py
+6
View File
@@ -1,3 +1,5 @@
"""Scanning helpers for discovering candidate files after download completion."""
from __future__ import annotations
import os
@@ -26,6 +28,7 @@ logger = setup_logger("shelfmark.download.postprocess.pipeline")
def get_supported_formats(content_type: str | None = None) -> list[str]:
"""Return supported file extensions for the requested content type."""
if check_audiobook(content_type):
return get_supported_audiobook_formats()
return get_book_formats()
@@ -60,6 +63,7 @@ def extract_archive_files(
*,
cleanup_archive: bool,
) -> tuple[list[Path], list[Path], list[Path], str | None]:
"""Extract an archive and classify the resulting files."""
content_type = task.content_type
try:
@@ -213,6 +217,7 @@ def collect_directory_files(
status_callback: Callable[[str, str | None], None] | None = None,
cleanup_archives: bool = False,
) -> tuple[list[Path], list[Path], list[Path], str | None]:
"""Collect supported files from a directory, extracting archives when allowed."""
content_type = task.content_type
book_files, rejected_files, archive_files, scan_error = scan_directory_tree(
directory, content_type
@@ -313,6 +318,7 @@ def collect_staged_files(
status_callback: Callable[[str, str | None], None] | None,
cleanup_archives: bool,
) -> tuple[list[Path], list[Path], list[Path], str | None]:
"""Collect supported files from a staged file or directory path."""
if run_blocking_io(working_path.is_dir):
if status_callback:
status_callback("resolving", "Processing download folder")
shelfmark/download/postprocess/steps.py
+4
View File
@@ -1,3 +1,5 @@
"""Helpers for recording debug steps in the post-processing pipeline."""
from __future__ import annotations
from shelfmark.core.logger import setup_logger
@@ -8,10 +10,12 @@ logger = setup_logger("shelfmark.download.postprocess.pipeline")
def record_step(steps: list[PlanStep], name: str, **details: object) -> None:
"""Append a named debug step to the processing plan."""
steps.append(PlanStep(name=name, details=details))
def log_plan_steps(task_id: str, steps: list[PlanStep]) -> None:
"""Log a compact summary of recorded post-processing steps."""
if not steps:
return
summary = " -> ".join(step.name for step in steps)
shelfmark/download/postprocess/transfer.py
+12 -7
View File
@@ -1,3 +1,5 @@
"""File transfer helpers for post-processing output delivery."""
from __future__ import annotations
import os
@@ -32,6 +34,7 @@ if TYPE_CHECKING:
from shelfmark.core.models import DownloadTask
logger = setup_logger("shelfmark.download.postprocess.pipeline")
_TRANSFER_PROCESS_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError)
def should_hardlink(task: DownloadTask) -> bool:
@@ -53,6 +56,7 @@ def should_hardlink(task: DownloadTask) -> bool:
def build_metadata_dict(task: DownloadTask) -> dict:
"""Build template metadata from a download task."""
return {
"Author": task.author,
"Title": task.title,
@@ -67,6 +71,7 @@ def build_metadata_dict(task: DownloadTask) -> dict:
def build_file_metadata(
task: DownloadTask, source_file: Path, part_number: str | None = None
) -> dict:
"""Build template metadata for a specific source file."""
metadata = build_metadata_dict(task)
metadata["OriginalName"] = source_file.stem
if part_number is not None:
@@ -121,10 +126,7 @@ def is_torrent_source(source_path: Path, task: DownloadTask) -> bool:
try:
return run_blocking_io(source_path.resolve) == run_blocking_io(original_path.resolve)
except OSError, ValueError:
try:
return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path))
except Exception:
return False
return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path))
def _max_attempts_for_batch(file_count: int, default: int = 100) -> int:
@@ -167,6 +169,7 @@ def transfer_book_files(
preserve_source: bool = False,
organization_mode: str | None = None,
) -> tuple[list[Path], str | None, dict[str, int]]:
"""Transfer discovered book files into their final destination layout."""
if not book_files:
return [], "No book files found", {"hardlink": 0, "copy": 0, "move": 0}
@@ -178,7 +181,7 @@ def transfer_book_files(
op_counts: dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0}
if organization_mode == "organize":
template = get_template(is_audiobook, "organize")
template = get_template(is_audiobook=is_audiobook, organization_mode="organize")
if len(book_files) == 1:
source_file = book_files[0]
@@ -239,7 +242,7 @@ def transfer_book_files(
if not task.format:
task.format = book_file.suffix.lower().lstrip(".")
template = get_template(is_audiobook, "rename")
template = get_template(is_audiobook=is_audiobook, organization_mode="rename")
metadata = build_file_metadata(task, book_file)
extension = book_file.suffix.lstrip(".") or task.format or ""
@@ -315,7 +318,7 @@ def process_directory(
processed_paths = final_paths
except Exception as exc:
except _TRANSFER_PROCESS_ERRORS as exc:
logger.error_trace(
"Task %s: error processing directory %s: %s", task.task_id, directory, exc
)
@@ -337,6 +340,7 @@ def transfer_file_to_library(
*,
use_hardlink: bool,
) -> str | None:
"""Transfer a single file into a library path derived from metadata."""
extension = source_path.suffix.lstrip(".") or task.format
template_metadata = dict(metadata)
template_metadata.setdefault("OriginalName", source_path.stem)
@@ -379,6 +383,7 @@ def transfer_directory_to_library(
*,
use_hardlink: bool,
) -> str | None:
"""Transfer a directory tree into a library path derived from metadata."""
content_type = task.content_type.lower() if task.content_type else None
source_files, _, _, scan_error = scan_directory_tree(source_dir, content_type)
if scan_error:
shelfmark/download/postprocess/types.py
+10
View File
@@ -1,3 +1,5 @@
"""Typed data containers used by the post-processing pipeline."""
from __future__ import annotations
from dataclasses import dataclass
@@ -11,6 +13,8 @@ if TYPE_CHECKING:
@dataclass(frozen=True)
class TransferPlan:
"""Plan describing how files should move from source to output."""
source_path: Path
use_hardlink: bool
allow_archive_extraction: bool
@@ -19,6 +23,8 @@ class TransferPlan:
@dataclass(frozen=True)
class OutputPlan:
"""Resolved output mode, staging strategy, and transfer settings."""
mode: str
stage_action: StageAction
staging_dir: Path
@@ -28,6 +34,8 @@ class OutputPlan:
@dataclass(frozen=True)
class PreparedFiles:
"""Prepared file set ready for transfer or output handling."""
output_plan: OutputPlan
working_path: Path
files: list[Path]
@@ -37,5 +45,7 @@ class PreparedFiles:
@dataclass(frozen=True)
class PlanStep:
"""Recorded post-processing step and its debug metadata."""
name: str
details: dict[str, Any]
shelfmark/download/postprocess/workspace.py
+6 -5
View File
@@ -1,6 +1,9 @@
"""Workspace helpers for managing mutable post-processing directories."""
from __future__ import annotations
import shutil
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
@@ -27,7 +30,7 @@ def is_within_tmp_dir(path: Path) -> bool:
# This is a *negative* check only; for potential TMP paths we still resolve to
# prevent symlink escapes from being treated as managed.
tmp_dir = _tmp_dir()
try:
with suppress(Exception):
if (
path.is_absolute()
and tmp_dir.is_absolute()
@@ -35,9 +38,6 @@ def is_within_tmp_dir(path: Path) -> bool:
and tmp_dir not in path.parents
):
return False
except Exception:
# Fall back to the slower resolve-based check below.
pass
try:
run_blocking_io(path.resolve).relative_to(run_blocking_io(tmp_dir.resolve))
@@ -48,7 +48,7 @@ def is_within_tmp_dir(path: Path) -> bool:
def is_managed_workspace_path(path: Path) -> bool:
"""True if Shelfmark should treat this path as mutable.
"""Return whether Shelfmark should treat this path as mutable.
The managed workspace is `TMP_DIR`. Anything outside it should be treated as
read-only for safety (e.g. torrent seeding directories).
@@ -90,6 +90,7 @@ def cleanup_output_staging(
task: DownloadTask,
cleanup_paths: list[Path] | None = None,
) -> None:
"""Clean up staging paths created for output processing."""
if output_plan.stage_action != STAGE_NONE:
cleanup_target = output_plan.staging_dir
if output_plan.staging_dir == _tmp_dir():
shelfmark/download/staging.py
+4 -2
View File
@@ -1,3 +1,5 @@
"""Helpers for staging downloaded files before post-processing."""
from __future__ import annotations
import hashlib
@@ -29,7 +31,7 @@ def get_staging_dir() -> Path:
def get_staging_path(task_id: str, extension: str) -> Path:
"""Get a staging path for a download."""
staging_dir = get_staging_dir()
safe_id = hashlib.md5(task_id.encode()).hexdigest()[:16]
safe_id = hashlib.blake2b(task_id.encode(), digest_size=8).hexdigest()
return staging_dir / f"{safe_id}.{extension.lstrip('.')}"
@@ -39,7 +41,7 @@ def build_staging_dir(prefix: str | None, task_id: str) -> Path:
if not prefix:
return base_dir
safe_id = hashlib.md5(task_id.encode()).hexdigest()[:8]
safe_id = hashlib.blake2b(task_id.encode(), digest_size=4).hexdigest()
staging_dir = base_dir / f"{prefix}_{safe_id}"
counter = 1
shelfmark/main.py
+128 -111
View File
@@ -1,5 +1,6 @@
"""Flask app - routes, WebSocket handlers, and middleware."""
import binascii
import io
import logging
import os
@@ -7,7 +8,7 @@ import re
import sqlite3
import time
from contextlib import suppress
from datetime import datetime, timedelta
from datetime import UTC, datetime, timedelta
from functools import wraps
from importlib import import_module
from pathlib import Path
@@ -31,8 +32,12 @@ from shelfmark.config.env import (
HIDE_LOCAL_AUTH,
OIDC_AUTO_REDIRECT,
RELEASE_VERSION,
SESSION_COOKIE_NAME,
SESSION_COOKIE_SECURE_ENV,
_is_config_dir_writable,
string_to_bool,
)
from shelfmark.config.security import _migrate_security_settings
from shelfmark.config.settings import _SUPPORTED_BOOK_LANGUAGE
from shelfmark.core.activity_view_state_service import ActivityViewStateService
from shelfmark.core.auth_modes import (
@@ -74,6 +79,7 @@ from shelfmark.core.requests_service import (
reopen_failed_request,
sync_delivery_states_from_queue_status,
)
from shelfmark.core.user_db import UserDB
from shelfmark.core.utils import normalize_base_path
from shelfmark.download import orchestrator as backend
from shelfmark.release_sources import (
@@ -87,6 +93,9 @@ if TYPE_CHECKING:
from shelfmark.metadata_providers import BookMetadata, MetadataProvider
logger = setup_logger(__name__)
FLASK_SECRET_KEY_MIN_BYTES = 32
_OPERATIONAL_ERRORS = (OSError, RuntimeError, TypeError, ValueError, sqlite3.Error)
_IMPORT_OPERATIONAL_ERRORS = (ImportError, *_OPERATIONAL_ERRORS)
def _raise_runtime_error(message: str) -> NoReturn:
@@ -102,7 +111,7 @@ BASE_PATH = normalize_base_path(app_config.get("URL_BASE", ""))
app = Flask(__name__)
app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 0 # Disable caching
app.config["APPLICATION_ROOT"] = BASE_PATH or "/"
app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore
app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore[assignment]
if BASE_PATH:
app.wsgi_app = PrefixMiddleware(app.wsgi_app, BASE_PATH, bypass_paths={"/api/health"})
@@ -149,17 +158,11 @@ except ImportError as e:
logger.warning("Failed to import plugin modules: %s", e)
# Migrate legacy security settings if needed
from shelfmark.config.security import _migrate_security_settings
_migrate_security_settings()
# Initialize user database and register multi-user routes
# If CONFIG_DIR doesn't exist or is read-only, multi-user features will be disabled
import os as _os
from shelfmark.core.user_db import UserDB
_user_db_path = str(Path(_os.environ.get("CONFIG_DIR", "/config")) / "users.db")
_user_db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db")
user_db: UserDB | None = None
download_history_service: DownloadHistoryService | None = None
activity_view_state_service: ActivityViewStateService | None = None
@@ -180,7 +183,7 @@ except (sqlite3.OperationalError, OSError) as e:
logger.warning(
"User database initialization failed: %s. Multi-user authentication features will be disabled. Ensure CONFIG_DIR (%s) exists and is writable.",
e,
_os.environ.get("CONFIG_DIR", "/config"),
os.environ.get("CONFIG_DIR", "/config"),
)
user_db = None
download_history_service = None
@@ -190,15 +193,16 @@ except (sqlite3.OperationalError, OSError) as e:
backend.start()
# Rate limiting for login attempts
# Structure: {username: {'count': int, 'lockout_until': datetime}}
# Map usernames to their failed-attempt counters and lockout timestamps.
failed_login_attempts: dict[str, dict[str, Any]] = {}
MAX_LOGIN_ATTEMPTS = 10
LOCKOUT_DURATION_MINUTES = 30
LOGIN_ATTEMPT_WARNING_THRESHOLD = 5
def cleanup_old_lockouts() -> None:
"""Remove expired lockout entries to prevent memory buildup."""
current_time = datetime.now()
current_time = datetime.now(UTC)
expired_users = [
username
for username, data in failed_login_attempts.items()
@@ -217,7 +221,7 @@ def is_account_locked(username: str) -> bool:
return False
lockout_until = failed_login_attempts[username].get("lockout_until")
return lockout_until is not None and datetime.now() < lockout_until
return lockout_until is not None and datetime.now(UTC) < lockout_until
def record_failed_login(username: str, ip_address: str) -> bool:
@@ -240,7 +244,7 @@ def record_failed_login(username: str, ip_address: str) -> bool:
)
if count >= MAX_LOGIN_ATTEMPTS:
lockout_until = datetime.now() + timedelta(minutes=LOCKOUT_DURATION_MINUTES)
lockout_until = datetime.now(UTC) + timedelta(minutes=LOCKOUT_DURATION_MINUTES)
failed_login_attempts[username]["lockout_until"] = lockout_until
logger.warning(
"Account locked for user '%s' until %s due to %s failed login attempts",
@@ -497,7 +501,7 @@ if user_db is not None:
emit_request_updates=_emit_request_updates,
ws_manager=ws_manager,
)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.warning("Failed to register request routes: %s", e)
@@ -525,6 +529,7 @@ class LogNoiseFilter(logging.Filter):
"""
def filter(self, record: logging.LogRecord) -> bool:
"""Return whether a log record should be emitted."""
message = record.getMessage() if hasattr(record, "getMessage") else str(record.msg)
# Exclude GET /api/status requests (polling noise)
@@ -564,8 +569,6 @@ werkzeug_logger.setLevel(logger.level)
werkzeug_logger.addFilter(LogNoiseFilter())
# Set up authentication defaults
from shelfmark.config.env import SESSION_COOKIE_NAME, SESSION_COOKIE_SECURE_ENV, string_to_bool
SESSION_COOKIE_SECURE = string_to_bool(SESSION_COOKIE_SECURE_ENV)
@@ -576,7 +579,7 @@ def _load_or_create_secret_key() -> bytes:
try:
if secret_path.exists():
secret_key = secret_path.read_bytes()
if len(secret_key) >= 32:
if len(secret_key) >= FLASK_SECRET_KEY_MIN_BYTES:
return secret_key
logger.warning(
"Invalid persisted Flask secret key at %s (length=%s). Regenerating.",
@@ -730,7 +733,7 @@ def proxy_auth_middleware() -> Response | tuple[Response, int] | None:
session["db_user_id"] = db_user["id"]
session.permanent = False
except Exception:
except _OPERATIONAL_ERRORS:
logger.exception("Proxy auth middleware error")
return jsonify({"error": "Authentication error"}), 500
else:
@@ -753,8 +756,10 @@ def set_security_headers(response: Response) -> Response:
def login_required(
f: Callable[..., Response | tuple[Response, int]],
) -> Callable[..., Response | tuple[Response, int]]:
"""Require authentication for a Flask route."""
@wraps(f)
def decorated_function(*args, **kwargs) -> Response | tuple[Response, int]:
def decorated_function(*args: object, **kwargs: object) -> Response | tuple[Response, int]:
auth_mode = get_auth_mode()
# If no authentication is configured, allow access
@@ -778,7 +783,7 @@ def login_required(
):
return jsonify({"error": "Admin access required"}), 403
except Exception:
except RuntimeError, TypeError, ValueError:
logger.exception("Admin access check error")
return jsonify({"error": "Internal Server Error"}), 500
@@ -821,6 +826,7 @@ def serve_frontend_assets(filename: str) -> Response:
@app.route("/")
def index() -> Response:
"""Serve the React frontend application.
Authentication is handled by the React app itself.
"""
return _serve_index_html()
@@ -859,9 +865,9 @@ if DEBUG:
@app.route("/api/debug", methods=["GET"])
@login_required
def debug() -> Response | tuple[Response, int]:
"""This will run the /app/genDebug.sh script, which will generate a debug zip with all the logs
The file will be named /tmp/shelfmark-debug.zip
And then return it to the user
"""Run `/app/genDebug.sh`, generate a debug zip, and return it.
The file is written to `/tmp/shelfmark-debug.zip` before being returned.
"""
try:
logger.info("Debug endpoint called, stopping GUI and generating debug info...")
@@ -888,14 +894,14 @@ if DEBUG:
except subprocess.CalledProcessError as e:
logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}")
return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Debug endpoint error: {e}")
return jsonify({"error": str(e)}), 500
@app.route("/api/restart", methods=["GET"])
@login_required
def restart() -> Response | tuple[Response, int]:
"""Restart the application"""
"""Restart the application."""
os._exit(0)
@@ -991,7 +997,7 @@ def api_download_release() -> Response | tuple[Response, int]:
"""
try:
data = request.get_json()
data = request.get_json(silent=True)
if not data:
return jsonify({"error": "No data provided"}), 400
@@ -1035,7 +1041,7 @@ def api_download_release() -> Response | tuple[Response, int]:
if success:
return jsonify({"status": "queued", "priority": priority})
return jsonify({"error": error_msg or "Failed to queue release"}), 500
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Release download error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1124,7 +1130,7 @@ def api_config() -> Response | tuple[Response, int]:
), # For universal mode
}
return jsonify(config)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Config error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1132,6 +1138,7 @@ def api_config() -> Response | tuple[Response, int]:
@app.route("/api/health", methods=["GET"])
def api_health() -> Response | tuple[Response, int]:
"""Health check endpoint for container orchestration.
No authentication required.
Returns:
@@ -1215,7 +1222,7 @@ def _notify_admin_for_terminal_download_status(
)
try:
notify_admin(event, context)
except Exception as exc:
except (RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"Failed to trigger admin notification for download %s (%s): %s",
task_id,
@@ -1226,7 +1233,7 @@ def _notify_admin_for_terminal_download_status(
return
try:
notify_user(owner_user_id, event, context)
except Exception as exc:
except (RuntimeError, TypeError, ValueError) as exc:
logger.warning(
"Failed to trigger user notification for download %s (%s, user_id=%s): %s",
task_id,
@@ -1264,10 +1271,7 @@ def _record_download_queued(task_id: str, task: Any) -> None:
origin = "requested" if request_id else "direct"
source_name = normalize_source(getattr(task, "source", None))
try:
source_display = get_source_display_name(source_name)
except Exception:
source_display = None
source_display = get_source_display_name(source_name)
try:
download_history_service.record_download(
@@ -1279,14 +1283,14 @@ def _record_download_queued(task_id: str, task: Any) -> None:
source_display_name=source_display,
title=str(getattr(task, "title", "Unknown title") or "Unknown title"),
author=normalize_optional_text(getattr(task, "author", None)),
format=normalize_optional_text(getattr(task, "format", None)),
file_format=normalize_optional_text(getattr(task, "format", None)),
size=normalize_optional_text(getattr(task, "size", None)),
preview=normalize_optional_text(getattr(task, "preview", None)),
content_type=normalize_optional_text(getattr(task, "content_type", None)),
origin=origin,
retry_payload=backend.serialize_task_for_retry(task),
)
except Exception as exc:
except _OPERATIONAL_ERRORS as exc:
logger.warning("Failed to record download at queue time for task %s: %s", task_id, exc)
return
@@ -1312,7 +1316,7 @@ def _record_download_queued(task_id: str, task: Any) -> None:
"task_id": task_id,
},
)
except Exception as exc:
except _OPERATIONAL_ERRORS as exc:
logger.warning("Failed to reset activity viewer state for task %s: %s", task_id, exc)
@@ -1334,7 +1338,7 @@ def _record_download_terminal_snapshot(task_id: str, status: QueueStatus, task:
retry_payload=backend.serialize_task_for_retry(task),
)
finalized_download = True
except Exception as exc:
except _OPERATIONAL_ERRORS as exc:
logger.warning("Failed to finalize download history for task %s: %s", task_id, exc)
if finalized_download:
@@ -1375,7 +1379,7 @@ def _record_download_terminal_snapshot(task_id: str, status: QueueStatus, task:
item_key=f"request:{request_id}",
)
_emit_request_update_events([reopened_request])
except Exception as exc:
except _OPERATIONAL_ERRORS as exc:
logger.warning(
"Failed to reopen request %s after terminal download error %s: %s",
request_id,
@@ -1429,23 +1433,25 @@ def _emit_request_update_events(updated_requests: list[dict[str, Any]]) -> None:
if not updated_requests or ws_manager is None:
return
try:
socketio_ref = getattr(ws_manager, "socketio", None)
is_enabled = getattr(ws_manager, "is_enabled", None)
if socketio_ref is None or not callable(is_enabled) or not is_enabled():
return
for updated in updated_requests:
payload = {
"request_id": updated["id"],
"status": updated["status"],
"delivery_state": updated.get("delivery_state"),
"title": (updated.get("book_data") or {}).get("title") or "Unknown title",
}
socketio_ref.emit("request_update", payload, to=f"user_{updated['user_id']}")
socketio_ref.emit("request_update", payload, to="admins")
except Exception as exc:
logger.warning("Failed to emit delivery request_update events: %s", exc)
for updated in updated_requests:
payload = {
"request_id": updated["id"],
"status": updated["status"],
"delivery_state": updated.get("delivery_state"),
"title": (updated.get("book_data") or {}).get("title") or "Unknown title",
}
emit_ws_event(
ws_manager,
event_name="request_update",
room=f"user_{updated['user_id']}",
payload=payload,
)
emit_ws_event(
ws_manager,
event_name="request_update",
room="admins",
payload=payload,
)
@app.route("/api/status", methods=["GET"])
@@ -1472,7 +1478,7 @@ def api_status() -> Response | tuple[Response, int]:
)
_emit_request_update_events(updated_requests)
return jsonify(status)
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Status error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1521,7 +1527,7 @@ def api_local_download() -> Response | tuple[Response, int]:
data = io.BytesIO(file_data)
return send_file(data, download_name=file_name, as_attachment=True)
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Local download error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1571,7 +1577,7 @@ def api_cover(cover_id: str) -> Response | tuple[Response, int]:
try:
original_url = base64.urlsafe_b64decode(encoded_url).decode()
except Exception as e:
except (binascii.Error, UnicodeDecodeError) as e:
logger.warning("Failed to decode cover URL: %s", e)
return jsonify({"error": "Invalid cover URL encoding"}), 400
@@ -1584,7 +1590,7 @@ def api_cover(cover_id: str) -> Response | tuple[Response, int]:
response = app.response_class(response=image_data, status=200, mimetype=content_type)
response.headers["Cache-Control"] = "public, max-age=86400"
response.headers["X-Cache"] = "MISS"
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Cover fetch error: {e}")
return jsonify({"error": str(e)}), 500
else:
@@ -1633,7 +1639,7 @@ def api_cancel_download(book_id: str) -> Response | tuple[Response, int]:
if success:
return jsonify({"status": "cancelled", "book_id": book_id})
return jsonify({"error": "Failed to cancel download or book not found"}), 404
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Cancel download error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1683,7 +1689,9 @@ def api_retry_download(book_id: str) -> Response | tuple[Response, int]:
), 403
success, error = backend.retry_download(book_id)
else:
assert history_row is not None
if history_row is None:
logger.error("Download history row disappeared while retrying task %s", book_id)
return jsonify({"error": "Download history not found"}), 404
request_id = normalize_positive_int(history_row.get("request_id"))
retry_payload = history_row.get("retry_payload")
final_status = history_row.get("final_status")
@@ -1705,7 +1713,7 @@ def api_retry_download(book_id: str) -> Response | tuple[Response, int]:
return jsonify({"error": error}), 404
return jsonify({"error": error or "Download cannot be retried"}), 409
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Retry download error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1726,7 +1734,7 @@ def api_set_priority(book_id: str) -> Response | tuple[Response, int]:
"""
try:
data = request.get_json()
data = request.get_json(silent=True)
if not data or "priority" not in data:
return jsonify({"error": "Priority not provided"}), 400
@@ -1738,7 +1746,7 @@ def api_set_priority(book_id: str) -> Response | tuple[Response, int]:
return jsonify({"error": "Failed to update priority or book not found"}), 404
except ValueError:
return jsonify({"error": "Invalid priority value"}), 400
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Set priority error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1756,7 +1764,7 @@ def api_reorder_queue() -> Response | tuple[Response, int]:
"""
try:
data = request.get_json()
data = request.get_json(silent=True)
if not data or "book_priorities" not in data:
return jsonify({"error": "book_priorities not provided"}), 400
@@ -1774,7 +1782,7 @@ def api_reorder_queue() -> Response | tuple[Response, int]:
if success:
return jsonify({"status": "reordered", "updated_count": len(book_priorities)})
return jsonify({"error": "Failed to reorder queue"}), 500
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Reorder queue error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1791,7 +1799,7 @@ def api_queue_order() -> Response | tuple[Response, int]:
try:
queue_order = backend.get_queue_order()
return jsonify({"queue": queue_order})
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Queue order error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1808,7 +1816,7 @@ def api_active_downloads() -> Response | tuple[Response, int]:
try:
active_downloads = backend.get_active_downloads()
return jsonify({"active_downloads": active_downloads})
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Active downloads error: {e}")
return jsonify({"error": str(e)}), 500
@@ -1855,7 +1863,7 @@ def _failed_login_response(username: str, ip_address: str) -> tuple[Response, in
), 429
attempts_remaining = MAX_LOGIN_ATTEMPTS - failed_login_attempts[username]["count"]
if attempts_remaining <= 5:
if attempts_remaining <= LOGIN_ATTEMPT_WARNING_THRESHOLD:
return jsonify(
{"error": f"Invalid username or password. {attempts_remaining} attempts remaining."}
), 401
@@ -1866,6 +1874,7 @@ def _failed_login_response(username: str, ip_address: str) -> tuple[Response, in
@app.route("/api/auth/login", methods=["POST"])
def api_login() -> Response | tuple[Response, int]:
"""Login endpoint that validates credentials and creates a session.
Supports both built-in credentials and CWA database authentication.
Includes rate limiting: 10 failed attempts = 30 minute lockout.
@@ -1880,7 +1889,7 @@ def api_login() -> Response | tuple[Response, int]:
"""
try:
ip_address = get_client_ip()
data = request.get_json()
data = request.get_json(silent=True)
if not data:
return jsonify({"error": "No data provided"}), 400
@@ -1901,7 +1910,7 @@ def api_login() -> Response | tuple[Response, int]:
# Check if account is locked due to failed login attempts
if is_account_locked(username):
lockout_until = failed_login_attempts[username].get("lockout_until")
remaining_time = (lockout_until - datetime.now()).total_seconds() / 60
remaining_time = (lockout_until - datetime.now(UTC)).total_seconds() / 60
logger.warning(
"Login attempt blocked for locked account '%s' from IP %s", username, ip_address
)
@@ -1960,7 +1969,7 @@ def api_login() -> Response | tuple[Response, int]:
return _failed_login_response(username, ip_address)
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Built-in auth error: {e}")
return jsonify({"error": "Authentication system error"}), 500
@@ -2017,14 +2026,14 @@ def api_login() -> Response | tuple[Response, int]:
)
return jsonify({"success": True})
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"CWA database error during login: {e}")
return jsonify({"error": "Authentication system error"}), 500
# Should not reach here, but handle gracefully
return jsonify({"error": "Unknown authentication mode"}), 500
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Login error: {e}")
return jsonify({"error": "Login failed"}), 500
@@ -2032,6 +2041,7 @@ def api_login() -> Response | tuple[Response, int]:
@app.route("/api/auth/logout", methods=["POST"])
def api_logout() -> Response | tuple[Response, int]:
"""Logout endpoint that clears the session.
For proxy auth, returns the logout URL if configured.
Returns:
@@ -2052,7 +2062,7 @@ def api_logout() -> Response | tuple[Response, int]:
return jsonify({"success": True, "logout_url": logout_url})
return jsonify({"success": True})
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Logout error: {e}")
return jsonify({"error": "Logout failed"}), 500
@@ -2091,8 +2101,8 @@ def api_auth_check() -> Response | tuple[Response, int]:
db_user = user_db.get_user(user_id=session["db_user_id"])
if db_user:
display_name = db_user.get("display_name") or None
except Exception:
pass
except (sqlite3.Error, TypeError, ValueError) as exc:
logger.debug("Could not load display name for session user: %s", exc)
response_data = {
"authenticated": is_authenticated,
@@ -2120,7 +2130,7 @@ def api_auth_check() -> Response | tuple[Response, int]:
response_data["oidc_auto_redirect"] = True
return jsonify(response_data)
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Auth check error: {e}")
return jsonify(
{
@@ -2182,8 +2192,12 @@ def api_metadata_providers() -> Response | tuple[Response, int]:
kwargs = get_provider_kwargs(info["name"])
provider = get_provider(info["name"], **kwargs)
provider_info["available"] = provider.is_available()
except Exception:
pass
except _OPERATIONAL_ERRORS as exc:
logger.debug(
"Metadata provider %s availability check failed: %s",
info["name"],
exc,
)
providers.append(provider_info)
@@ -2195,7 +2209,7 @@ def api_metadata_providers() -> Response | tuple[Response, int]:
"configured_provider_combined": configured_combined_metadata_provider or None,
}
)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Metadata providers error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2264,7 +2278,7 @@ def api_metadata_config() -> Response | tuple[Response, int]:
"default_sort": get_provider_default_sort(provider_name, user_id=db_user_id),
}
)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Metadata config error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2409,7 +2423,7 @@ def api_metadata_search() -> Response | tuple[Response, int]:
if search_result.source_title:
response_data["source_title"] = search_result.source_title
return jsonify(response_data)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Metadata search error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2450,7 +2464,7 @@ def api_metadata_field_options() -> Response:
options = provider.get_search_field_options(field_key, query=query_text or None)
return jsonify({"options": options})
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.warning("Metadata field options endpoint error: %s", e)
return jsonify({"options": []})
@@ -2467,13 +2481,15 @@ def _resolve_metadata_provider(provider_name: str) -> MetadataProvider:
)
if not is_provider_registered(provider_name):
raise ValueError(f"Unknown metadata provider: {provider_name}")
msg = f"Unknown metadata provider: {provider_name}"
raise ValueError(msg)
kwargs = get_provider_kwargs(provider_name)
prov = get_provider(provider_name, **kwargs)
if not prov.is_available():
raise RuntimeError(f"Provider '{provider_name}' is not available")
msg = f"Provider '{provider_name}' is not available"
raise RuntimeError(msg)
return prov
@@ -2514,7 +2530,7 @@ def api_metadata_book(provider: str, book_id: str) -> Response | tuple[Response,
return jsonify({"error": str(e)}), 400
except RuntimeError as e:
return jsonify({"error": str(e)}), 503
except Exception as e:
except (OSError, TypeError, sqlite3.Error) as e:
logger.error_trace(f"Metadata book error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2524,20 +2540,20 @@ def _handle_target_errors(
) -> Callable[
[Callable[..., Response | tuple[Response, int]]], Callable[..., Response | tuple[Response, int]]
]:
"""Decorator that wraps a metadata-target route with standard error handling."""
"""Wrap a metadata-target route with standard error handling."""
def decorator(
fn: Callable[..., Response | tuple[Response, int]],
) -> Callable[..., Response | tuple[Response, int]]:
@wraps(fn)
def wrapper(*args, **kwargs) -> Response | tuple[Response, int]:
def wrapper(*args: object, **kwargs: object) -> Response | tuple[Response, int]:
try:
return fn(*args, **kwargs)
except (NotImplementedError, ValueError) as e:
return jsonify({"error": str(e)}), 400
except RuntimeError as e:
return jsonify({"error": str(e)}), 502
except Exception as e:
except (OSError, TypeError, sqlite3.Error) as e:
logger.error_trace(f"{fallback_message}: {e}")
return jsonify({"error": fallback_message}), 500
@@ -2589,7 +2605,7 @@ def api_metadata_book_targets_update(
if not isinstance(selected, bool):
return jsonify({"error": "selected must be a boolean"}), 400
result = prov.set_book_target_state(book_id, target, selected)
result = prov.set_book_target_state(book_id, target, selected=selected)
response: dict = {
"success": True,
"changed": bool(result.get("changed", True)),
@@ -2680,7 +2696,7 @@ def api_releases() -> Response | tuple[Response, int]:
)
except ValueError:
return None, [], f"Unknown source: {source_name}"
except Exception as e:
except (SourceUnavailableError, *_OPERATIONAL_ERRORS) as e:
logger.warning("Release search failed for source %s: %s", source_name, e)
return None, [], f"{source_name}: {e!s}"
else:
@@ -2809,7 +2825,7 @@ def api_releases() -> Response | tuple[Response, int]:
try:
first_source = source_instances[sources_to_search[0]]
column_config = serialize_column_config(first_source.get_column_config())
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.warning("Failed to get column config: %s", e)
# Convert book to dict and transform cover_url
@@ -2851,7 +2867,7 @@ def api_releases() -> Response | tuple[Response, int]:
except SourceUnavailableError as e:
logger.warning("Release search unavailable: %s", e)
return jsonify({"error": str(e)}), 503
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Releases search error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2870,7 +2886,7 @@ def api_release_sources() -> Response | tuple[Response, int]:
sources = list_available_sources()
return jsonify(sources)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Release sources error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2892,7 +2908,7 @@ def api_release_source_record(source_name: str, record_id: str) -> Response | tu
except SourceUnavailableError as e:
logger.warning("Release source record unavailable: %s", e)
return jsonify({"error": str(e)}), 503
except Exception as e:
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Release source record error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2918,7 +2934,7 @@ def api_settings_get_all() -> Response | tuple[Response, int]:
data = serialize_all_settings(include_values=True)
return jsonify(data)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Settings get error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2952,7 +2968,7 @@ def api_settings_get_tab(tab_name: str) -> Response | tuple[Response, int]:
return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404
return jsonify(serialize_tab(tab, include_values=True))
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Settings get tab error: {e}")
return jsonify({"error": str(e)}), 500
@@ -2988,7 +3004,7 @@ def api_settings_update_tab(tab_name: str) -> Response | tuple[Response, int]:
if not tab:
return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404
values = request.get_json()
values = request.get_json(silent=True)
if values is None or not isinstance(values, dict):
return jsonify({"error": "Request body must be a JSON object"}), 400
@@ -3001,7 +3017,7 @@ def api_settings_update_tab(tab_name: str) -> Response | tuple[Response, int]:
if result["success"]:
return jsonify(result)
return jsonify(result), 400
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Settings update error: {e}")
return jsonify({"error": str(e)}), 500
@@ -3039,7 +3055,7 @@ def api_settings_execute_action(tab_name: str, action_key: str) -> Response | tu
if result["success"]:
return jsonify(result)
return jsonify(result), 400
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Settings action error: {e}")
return jsonify({"error": str(e)}), 500
@@ -3065,7 +3081,7 @@ def api_onboarding_get() -> Response | tuple[Response, int]:
config = get_onboarding_config()
return jsonify(config)
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Onboarding get error: {e}")
return jsonify({"error": str(e)}), 500
@@ -3087,7 +3103,7 @@ def api_onboarding_save() -> Response | tuple[Response, int]:
import_module("shelfmark.config.settings")
from shelfmark.core.onboarding import save_onboarding_settings
data = request.get_json()
data = request.get_json(silent=True)
if not data:
return jsonify({"success": False, "message": "No data provided"}), 400
@@ -3096,7 +3112,7 @@ def api_onboarding_save() -> Response | tuple[Response, int]:
if result["success"]:
return jsonify(result)
return jsonify(result), 400
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Onboarding save error: {e}")
return jsonify({"error": str(e)}), 500
@@ -3115,7 +3131,7 @@ def api_onboarding_skip() -> Response | tuple[Response, int]:
mark_onboarding_complete()
return jsonify({"success": True, "message": "Onboarding skipped"})
except Exception as e:
except _IMPORT_OPERATIONAL_ERRORS as e:
logger.error_trace(f"Onboarding skip error: {e}")
return jsonify({"error": str(e)}), 500
@@ -3125,6 +3141,7 @@ def api_onboarding_skip() -> Response | tuple[Response, int]:
@app.route("/<path:path>")
def catch_all(path: str) -> Response:
"""Serve the React app for any route not matched by API endpoints.
This allows React Router to handle client-side routing.
Authentication is handled by the React app itself.
"""
@@ -3146,7 +3163,7 @@ def handle_connect() -> None:
# Join appropriate room based on authenticated user session
is_admin, db_user_id, can_access_status = _resolve_status_scope()
ws_manager.join_user_room(request.sid, is_admin, db_user_id)
ws_manager.join_user_room(request.sid, is_admin=is_admin, db_user_id=db_user_id)
# Send initial status to the newly connected client (filtered)
try:
@@ -3157,7 +3174,7 @@ def handle_connect() -> None:
user_id = None if is_admin else db_user_id
status = backend.queue_status(user_id=user_id)
emit("status_update", status)
except Exception:
except _OPERATIONAL_ERRORS:
logger.exception("Error sending initial status")
@@ -3178,7 +3195,7 @@ def handle_status_request() -> None:
"""Handle manual status request from client."""
try:
is_admin, db_user_id, can_access_status = _resolve_status_scope()
ws_manager.sync_user_room(request.sid, is_admin, db_user_id)
ws_manager.sync_user_room(request.sid, is_admin=is_admin, db_user_id=db_user_id)
if not can_access_status:
emit("status_update", {})
@@ -3187,7 +3204,7 @@ def handle_status_request() -> None:
user_id = None if is_admin else db_user_id
status = backend.queue_status(user_id=user_id)
emit("status_update", status)
except Exception:
except _OPERATIONAL_ERRORS:
logger.exception("Error handling status request")
emit("error", {"message": "Failed to get status"})
shelfmark/metadata_providers/__init__.py
+5 -4
View File
@@ -418,6 +418,7 @@ class MetadataProvider(ABC):
self,
book_id: str,
target: str,
*,
selected: bool,
) -> dict[str, Any]:
"""Set whether a book belongs to a provider-managed list or shelf.
@@ -436,7 +437,7 @@ _PROVIDER_KWARGS_FACTORIES: dict[str, Any] = {} # Callable[[], Dict]
def register_provider(
name: str,
) -> Callable[[type[MetadataProvider]], type[MetadataProvider]]:
"""Decorator to register a metadata provider."""
"""Register a metadata provider."""
def decorator(cls: type[MetadataProvider]) -> type[MetadataProvider]:
_PROVIDERS[name] = cls
@@ -448,7 +449,7 @@ def register_provider(
def register_provider_kwargs(
name: str,
) -> Callable[[Callable[[], dict[str, Any]]], Callable[[], dict[str, Any]]]:
"""Decorator to register a provider's kwargs factory.
"""Register a provider kwargs factory.
The decorated function should return a Dict of kwargs to pass to the
provider constructor. This allows each provider to define its own
@@ -469,8 +470,8 @@ def register_provider_kwargs(
return decorator
def get_provider(name: str, **kwargs) -> MetadataProvider:
"""Factory - instantiate any registered provider."""
def get_provider(name: str, **kwargs: object) -> MetadataProvider:
"""Instantiate a registered metadata provider."""
if name not in _PROVIDERS:
msg = f"Unknown metadata provider: {name}"
raise ValueError(msg)
shelfmark/metadata_providers/googlebooks.py
+2 -3
View File
@@ -117,7 +117,7 @@ class GoogleBooksProvider(MetadataProvider):
key_prefix="googlebooks:search",
)
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Cached search implementation."""
"""Return cached search results for Google Books."""
# Build query string with Google Books operators
author_value = options.fields.get("author", "").strip()
title_value = options.fields.get("title", "").strip()
@@ -275,7 +275,6 @@ class GoogleBooksProvider(MetadataProvider):
if not volume_id or not title:
return None
# Authors (list)
authors = volume_info.get("authors", [])
# ISBNs - extract from industryIdentifiers
@@ -353,7 +352,7 @@ class GoogleBooksProvider(MetadataProvider):
display_fields=display_fields,
)
except Exception as e:
except (AttributeError, TypeError, ValueError) as e:
logger.debug("Failed to parse Google Books volume: %s", e)
return None
shelfmark/metadata_providers/hardcover.py
+59 -30
View File
@@ -3,7 +3,8 @@
import re
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
from datetime import UTC, datetime
from http import HTTPStatus
from typing import Any, ClassVar
from urllib.parse import urlparse
@@ -42,6 +43,10 @@ logger = setup_logger(__name__)
HARDCOVER_API_URL = "https://api.hardcover.app/v1/graphql"
HARDCOVER_PAGE_SIZE = 25 # Hardcover API returns max 25 results per page
HARDCOVER_MIN_AUTHOR_PARTS = 2
HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH = 2
HARDCOVER_MAX_SERIES_OPTIONS = 7
HARDCOVER_API_KEY_MIN_LENGTH = 100
HARDCOVER_LIST_URL_PATTERN = re.compile(
r"^/(?:@([\w.-]+)/)?lists?/([\w-]+)/?$",
re.IGNORECASE,
@@ -693,11 +698,11 @@ def _simplify_author_for_search(author: str) -> str | None:
# Handle "Last, First ..." -> "First ... Last"
if "," in normalized:
parts = [p.strip() for p in normalized.split(",") if p.strip()]
if len(parts) >= 2:
if len(parts) >= HARDCOVER_MIN_AUTHOR_PARTS:
normalized = " ".join([*parts[1:], parts[0]]).strip()
tokens = normalized.split(" ")
if len(tokens) < 2:
if len(tokens) < HARDCOVER_MIN_AUTHOR_PARTS:
return None
keep_suffixes = {"jr", "jr.", "sr", "sr.", "ii", "iii", "iv", "v"}
@@ -1117,7 +1122,7 @@ class HardcoverProvider(MetadataProvider):
) -> list[dict[str, Any]]:
"""Run a Hardcover search request for field-level typeahead options."""
normalized_query = _normalize_search_text(query)
if not self.api_key or len(normalized_query) < 2:
if not self.api_key or len(normalized_query) < HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH:
return []
result = self._execute_query(
@@ -1226,7 +1231,7 @@ class HardcoverProvider(MetadataProvider):
exclude_compilations = app_config.get("HARDCOVER_EXCLUDE_COMPILATIONS", False)
exclude_unreleased = app_config.get("HARDCOVER_EXCLUDE_UNRELEASED", False)
current_year = datetime.now().year
current_year = datetime.now(UTC).year
options: list[dict[str, str]] = []
seen_labels: set[str] = set()
@@ -1327,7 +1332,7 @@ class HardcoverProvider(MetadataProvider):
if description:
option["description"] = description
options.append(option)
if len(options) >= 7:
if len(options) >= HARDCOVER_MAX_SERIES_OPTIONS:
break
return options
@@ -1423,7 +1428,7 @@ class HardcoverProvider(MetadataProvider):
str(series_data.get("name") or "").strip() if isinstance(series_data, dict) else ""
)
allow_split_parts = _series_allows_split_parts(series_name)
today = datetime.now().date()
today = datetime.now(UTC).date()
book_series_rows = (
series_data.get("book_series", []) if isinstance(series_data, dict) else []
@@ -1516,7 +1521,7 @@ class HardcoverProvider(MetadataProvider):
@cacheable(ttl=120, key_prefix="hardcover:user_lists")
def _get_user_lists_cached(self, _cache_user_id: str) -> list[dict[str, str]]:
"""Cached wrapper keyed by Hardcover user id to avoid cross-user cache leakage."""
"""Return cached user lists keyed by Hardcover user id."""
return self._fetch_user_lists()
def _fetch_current_user_books_by_status(
@@ -1544,7 +1549,7 @@ class HardcoverProvider(MetadataProvider):
page: int,
limit: int,
) -> SearchResult:
"""Cached wrapper keyed by Hardcover user id and status shelf."""
"""Return cached status-shelf books keyed by user id and shelf."""
return self._fetch_user_books_by_status(status_id, page, limit)
def _fetch_user_books_by_status(self, status_id: int, page: int, limit: int) -> SearchResult:
@@ -1706,7 +1711,8 @@ class HardcoverProvider(MetadataProvider):
book_id_int = coerce_int(book_id, 0)
if book_id_int < 1:
raise ValueError("book_id must be a valid Hardcover book id")
msg = "book_id must be a valid Hardcover book id"
raise ValueError(msg)
state = self._fetch_book_target_state(book_id_int)
options = [
@@ -1722,21 +1728,31 @@ class HardcoverProvider(MetadataProvider):
return options
def set_book_target_state(self, book_id: str, target: str, selected: bool) -> dict[str, Any]:
def set_book_target_state(
self,
book_id: str,
target: str,
*,
selected: bool,
) -> dict[str, Any]:
"""Set whether a Hardcover book belongs to a status shelf or user list."""
if not self.api_key:
raise ValueError("Hardcover is not configured")
msg = "Hardcover is not configured"
raise ValueError(msg)
book_id_int = coerce_int(book_id, 0)
if book_id_int < 1:
raise ValueError("book_id must be a valid Hardcover book id")
msg = "book_id must be a valid Hardcover book id"
raise ValueError(msg)
selected_target = str(target or "").strip()
if not selected_target:
raise ValueError("target is required")
msg = "target is required"
raise ValueError(msg)
if selected_target not in self._get_writable_targets():
raise ValueError("Unsupported Hardcover target")
msg = "Unsupported Hardcover target"
raise ValueError(msg)
state = self._fetch_book_target_state(book_id_int)
status_ids_to_invalidate: set[int] = set()
@@ -1769,7 +1785,8 @@ class HardcoverProvider(MetadataProvider):
if changed:
list_ids_to_invalidate.add(list_id)
else:
raise ValueError("Unsupported Hardcover target")
msg = "Unsupported Hardcover target"
raise ValueError(msg)
if changed:
self._invalidate_book_target_caches(
@@ -1787,13 +1804,15 @@ class HardcoverProvider(MetadataProvider):
def _unwrap_me_data(result: dict | None) -> dict:
"""Extract and validate the ``me`` payload from a GraphQL result."""
if not isinstance(result, dict):
raise HardcoverTargetPayloadError("Hardcover could not load book targets")
msg = "Hardcover could not load book targets"
raise HardcoverTargetPayloadError(msg)
me_data = result.get("me", {})
if isinstance(me_data, list) and me_data:
me_data = me_data[0]
if not isinstance(me_data, dict):
raise HardcoverTargetPayloadError("Hardcover returned an invalid target payload")
msg = "Hardcover returned an invalid target payload"
raise HardcoverTargetPayloadError(msg)
return me_data
def _fetch_book_target_state(self, book_id: int) -> HardcoverBookTargetState:
@@ -2052,7 +2071,8 @@ class HardcoverProvider(MetadataProvider):
try:
return int(value.split(":", 1)[1])
except (IndexError, ValueError) as exc:
raise ValueError(f"Invalid Hardcover {label}") from exc
msg = f"Invalid Hardcover {label}"
raise ValueError(msg) from exc
@staticmethod
def _check_mutation_result(result: Any, key: str, *, check_error: bool = True) -> None:
@@ -2071,7 +2091,8 @@ class HardcoverProvider(MetadataProvider):
raise ValueError(error_text)
if payload.get("id") is not None:
return
raise RuntimeError("Hardcover could not complete this action")
msg = "Hardcover could not complete this action"
raise RuntimeError(msg)
def search(self, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Search for books using Hardcover's search API."""
@@ -2140,7 +2161,7 @@ class HardcoverProvider(MetadataProvider):
@cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="hardcover:search")
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> SearchResult:
"""Cached search implementation."""
"""Return cached Hardcover search results."""
# Determine query and fields based on custom search fields
# Note: Hardcover API requires 'weights' when using 'fields' parameter
author_value = options.fields.get("author", "").strip()
@@ -2195,7 +2216,7 @@ class HardcoverProvider(MetadataProvider):
# Parse hits, filtering compilations and unreleased books if enabled
exclude_compilations = app_config.get("HARDCOVER_EXCLUDE_COMPILATIONS", False)
exclude_unreleased = app_config.get("HARDCOVER_EXCLUDE_UNRELEASED", False)
current_year = datetime.now().year
current_year = datetime.now(UTC).year
books = []
for hit in hits:
item = _unwrap_hit_document(hit)
@@ -2416,29 +2437,34 @@ class HardcoverProvider(MetadataProvider):
except requests.Timeout as e:
logger.warning("Hardcover API request timed out")
if raise_on_error:
raise RuntimeError("Hardcover API request timed out") from e
msg = "Hardcover API request timed out"
raise RuntimeError(msg) from e
return None
except requests.HTTPError as e:
if e.response.status_code == 401:
if e.response.status_code == HTTPStatus.UNAUTHORIZED:
logger.exception("Hardcover API key is invalid")
if raise_on_error:
raise RuntimeError("Hardcover API key is invalid") from e
msg = "Hardcover API key is invalid"
raise RuntimeError(msg) from e
else:
logger.exception("Hardcover API HTTP error")
if raise_on_error:
raise RuntimeError(f"Hardcover API HTTP error: {e}") from e
msg = f"Hardcover API HTTP error: {e}"
raise RuntimeError(msg) from e
return None
except HardcoverGraphQLError:
raise
except ValueError as e:
logger.exception("Hardcover API returned invalid JSON")
if raise_on_error:
raise RuntimeError("Hardcover API returned an invalid response") from e
msg = "Hardcover API returned an invalid response"
raise RuntimeError(msg) from e
return None
except (TypeError, requests.RequestException) as e:
logger.exception("Hardcover API request failed")
if raise_on_error:
raise RuntimeError("Hardcover API request failed") from e
msg = "Hardcover API request failed"
raise RuntimeError(msg) from e
return None
def _parse_search_result(self, item: dict) -> BookMetadata | None:
@@ -2705,10 +2731,13 @@ def _test_hardcover_connection(current_values: dict[str, Any] | None = None) ->
_save_connected_user(None, None)
return {"success": False, "message": "API key is required"}
if key_len < 100:
if key_len < HARDCOVER_API_KEY_MIN_LENGTH:
return {
"success": False,
"message": f"API key seems too short ({key_len} chars). Expected 500+ chars.",
"message": (
f"API key seems too short ({key_len} chars). "
f"Expected {HARDCOVER_API_KEY_MIN_LENGTH}+ chars."
),
}
connection_result = {"success": False, "message": "API request failed - check your API key"}
shelfmark/metadata_providers/openlibrary.py
+9 -6
View File
@@ -4,6 +4,7 @@ import re
import threading
import time
from collections import deque
from http import HTTPStatus
from typing import Any, ClassVar
import requests
@@ -39,6 +40,8 @@ COVERS_BASE_URL = "https://covers.openlibrary.org"
# We use a sliding window with 90 requests per 60 seconds for safety margin
RATE_LIMIT_REQUESTS = 90
RATE_LIMIT_WINDOW_SECONDS = 60
ISBN_10_LENGTH = 10
ISBN_13_LENGTH = 13
class RateLimiter:
@@ -148,7 +151,7 @@ class OpenLibraryProvider(MetadataProvider):
ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="openlibrary:search"
)
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> list[BookMetadata]:
"""Cached search implementation."""
"""Return cached Open Library search results."""
_rate_limiter.wait_if_needed()
# Build query params
@@ -210,7 +213,7 @@ class OpenLibraryProvider(MetadataProvider):
logger.warning("Open Library search timed out")
return []
except requests.HTTPError as e:
if e.response.status_code == 503:
if e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE:
logger.warning("Open Library service unavailable (503)")
else:
logger.exception("Open Library HTTP error")
@@ -249,7 +252,7 @@ class OpenLibraryProvider(MetadataProvider):
logger.warning("Open Library get_book timed out")
return None
except requests.HTTPError as e:
if e.response.status_code == 404:
if e.response.status_code == HTTPStatus.NOT_FOUND:
logger.debug("Open Library work not found: %s", book_id)
else:
logger.exception("Open Library HTTP error")
@@ -310,7 +313,7 @@ class OpenLibraryProvider(MetadataProvider):
return self._parse_edition(edition, clean_isbn)
except requests.HTTPError as e:
if e.response.status_code == 404:
if e.response.status_code == HTTPStatus.NOT_FOUND:
logger.debug("Open Library ISBN not found: %s", isbn)
else:
logger.exception("Open Library ISBN search HTTP error")
@@ -339,8 +342,8 @@ class OpenLibraryProvider(MetadataProvider):
# Get ISBNs - find first ISBN-10 and ISBN-13
isbns = doc.get("isbn", [])
isbn_10 = next((i for i in isbns if len(i) == 10), None)
isbn_13 = next((i for i in isbns if len(i) == 13), None)
isbn_10 = next((i for i in isbns if len(i) == ISBN_10_LENGTH), None)
isbn_13 = next((i for i in isbns if len(i) == ISBN_13_LENGTH), None)
# Get cover URL
cover_id = doc.get("cover_i")
shelfmark/release_sources/__init__.py
+28 -9
View File
@@ -3,6 +3,7 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from enum import StrEnum
from importlib import import_module
from typing import TYPE_CHECKING, Any, ClassVar, Literal
if TYPE_CHECKING:
@@ -382,7 +383,7 @@ class DownloadHandler(ABC):
"""Execute download and return a path to the downloaded payload."""
def post_process_cleanup(self, task: DownloadTask, *, success: bool) -> None:
"""Optional hook called after orchestrator post-processing.
"""Run optional cleanup after orchestrator post-processing.
This is primarily used for external download clients, where the handler may need
to trigger client-side cleanup only after Shelfmark has safely imported the files.
@@ -398,12 +399,30 @@ class DownloadHandler(ABC):
_SOURCES: dict[str, type[ReleaseSource]] = {}
_HANDLERS: dict[str, type[DownloadHandler]] = {}
_BUILTIN_SOURCE_MODULES = (
"shelfmark.release_sources.audiobookbay",
"shelfmark.release_sources.direct_download",
"shelfmark.release_sources.irc",
"shelfmark.release_sources.prowlarr",
)
_builtin_source_state = {"loaded": False}
def _ensure_builtin_sources_registered() -> None:
"""Import built-in source modules once to populate source registries."""
if _builtin_source_state["loaded"]:
return
for module_name in _BUILTIN_SOURCE_MODULES:
import_module(module_name)
_builtin_source_state["loaded"] = True
def register_source(
name: str,
) -> Callable[[type[ReleaseSource]], type[ReleaseSource]]:
"""Decorator to register a release source."""
"""Register a release source."""
def decorator(cls: type[ReleaseSource]) -> type[ReleaseSource]:
_SOURCES[name] = cls
@@ -415,7 +434,7 @@ def register_source(
def register_handler(
name: str,
) -> Callable[[type[DownloadHandler]], type[DownloadHandler]]:
"""Decorator to register a download handler."""
"""Register a download handler."""
def decorator(cls: type[DownloadHandler]) -> type[DownloadHandler]:
_HANDLERS[name] = cls
@@ -426,6 +445,7 @@ def register_handler(
def get_source(name: str) -> ReleaseSource:
"""Get a release source instance by name."""
_ensure_builtin_sources_registered()
if name not in _SOURCES:
msg = f"Unknown release source: {name}"
raise ValueError(msg)
@@ -434,6 +454,7 @@ def get_source(name: str) -> ReleaseSource:
def get_handler(name: str) -> DownloadHandler:
"""Get a download handler instance by name."""
_ensure_builtin_sources_registered()
if name not in _HANDLERS:
msg = f"Unknown download handler: {name}"
raise ValueError(msg)
@@ -442,6 +463,7 @@ def get_handler(name: str) -> DownloadHandler:
def list_available_sources() -> list[dict]:
"""List all registered sources with their availability status."""
_ensure_builtin_sources_registered()
result = []
for name, src_class in _SOURCES.items():
instance = src_class()
@@ -462,6 +484,7 @@ def list_available_sources() -> list[dict]:
def get_source_display_name(name: str) -> str:
"""Get display name for a source by its identifier."""
_ensure_builtin_sources_registered()
if name in _SOURCES:
return _SOURCES[name]().display_name
return name.replace("_", " ").title()
@@ -505,14 +528,10 @@ def browse_record_to_book_metadata(
def source_results_are_releases(name: str) -> bool:
"""Whether a source's browse/search results already map to concrete releases."""
_ensure_builtin_sources_registered()
if name not in _SOURCES:
return False
return _SOURCES[name]().search_results_are_releases()
# Import source implementations to trigger registration
# These must be imported AFTER the base classes and registry are defined
from shelfmark.release_sources import audiobookbay as audiobookbay
from shelfmark.release_sources import direct_download as direct_download
from shelfmark.release_sources import irc as irc
from shelfmark.release_sources import prowlarr as prowlarr
_ensure_builtin_sources_registered()
shelfmark/release_sources/audiobookbay/source.py
+7 -2
View File
@@ -25,6 +25,7 @@ from shelfmark.release_sources.audiobookbay import scraper
from shelfmark.release_sources.audiobookbay.utils import normalize_hostname, parse_size
logger = setup_logger(__name__)
MIN_RELEVANCE_QUERY_WORD_LENGTH = 2
# Map language names to ISO 639-1 codes (matching frontend color maps)
@@ -133,7 +134,7 @@ def _parse_bitrate_to_kbps(bitrate: str | None) -> int | None:
def _generate_source_id(detail_url: str) -> str:
"""Generate a unique source ID from detail URL."""
return hashlib.md5(detail_url.encode()).hexdigest()
return hashlib.blake2b(detail_url.encode(), digest_size=16).hexdigest()
@register_source("audiobookbay")
@@ -246,7 +247,11 @@ class AudiobookBaySource(ReleaseSource):
)
# Extract query words for relevance checking
query_words = {word.lower() for word in query_lower.split() if len(word) > 2}
query_words = {
word.lower()
for word in query_lower.split()
if len(word) > MIN_RELEVANCE_QUERY_WORD_LENGTH
}
releases = []
for result in results:
shelfmark/release_sources/direct_download.py
+40 -36
View File
@@ -5,6 +5,7 @@ import json
import re
import time
from dataclasses import replace
from http import HTTPStatus
from typing import TYPE_CHECKING, ClassVar, NoReturn
from urllib.parse import quote
@@ -68,6 +69,7 @@ _DOWNLOAD_SOURCES = [
_SOURCE_FAILURE_THRESHOLD = 4
_MIN_VALID_FILE_SIZE = 10 * 1024
_AA_COUNTDOWN_MAX_SECONDS = 300
# Sources that require Cloudflare bypass
_CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "welib"})
@@ -163,7 +165,7 @@ def _normalize_size(size_str: str) -> str:
return _SIZE_UNIT_PATTERN.sub(lambda m: m.group(1).upper(), size_str.strip())
class SearchUnavailable(SourceUnavailableError):
class SearchUnavailableError(SourceUnavailableError):
"""Raised when Anna's Archive cannot be reached via any mirror/DNS."""
@@ -178,7 +180,7 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
List[BrowseRecord]: List of matching books
Raises:
SearchUnavailable: If Anna's Archive cannot be reached
SearchUnavailableError: If Anna's Archive cannot be reached
Exception: If parsing fails
"""
@@ -224,9 +226,8 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
if not html:
# Network/mirror exhaustion path bubbles up so API can notify clients
raise SearchUnavailable(
"Unable to reach download source. Network restricted or mirrors are blocked."
)
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
raise SearchUnavailableError(msg)
if "No files found." in html:
logger.info("No books found for query: %s", query)
@@ -237,7 +238,8 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
if not tbody:
logger.warning("No results table found for query: %s", query)
raise RuntimeError("No books found. Please try another query.")
msg = "No books found. Please try another query."
raise RuntimeError(msg)
books = []
if isinstance(tbody, Tag):
@@ -274,9 +276,8 @@ def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseR
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
if not html:
raise SearchUnavailable(
"Unable to reach download source. Network restricted or mirrors are blocked."
)
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
raise SearchUnavailableError(msg)
soup = BeautifulSoup(html, "html.parser")
@@ -320,7 +321,8 @@ def _parse_book_info_page(
data = soup.select_one("body > main > div:nth-of-type(1)")
if not data:
raise RuntimeError(f"Failed to parse book info for ID: {book_id}")
msg = f"Failed to parse book info for ID: {book_id}"
raise RuntimeError(msg)
preview: str = ""
@@ -381,15 +383,15 @@ def _parse_book_info_page(
divs = [div for div in divs if div.text.strip() != ""]
all_details = _find_in_divs(divs, " · ")
format = ""
file_format = ""
size = ""
content = ""
for _details in all_details:
_details = _details.split(" · ")
for f in _details:
if format == "" and f.strip().lower() in config.SUPPORTED_FORMATS:
format = f.strip().lower()
if file_format == "" and f.strip().lower() in config.SUPPORTED_FORMATS:
file_format = f.strip().lower()
if size == "" and any(u in f.strip().lower() for u in ("mb", "kb", "gb")):
size = _normalize_size(f)
if content == "":
@@ -397,11 +399,11 @@ def _parse_book_info_page(
if ct in f.strip().lower():
content = ct
break
if format == "" or size == "":
if file_format == "" or size == "":
for f in _details:
stripped = f.strip().lower()
if format == "" and stripped and " " not in stripped:
format = stripped
if file_format == "" and stripped and " " not in stripped:
file_format = stripped
if size == "" and "." in stripped:
size = _normalize_size(f)
@@ -418,7 +420,7 @@ def _parse_book_info_page(
content=content,
publisher=(_find_in_divs(divs, "icon-[mdi--company]", is_class=True) or [""])[0],
author=(_find_in_divs(divs, "icon-[mdi--user-edit]", is_class=True) or [""])[0],
format=format,
format=file_format,
size=size,
description=description,
download_urls=urls,
@@ -438,7 +440,7 @@ def _parse_book_info_page(
if "downloads_total" in summary_data:
info["Downloads"] = [str(summary_data["downloads_total"])]
except (
SearchUnavailable,
SearchUnavailableError,
RuntimeError,
json.JSONDecodeError,
TypeError,
@@ -592,7 +594,7 @@ def _fetch_aa_page_urls(book_info: BrowseRecord, urls_by_source: dict[str, list[
try:
fresh_book_info = get_book_info(book_info.id, fetch_download_count=False)
_group_urls_by_source(fresh_book_info.download_urls, urls_by_source)
except (SearchUnavailable, RuntimeError, TypeError, AttributeError) as e:
except (SearchUnavailableError, RuntimeError, TypeError, AttributeError) as e:
logger.warning("Failed to fetch AA page: %s", e)
@@ -740,7 +742,7 @@ def _get_download_urls_from_welib(
status_callback=status_callback,
)
except (
SearchUnavailable,
SearchUnavailableError,
requests.exceptions.RequestException,
RuntimeError,
ValueError,
@@ -780,7 +782,7 @@ def _extract_libgen_download_url(link: str, cancel_flag: Event | None = None) ->
verify=network.get_ssl_verify(link),
)
if response.status_code != 200:
if response.status_code != HTTPStatus.OK:
logger.debug("Libgen fast: %s returned %s", link, response.status_code)
return ""
@@ -1058,13 +1060,13 @@ def _extract_slow_download_url(
countdown_seconds = _extract_countdown_seconds(soup, html_str)
if countdown_seconds > 0:
MAX_COUNTDOWN_SECONDS = 600
sleep_time = min(countdown_seconds, MAX_COUNTDOWN_SECONDS)
if countdown_seconds > MAX_COUNTDOWN_SECONDS:
max_countdown_seconds = 600
sleep_time = min(countdown_seconds, max_countdown_seconds)
if countdown_seconds > max_countdown_seconds:
logger.warning(
"Countdown %ss exceeds max, capping at %ss",
countdown_seconds,
MAX_COUNTDOWN_SECONDS,
max_countdown_seconds,
)
logger.info("AA waitlist: %ss for %s", sleep_time, title)
@@ -1114,36 +1116,36 @@ def _extract_countdown_seconds(soup: BeautifulSoup, html_str: str) -> int:
countdown_attr = re.search(r'data-countdown=["\'](\d+)["\']', html_str)
if countdown_attr:
seconds = int(countdown_attr.group(1))
if 0 < seconds < 300:
if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS:
return seconds
js_countdown = re.search(r"countdown:\s*(\d+)", html_str)
if js_countdown:
seconds = int(js_countdown.group(1))
if 0 < seconds < 300:
if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS:
return seconds
js_var = re.search(r"(?:var|let|const)\s+countdown\s*=\s*(\d+)", html_str)
if js_var:
seconds = int(js_var.group(1))
if 0 < seconds < 300:
if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS:
return seconds
countdown_secs = re.search(r"countdownSeconds\s*=\s*(\d+)", html_str)
if countdown_secs:
seconds = int(countdown_secs.group(1))
if 0 < seconds < 300:
if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS:
return seconds
json_countdown = re.search(r'["\']countdown[_-]?seconds["\']\s*:\s*(\d+)', html_str)
if json_countdown:
seconds = int(json_countdown.group(1))
if 0 < seconds < 300:
if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS:
return seconds
wait_text = re.search(r"wait\s+(\d+)\s+seconds", html_str, re.IGNORECASE)
if wait_text:
seconds = int(wait_text.group(1))
if 0 < seconds < 300:
if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS:
return seconds
return 0
@@ -1156,7 +1158,7 @@ def _parse_countdown_seconds_from_element(element: Tag) -> int | None:
except ValueError, TypeError:
return None
if 0 < seconds < 300:
if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS:
return seconds
return None
@@ -1204,6 +1206,7 @@ class DirectDownloadSource(ReleaseSource):
supported_content_types: ClassVar[list[str]] = ["ebook"] # Direct downloads only support ebooks
def __init__(self) -> None:
"""Initialize per-instance search state for direct downloads."""
# Tracks which search method was used in the last search() call
# "isbn" = ISBN search returned results, "title_author" = title+author was used
self._last_search_type: str = "title_author"
@@ -1307,6 +1310,7 @@ class DirectDownloadSource(ReleaseSource):
Args:
book: Book metadata from provider
plan: Precomputed search plan with normalized queries and filters.
expand_search: If True, skip ISBN and use title+author directly
languages: Language codes to filter by (overrides book.language/config)
content_type: Ignored - Direct download uses format filtering instead
@@ -1347,7 +1351,7 @@ class DirectDownloadSource(ReleaseSource):
self._last_search_type = "isbn"
return [_browse_record_to_release(record) for record in results]
logger.debug("No ISBN results, falling back to title+author")
except SearchUnavailable:
except SearchUnavailableError:
raise
except (ValueError, TypeError, AttributeError, RuntimeError) as e:
logger.warning("ISBN search failed: %s", e)
@@ -1372,7 +1376,7 @@ class DirectDownloadSource(ReleaseSource):
if bi.id not in seen_ids:
seen_ids.add(bi.id)
all_results.append(bi)
except SearchUnavailable:
except SearchUnavailableError:
raise
except Exception:
logger.exception("Search error")
@@ -1392,7 +1396,7 @@ class DirectDownloadSource(ReleaseSource):
if bi.id not in seen_ids:
seen_ids.add(bi.id)
all_results.append(bi)
except SearchUnavailable:
except SearchUnavailableError:
raise
except Exception:
logger.exception("Search error")
@@ -1477,7 +1481,7 @@ class DirectDownloadHandler(DownloadHandler):
progress_callback: Callable[[float], None],
status_callback: Callable[[str, str | None], None],
) -> str | None:
"""Internal method to execute the download with fetched browse record.
"""Execute the direct-download flow with a fetched browse record.
This contains the core download logic: cascade through sources,
handle bypass, move to final location.
shelfmark/release_sources/irc/client.py
+6 -3
View File
@@ -10,7 +10,7 @@ import time
from contextlib import suppress
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Self
from shelfmark.core.logger import setup_logger
@@ -78,6 +78,7 @@ class IRCClient:
use_tls: bool = True,
version: str = "Shelfmark 1.0",
) -> None:
"""Initialize the IRC client with connection settings and defaults."""
if not nick:
msg = "IRC nickname is required"
raise IRCError(msg)
@@ -456,9 +457,11 @@ class IRCClient:
"""Check if currently connected."""
return self._connected and self._socket is not None
def __enter__(self) -> IRCClient:
def __enter__(self) -> Self:
"""Connect and return the IRC client for context-manager usage."""
self.connect()
return self
def __exit__(self, *args) -> None:
def __exit__(self, *args: object) -> None:
"""Disconnect the IRC client when leaving a context manager."""
self.disconnect()
shelfmark/release_sources/irc/connection_manager.py
+9 -6
View File
@@ -6,15 +6,17 @@ Maintains persistent IRC connections to avoid reconnecting between search and do
import threading
import time
from contextlib import suppress
from typing import Self
from shelfmark.core.logger import setup_logger
from .client import IRCClient
from .client import IRCClient, IRCError
logger = setup_logger(__name__)
# How long to keep an idle connection before closing it
IDLE_TIMEOUT = 300.0 # 5 minutes
_IRC_CONNECTION_ERRORS = (IRCError, OSError, RuntimeError)
class IRCConnectionManager:
@@ -28,7 +30,7 @@ class IRCConnectionManager:
_instance: IRCConnectionManager | None = None
_lock = threading.Lock()
def __new__(cls) -> IRCConnectionManager:
def __new__(cls) -> Self:
"""Singleton pattern - only one connection manager."""
if cls._instance is None:
with cls._lock:
@@ -38,6 +40,7 @@ class IRCConnectionManager:
return cls._instance
def __init__(self) -> None:
"""Initialize connection caches for the singleton manager."""
if self._initialized:
return
@@ -87,7 +90,7 @@ class IRCConnectionManager:
logger.info("Closing idle IRC connection: %s", key)
try:
client.disconnect()
except Exception as e:
except _IRC_CONNECTION_ERRORS as e:
logger.debug("Error closing idle connection: %s", e)
def get_connection(
@@ -183,7 +186,7 @@ class IRCConnectionManager:
self._last_used[key] = time.time()
self._channels[key] = channel
self._connecting.pop(key, None)
except Exception:
except _IRC_CONNECTION_ERRORS:
# Clear connecting flag on failure
with self._conn_lock:
self._connecting.pop(key, None)
@@ -220,7 +223,7 @@ class IRCConnectionManager:
try:
client.disconnect()
except Exception as e:
except _IRC_CONNECTION_ERRORS as e:
logger.debug("Error closing connection: %s", e)
logger.debug("Closed IRC connection: %s", key)
@@ -242,7 +245,7 @@ class IRCConnectionManager:
"""Disconnect one IRC client and log failures."""
try:
client.disconnect()
except Exception as e:
except _IRC_CONNECTION_ERRORS as e:
logger.debug("Error closing connection %s: %s", key, e)
shelfmark/release_sources/irc/source.py
+1
View File
@@ -75,6 +75,7 @@ class IRCReleaseSource(ReleaseSource):
can_be_default = False # Exclude from default source options (requires deliberate selection)
def __init__(self) -> None:
"""Initialize per-search IRC source state."""
# Track online servers from most recent search
self._online_servers: set[str] | None = None
shelfmark/release_sources/prowlarr/api.py
+13 -8
View File
@@ -1,5 +1,6 @@
"""Prowlarr API client for connection testing, indexer listing, and search."""
from contextlib import suppress
from http import HTTPStatus
from typing import Any
@@ -15,12 +16,20 @@ logger = setup_logger(__name__)
_HTTP_STATUS_UNAUTHORIZED = HTTPStatus.UNAUTHORIZED
_BOOK_CATEGORY_RANGE_START = 7000
_BOOK_CATEGORY_RANGE_END = 8000
_PROWLARR_CLIENT_ERRORS = (
requests.exceptions.RequestException,
OSError,
RuntimeError,
TypeError,
ValueError,
)
class ProwlarrClient:
"""Client for interacting with the Prowlarr API."""
def __init__(self, url: str, api_key: str, timeout: int = 30) -> None:
"""Initialize the API client with base URL, key, and timeout."""
self.base_url = normalize_http_url(url)
self.api_key = api_key
self.timeout = timeout
@@ -54,11 +63,9 @@ class ProwlarrClient:
)
if not response.ok:
try:
with suppress(Exception):
error_body = response.text[:500]
logger.error("Prowlarr API error response: %s", error_body)
except Exception:
pass
response.raise_for_status()
return response.json()
@@ -91,7 +98,7 @@ class ProwlarrClient:
if e.response is not None and e.response.status_code == _HTTP_STATUS_UNAUTHORIZED:
return False, "Invalid API key"
return False, f"HTTP error {status}"
except Exception as e:
except _PROWLARR_CLIENT_ERRORS as e:
return False, f"Connection failed: {e!s}"
else:
logger.info("Prowlarr connection successful: version %s", version)
@@ -101,7 +108,7 @@ class ProwlarrClient:
"""Get all configured indexers."""
try:
return self._request("GET", "/api/v1/indexer")
except Exception:
except _PROWLARR_CLIENT_ERRORS:
logger.exception("Failed to get indexers")
return []
@@ -215,11 +222,9 @@ class ProwlarrClient:
verify=get_ssl_verify(url),
)
if not response.ok:
try:
with suppress(Exception):
error_body = response.text[:500]
logger.error("Prowlarr Torznab error response: %s", error_body)
except Exception:
pass
response.raise_for_status()
results = parse_torznab_xml(response.text)
shelfmark/release_sources/prowlarr/handler.py
+8 -1
View File
@@ -2,7 +2,7 @@
from typing import TYPE_CHECKING
from shelfmark.core.config import config as config
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import normalize_optional_text
from shelfmark.download.clients import (
@@ -36,6 +36,13 @@ if TYPE_CHECKING:
from shelfmark.core.models import DownloadTask
logger = setup_logger(__name__)
__all__ = [
"ProwlarrHandler",
"POLL_INTERVAL",
"COMPLETED_PATH_RETRY_INTERVAL",
"COMPLETED_PATH_MAX_ATTEMPTS",
"config",
]
# Backwards-compat constants for tests patching this module.
POLL_INTERVAL = _DEFAULT_POLL_INTERVAL
shelfmark/release_sources/prowlarr/settings.py
+13 -2
View File
@@ -2,6 +2,8 @@
from typing import Any
import requests
from shelfmark.core.settings_registry import (
ActionButton,
CheckboxField,
@@ -16,6 +18,15 @@ from shelfmark.core.utils import normalize_http_url
# ==================== Dynamic Options Loaders ====================
_PROWLARR_SETTINGS_ERRORS = (
requests.exceptions.RequestException,
AttributeError,
OSError,
RuntimeError,
TypeError,
ValueError,
)
def _get_indexer_options() -> list[dict[str, str]]:
"""Fetch available indexers from Prowlarr for the multi-select field.
@@ -62,7 +73,7 @@ def _get_indexer_options() -> list[dict[str, str]]:
}
)
except Exception:
except _PROWLARR_SETTINGS_ERRORS:
logger.exception("Failed to fetch Prowlarr indexers")
return []
@@ -95,7 +106,7 @@ def _test_prowlarr_connection(current_values: dict[str, Any] | None = None) -> d
try:
client = ProwlarrClient(url, api_key)
success, message = client.test_connection()
except Exception as e:
except _PROWLARR_SETTINGS_ERRORS as e:
return {"success": False, "message": f"Connection failed: {e!s}"}
else:
return {"success": success, "message": message}
shelfmark/release_sources/prowlarr/source.py
+5 -3
View File
@@ -38,6 +38,7 @@ logger = setup_logger(__name__)
_SIZE_UNIT_BASE = 1024
_TWO_FORMATS = 2
_PROWLARR_SOURCE_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, ValueError)
def _raise_timeout_error(message: str) -> NoReturn:
@@ -443,6 +444,7 @@ class ProwlarrSource(ReleaseSource):
] # Explicitly declare support for both
def __init__(self) -> None:
"""Initialize per-instance search state for Prowlarr."""
self.last_search_type: str | None = None
def get_column_config(self) -> ReleaseColumnConfig:
@@ -482,7 +484,7 @@ class ProwlarrSource(ReleaseSource):
default_indexers = (
sorted(selected_indexer_names) if selected_indexer_names else None
)
except Exception as e:
except _PROWLARR_SOURCE_ERRORS as e:
logger.warning("Failed to fetch indexer list for column config: %s", e)
return ReleaseColumnConfig(
@@ -617,7 +619,7 @@ class ProwlarrSource(ReleaseSource):
if idx_id is not None:
with suppress(TypeError, ValueError):
ids.append(int(idx_id))
except Exception as e:
except _PROWLARR_SOURCE_ERRORS as e:
logger.warning("Failed to resolve indexer names to IDs: %s", e)
return None
else:
@@ -635,7 +637,7 @@ class ProwlarrSource(ReleaseSource):
try:
enabled_indexers = client.get_enabled_indexers_detailed()
except Exception as e:
except _PROWLARR_SOURCE_ERRORS as e:
logger.warning("Failed to load enabled Prowlarr indexers: %s", e)
return []
shelfmark/release_sources/prowlarr/torznab.py
+6 -3
View File
@@ -44,7 +44,9 @@ def _coerce_float(value: str | None) -> float | None:
def _strip_author_from_title(title: str, author: str | None) -> str:
"""Prowlarr's MyAnonamouse parser appends " by {author}" into the title while
"""Strip duplicate trailing author text from a Torznab title.
Prowlarr's MyAnonamouse parser appends " by {author}" into the title while
also emitting author/booktitle fields. Shelfmark's UI shows author
separately, so strip the duplicated " by author" segment when present.
"""
@@ -59,8 +61,9 @@ def _strip_author_from_title(title: str, author: str | None) -> str:
def parse_torznab_xml(xml_text: str) -> list[dict[str, Any]]:
"""Parse a Torznab/Newznab XML response into a list of dicts that roughly match
Prowlarr's JSON search results shape.
"""Parse a Torznab/Newznab XML response into Prowlarr-like result dicts.
This keeps the parsed shape close to Prowlarr's JSON search results.
"""
if not xml_text or not xml_text.strip():
return []
tests/bypass/test_internal_bypasser.py
+89
View File
@@ -38,6 +38,7 @@ def test_bypass_tries_all_methods_before_abort(monkeypatch):
def test_extract_cookies_from_cdp_filters_and_stores_ua():
import time
import shelfmark.bypass.internal_bypasser as internal_bypasser
class FakeCookie:
@@ -80,6 +81,7 @@ def test_extract_cookies_from_cdp_filters_and_stores_ua():
def test_extract_cookies_from_cdp_normalizes_session_expiry():
import time
import shelfmark.bypass.internal_bypasser as internal_bypasser
class FakeCookie:
@@ -121,3 +123,90 @@ def test_extract_cookies_from_cdp_normalizes_session_expiry():
# Verify fallback to "expires" key for expiry checks
internal_bypasser._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10
assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {}
def test_get_page_info_returns_safe_defaults_on_cdp_errors():
from seleniumbase.undetected.cdp_driver.connection import ProtocolException
import shelfmark.bypass.internal_bypasser as internal_bypasser
class FakePage:
async def get_title(self):
raise ProtocolException("no title")
async def evaluate(self, _expr):
raise ProtocolException("no body")
async def get_current_url(self):
raise ProtocolException("no url")
title, body, current_url = asyncio.run(internal_bypasser._get_page_info(FakePage()))
assert title == ""
assert body == ""
assert current_url == ""
def test_try_with_cached_cookies_returns_none_on_request_exception(monkeypatch):
import time
import requests
import shelfmark.bypass.internal_bypasser as internal_bypasser
internal_bypasser.clear_cf_cookies()
internal_bypasser._cf_cookies["example.com"] = {
"cf_clearance": {
"value": "abc",
"domain": "example.com",
"path": "/",
"expiry": int(time.time()) + 3600,
"secure": True,
"httpOnly": True,
}
}
def _raise(*_args, **_kwargs):
raise requests.RequestException("boom")
monkeypatch.setattr(internal_bypasser.requests, "get", _raise)
assert internal_bypasser._try_with_cached_cookies("https://example.com", "example.com") is None
def test_get_bypassed_page_retries_next_mirror_after_runtime_error(monkeypatch):
import shelfmark.bypass.internal_bypasser as internal_bypasser
class FakeSelector:
def __init__(self):
self.urls = ["https://mirror-one.example/book", "https://mirror-two.example/book"]
self.index = 0
def rewrite(self, _url):
return self.urls[self.index]
def next_mirror_or_rotate_dns(self, *, allow_dns=True):
del allow_dns
self.index = 1
return "https://mirror-two.example", "mirror"
calls: list[str] = []
def _fake_get(url, retry=None, cancel_flag=None):
del retry, cancel_flag
calls.append(url)
if len(calls) == 1:
raise RuntimeError("browser hiccup")
return "<html>ok</html>"
monkeypatch.setattr(internal_bypasser, "_try_with_cached_cookies", lambda *_args, **_kwargs: None)
monkeypatch.setattr(internal_bypasser, "get", _fake_get)
selector = FakeSelector()
result = internal_bypasser.get_bypassed_page("https://orig.example/book", selector=selector)
assert result == "<html>ok</html>"
assert calls == [
"https://mirror-one.example/book",
"https://mirror-two.example/book",
]
tests/config/test_dns_settings_live_apply.py
+38
View File
@@ -0,0 +1,38 @@
def test_update_settings_network_logs_dns_apply_failure(monkeypatch):
import shelfmark.config.settings # noqa: F401
import shelfmark.core.settings_registry as registry
from shelfmark.core.config import config as config_obj
from shelfmark.core.settings_registry import update_settings
monkeypatch.setattr("shelfmark.core.settings_registry.save_config_file", lambda _tab, _values: True)
monkeypatch.setattr(config_obj, "refresh", lambda: None)
import shelfmark.download.network as network
def failing_set_dns_provider(*args, **kwargs) -> None:
raise RuntimeError("dns apply failed")
monkeypatch.setattr(network, "set_dns_provider", failing_set_dns_provider)
warnings: list[tuple[str, tuple[object, ...]]] = []
monkeypatch.setattr(
registry.logger,
"warning",
lambda message, *args: warnings.append((str(message), args)),
)
result = update_settings(
"network",
{
"CUSTOM_DNS": "manual",
"CUSTOM_DNS_MANUAL": "1.1.1.1,8.8.8.8",
},
)
assert result["success"] is True
assert len(warnings) == 1
message, args = warnings[0]
assert message == "Failed to apply DNS settings: %s"
assert len(args) == 1
assert isinstance(args[0], RuntimeError)
assert str(args[0]) == "dns apply failed"
tests/config/test_mirror_settings_live_apply.py
+34 -1
View File
@@ -1,7 +1,6 @@
def test_update_settings_mirrors_applies_aa_changes_live(monkeypatch):
# Ensure settings tabs are registered (mirrors tab lives here).
import shelfmark.config.settings # noqa: F401
from shelfmark.core.config import config as config_obj
from shelfmark.core.settings_registry import update_settings
@@ -26,3 +25,37 @@ def test_update_settings_mirrors_applies_aa_changes_live(monkeypatch):
assert result["success"] is True
assert called["force"] is True
def test_update_settings_mirrors_logs_live_apply_failure(monkeypatch):
import shelfmark.config.settings # noqa: F401
import shelfmark.core.settings_registry as registry
from shelfmark.core.config import config as config_obj
from shelfmark.core.settings_registry import update_settings
monkeypatch.setattr("shelfmark.core.settings_registry.save_config_file", lambda _tab, _values: True)
monkeypatch.setattr(config_obj, "refresh", lambda: None)
import shelfmark.download.network as network
def failing_init_aa(*, force: bool = False) -> None:
del force
raise RuntimeError("mirror apply failed")
monkeypatch.setattr(network, "init_aa", failing_init_aa)
warnings: list[tuple[str, tuple[object, ...]]] = []
monkeypatch.setattr(
registry.logger,
"warning",
lambda message, *args: warnings.append((str(message), args)),
)
result = update_settings("mirrors", {"AA_BASE_URL": "https://annas-archive.li"})
assert result["success"] is True
assert len(warnings) == 1
message, args = warnings[0]
assert message == "Failed to apply AA mirror settings: %s"
assert len(args) == 1
assert isinstance(args[0], RuntimeError)
assert str(args[0]) == "mirror apply failed"
tests/core/test_activity_routes_api.py
+37 -8
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import importlib
import sqlite3
import uuid
from types import SimpleNamespace
from unittest.mock import ANY, patch
@@ -66,7 +67,7 @@ def _record_terminal_download(
source_display_name=source_display_name,
title=title,
author=author,
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -390,7 +391,7 @@ class TestActivityRoutes:
source_display_name="Direct Download",
title="Active Dismiss Task",
author="Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -554,7 +555,7 @@ class TestActivityRoutes:
source_display_name="Direct Download",
title="Stale Active Download",
author="Stale Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -605,7 +606,7 @@ class TestActivityRoutes:
source_display_name="Prowlarr",
title="Interrupted Requested Download",
author="Stale Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -782,6 +783,34 @@ class TestActivityRoutes:
assert response.status_code == 403
assert response.json["code"] == "user_identity_unavailable"
def test_dismiss_many_with_user_db_lookup_failure_returns_identity_unavailable(
self, main_module, client
):
user = _create_user(main_module, prefix="reader")
_set_session(client, user_id=user["username"], db_user_id=user["id"], is_admin=False)
with (
patch.object(main_module, "get_auth_mode", return_value="builtin"),
patch.object(
main_module.user_db,
"get_user",
side_effect=sqlite3.OperationalError("database is locked"),
),
patch("shelfmark.core.activity_routes.logger.warning") as mock_warning,
):
response = client.post(
"/api/activity/dismiss-many",
json={"items": [{"item_type": "download", "item_key": "download:test-db-error"}]},
)
assert response.status_code == 403
assert response.json["code"] == "user_identity_unavailable"
mock_warning.assert_any_call(
"Failed to validate activity db identity %s: %s",
user["id"],
ANY,
)
def test_clear_history_logs_identity_failure(self, main_module, client):
admin = _create_user(main_module, prefix="admin", role="admin")
_set_session(client, user_id=admin["username"], db_user_id=None, is_admin=True)
@@ -896,7 +925,7 @@ class TestActivityRoutes:
source_display_name="Direct Download",
title="Stale Active Task",
author="Stale Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -938,7 +967,7 @@ class TestActivityRoutes:
source_display_name="Prowlarr",
title="Stale Active Requested Task",
author="Stale Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -1027,7 +1056,7 @@ class TestActivityRoutes:
source_display_name="Prowlarr",
title="Retry Gone Request",
author="Retry Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -1068,7 +1097,7 @@ class TestActivityRoutes:
source_display_name="Direct Download",
title="Active Downloading Task",
author="Active Author",
format="epub",
file_format="epub",
size="2 MB",
preview=None,
content_type="ebook",
tests/core/test_admin_users_api.py
+18
View File
@@ -700,6 +700,24 @@ class TestAdminUserUpdateEndpoint:
assert resp.json["error"] == "Invalid settings payload"
assert any("Unknown setting: destination" in msg for msg in resp.json["details"])
def test_update_user_settings_warns_when_runtime_refresh_fails(self, admin_client, user_db):
user = user_db.create_user(username="alice")
with (
patch("shelfmark.core.admin_routes.app_config.refresh", side_effect=RuntimeError("boom")),
patch("shelfmark.core.admin_routes.logger.warning") as mock_warning,
):
resp = admin_client.put(
f"/api/admin/users/{user['id']}",
json={"settings": {"DESTINATION": "/books/alice"}},
)
assert resp.status_code == 200
settings = user_db.get_user_settings(user["id"])
assert settings["DESTINATION"] == "/books/alice"
mock_warning.assert_called_once()
assert "failed to refresh runtime config" in mock_warning.call_args[0][0]
def test_update_response_excludes_password_hash(self, admin_client, user_db):
user = user_db.create_user(username="alice", password_hash="secret")
tests/core/test_download_api_guardrails.py
+5 -5
View File
@@ -151,7 +151,7 @@ class TestReleaseDownloadEndpointGuardrails:
assert captured["release_data"] == {**payload, "content_type": "audiobook"}
assert captured["priority"] == 1
def test_non_json_payload_returns_500_current_behavior(self, main_module, client):
def test_non_json_payload_returns_400(self, main_module, client):
with patch.object(main_module, "get_auth_mode", return_value="none"):
with patch.object(main_module.backend, "queue_release") as mock_queue_release:
resp = client.post(
@@ -161,8 +161,8 @@ class TestReleaseDownloadEndpointGuardrails:
)
body = resp.get_json()
assert resp.status_code == 500
assert "Unsupported Media Type" in body["error"]
assert resp.status_code == 400
assert body == {"error": "No data provided"}
mock_queue_release.assert_not_called()
def test_admin_can_queue_release_on_behalf_of_another_user(self, main_module, client):
@@ -520,7 +520,7 @@ class TestRetryDownloadEndpointGuardrails:
source_display_name="Direct Download",
title="Persisted Direct Task",
author="Direct Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
@@ -705,7 +705,7 @@ class TestRetryDownloadEndpointGuardrails:
source_display_name="Prowlarr",
title="Persisted Requested Book",
author="Request Author",
format="epub",
file_format="epub",
size="1 MB",
preview=None,
content_type="ebook",
tests/core/test_download_history_service.py
+1 -1
View File
@@ -30,7 +30,7 @@ def test_record_download_stores_utc_iso_timestamps():
source_display_name="Direct Download",
title="Example",
author=None,
format=None,
file_format=None,
size=None,
preview=None,
content_type="ebook",
tests/core/test_download_processing.py
+1 -1
View File
@@ -506,7 +506,7 @@ class TestProcessDirectory:
with patch('shelfmark.core.config.config') as mock_config, \
patch('shelfmark.config.env.TMP_DIR', temp_dirs["staging"]), \
patch('shelfmark.download.postprocess.transfer.atomic_move', side_effect=Exception("Move failed")):
patch('shelfmark.download.postprocess.transfer.atomic_move', side_effect=RuntimeError("Move failed")):
mock_config.USE_BOOK_TITLE = False
mock_config.get = MagicMock(side_effect=lambda key, default=None, **_kwargs: {
tests/core/test_hardlink.py
+28
View File
@@ -1345,6 +1345,34 @@ class TestTorrentSourceCleanupProtection:
assert is_torrent_source(torrent_path, task) is True
assert is_torrent_source(staging_path, task) is False
def testis_torrent_source_falls_back_to_normalized_paths(self, tmp_path, monkeypatch):
"""If resolve() fails, path comparison should still fall back safely."""
import shelfmark.download.postprocess.transfer as transfer_module
from shelfmark.download.postprocess.pipeline import is_torrent_source
from shelfmark.core.models import DownloadTask, SearchMode
torrent_path = tmp_path / "downloads" / "book.epub"
fallback_path = tmp_path / "downloads" / ".." / "downloads" / "book.epub"
task = DownloadTask(
task_id="test",
source="prowlarr",
title="Test",
author="Author",
format="epub",
search_mode=SearchMode.UNIVERSAL,
original_download_path=str(torrent_path),
)
monkeypatch.setattr(
transfer_module,
"run_blocking_io",
lambda _func, *_args, **_kwargs: (_ for _ in ()).throw(OSError("resolve failed")),
)
assert is_torrent_source(fallback_path, task) is True
class TestEdgeCases:
"""Edge cases and error handling."""
tests/core/test_image_cache.py
+22
View File
@@ -0,0 +1,22 @@
"""Tests for targeted image cache safety and fetch fallbacks."""
import requests
from shelfmark.core.image_cache import ImageCacheService
def test_is_safe_url_rejects_invalid_ipv6_url() -> None:
assert ImageCacheService._is_safe_url("http://[") is False
def test_fetch_and_cache_returns_none_on_request_exception(tmp_path, monkeypatch) -> None:
cache = ImageCacheService(tmp_path)
monkeypatch.setattr(cache, "_is_safe_url", lambda _url: True)
def fake_get(*args, **kwargs):
raise requests.exceptions.TooManyRedirects("too many redirects")
monkeypatch.setattr("shelfmark.core.image_cache.requests.get", fake_get)
assert cache.fetch_and_cache("cover-1", "https://example.com/cover.jpg") is None
assert "cover-1" not in cache._index
tests/core/test_oidc_routes.py
+23
View File
@@ -167,6 +167,15 @@ class TestOIDCLoginEndpoint:
class TestOIDCCallbackEndpoint:
def test_normalize_claims_returns_empty_dict_for_invalid_mapping(self):
from shelfmark.core.oidc_routes import _normalize_claims
class BadClaims:
def __iter__(self):
raise TypeError("bad claims")
assert _normalize_claims(BadClaims()) == {}
@patch("shelfmark.core.oidc_routes._get_oidc_client")
def test_callback_creates_session(self, mock_get_client, client):
fake_client = Mock()
@@ -339,6 +348,20 @@ class TestOIDCCallbackEndpoint:
assert error is not None
assert "issuer validation failed" in error
@patch("shelfmark.core.oidc_routes._get_oidc_client")
def test_callback_tolerates_metadata_lookup_failure_during_claim_diagnostics(
self, mock_get_client, client
):
fake_client = Mock()
fake_client.authorize_access_token.side_effect = InvalidClaimError("iss")
fake_client.load_server_metadata.side_effect = RuntimeError("metadata failed")
mock_get_client.return_value = (fake_client, MOCK_OIDC_CONFIG)
resp = client.get("/api/auth/oidc/callback?code=abc123&state=test-state")
error = _get_oidc_error(resp)
assert error is not None
assert "issuer validation failed" in error
@patch("shelfmark.core.oidc_routes._get_oidc_client")
def test_callback_redirects_when_auto_provision_disabled_and_no_email_match(
self, mock_get_client, client
tests/core/test_original_name_template_variable.py
+4 -2
View File
@@ -32,7 +32,9 @@ class TestOriginalNameTransferTemplates:
monkeypatch.setattr(
"shelfmark.download.postprocess.transfer.get_template",
lambda _is_audiobook, mode: "{OriginalName}" if mode == "rename" else "{Author}/{Title}",
lambda *, is_audiobook, organization_mode: (
"{OriginalName}" if organization_mode == "rename" else "{Author}/{Title}"
),
)
task = DownloadTask(
@@ -70,7 +72,7 @@ class TestOriginalNameTransferTemplates:
monkeypatch.setattr(
"shelfmark.download.postprocess.transfer.get_template",
lambda _is_audiobook, mode: "{Author}/{Title}/{OriginalName}",
lambda *, is_audiobook, organization_mode: "{Author}/{Title}/{OriginalName}",
)
task = DownloadTask(
tests/core/test_per_user_downloads.py
+48
View File
@@ -5,6 +5,8 @@ Tests that DownloadTask has a user_id field and that the queue
can be filtered by user.
"""
import sqlite3
from shelfmark.core.models import DownloadTask, QueueStatus
from shelfmark.core.queue import BookQueue
@@ -277,6 +279,52 @@ class TestUserDestinationTemplate:
result = get_destination(is_audiobook=True, user_id=42, username="alice")
assert result == Path("/audiobooks/alice")
def test_get_destination_looks_up_username_from_user_db(self, monkeypatch, tmp_path):
from pathlib import Path
from shelfmark.core.config import config
from shelfmark.core.user_db import UserDB
from shelfmark.core.utils import get_destination
monkeypatch.setenv("CONFIG_DIR", str(tmp_path))
user_db = UserDB(str(tmp_path / "users.db"))
user_db.initialize()
user = user_db.create_user(username="alice")
def fake_config_get(key, default=None, user_id=None):
if key == "DESTINATION":
return "/books/{User}"
if key == "INGEST_DIR":
return "/books"
return default
monkeypatch.setattr(config, "get", fake_config_get)
result = get_destination(is_audiobook=False, user_id=user["id"], username=None)
assert result == Path("/books/alice")
def test_get_destination_falls_back_when_user_db_lookup_fails(self, monkeypatch):
from pathlib import Path
from shelfmark.core.config import config
from shelfmark.core.utils import get_destination
def fake_config_get(key, default=None, user_id=None):
if key == "DESTINATION":
return "/books/{User}"
if key == "INGEST_DIR":
return "/books"
return default
monkeypatch.setattr(config, "get", fake_config_get)
monkeypatch.setattr(
"shelfmark.core.user_db.UserDB.get_user",
lambda self, **kwargs: (_ for _ in ()).throw(sqlite3.OperationalError("locked")),
)
result = get_destination(is_audiobook=False, user_id=42, username=None)
assert result == Path("/books")
class TestTaskToDictUsername:
"""Tests that _task_to_dict includes username for frontend display."""

Some files were not shown because too many files have changed in this diff Show More