diff --git a/pyproject.toml b/pyproject.toml index 33bf984c..1e8d069c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,17 +66,40 @@ extend-exclude = [".local"] [tool.ruff.lint] select = [ "F", "I", "UP", "B", "C4", "SIM", "PTH", "RET", "PIE", "FURB", "PERF", "TRY", - "ANN001", "ANN201", "ANN202", "ANN204", + "A", "DTZ", "N", + "BLE001", + "ANN001", "ANN002", "ANN003", "ANN201", "ANN202", "ANN204", + "E402", + "ERA001", "E731", - "FBT002", "FBT003", + "S101", + "S110", + "S105", "S108", + "S311", "S324", + "S607", "S608", "G003", "G004", + "PGH003", + "PLC0414", "PLR1714", + "PLW1510", + "PLW2901", "PLW0108", + "PT028", + "PYI034", "Q000", "RUF005", "RUF012", "RUF013", "RUF059", "RUF100", - "TC001", "TC003", + "TC001", "TC002", "TC003", +] +ignore = ["D", "EM", "FBT", "PLR2004", "UP035", "TRY003", "E501", "TD002", "S104", "S603"] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "ANN", + "S101", + "S105", + "S108", + "S311", ] -ignore = ["UP035", "TRY003", "E501"] [tool.basedpyright] include = ["shelfmark"] diff --git a/shelfmark/api/websocket.py b/shelfmark/api/websocket.py index 75b382f1..5753609d 100644 --- a/shelfmark/api/websocket.py +++ b/shelfmark/api/websocket.py @@ -1,15 +1,18 @@ """WebSocket manager for real-time status updates.""" +from __future__ import annotations + import logging import threading from typing import TYPE_CHECKING, Any -from flask import Flask from flask_socketio import SocketIO, join_room, leave_room if TYPE_CHECKING: from collections.abc import Callable + from flask import Flask + logger = logging.getLogger(__name__) @@ -17,6 +20,7 @@ class WebSocketManager: """Manages WebSocket connections and broadcasts.""" def __init__(self) -> None: + """Initialize in-memory connection and room tracking.""" self.socketio: SocketIO | None = None self._enabled = False self._connection_count = 0 diff --git a/shelfmark/bypass/__init__.py b/shelfmark/bypass/__init__.py index 2c012567..2986e3a9 100644 --- a/shelfmark/bypass/__init__.py +++ b/shelfmark/bypass/__init__.py @@ -1,5 +1,5 @@ """Cloudflare bypass utilities.""" -class BypassCancelledException(Exception): +class BypassCancelledError(Exception): """Raised when a bypass operation is cancelled.""" diff --git a/shelfmark/bypass/external_bypasser.py b/shelfmark/bypass/external_bypasser.py index 6b9c66a7..2c827984 100644 --- a/shelfmark/bypass/external_bypasser.py +++ b/shelfmark/bypass/external_bypasser.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING import requests -from shelfmark.bypass import BypassCancelledException +from shelfmark.bypass import BypassCancelledError from shelfmark.core.config import config from shelfmark.core.logger import setup_logger from shelfmark.core.utils import normalize_http_url @@ -18,6 +18,7 @@ if TYPE_CHECKING: from shelfmark.download import network logger = setup_logger(__name__) +_RNG = random.SystemRandom() # Timeout constants (seconds) CONNECT_TIMEOUT = 10 @@ -102,7 +103,7 @@ def _check_cancelled(cancel_flag: Event | None, context: str) -> None: if cancel_flag and cancel_flag.is_set(): logger.info("External bypasser cancelled %s", context) msg = "Bypass cancelled" - raise BypassCancelledException(msg) + raise BypassCancelledError(msg) def _sleep_with_cancellation(seconds: float, cancel_flag: Event | None) -> None: @@ -136,7 +137,7 @@ def get_bypassed_page( if attempt == MAX_RETRY: break - delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random() + delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + _RNG.random() logger.info( "External bypasser attempt %s/%s failed, retrying in %.1fs", attempt, diff --git a/shelfmark/bypass/fingerprint.py b/shelfmark/bypass/fingerprint.py index c1c64562..e14ebe13 100644 --- a/shelfmark/bypass/fingerprint.py +++ b/shelfmark/bypass/fingerprint.py @@ -21,9 +21,11 @@ COMMON_RESOLUTIONS = [ # Current screen size (module-level singleton) _current_screen_size: tuple[int, int] | None = None +_RNG = random.SystemRandom() def get_screen_size() -> tuple[int, int]: + """Return the current synthetic screen size, generating one if needed.""" global _current_screen_size if _current_screen_size is None: _current_screen_size = _generate_screen_size() @@ -36,6 +38,7 @@ def get_screen_size() -> tuple[int, int]: def rotate_screen_size() -> tuple[int, int]: + """Rotate to a new synthetic screen size and return it.""" global _current_screen_size old_size = _current_screen_size _current_screen_size = _generate_screen_size() @@ -56,6 +59,7 @@ def rotate_screen_size() -> tuple[int, int]: def clear_screen_size() -> None: + """Clear the cached synthetic screen size.""" global _current_screen_size _current_screen_size = None @@ -63,4 +67,4 @@ def clear_screen_size() -> None: def _generate_screen_size() -> tuple[int, int]: resolutions = [(w, h) for w, h, _ in COMMON_RESOLUTIONS] weights = [weight for _, _, weight in COMMON_RESOLUTIONS] - return random.choices(resolutions, weights=weights)[0] + return _RNG.choices(resolutions, weights=weights)[0] diff --git a/shelfmark/bypass/internal_bypasser.py b/shelfmark/bypass/internal_bypasser.py index 2de89114..460d0760 100644 --- a/shelfmark/bypass/internal_bypasser.py +++ b/shelfmark/bypass/internal_bypasser.py @@ -1,15 +1,20 @@ +"""Internal Cloudflare bypass implementation using SeleniumBase and CDP helpers.""" + import asyncio import os import random +import shutil import signal import socket import stat import subprocess +import tempfile import threading import time import traceback from contextlib import suppress -from datetime import datetime +from datetime import UTC, datetime +from http import HTTPStatus from pathlib import Path from threading import Event from typing import Any @@ -17,8 +22,9 @@ from urllib.parse import urlparse import requests from seleniumbase import cdp_driver +from seleniumbase.undetected.cdp_driver.connection import ProtocolException -from shelfmark.bypass import BypassCancelledException +from shelfmark.bypass import BypassCancelledError from shelfmark.bypass.fingerprint import get_screen_size from shelfmark.config import env from shelfmark.config.env import LOG_DIR @@ -30,8 +36,12 @@ from shelfmark.download.network import get_proxies, get_ssl_verify logger = setup_logger(__name__) -SELENIUMBASE_RUNTIME_ROOT = Path("/tmp/shelfmark/seleniumbase") +SELENIUMBASE_RUNTIME_ROOT = Path(tempfile.gettempdir()) / "shelfmark" / "seleniumbase" SELENIUMBASE_DOWNLOADS_DIR = SELENIUMBASE_RUNTIME_ROOT / "downloaded_files" +_BYPASSED_BODY_LENGTH_MIN = 100_000 +_BYPASS_EMOJI_MATCH_MIN = 3 +_LOADING_BODY_LENGTH_MAX = 50 +_PAGE_BODY_PREVIEW_CHARS = 500 # Challenge detection indicators CLOUDFLARE_INDICATORS = [ @@ -54,6 +64,35 @@ DISPLAY = { "ffmpeg_output": None, } LOCKED = threading.Lock() +_PGREP_PATH = shutil.which("pgrep") +_PKILL_PATH = shutil.which("pkill") +_RNG = random.SystemRandom() + +_CDP_OPERATION_ERRORS = ( + asyncio.TimeoutError, + AttributeError, + NameError, + OSError, + ProtocolException, + RuntimeError, + TypeError, + ValueError, +) +_PATH_INSPECTION_ERRORS = (OSError, RuntimeError, TypeError, ValueError) +_REQUEST_OPERATION_ERRORS = ( + OSError, + RuntimeError, + TypeError, + ValueError, + requests.RequestException, +) +_SUBPROCESS_OPERATION_ERRORS = ( + OSError, + RuntimeError, + TypeError, + ValueError, + subprocess.SubprocessError, +) def _describe_runtime_path(path: str | Path) -> str: @@ -68,7 +107,7 @@ def _describe_runtime_path(path: str | Path) -> str: return f"{path}{link_target} exists uid={st.st_uid} gid={st.st_gid} mode={oct(mode)}" except FileNotFoundError: return f"{path} missing" - except Exception as e: + except _PATH_INSPECTION_ERRORS as e: return f"{path} error={type(e).__name__}: {e}" @@ -85,16 +124,13 @@ class _CdpWorker: self._loop = loop self._ready.set() loop.run_forever() - try: + with suppress(Exception): pending = asyncio.all_tasks(loop) for task in pending: task.cancel() if pending: loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) - except Exception: - pass - finally: - loop.close() + loop.close() def start(self) -> None: with self._lock: @@ -108,12 +144,14 @@ class _CdpWorker: ) self._thread.start() if not self._ready.wait(timeout=10): - raise RuntimeError("CDP worker loop failed to start") + msg = "CDP worker loop failed to start" + raise RuntimeError(msg) def run(self, coro: Any, timeout: float | None = None) -> Any: self.start() if not self._loop or self._loop.is_closed(): - raise RuntimeError("CDP worker loop not available") + msg = "CDP worker loop not available" + raise RuntimeError(msg) future = asyncio.run_coroutine_threadsafe(coro, self._loop) return future.result(timeout=timeout) @@ -121,7 +159,7 @@ class _CdpWorker: _CDP_WORKER = _CdpWorker() # Cookie storage - shared with requests library for Cloudflare bypass -# Structure: {domain: {cookie_name: {value, expiry, ...}}} +# Nested mapping of domain to cookie name to cookie metadata. _cf_cookies: dict[str, dict] = {} _cf_cookies_lock = threading.Lock() @@ -212,18 +250,18 @@ async def _extract_cookies_from_cdp(driver: Any, page: Any, url: str) -> None: try: try: all_cookies = await driver.cookies.get_all(requests_cookie_format=True) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Failed to get cookies via CDP: %s", e) return try: user_agent = await page.evaluate("navigator.userAgent") - except Exception: + except _CDP_OPERATION_ERRORS: user_agent = None _store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Failed to extract cookies: %s", e) @@ -290,10 +328,18 @@ def _cleanup_orphan_processes() -> int: logger.debug("Checking for orphan processes...") logger.log_resource_usage() + if _PGREP_PATH is None or _PKILL_PATH is None: + logger.warning("Skipping orphan-process cleanup because pgrep/pkill are unavailable") + return 0 + for proc_name in processes_to_kill: try: result = subprocess.run( - ["pgrep", "-f", proc_name], capture_output=True, text=True, timeout=5 + [_PGREP_PATH, "-f", proc_name], + capture_output=True, + check=False, + text=True, + timeout=5, ) if result.returncode != 0 or not result.stdout.strip(): continue @@ -303,7 +349,10 @@ def _cleanup_orphan_processes() -> int: logger.info("Found %s orphan %s process(es), killing...", count, proc_name) kill_result = subprocess.run( - ["pkill", "-9", "-f", proc_name], capture_output=True, timeout=5 + [_PKILL_PATH, "-9", "-f", proc_name], + capture_output=True, + check=False, + timeout=5, ) if kill_result.returncode == 0: total_killed += count @@ -312,7 +361,7 @@ def _cleanup_orphan_processes() -> int: except subprocess.TimeoutExpired: logger.warning("Timeout while checking for %s processes", proc_name) - except Exception as e: + except _SUBPROCESS_OPERATION_ERRORS as e: logger.debug("Error checking for %s processes: %s", proc_name, e) if total_killed > 0: @@ -329,16 +378,16 @@ async def _get_page_info(page: Any) -> tuple[str, str, str]: """Extract page title, body text, and current URL safely.""" try: title = (await page.get_title() or "").lower() - except Exception: + except _CDP_OPERATION_ERRORS: title = "" try: body = await page.evaluate("document.body ? document.body.innerText : ''") body = (body or "").lower() - except Exception: + except _CDP_OPERATION_ERRORS: body = "" try: current_url = await page.get_current_url() or "" - except Exception: + except _CDP_OPERATION_ERRORS: current_url = "" return title, body, current_url @@ -358,85 +407,75 @@ def _has_cloudflare_patterns(body: str, url: str) -> bool: async def _detect_challenge_type(page: Any) -> str: """Detect challenge type: 'cloudflare', 'ddos_guard', or 'none'.""" - try: - title, body, current_url = await _get_page_info(page) - except Exception as e: - logger.warning("Error detecting challenge type: %s", e) - return "none" - else: - # DDOS-Guard indicators - if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS): - logger.debug("DDOS-Guard indicator found: '%s'", found) - return "ddos_guard" + title, body, current_url = await _get_page_info(page) - # Cloudflare indicators - if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS): - logger.debug("Cloudflare indicator found: '%s'", found) - return "cloudflare" + # DDOS-Guard indicators + if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS): + logger.debug("DDOS-Guard indicator found: '%s'", found) + return "ddos_guard" - # Check URL patterns - if _has_cloudflare_patterns(body, current_url): - return "cloudflare" + # Cloudflare indicators + if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS): + logger.debug("Cloudflare indicator found: '%s'", found) + return "cloudflare" - return "none" + # Check URL patterns + if _has_cloudflare_patterns(body, current_url): + return "cloudflare" + + return "none" async def _is_bypassed(page: Any, *, escape_emojis: bool = True) -> bool: """Check if the protection has been bypassed.""" - try: - title, body, current_url = await _get_page_info(page) - except Exception as e: - logger.warning("Error checking bypass status: %s", e) - return False + title, body, current_url = await _get_page_info(page) + body_len = len(body.strip()) - else: - body_len = len(body.strip()) + # Long page content = probably bypassed + if body_len > _BYPASSED_BODY_LENGTH_MIN: + logger.debug("Page content too long, probably bypassed (len: %s)", body_len) + return True - # Long page content = probably bypassed - if body_len > 100000: - logger.debug("Page content too long, probably bypassed (len: %s)", body_len) + # Multiple emojis = probably real content + if escape_emojis: + import emoji + + if len(emoji.emoji_list(body)) >= _BYPASS_EMOJI_MATCH_MIN: + logger.debug("Detected emojis in page, probably bypassed") return True - # Multiple emojis = probably real content - if escape_emojis: - import emoji + # Check for protection indicators (means NOT bypassed) + if _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS): + return False - if len(emoji.emoji_list(body)) >= 3: - logger.debug("Detected emojis in page, probably bypassed") - return True + # Cloudflare URL patterns + if _has_cloudflare_patterns(body, current_url): + logger.debug("Cloudflare patterns detected in page") + return False - # Check for protection indicators (means NOT bypassed) - if _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS): - return False + # Page too short = still loading + if body_len < _LOADING_BODY_LENGTH_MAX: + logger.debug("Page content too short, might still be loading") + return False - # Cloudflare URL patterns - if _has_cloudflare_patterns(body, current_url): - logger.debug("Cloudflare patterns detected in page") - return False - - # Page too short = still loading - if body_len < 50: - logger.debug("Page content too short, might still be loading") - return False - - logger.debug("Bypass check passed - Title: '%s', Body length: %s", title[:100], body_len) - return True + logger.debug("Bypass check passed - Title: '%s', Body length: %s", title[:100], body_len) + return True async def _bypass_method_humanlike(page: Any) -> bool: """Human-like behavior with scroll, wait, and reload.""" try: logger.debug("Attempting bypass: human-like interaction") - await asyncio.sleep(random.uniform(6, 10)) + await asyncio.sleep(_RNG.uniform(6, 10)) try: await page.evaluate("window.scrollTo(0, 10000);") await page.wait() - await asyncio.sleep(random.uniform(1, 2)) + await asyncio.sleep(_RNG.uniform(1, 2)) await page.evaluate("window.scrollTo(0, 0);") await page.wait() - await asyncio.sleep(random.uniform(2, 3)) - except Exception as e: + await asyncio.sleep(_RNG.uniform(2, 3)) + except _CDP_OPERATION_ERRORS as e: logger.debug("Scroll behavior failed: %s", e) if await _is_bypassed(page): @@ -444,19 +483,19 @@ async def _bypass_method_humanlike(page: Any) -> bool: logger.debug("Trying page refresh...") await page.reload(ignore_cache=True) - await asyncio.sleep(random.uniform(5, 8)) + await asyncio.sleep(_RNG.uniform(5, 8)) if await _is_bypassed(page): return True try: await page.solve_captcha() - await asyncio.sleep(random.uniform(3, 5)) - except Exception as e: + await asyncio.sleep(_RNG.uniform(3, 5)) + except _CDP_OPERATION_ERRORS as e: logger.debug("Final captcha click failed: %s", e) return await _is_bypassed(page) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Human-like method failed: %s", e) return False @@ -466,9 +505,9 @@ async def _bypass_method_cdp_solve(page: Any) -> bool: try: logger.debug("Attempting bypass: CDP solve_captcha") await page.solve_captcha() - await asyncio.sleep(random.uniform(3, 5)) + await asyncio.sleep(_RNG.uniform(3, 5)) return await _is_bypassed(page) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("CDP solve_captcha failed: %s", e) return False @@ -495,15 +534,15 @@ async def _bypass_method_cdp_click(page: Any) -> bool: logger.debug("CDP clicking: %s", selector) await page.click(selector) - await asyncio.sleep(random.uniform(2, 4)) + await asyncio.sleep(_RNG.uniform(2, 4)) if await _is_bypassed(page): return True - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("CDP click on '%s' failed: %s", selector, e) return await _is_bypassed(page) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("CDP Mode click failed: %s", e) return False @@ -525,11 +564,11 @@ async def _bypass_method_cdp_gui_click(page: Any) -> bool: try: logger.debug("Trying solve_captcha()") await page.solve_captcha() - await asyncio.sleep(random.uniform(3, 5)) + await asyncio.sleep(_RNG.uniform(3, 5)) if await _is_bypassed(page): return True - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("solve_captcha() failed: %s", e) for selector in CDP_GUI_CLICK_SELECTORS: @@ -539,15 +578,15 @@ async def _bypass_method_cdp_gui_click(page: Any) -> bool: logger.debug("CDP click_with_offset: %s", selector) await page.click_with_offset(selector, 0, 0, center=True) - await asyncio.sleep(random.uniform(3, 5)) + await asyncio.sleep(_RNG.uniform(3, 5)) if await _is_bypassed(page): return True - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("CDP gui_click on '%s' failed: %s", selector, e) return await _is_bypassed(page) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("CDP Mode gui_click failed: %s", e) return False @@ -566,7 +605,8 @@ def _check_cancellation(cancel_flag: Event | None, message: str) -> None: """Check if cancellation was requested and raise if so.""" if cancel_flag and cancel_flag.is_set(): logger.info(message) - raise BypassCancelledException("Bypass cancelled") + msg = "Bypass cancelled" + raise BypassCancelledError(msg) async def _bypass( @@ -594,17 +634,17 @@ async def _bypass( # No challenge detected but page doesn't look bypassed - wait and retry if challenge_type == "none": logger.info("No challenge detected, waiting for page to settle...") - await asyncio.sleep(random.uniform(2, 3)) + await asyncio.sleep(_RNG.uniform(2, 3)) if await _is_bypassed(page): return True # Try a simple refresh instead of captcha methods try: await page.reload(ignore_cache=True) - await asyncio.sleep(random.uniform(1, 2)) + await asyncio.sleep(_RNG.uniform(1, 2)) if await _is_bypassed(page): logger.info("Bypass successful after refresh") return True - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Refresh during no-challenge wait failed: %s", e) continue @@ -625,7 +665,7 @@ async def _bypass( logger.info("Bypass attempt %s/%s using %s", try_count + 1, max_retries, method.__name__) if try_count > 0: - wait_time = min(random.uniform(2, 4) * try_count, 12) + wait_time = min(_RNG.uniform(2, 4) * try_count, 12) logger.info("Waiting %0.1fs before trying...", wait_time) for _ in range(int(wait_time)): _check_cancellation(cancel_flag, "Bypass cancelled during wait") @@ -636,9 +676,9 @@ async def _bypass( if await method(page): logger.info("Bypass successful using %s", method.__name__) return True - except BypassCancelledException: + except BypassCancelledError: raise - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.warning("Exception in %s: %s", method.__name__, e) logger.info("Bypass method %s failed.", method.__name__) @@ -700,7 +740,7 @@ def _build_host_resolver_rules() -> list[str]: logger.warning("Chrome: No addresses returned for %s", hostname) except socket.gaierror as e: logger.warning("Chrome: Could not pre-resolve %s: %s", hostname, e) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error_trace(f"Error pre-resolving hostnames for Chrome: {e}") return host_rules @@ -726,7 +766,7 @@ async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str: current_url = await page.get_current_url() title = await page.get_title() logger.debug("Page loaded - URL: %s, Title: %s", current_url, title) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Could not get page info: %s", e) logger.debug("Starting bypass process...") @@ -738,9 +778,12 @@ async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str: try: body = await page.evaluate("document.body ? document.body.innerText : ''") if body: - logger.debug(f"Page content: {body[:500]}..." if len(body) > 500 else body) - except Exception: - pass + preview = body + if len(body) > _PAGE_BODY_PREVIEW_CHARS: + preview = body[:_PAGE_BODY_PREVIEW_CHARS] + "..." + logger.debug("Page content: %s", preview) + except _CDP_OPERATION_ERRORS as exc: + logger.debug("Could not inspect protected page body: %s", exc) return "" @@ -767,9 +810,9 @@ def get(url: str, retry: int | None = None, cancel_flag: Event | None = None) -> result = await _get(url, driver, cancel_flag) if result: return result - except BypassCancelledException: + except BypassCancelledError: raise - except Exception as e: + except _CDP_OPERATION_ERRORS as e: error_details = f"{type(e).__name__}: {e}" logger.warning( "Bypass failed (attempt %s/%s): %s", attempt + 1, retry, error_details @@ -824,7 +867,7 @@ async def _create_cdp_browser(url: str) -> Any: proxy=proxy, browser_args=browser_args, ) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.warning("Pure CDP browser startup failed: %s: %s", type(e).__name__, e) logger.warning( "SeleniumBase runtime paths: cwd=%s; %s; %s; %s; %s", @@ -832,13 +875,13 @@ async def _create_cdp_browser(url: str) -> Any: _describe_runtime_path(SELENIUMBASE_DOWNLOADS_DIR), _describe_runtime_path("/app/downloaded_files"), _describe_runtime_path("downloaded_files"), - _describe_runtime_path("/tmp"), + _describe_runtime_path(tempfile.gettempdir()), ) raise try: await driver.page.set_window_rect(0, 0, screen_width, screen_height) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Failed to set window size: %s", e) # Start FFmpeg recording if debug mode (record each bypass session) @@ -868,13 +911,13 @@ async def _close_cdp_driver(driver: Any) -> None: connections.extend(driver.targets) for conn in connections: await _close_websocket_connection(conn) - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Error during connection cleanup: %s", e) try: driver.stop() logger.debug("Stopped CDP browser") - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("CDP stop: %s", e) if env.DOCKERMODE: @@ -898,9 +941,9 @@ async def _close_cdp_driver(driver: Any) -> None: if _pid_alive(pid): os.kill(pid, signal.SIGKILL) logger.debug("Killed Chrome pid %s", pid) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.debug("Failed to kill Chrome pid %s: %s", pid, e) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.debug("Process cleanup failed: %s", e) logger.log_resource_usage() @@ -910,7 +953,7 @@ async def _close_websocket_connection(conn: Any) -> None: """Close one websocket-like connection, ignoring best-effort failures.""" try: await conn.aclose() - except Exception as e: + except _CDP_OPERATION_ERRORS as e: logger.debug("Failed to close websocket connection: %s", e) @@ -918,7 +961,7 @@ def _start_ffmpeg_recording(display: str) -> None: """Start FFmpeg screen recording for debug mode.""" global DISPLAY RECORDING_DIR.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now().strftime("%y%m%d-%H%M%S") + timestamp = datetime.now(UTC).strftime("%y%m%d-%H%M%S") output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4" screen_width, screen_height = get_screen_size() @@ -981,7 +1024,7 @@ def _stop_ffmpeg_recording() -> None: proc.send_signal(signal.SIGINT) proc.wait(timeout=5) logger.debug("Stopped ffmpeg recording") - except Exception as e: + except _SUBPROCESS_OPERATION_ERRORS as e: logger.debug("ffmpeg stop: %s", e) with suppress(Exception): proc.terminate() @@ -1013,11 +1056,11 @@ def _try_with_cached_cookies(url: str, hostname: str) -> str | None: timeout=(5, 10), verify=get_ssl_verify(url), ) - if response.status_code == 200: + if response.status_code == HTTPStatus.OK: logger.debug("Cached cookies worked, skipped Chrome bypass") return response.text - except Exception: - pass + except _REQUEST_OPERATION_ERRORS as exc: + logger.debug("Cached cookie retry failed for %s: %s", url, exc) return None @@ -1036,9 +1079,9 @@ def get_bypassed_page( try: response_html = get(attempt_url, cancel_flag=cancel_flag) - except BypassCancelledException: + except BypassCancelledError: raise - except Exception: + except _CDP_OPERATION_ERRORS + _REQUEST_OPERATION_ERRORS: _check_cancellation(cancel_flag, "Bypass cancelled") new_base, action = sel.next_mirror_or_rotate_dns() if action in ("mirror", "dns") and new_base: @@ -1048,6 +1091,7 @@ def get_bypassed_page( raise if not response_html.strip(): - raise requests.exceptions.RequestException("Failed to bypass Cloudflare") + msg = "Failed to bypass Cloudflare" + raise requests.exceptions.RequestException(msg) return response_html diff --git a/shelfmark/config/booklore_settings.py b/shelfmark/config/booklore_settings.py index a823eae9..94dd1441 100644 --- a/shelfmark/config/booklore_settings.py +++ b/shelfmark/config/booklore_settings.py @@ -1,3 +1,5 @@ +"""Helpers for Booklore settings validation, option loading, and connection tests.""" + from __future__ import annotations from typing import Any @@ -164,7 +166,7 @@ def get_booklore_path_options() -> list[dict[str, Any]]: return path_options -def test_booklore_connection( +def check_booklore_connection( current_values: dict[str, Any] | None = None, ) -> dict[str, Any]: """Test the Booklore connection using current form values.""" diff --git a/shelfmark/config/email_settings.py b/shelfmark/config/email_settings.py index 8753220b..23512275 100644 --- a/shelfmark/config/email_settings.py +++ b/shelfmark/config/email_settings.py @@ -1,5 +1,8 @@ +"""Helpers for email settings validation and SMTP connection tests.""" + from __future__ import annotations +import smtplib from typing import Any from shelfmark.core.config import config @@ -10,7 +13,7 @@ from shelfmark.download.outputs.email import ( ) -def test_email_connection( +def check_email_connection( current_values: dict[str, Any] | None = None, ) -> dict[str, Any]: """Test SMTP connectivity using current form values (including unsaved changes).""" @@ -41,7 +44,7 @@ def test_email_connection( test_smtp_connection(smtp_config) except EmailOutputError as exc: return {"success": False, "message": str(exc)} - except Exception as exc: + except (OSError, smtplib.SMTPException) as exc: return {"success": False, "message": f"SMTP test failed: {exc}"} else: return {"success": True, "message": "Connected to SMTP server"} diff --git a/shelfmark/config/env.py b/shelfmark/config/env.py index ed131362..3f07ba67 100644 --- a/shelfmark/config/env.py +++ b/shelfmark/config/env.py @@ -3,6 +3,7 @@ import json import os import shutil +import tempfile from pathlib import Path @@ -89,7 +90,7 @@ CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config")) LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/")) LOG_DIR = LOG_ROOT / "shelfmark" LOG_FILE = LOG_DIR / "shelfmark.log" -TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/shelfmark")) +TMP_DIR = Path(os.getenv("TMP_DIR", (Path(tempfile.gettempdir()) / "shelfmark").as_posix())) INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books")) diff --git a/shelfmark/config/notifications_settings.py b/shelfmark/config/notifications_settings.py index 20edd733..681a0c2e 100644 --- a/shelfmark/config/notifications_settings.py +++ b/shelfmark/config/notifications_settings.py @@ -143,6 +143,7 @@ def _extract_unique_route_urls(routes: list[dict[str, Any]]) -> list[str]: def build_notification_test_result(routes_input: Any, *, scope_label: str) -> dict[str, Any]: + """Validate routes and return a test-notification result payload.""" invalid_event_count = _count_invalid_route_events(routes_input) if invalid_event_count: return { diff --git a/shelfmark/config/security.py b/shelfmark/config/security.py index 36d2cc9a..eeab80e6 100644 --- a/shelfmark/config/security.py +++ b/shelfmark/config/security.py @@ -4,8 +4,8 @@ from typing import TYPE_CHECKING, Any from shelfmark.config.migrations import migrate_security_settings from shelfmark.config.security_handlers import ( + check_oidc_connection, on_save_security, - test_oidc_connection, ) from shelfmark.core.config import config as app_config from shelfmark.core.logger import setup_logger @@ -61,7 +61,7 @@ def _on_save_security(values: dict[str, Any]) -> dict[str, Any]: def _test_oidc_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]: - return test_oidc_connection( + return check_oidc_connection( load_security_config=lambda: { "OIDC_DISCOVERY_URL": app_config.get("OIDC_DISCOVERY_URL", ""), }, diff --git a/shelfmark/config/security_handlers.py b/shelfmark/config/security_handlers.py index 660485d5..17cef375 100644 --- a/shelfmark/config/security_handlers.py +++ b/shelfmark/config/security_handlers.py @@ -50,7 +50,7 @@ def on_save_security( return {"error": False, "values": normalized_values} -def test_oidc_connection( +def check_oidc_connection( *, load_security_config: Callable[[], dict[str, Any]], current_values: dict[str, Any] | None = None, diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index b94371d5..0f429f54 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -4,6 +4,33 @@ import json from pathlib import Path from typing import Any +from shelfmark.config import env +from shelfmark.config.booklore_settings import ( + check_booklore_connection, + get_booklore_library_options, + get_booklore_path_options, +) +from shelfmark.config.email_settings import check_email_connection +from shelfmark.core.logger import setup_logger +from shelfmark.core.settings_registry import ( + ActionButton, + CheckboxField, + HeadingField, + MultiSelectField, + NumberField, + OrderableListField, + PasswordField, + SelectField, + SettingsField, + TableField, + TagListField, + TextField, + load_config_file, + register_group, + register_on_save, + register_settings, +) + def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]: """Validate advanced settings before persisting.""" @@ -64,16 +91,9 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]: return {"error": False, "values": values} -from shelfmark.config import env -from shelfmark.config.booklore_settings import ( - get_booklore_library_options, - get_booklore_path_options, - test_booklore_connection, -) -from shelfmark.config.email_settings import test_email_connection -from shelfmark.core.logger import setup_logger - logger = setup_logger(__name__) +_SMTP_PORT_MAX = 65535 +_EMAIL_ATTACHMENT_LIMIT_MB_MAX = 600 # Log bootstrap configuration values at DEBUG level logger.debug("Bootstrap configuration:") @@ -117,25 +137,6 @@ def _log_external_bypasser_warning() -> None: ) -from shelfmark.core.settings_registry import ( - ActionButton, - CheckboxField, - HeadingField, - MultiSelectField, - NumberField, - OrderableListField, - PasswordField, - SelectField, - SettingsField, - TableField, - TagListField, - TextField, - load_config_file, - register_group, - register_on_save, - register_settings, -) - register_group("direct_download", "Direct Download", icon="download", order=20) register_group( @@ -301,8 +302,8 @@ def _get_zlib_mirror_options() -> list[dict[str, str]]: # Add custom mirrors additional = config.get("ZLIB_ADDITIONAL_URLS", "") if additional: - for url in additional.split(","): - url = url.strip() + for raw_url in additional.split(","): + url = raw_url.strip() if url and url not in DEFAULT_ZLIB_MIRRORS: domain = url.replace("https://", "").replace("http://", "").split("/")[0] options.append({"value": url, "label": f"{domain} (custom)"}) @@ -325,8 +326,8 @@ def _get_welib_mirror_options() -> list[dict[str, str]]: # Add custom mirrors additional = config.get("WELIB_ADDITIONAL_URLS", "") if additional: - for url in additional.split(","): - url = url.strip() + for raw_url in additional.split(","): + url = raw_url.strip() if url and url not in DEFAULT_WELIB_MIRRORS: domain = url.replace("https://", "").replace("http://", "").split("/")[0] options.append({"value": url, "label": f"{domain} (custom)"}) @@ -777,10 +778,10 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]: except TypeError, ValueError: return {"error": True, "message": "SMTP port must be a number", "values": values} - if port < 1 or port > 65535: + if port < 1 or port > _SMTP_PORT_MAX: return { "error": True, - "message": "SMTP port must be between 1 and 65535", + "message": f"SMTP port must be between 1 and {_SMTP_PORT_MAX}", "values": values, } @@ -818,10 +819,13 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]: "values": values, } - if attachment_limit_mb < 1 or attachment_limit_mb > 600: + if attachment_limit_mb < 1 or attachment_limit_mb > _EMAIL_ATTACHMENT_LIMIT_MB_MAX: return { "error": True, - "message": "Attachment size limit (MB) must be between 1 and 600", + "message": ( + "Attachment size limit (MB) must be between 1 and " + f"{_EMAIL_ATTACHMENT_LIMIT_MB_MAX}" + ), "values": values, } @@ -1049,7 +1053,7 @@ def download_settings() -> list[SettingsField]: label="Test Connection", description="Verify your Grimmory configuration", style="primary", - callback=test_booklore_connection, + callback=check_booklore_connection, show_when={"field": "BOOKS_OUTPUT_MODE", "value": "booklore"}, ), HeadingField( @@ -1157,7 +1161,7 @@ def download_settings() -> list[SettingsField]: label="Test SMTP Connection", description="Verify your SMTP configuration (connect + optional login).", style="primary", - callback=test_email_connection, + callback=check_email_connection, show_when={"field": "BOOKS_OUTPUT_MODE", "value": "email"}, ), # === AUDIOBOOKS SECTION === @@ -1358,7 +1362,7 @@ def _get_slow_source_defaults() -> list[dict[str, str | bool]]: "download_sources", "Download Sources", icon="download", order=21, group="direct_download" ) def download_source_settings() -> list[SettingsField]: - """Settings for download source behavior.""" + """Return settings for download source behavior.""" return [ PasswordField( key="AA_DONATOR_KEY", @@ -1466,7 +1470,7 @@ def download_source_settings() -> list[SettingsField]: "cloudflare_bypass", "Cloudflare Bypass", icon="shield", order=22, group="direct_download" ) def cloudflare_bypass_settings() -> list[SettingsField]: - """Settings for Cloudflare bypass behavior.""" + """Return settings for Cloudflare bypass behavior.""" return [ CheckboxField( key="USE_CF_BYPASS", diff --git a/shelfmark/core/activity_routes.py b/shelfmark/core/activity_routes.py index 617a56c3..4f453c0b 100644 --- a/shelfmark/core/activity_routes.py +++ b/shelfmark/core/activity_routes.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sqlite3 from typing import TYPE_CHECKING, Any, NamedTuple from flask import Flask, Response, jsonify, request, session @@ -37,6 +38,7 @@ if TYPE_CHECKING: from shelfmark.core.user_db import UserDB logger = setup_logger(__name__) +_USER_DB_IDENTITY_ERRORS = (sqlite3.Error, OSError) def _normalize_log_field(value: object) -> str: @@ -192,7 +194,7 @@ def _resolve_db_user_id( if user_db is not None: try: db_user = user_db.get_user(user_id=parsed_db_user_id) - except Exception as exc: + except _USER_DB_IDENTITY_ERRORS as exc: logger.warning("Failed to validate activity db identity %s: %s", parsed_db_user_id, exc) db_user = None if db_user is None: diff --git a/shelfmark/core/activity_view_state_service.py b/shelfmark/core/activity_view_state_service.py index c7401946..3aeec4c8 100644 --- a/shelfmark/core/activity_view_state_service.py +++ b/shelfmark/core/activity_view_state_service.py @@ -15,6 +15,7 @@ USER_VIEWER_SCOPE_PREFIX = "user:" def user_viewer_scope(user_id: int) -> str: + """Build the persisted viewer scope string for a specific user.""" if not isinstance(user_id, int) or user_id < 1: msg = "user_id must be a positive integer" raise ValueError(msg) @@ -22,6 +23,7 @@ def user_viewer_scope(user_id: int) -> str: def normalize_viewer_scope(viewer_scope: object) -> str: + """Validate and normalize a persisted viewer scope string.""" if not isinstance(viewer_scope, str) or not viewer_scope.strip(): msg = "viewer_scope must be a non-empty string" raise ValueError(msg) @@ -75,6 +77,7 @@ class ActivityViewStateService: """Service for per-viewer activity dismissal and history visibility.""" def __init__(self, db_path: str) -> None: + """Initialize the service with the SQLite state database path.""" self._db_path = db_path self._lock = threading.Lock() @@ -90,6 +93,7 @@ class ActivityViewStateService: viewer_scope: str, limit: int | None = None, ) -> list[dict[str, Any]]: + """Return dismissed rows for a viewer, including cleared history entries.""" normalized_scope = normalize_viewer_scope(viewer_scope) normalized_limit = None if limit is None else max(1, int(limit)) query = """ @@ -118,6 +122,7 @@ class ActivityViewStateService: limit: int = 50, offset: int = 0, ) -> list[dict[str, Any]]: + """Return active dismissal history rows for a viewer.""" normalized_scope = normalize_viewer_scope(viewer_scope) normalized_limit = max(1, min(int(limit), 5000)) normalized_offset = max(0, int(offset)) @@ -147,6 +152,7 @@ class ActivityViewStateService: item_type: str, item_key: str, ) -> int: + """Mark a single activity item as dismissed for a viewer.""" normalized_scope = normalize_viewer_scope(viewer_scope) normalized_type = _normalize_item_type(item_type) normalized_key = _normalize_item_key(item_key, item_type=normalized_type) @@ -183,6 +189,7 @@ class ActivityViewStateService: viewer_scope: str, items: list[dict[str, str]], ) -> int: + """Mark multiple activity items as dismissed for a viewer.""" normalized_scope = normalize_viewer_scope(viewer_scope) if not items: return 0 @@ -236,6 +243,7 @@ class ActivityViewStateService: conn.close() def clear_history(self, *, viewer_scope: str) -> int: + """Mark all dismissed items as cleared for a viewer.""" normalized_scope = normalize_viewer_scope(viewer_scope) cleared_at = now_utc_iso() @@ -259,6 +267,7 @@ class ActivityViewStateService: conn.close() def clear_item_for_all_viewers(self, *, item_type: str, item_key: str) -> int: + """Delete a dismissed item record for every viewer.""" normalized_type = _normalize_item_type(item_type) normalized_key = _normalize_item_key(item_key, item_type=normalized_type) @@ -279,6 +288,7 @@ class ActivityViewStateService: conn.close() def delete_viewer_scope(self, *, viewer_scope: str) -> int: + """Delete all activity-view state rows for a viewer scope.""" normalized_scope = normalize_viewer_scope(viewer_scope) with self._lock: @@ -295,6 +305,7 @@ class ActivityViewStateService: conn.close() def delete_items(self, *, item_type: str, item_keys: list[str]) -> int: + """Delete multiple dismissed item records for a given item type.""" normalized_type = _normalize_item_type(item_type) normalized_keys = [ _normalize_item_key(item_key, item_type=normalized_type) for item_key in item_keys @@ -302,16 +313,12 @@ class ActivityViewStateService: if not normalized_keys: return 0 - placeholders = ",".join("?" for _ in normalized_keys) with self._lock: conn = self._connect() try: - cursor = conn.execute( - f""" - DELETE FROM activity_view_state - WHERE item_type = ? AND item_key IN ({placeholders}) - """, - (normalized_type, *normalized_keys), + cursor = conn.executemany( + "DELETE FROM activity_view_state WHERE item_type = ? AND item_key = ?", + [(normalized_type, normalized_key) for normalized_key in normalized_keys], ) conn.commit() rowcount = int(cursor.rowcount) if cursor.rowcount is not None else 0 diff --git a/shelfmark/core/admin_routes.py b/shelfmark/core/admin_routes.py index 70b201a5..ecdf7112 100644 --- a/shelfmark/core/admin_routes.py +++ b/shelfmark/core/admin_routes.py @@ -40,6 +40,8 @@ if TYPE_CHECKING: from shelfmark.core.user_db import UserDB logger = setup_logger(__name__) +MIN_PASSWORD_LENGTH = 4 +_CONFIG_REFRESH_ERRORS = (ImportError, OSError, RuntimeError, TypeError, ValueError) __all__ = [ "get_booklore_library_options", @@ -122,7 +124,8 @@ def _serialize_user( def _sync_all_cwa_users(user_db: UserDB) -> dict[str, int]: """Sync all users from the Calibre-Web database into users.db.""" if not CWA_DB_PATH or not CWA_DB_PATH.exists(): - raise FileNotFoundError("Calibre-Web database is not available") + msg = "Calibre-Web database is not available" + raise FileNotFoundError(msg) db_path = os.fspath(CWA_DB_PATH) db_uri = f"file:{db_path}?mode=ro&immutable=1" @@ -143,7 +146,7 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None: def _require_admin( f: Callable[..., Response | tuple[Response, int]], ) -> Callable[..., Response | tuple[Response, int]]: - """Decorator to require admin session for admin routes. + """Require an admin session for admin routes. In no-auth mode, everyone has access (is_admin defaults True). In auth-required modes, requires an authenticated session with admin role. @@ -151,7 +154,7 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None: """ @wraps(f) - def decorated(*args, **kwargs) -> Response | tuple[Response, int]: + def decorated(*args: object, **kwargs: object) -> Response | tuple[Response, int]: auth_mode = load_active_auth_mode(CWA_DB_PATH, user_db=user_db) g.auth_mode = auth_mode if auth_mode != "none": @@ -197,8 +200,10 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None: if not username: return jsonify({"error": "Username is required"}), 400 - if not password or len(password) < 4: - return jsonify({"error": "Password must be at least 4 characters"}), 400 + if not password or len(password) < MIN_PASSWORD_LENGTH: + return jsonify( + {"error": f"Password must be at least {MIN_PASSWORD_LENGTH} characters"} + ), 400 if role not in ("admin", "user"): return jsonify({"error": "Role must be 'admin' or 'user'"}), 400 @@ -277,8 +282,10 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None: "message": "Password authentication is only available for local users.", } ), 400 - if len(password) < 4: - return jsonify({"error": "Password must be at least 4 characters"}), 400 + if len(password) < MIN_PASSWORD_LENGTH: + return jsonify( + {"error": f"Password must be at least {MIN_PASSWORD_LENGTH} characters"} + ), 400 user_db.update_user(user_id, password_hash=generate_password_hash(password)) # Update user fields @@ -365,11 +372,13 @@ def register_admin_routes(app: Flask, user_db: UserDB) -> None: user_db.set_user_settings(user_id, validated_settings) # Ensure runtime reads see updated per-user overrides immediately. try: - from shelfmark.core.config import config as app_config - app_config.refresh(force=True) - except Exception: - pass + except _CONFIG_REFRESH_ERRORS as exc: + logger.warning( + "Updated settings for user %s but failed to refresh runtime config: %s", + user_id, + exc, + ) updated = user_db.get_user(user_id=user_id) result = _serialize_user( diff --git a/shelfmark/core/admin_settings_routes.py b/shelfmark/core/admin_settings_routes.py index bdb35b76..a0b4bb17 100644 --- a/shelfmark/core/admin_settings_routes.py +++ b/shelfmark/core/admin_settings_routes.py @@ -32,6 +32,7 @@ if TYPE_CHECKING: def validate_user_settings( settings: dict[str, Any], ) -> tuple[dict[str, Any], list[str]]: + """Validate and normalize per-user settings overrides.""" settings_registry = _get_settings_registry() field_map = settings_registry.get_settings_field_map() overridable_map = settings_registry.get_user_overridable_fields() @@ -132,6 +133,7 @@ def build_user_notification_test_response( user_id: int, payload: object, ) -> tuple[dict[str, Any], int]: + """Build a notification test response using effective per-user routes.""" from shelfmark.core.config import config as app_config routes_input = app_config.get("USER_NOTIFICATION_ROUTES", [], user_id=user_id) @@ -151,6 +153,8 @@ def register_admin_settings_routes( user_db: UserDB, require_admin: Callable[[Callable[..., object]], Callable[..., object]], ) -> None: + """Register admin endpoints for user-specific settings and defaults.""" + @app.route("/api/admin/download-defaults", methods=["GET"]) @require_admin def admin_download_defaults() -> Response | tuple[Response, int]: diff --git a/shelfmark/core/auth_modes.py b/shelfmark/core/auth_modes.py index f82501f3..a8f0d475 100644 --- a/shelfmark/core/auth_modes.py +++ b/shelfmark/core/auth_modes.py @@ -1,6 +1,7 @@ """Authentication mode, auth-source normalization, and admin access policy helpers.""" import os +import sqlite3 from pathlib import Path from typing import TYPE_CHECKING, Any @@ -33,7 +34,7 @@ def has_local_password_admin(user_db: object | None = None) -> bool: db.initialize() return db.has_admin_with_password() - except Exception: + except AttributeError, ImportError, OSError, RuntimeError, TypeError, ValueError, sqlite3.Error: return False @@ -99,7 +100,7 @@ def load_active_auth_mode( cwa_db_path, has_local_admin=has_local_password_admin(user_db), ) - except Exception: + except ImportError, OSError, RuntimeError, TypeError, ValueError, sqlite3.Error: return "none" diff --git a/shelfmark/core/cache.py b/shelfmark/core/cache.py index f36c5ddb..399da27f 100644 --- a/shelfmark/core/cache.py +++ b/shelfmark/core/cache.py @@ -113,7 +113,7 @@ def get_metadata_cache() -> CacheService: return _metadata_cache -def cache_key(*args, **kwargs) -> str: +def cache_key(*args: object, **kwargs: object) -> str: """Generate cache key from arguments.""" parts = [str(arg) for arg in args] parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items())) @@ -126,7 +126,7 @@ def cacheable( ttl_default: int = 300, key_prefix: str = "", ) -> Callable[[Callable[P, R]], Callable[P, R]]: - """Decorator for caching function results. Use ttl (static) or ttl_key (from config).""" + """Cache function results with a static or config-backed TTL.""" def decorator(func: Callable[P, R]) -> Callable[P, R]: @wraps(func) diff --git a/shelfmark/core/config.py b/shelfmark/core/config.py index caa318f7..278583af 100644 --- a/shelfmark/core/config.py +++ b/shelfmark/core/config.py @@ -6,7 +6,7 @@ import time from importlib import import_module from pathlib import Path from threading import Lock -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self if TYPE_CHECKING: from types import ModuleType @@ -61,7 +61,8 @@ class Config: _instance: Config | None = None _lock = Lock() - def __new__(cls) -> Config: + def __new__(cls) -> Self: + """Return the shared configuration singleton instance.""" if cls._instance is None: with cls._lock: if cls._instance is None: @@ -70,6 +71,7 @@ class Config: return cls._instance def __init__(self) -> None: + """Initialize caches and backing stores for the singleton.""" if self._initialized: return self._cache: dict[str, Any] = {} @@ -161,7 +163,7 @@ class Config: db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db") user_db = user_db_cls(db_path) user_db.initialize() - except Exception: + except ImportError, OSError, sqlite3.Error: # Multi-user support is optional; fall back to global config when unavailable. return None else: diff --git a/shelfmark/core/download_history_service.py b/shelfmark/core/download_history_service.py index 20d889f8..4780971c 100644 --- a/shelfmark/core/download_history_service.py +++ b/shelfmark/core/download_history_service.py @@ -75,6 +75,7 @@ class DownloadHistoryService: """Service for persisted canonical download activity rows.""" def __init__(self, db_path: str) -> None: + """Initialize the service with the SQLite history database path.""" self._db_path = db_path self._lock = threading.Lock() @@ -146,6 +147,7 @@ class DownloadHistoryService: @staticmethod def is_retry_available(row: dict[str, Any]) -> bool: + """Return whether a persisted download row can be retried.""" final_status = ( str(row.get("retry_final_status") or row.get("final_status") or "").strip().lower() ) @@ -175,6 +177,7 @@ class DownloadHistoryService: @staticmethod def to_download_payload(row: dict[str, Any]) -> dict[str, Any]: + """Build the sidebar/history download payload for a persisted row.""" return { "id": row.get("task_id"), "title": row.get("title"), @@ -211,6 +214,7 @@ class DownloadHistoryService: @classmethod def to_history_row(cls, row: dict[str, Any], *, dismissed_at: str) -> dict[str, Any]: + """Build the activity-history payload for a persisted download row.""" task_id = str(row.get("task_id") or "").strip() item_key = cls._to_item_key(task_id) download_payload = cls.to_download_payload(row) @@ -245,7 +249,7 @@ class DownloadHistoryService: source_display_name: str | None, title: str, author: str | None, - format: str | None, + file_format: str | None, size: str | None, preview: str | None, content_type: str | None, @@ -303,7 +307,7 @@ class DownloadHistoryService: normalize_optional_text(source_display_name), normalized_title, normalize_optional_text(author), - normalize_optional_text(format), + normalize_optional_text(file_format), normalize_optional_text(size), normalize_optional_text(preview), normalize_optional_text(content_type), @@ -368,6 +372,7 @@ class DownloadHistoryService: conn.close() def get_by_task_id(self, task_id: str) -> dict[str, Any] | None: + """Return a persisted download row for the given task id.""" normalized_task_id = _normalize_task_id(task_id) conn = self._connect() try: @@ -385,6 +390,7 @@ class DownloadHistoryService: user_id: int | None, limit: int = 200, ) -> list[dict[str, Any]]: + """Return recent persisted download rows, optionally scoped to one user.""" normalized_user_id = normalize_optional_positive_int(user_id, "user_id") normalized_limit = _normalize_limit(limit, default=200, minimum=1, maximum=1000) query = "SELECT * FROM download_history" diff --git a/shelfmark/core/external_user_linking.py b/shelfmark/core/external_user_linking.py index 4323a426..5f23fdeb 100644 --- a/shelfmark/core/external_user_linking.py +++ b/shelfmark/core/external_user_linking.py @@ -60,6 +60,7 @@ def _get_by_subject( def find_unique_user_by_email(user_db: UserDB, email: str | None) -> dict[str, Any] | None: + """Return the unique local user matching an email address, if any.""" key = _email_key(_normalize_email(email)) if not key: return None diff --git a/shelfmark/core/image_cache.py b/shelfmark/core/image_cache.py index 7dc7c069..a544971e 100644 --- a/shelfmark/core/image_cache.py +++ b/shelfmark/core/image_cache.py @@ -485,13 +485,12 @@ class ImageCacheService: """Check that a URL is safe to fetch (no SSRF to internal resources).""" try: parsed = urlparse(url) - except Exception: + hostname = parsed.hostname + except ValueError: return False if parsed.scheme not in ("http", "https"): return False - - hostname = parsed.hostname if not hostname: return False @@ -570,7 +569,7 @@ class ImageCacheService: is_404 = e.response is not None and e.response.status_code == HTTP_NOT_FOUND self.put_negative(cache_id, transient=not is_404) return None - except Exception: + except requests.exceptions.RequestException: return None else: return cached_data diff --git a/shelfmark/core/logger.py b/shelfmark/core/logger.py index 592226fc..be5f024e 100644 --- a/shelfmark/core/logger.py +++ b/shelfmark/core/logger.py @@ -28,9 +28,14 @@ class CustomLogger(logging.Logger): self.debug(msg, *args, exc_info=has_exception, **kwargs) def log_resource_usage(self) -> None: - # Best-effort only; this should never raise during exception logging. + """Log best-effort CPU and memory usage for the current container.""" try: import psutil + except ImportError: + return + + # Best-effort only; this should never raise during exception logging. + try: def _get_process_rss_mb(proc: object) -> float | None: try: @@ -57,7 +62,7 @@ class CustomLogger(logging.Logger): except PermissionError, psutil.AccessDenied, OSError: try: app_memory_mb = psutil.Process().memory_info().rss / (1024 * 1024) - except Exception: + except AttributeError, OSError, psutil.Error: app_memory_mb = 0.0 memory = psutil.virtual_memory() @@ -68,7 +73,7 @@ class CustomLogger(logging.Logger): f"Container Memory: App={app_memory_mb:.2f} MB, System={system_used_mb:.2f} MB, " f"Available={available_mb:.2f} MB, CPU: {cpu_percent:.2f}%" ) - except Exception: + except AttributeError, OSError, psutil.Error: # Avoid breaking the original log call if psutil is missing or restricted. return @@ -124,7 +129,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger: ) file_handler.setFormatter(formatter) logger.addHandler(file_handler) - except Exception as e: + except (OSError, TypeError, ValueError) as e: logger.error_trace(f"Failed to create log file: {e}", exc_info=True) return logger diff --git a/shelfmark/core/models.py b/shelfmark/core/models.py index 3cec667c..d6461156 100644 --- a/shelfmark/core/models.py +++ b/shelfmark/core/models.py @@ -14,6 +14,7 @@ def build_filename( year: str | None = None, fmt: str | None = None, ) -> str: + """Build a filesystem-safe filename from book metadata.""" parts = [] if author: parts.append(author) @@ -62,6 +63,8 @@ ACTIVE_QUEUE_STATUSES: frozenset[QueueStatus] = frozenset( class SearchMode(StrEnum): + """Search modes supported by the Shelfmark UI and API.""" + DIRECT = "direct" UNIVERSAL = "universal" @@ -83,6 +86,8 @@ class QueueItem: @dataclass class DownloadTask: + """Mutable download task state tracked throughout the pipeline.""" + task_id: str # Unique ID (e.g., AA MD5 hash, Prowlarr GUID) source: str # Handler name ("direct_download", "prowlarr") title: str # Display title for queue sidebar diff --git a/shelfmark/core/naming.py b/shelfmark/core/naming.py index ccfa1122..23018cef 100644 --- a/shelfmark/core/naming.py +++ b/shelfmark/core/naming.py @@ -54,6 +54,7 @@ sanitize_path_component = sanitize_filename def format_series_position(position: str | float | None) -> str: + """Format a series position for naming templates.""" if position is None: return "" @@ -95,36 +96,37 @@ def parse_naming_template( *, allow_path_separators: bool = True, ) -> str: + """Render a naming template with Shelfmark metadata placeholders.""" if not template: return "" # Normalize metadata keys to lowercase for case-insensitive matching normalized = {k.lower(): v for k, v in metadata.items()} - def find_token(content: str) -> tuple[str | None, int]: + def find_placeholder(content: str) -> tuple[str | None, int]: content_lower = content.lower() - for token in KNOWN_TOKENS: - idx = content_lower.find(token) + for placeholder_name in KNOWN_TOKENS: + idx = content_lower.find(placeholder_name) if idx != -1: - return token, idx + return placeholder_name, idx return None, -1 - def token_value(token: str) -> str: - value = normalized.get(token) - if token == "seriesposition": + def placeholder_value(placeholder_name: str) -> str: + value = normalized.get(placeholder_name) + if placeholder_name == "seriesposition": value = format_series_position(value) if value is None: return "" return str(value).strip() def render_block(content: str) -> str | None: - token, idx = find_token(content) - if token is None: + placeholder_name, idx = find_placeholder(content) + if placeholder_name is None: return None prefix = content[:idx] - suffix = content[idx + len(token) :] - value = token_value(token) + suffix = content[idx + len(placeholder_name) :] + value = placeholder_value(placeholder_name) if not value: return "" @@ -153,10 +155,10 @@ def parse_naming_template( include_literal = False if idx + 1 < len(matches) and match.end() == matches[idx + 1].start(): next_content = matches[idx + 1].group(1) - next_token, _next_idx = find_token(next_content) - if next_token is not None: + next_placeholder_name, _next_idx = find_placeholder(next_content) + if next_placeholder_name is not None: conditional_literal = True - include_literal = bool(token_value(next_token)) + include_literal = bool(placeholder_value(next_placeholder_name)) if include_literal: parts.append(content) elif not conditional_literal and re.search(r"\s", content): @@ -194,6 +196,7 @@ def build_library_path( metadata: Mapping[str, str | int | float | None], extension: str | None = None, ) -> Path: + """Build a final library path from a template and metadata.""" relative = parse_naming_template(template, metadata, allow_path_separators=True) if not relative: diff --git a/shelfmark/core/notifications.py b/shelfmark/core/notifications.py index 33489831..e5584a94 100644 --- a/shelfmark/core/notifications.py +++ b/shelfmark/core/notifications.py @@ -33,6 +33,7 @@ _APPRISE_LOGO_URL = ( "https://raw.githubusercontent.com/calibrain/shelfmark/main/src/frontend/public/logo.png" ) _APPRISE_LOGGER_NAME = "apprise" +_APPRISE_DISPATCH_ERRORS = (RuntimeError, TypeError, ValueError) class NotificationEvent(StrEnum): @@ -401,7 +402,7 @@ def _dispatch_to_apprise( with _capture_apprise_logs(min_level=logging.INFO) as apprise_records: try: plugin = apprise.Apprise.instantiate(url, asset=getattr(apobj, "asset", None)) - except Exception as exc: + except _APPRISE_DISPATCH_ERRORS as exc: logger.warning( "Failed to register notification route URL for scheme '%s': %s", scheme, @@ -435,7 +436,7 @@ def _dispatch_to_apprise( try: delivered = bool(apobj.notify(title=title, body=body, notify_type=notify_type)) - except Exception as exc: + except _APPRISE_DISPATCH_ERRORS as exc: _log_apprise_records(apprise_records) failed_delivery_urls += 1 logger.warning( @@ -526,16 +527,17 @@ def _create_apprise_client() -> object: ) except TypeError: # Support older Apprise versions that do not expose image_url_logo. - asset = apprise_asset_cls( - app_id=_APPRISE_APP_ID, - app_desc=_APPRISE_APP_DESC, - ) - except Exception: - return apprise_cls() + try: + asset = apprise_asset_cls( + app_id=_APPRISE_APP_ID, + app_desc=_APPRISE_APP_DESC, + ) + except TypeError: + return apprise_cls() try: return apprise_cls(asset=asset) - except Exception: + except TypeError: return apprise_cls() @@ -556,7 +558,7 @@ def notify_admin(event: NotificationEvent, context: NotificationContext) -> None try: _executor.submit(_dispatch_admin_async, event, context, urls) - except Exception as exc: + except RuntimeError as exc: logger.warning("Failed to queue admin notification '%s': %s", event.value, exc) @@ -575,7 +577,7 @@ def notify_user( try: _executor.submit(_dispatch_user_async, normalized_user_id, event, context, urls) - except Exception as exc: + except RuntimeError as exc: logger.warning( "Failed to queue user notification '%s' for user_id=%s: %s", event.value, diff --git a/shelfmark/core/oidc_routes.py b/shelfmark/core/oidc_routes.py index 388679aa..89634c0b 100644 --- a/shelfmark/core/oidc_routes.py +++ b/shelfmark/core/oidc_routes.py @@ -7,6 +7,7 @@ Business logic remains in oidc_auth.py. from typing import TYPE_CHECKING, Any from urllib.parse import urlencode, urlsplit, urlunsplit +from authlib.integrations.base_client.errors import OAuthError from authlib.integrations.flask_client import OAuth from authlib.jose.errors import InvalidClaimError from flask import Flask, Response, jsonify, redirect, request, session @@ -26,6 +27,7 @@ if TYPE_CHECKING: logger = setup_logger(__name__) oauth = OAuth() _RETURN_TO_SESSION_KEY = "oidc_return_to" +_OIDC_CLIENT_ERRORS = (OAuthError, OSError, RuntimeError, TypeError, ValueError) def _normalize_claims(raw_claims: object) -> dict[str, Any]: @@ -38,7 +40,7 @@ def _normalize_claims(raw_claims: object) -> dict[str, Any]: return raw_claims.to_dict() # type: ignore[no-any-return] try: return dict(raw_claims) - except Exception: + except TypeError, ValueError: return {} @@ -214,7 +216,7 @@ def register_oidc_routes(app: Flask, user_db: UserDB) -> None: metadata = client.load_server_metadata() if isinstance(metadata, dict): provider_issuer = str(metadata.get("issuer", "")) - except Exception as metadata_error: + except _OIDC_CLIENT_ERRORS as metadata_error: logger.debug( "OIDC metadata lookup failed during claim diagnostics: %s", metadata_error, diff --git a/shelfmark/core/onboarding.py b/shelfmark/core/onboarding.py index 51ae2ff0..3b7f5c30 100644 --- a/shelfmark/core/onboarding.py +++ b/shelfmark/core/onboarding.py @@ -86,7 +86,7 @@ def _get_field_from_tab(tab_name: str, field_key: str) -> SettingsField | None: return None -def _clone_field_with_overrides(field: SettingsField, **overrides) -> SettingsField: +def _clone_field_with_overrides(field: SettingsField, **overrides: object) -> SettingsField: """Clone a field with optional attribute overrides. Useful for customizing labels, descriptions, or defaults for onboarding context. diff --git a/shelfmark/core/path_mappings.py b/shelfmark/core/path_mappings.py index afe8ca2a..61578510 100644 --- a/shelfmark/core/path_mappings.py +++ b/shelfmark/core/path_mappings.py @@ -21,6 +21,8 @@ _WINDOWS_DRIVE_PREFIX_LENGTH = 2 @dataclass(frozen=True) class RemotePathMapping: + """Mapping from a remote path prefix to a local path prefix.""" + host: str remote_path: str local_path: str @@ -49,6 +51,7 @@ def _normalize_host(host: str) -> str: def parse_remote_path_mappings(value: object) -> list[RemotePathMapping]: + """Parse configured remote-path mapping rows into normalized mappings.""" if not value or not isinstance(value, list): return [] @@ -79,6 +82,7 @@ def remap_remote_to_local_with_match( host: str, remote_path: str | Path, ) -> tuple[Path, bool]: + """Remap a remote path and report whether a configured mapping matched.""" host_normalized = _normalize_host(host) remote_normalized = _normalize_prefix(str(remote_path)) @@ -124,6 +128,7 @@ def remap_remote_to_local_with_match( def remap_remote_to_local( *, mappings: Iterable[RemotePathMapping], host: str, remote_path: str | Path ) -> Path: + """Remap a remote path to a local path using the configured mappings.""" remapped, _ = remap_remote_to_local_with_match( mappings=mappings, host=host, diff --git a/shelfmark/core/prefix_middleware.py b/shelfmark/core/prefix_middleware.py index 94868779..54434e8c 100644 --- a/shelfmark/core/prefix_middleware.py +++ b/shelfmark/core/prefix_middleware.py @@ -17,11 +17,13 @@ class PrefixMiddleware: prefix: str, bypass_paths: Iterable[str] | None = None, ) -> None: + """Initialize the middleware with a prefix and optional bypass paths.""" self.app = app self.prefix = prefix.rstrip("/") self.bypass_paths = set(bypass_paths or []) def __call__(self, environ: dict[str, object], start_response: Callable[..., object]) -> object: + """Rewrite prefixed requests before handing them to the wrapped app.""" path = environ.get("PATH_INFO", "") or "" if path in self.bypass_paths: diff --git a/shelfmark/core/queue.py b/shelfmark/core/queue.py index 254e4898..35246eb5 100644 --- a/shelfmark/core/queue.py +++ b/shelfmark/core/queue.py @@ -2,7 +2,7 @@ import queue import time -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from pathlib import Path from threading import Event, Lock from typing import TYPE_CHECKING, Any @@ -20,12 +20,14 @@ if TYPE_CHECKING: from collections.abc import Callable logger = setup_logger(__name__) +_QUEUE_HOOK_ERRORS = (OSError, RuntimeError, TypeError, ValueError) class BookQueue: """Thread-safe download queue manager with priority support and cancellation.""" def __init__(self) -> None: + """Initialize queue state, locks, and lifecycle hooks.""" self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue() self._lock = Lock() self._status: dict[str, QueueStatus] = {} @@ -67,7 +69,7 @@ class BookQueue: if hook is not None: try: hook(task_id, task) - except Exception as exc: + except _QUEUE_HOOK_ERRORS as exc: logger.warning("Queue hook failed while adding task %s: %s", task_id, exc) return True @@ -104,9 +106,9 @@ class BookQueue: return self._status.get(task_id) def _update_status(self, book_id: str, status: QueueStatus) -> None: - """Internal method to update status and timestamp.""" + """Update the status and timestamp for a task.""" self._status[book_id] = status - self._status_timestamps[book_id] = datetime.now() + self._status_timestamps[book_id] = datetime.now(UTC) def set_terminal_status_hook( self, @@ -306,7 +308,7 @@ class BookQueue: if hook is not None and hook_task is not None: try: hook(task_id, hook_task) - except Exception as exc: + except _QUEUE_HOOK_ERRORS as exc: logger.warning("Queue hook failed while requeueing task %s: %s", task_id, exc) return True @@ -349,7 +351,7 @@ class BookQueue: """Remove any tasks that are done downloading or have stale status.""" terminal_statuses = TERMINAL_QUEUE_STATUSES with self._lock: - current_time = datetime.now() + current_time = datetime.now(UTC) to_remove = [] for task_id, status in self._status.items(): diff --git a/shelfmark/core/request_helpers.py b/shelfmark/core/request_helpers.py index 4ac93995..b88ae6cb 100644 --- a/shelfmark/core/request_helpers.py +++ b/shelfmark/core/request_helpers.py @@ -32,7 +32,7 @@ def emit_ws_event( if socketio is None or not callable(is_enabled) or not is_enabled(): return socketio.emit(event_name, payload, to=room) - except Exception as exc: + except (AttributeError, RuntimeError, TypeError, ValueError) as exc: _logger.warning( "Failed to emit WebSocket event '%s' to room '%s': %s", event_name, diff --git a/shelfmark/core/request_routes.py b/shelfmark/core/request_routes.py index 46371642..b922632a 100644 --- a/shelfmark/core/request_routes.py +++ b/shelfmark/core/request_routes.py @@ -48,6 +48,7 @@ if TYPE_CHECKING: from shelfmark.core.user_db import UserDB logger = setup_logger(__name__) +_NOTIFICATION_TRIGGER_ERRORS = (RuntimeError, TypeError, ValueError) def _error_response( @@ -508,7 +509,7 @@ def _notify_admin_for_request_event( owner_user_id = normalize_positive_int(request_row.get("user_id")) try: notify_admin(event, context) - except Exception as exc: + except _NOTIFICATION_TRIGGER_ERRORS as exc: logger.warning( "Failed to trigger admin notification for request event '%s': %s", event.value, @@ -518,7 +519,7 @@ def _notify_admin_for_request_event( return try: notify_user(owner_user_id, event, context) - except Exception as exc: + except _NOTIFICATION_TRIGGER_ERRORS as exc: logger.warning( "Failed to trigger user notification for request event '%s' (user_id=%s): %s", event.value, diff --git a/shelfmark/core/requests_service.py b/shelfmark/core/requests_service.py index 7cd03c1e..20a0596f 100644 --- a/shelfmark/core/requests_service.py +++ b/shelfmark/core/requests_service.py @@ -40,6 +40,7 @@ class RequestServiceError(ValueError): code: str | None = None, required_mode: str | None = None, ) -> None: + """Initialize the error with HTTP metadata for API callers.""" super().__init__(message) self.status_code = status_code self.code = code diff --git a/shelfmark/core/search_plan.py b/shelfmark/core/search_plan.py index 80de75b8..ef2e33d2 100644 --- a/shelfmark/core/search_plan.py +++ b/shelfmark/core/search_plan.py @@ -1,10 +1,10 @@ +"""Helpers for building release search plans from metadata and user input.""" + from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING -MANUAL_QUERY_MAX_LEN = 256 - from shelfmark.core.config import config from shelfmark.metadata_providers import ( BookMetadata, @@ -15,6 +15,8 @@ from shelfmark.metadata_providers import ( if TYPE_CHECKING: from shelfmark.core.models import SearchFilters +MANUAL_QUERY_MAX_LEN = 256 + @dataclass(frozen=True) class ReleaseSearchVariant: @@ -26,6 +28,7 @@ class ReleaseSearchVariant: @property def query(self) -> str: + """Return the combined title-and-author query for this variant.""" return " ".join(part for part in [self.title, self.author] if part).strip() @@ -44,6 +47,7 @@ class ReleaseSearchPlan: @property def primary_query(self) -> str: + """Return the first expanded title query, if one exists.""" return self.title_variants[0].query if self.title_variants else "" @@ -94,6 +98,7 @@ def build_release_search_plan( indexers: list[str] | None = None, source_filters: SearchFilters | None = None, ) -> ReleaseSearchPlan: + """Build normalized search variants shared across release sources.""" resolved_languages = _normalize_languages(languages) resolved_manual_query = None diff --git a/shelfmark/core/self_user_routes.py b/shelfmark/core/self_user_routes.py index 7263360a..635b4e8d 100644 --- a/shelfmark/core/self_user_routes.py +++ b/shelfmark/core/self_user_routes.py @@ -1,5 +1,6 @@ """Self-service user account routes.""" +import sqlite3 from functools import wraps from typing import TYPE_CHECKING, Any @@ -20,6 +21,7 @@ from shelfmark.core.auth_modes import ( load_active_auth_mode, normalize_auth_source, ) +from shelfmark.core.config import config as app_config from shelfmark.core.logger import setup_logger from shelfmark.core.settings_registry import load_config_file from shelfmark.core.user_settings_overrides import ( @@ -47,6 +49,8 @@ _VALID_SELF_SETTINGS_SECTIONS = ( _SELF_SETTINGS_SECTION_NOTIFICATIONS, ) _DEFAULT_VISIBLE_SELF_SETTINGS_SECTIONS = list(_VALID_SELF_SETTINGS_SECTIONS) +_USER_PREFERENCES_FALLBACK_ERRORS = (ImportError, OSError, RuntimeError, TypeError, sqlite3.Error) +_CONFIG_REFRESH_ERRORS = (ImportError, OSError, RuntimeError, TypeError, ValueError) def _get_current_user( @@ -91,6 +95,36 @@ def _serialize_self_user(user: Mapping[str, Any], auth_mode: str) -> dict[str, A return payload +def _build_optional_user_preferences( + user_db: UserDB, + *, + user_id: int, + tab_name: str, + missing_tab_error: str, + preference_label: str, +) -> tuple[dict[str, Any] | None, tuple[Response, int] | None]: + try: + return _build_user_preferences_payload(user_db, user_id, tab_name), None + except ValueError as exc: + if str(exc) == missing_tab_error: + return None, (jsonify({"error": missing_tab_error}), 500) + logger.warning( + "Failed to build user %s preferences for user_id=%s: %s", + preference_label, + user_id, + exc, + ) + return None, None + except _USER_PREFERENCES_FALLBACK_ERRORS as exc: + logger.warning( + "Failed to build user %s preferences for user_id=%s: %s", + preference_label, + user_id, + exc, + ) + return None, None + + def _normalize_visible_self_settings_sections(raw_sections: object) -> list[str]: """Normalize users.VISIBLE_SELF_SETTINGS_SECTIONS to a safe ordered list.""" if raw_sections is None: @@ -147,13 +181,13 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None: def _require_authenticated_user( f: Callable[..., Response | tuple[Response, int]], ) -> Callable[..., Response | tuple[Response, int]]: - """Decorator requiring an authenticated session linked to a local user row. + """Require an authenticated session linked to a local user row. Caches the resolved auth_mode in ``g.auth_mode`` for the request. """ @wraps(f) - def decorated(*args, **kwargs) -> Response | tuple[Response, int]: + def decorated(*args: object, **kwargs: object) -> Response | tuple[Response, int]: auth_mode = load_active_auth_mode(CWA_DB_PATH, user_db=user_db) g.auth_mode = auth_mode if auth_mode != "none" and "user_id" not in session: @@ -179,51 +213,39 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None: delivery_preferences = None if _SELF_SETTINGS_SECTION_DELIVERY in visible_self_settings_sections: - try: - delivery_preferences = _build_user_preferences_payload( - user_db, user_id, "downloads" - ) - except ValueError: - return jsonify({"error": "Downloads settings tab not found"}), 500 - except Exception as exc: - logger.warning( - "Failed to build user delivery preferences for user_id=%s: %s", - user_id, - exc, - ) - delivery_preferences = None + delivery_preferences, error_response = _build_optional_user_preferences( + user_db, + user_id=user_id, + tab_name="downloads", + missing_tab_error="Downloads settings tab not found", + preference_label="delivery", + ) + if error_response: + return error_response search_preferences = None if _SELF_SETTINGS_SECTION_SEARCH in visible_self_settings_sections: - try: - search_preferences = _build_user_preferences_payload( - user_db, user_id, "search_mode" - ) - except ValueError: - return jsonify({"error": "Search mode settings tab not found"}), 500 - except Exception as exc: - logger.warning( - "Failed to build user search preferences for user_id=%s: %s", - user_id, - exc, - ) - search_preferences = None + search_preferences, error_response = _build_optional_user_preferences( + user_db, + user_id=user_id, + tab_name="search_mode", + missing_tab_error="Search mode settings tab not found", + preference_label="search", + ) + if error_response: + return error_response notification_preferences = None if _SELF_SETTINGS_SECTION_NOTIFICATIONS in visible_self_settings_sections: - try: - notification_preferences = _build_user_preferences_payload( - user_db, user_id, "notifications" - ) - except ValueError: - return jsonify({"error": "Notifications settings tab not found"}), 500 - except Exception as exc: - logger.warning( - "Failed to build user notification preferences for user_id=%s: %s", - user_id, - exc, - ) - notification_preferences = None + notification_preferences, error_response = _build_optional_user_preferences( + user_db, + user_id=user_id, + tab_name="notifications", + missing_tab_error="Notifications settings tab not found", + preference_label="notification", + ) + if error_response: + return error_response user_overridable_keys = sorted( set(delivery_preferences.get("keys", []) if delivery_preferences else []) @@ -370,11 +392,13 @@ def register_self_user_routes(app: Flask, user_db: UserDB) -> None: user_db.set_user_settings(user_id, validated_settings) try: - from shelfmark.core.config import config as app_config - app_config.refresh(force=True) - except Exception: - pass + except _CONFIG_REFRESH_ERRORS as exc: + logger.warning( + "Updated settings for user %s but failed to refresh runtime config: %s", + user_id, + exc, + ) updated = user_db.get_user(user_id=user_id) if not updated: diff --git a/shelfmark/core/settings_registry.py b/shelfmark/core/settings_registry.py index 6c8e765a..a9af8235 100644 --- a/shelfmark/core/settings_registry.py +++ b/shelfmark/core/settings_registry.py @@ -12,6 +12,7 @@ from werkzeug.utils import secure_filename from shelfmark.core.logger import setup_logger logger = setup_logger(__name__) +_SETTINGS_LIVE_APPLY_ERRORS = (OSError, RuntimeError, TypeError, ValueError) if TYPE_CHECKING: from collections.abc import Callable, Iterator @@ -114,6 +115,8 @@ class TagListField(FieldBase): @dataclass class OrderableListField(FieldBase): + """Settings field for ordered, toggleable option lists.""" + # Options can be a list or a callable that returns a list (for lazy evaluation) # Each option: {id, label, description?, disabledReason?, isLocked?, section?, isPinned?} # - isLocked: toggle is disabled (can't enable/disable) @@ -154,9 +157,11 @@ class CustomComponentField: universal_only: bool = False def get_field_type(self) -> str: + """Return the serialized field type for this custom component.""" return "CustomComponentField" def get_bind_keys(self) -> list[str]: + """Return the config keys this custom component reads or writes.""" if self.bind_keys: return self.bind_keys return [f.key for f in self.value_fields if getattr(f, "key", None)] @@ -164,6 +169,8 @@ class CustomComponentField: @dataclass class ActionButton: + """Definition for a custom action button in the settings UI.""" + key: str # Action identifier label: str # Button text description: str = "" # Help text @@ -181,6 +188,7 @@ class ActionButton: ) def get_field_type(self) -> str: + """Return the serialized field type for this action button.""" return "ActionButton" @@ -206,6 +214,7 @@ class HeadingField: universal_only: bool = False # Only show in Universal search mode (hide in Direct mode) def get_field_type(self) -> str: + """Return the serialized field type for this heading field.""" return "HeadingField" @@ -255,6 +264,7 @@ _REGISTRY_LOCK = Lock() def register_group(name: str, display_name: str, icon: str | None = None, order: int = 100) -> None: + """Register a settings group used to organize tabs in the UI.""" with _REGISTRY_LOCK: group = SettingsGroup( name=name, @@ -273,6 +283,8 @@ def register_settings( order: int = 100, group: str | None = None, ) -> Callable[[Callable[[], list[SettingsField]]], Callable[[], list[SettingsField]]]: + """Register a settings tab and its field factory.""" + def decorator(func: Callable[[], list[SettingsField]]) -> Callable[[], list[SettingsField]]: with _REGISTRY_LOCK: fields = func() @@ -297,6 +309,7 @@ def register_settings( def register_on_save(tab_name: str, handler: Callable[[dict[str, Any]], dict[str, Any]]) -> None: + """Register an on-save hook for a settings tab.""" with _REGISTRY_LOCK: _ON_SAVE_HANDLERS[tab_name] = handler logger.debug("Registered on_save handler for tab: %s", tab_name) @@ -407,6 +420,7 @@ def _ensure_config_dir(tab_name: str) -> None: def load_config_file(tab_name: str) -> dict[str, Any]: + """Load a settings tab config file, returning an empty dict on failure.""" config_path = _get_config_file_path(tab_name) if not config_path.exists(): @@ -421,6 +435,7 @@ def load_config_file(tab_name: str) -> dict[str, Any]: def save_config_file(tab_name: str, values: dict[str, Any]) -> bool: + """Merge and save persisted settings values for a tab.""" try: _ensure_config_dir(tab_name) config_path = _get_config_file_path(tab_name) @@ -506,6 +521,7 @@ def initialize_default_configs() -> bool: def sync_env_to_config() -> None: + """Sync supported environment-backed settings into config files.""" # Initialize default configs first (for fresh installs) initialize_default_configs() @@ -821,6 +837,7 @@ def migrate_download_to_browser_settings() -> None: def get_setting_value(field: SettingsField, tab_name: str) -> object: + """Resolve the effective value for a settings field.""" if isinstance(field, (ActionButton, HeadingField, CustomComponentField)): return None # Actions and headings don't have values @@ -1162,7 +1179,7 @@ def _apply_dns_settings(config: Config) -> None: network.set_dns_provider(provider, manual_servers, use_doh=use_doh) except ImportError: pass # Network module not available - except Exception as e: + except _SETTINGS_LIVE_APPLY_ERRORS as e: logger.warning("Failed to apply DNS settings: %s", e) @@ -1179,11 +1196,12 @@ def _apply_aa_mirror_settings(config: Config) -> None: network.init_aa(force=True) except ImportError: pass # Network module not available - except Exception as e: + except _SETTINGS_LIVE_APPLY_ERRORS as e: logger.warning("Failed to apply AA mirror settings: %s", e) def update_settings(tab_name: str, values: dict[str, Any]) -> dict[str, Any]: + """Validate, persist, and post-process updates for a settings tab.""" tab = get_settings_tab(tab_name) if not tab: return { @@ -1290,7 +1308,7 @@ def update_settings(tab_name: str, values: dict[str, Any]) -> dict[str, Any]: ) _apply_ssl_warning_suppression() - except Exception as e: + except _SETTINGS_LIVE_APPLY_ERRORS as e: logger.warning("Failed to apply certificate validation setting: %s", e) # Apply AA mirror settings changes live (mirrors tab) diff --git a/shelfmark/core/user_db.py b/shelfmark/core/user_db.py index 09b12c05..34f4714a 100644 --- a/shelfmark/core/user_db.py +++ b/shelfmark/core/user_db.py @@ -175,6 +175,7 @@ class UserDB: _VALID_AUTH_SOURCES: ClassVar[frozenset[str]] = frozenset(AUTH_SOURCE_SET) def __init__(self, db_path: str) -> None: + """Initialize the user database wrapper for the given SQLite path.""" self._db_path = db_path self._lock = threading.Lock() @@ -273,7 +274,8 @@ class UserDB: ) -> dict[str, Any]: """Create a new user. Raises ValueError if username or oidc_subject already exists.""" if auth_source not in self._VALID_AUTH_SOURCES: - raise ValueError(f"Invalid auth_source: {auth_source}") + msg = f"Invalid auth_source: {auth_source}" + raise ValueError(msg) with self._lock: conn = self._connect() try: @@ -296,7 +298,8 @@ class UserDB: user_id = cursor.lastrowid return self._get_user_by_id(conn, user_id) except sqlite3.IntegrityError as e: - raise ValueError(f"User already exists: {e}") from e + msg = f"User already exists: {e}" + raise ValueError(msg) from e finally: conn.close() @@ -337,25 +340,35 @@ class UserDB: "role", } ) + _USER_UPDATE_STATEMENTS: ClassVar[dict[str, str]] = { + "email": "UPDATE users SET email = ? WHERE id = ?", + "display_name": "UPDATE users SET display_name = ? WHERE id = ?", + "password_hash": "UPDATE users SET password_hash = ? WHERE id = ?", + "oidc_subject": "UPDATE users SET oidc_subject = ? WHERE id = ?", + "auth_source": "UPDATE users SET auth_source = ? WHERE id = ?", + "role": "UPDATE users SET role = ? WHERE id = ?", + } - def update_user(self, user_id: int, **kwargs) -> None: + def update_user(self, user_id: int, **kwargs: object) -> None: """Update user fields. Raises ValueError if user not found or invalid column.""" if not kwargs: return for k in kwargs: if k not in self._ALLOWED_UPDATE_COLUMNS: - raise ValueError(f"Invalid column: {k}") + msg = f"Invalid column: {k}" + raise ValueError(msg) if "auth_source" in kwargs and kwargs["auth_source"] not in self._VALID_AUTH_SOURCES: - raise ValueError(f"Invalid auth_source: {kwargs['auth_source']}") + msg = f"Invalid auth_source: {kwargs['auth_source']}" + raise ValueError(msg) with self._lock: conn = self._connect() try: # Verify user exists if not self._get_user_by_id(conn, user_id): - raise ValueError(f"User {user_id} not found") - sets = ", ".join(f"{k} = ?" for k in kwargs) - values = [*list(kwargs.values()), user_id] - conn.execute(f"UPDATE users SET {sets} WHERE id = ?", values) + msg = f"User {user_id} not found" + raise ValueError(msg) + for column, value in kwargs.items(): + conn.execute(self._USER_UPDATE_STATEMENTS[column], (value, user_id)) conn.commit() finally: conn.close() @@ -371,14 +384,9 @@ class UserDB: ).fetchall() request_item_keys = [f"request:{row['id']}" for row in request_rows] if request_item_keys: - placeholders = ",".join("?" for _ in request_item_keys) - conn.execute( - f""" - DELETE FROM activity_view_state - WHERE item_type = 'request' - AND item_key IN ({placeholders}) - """, - request_item_keys, + conn.executemany( + "DELETE FROM activity_view_state WHERE item_type = 'request' AND item_key = ?", + [(item_key,) for item_key in request_item_keys], ) conn.execute( "DELETE FROM activity_view_state WHERE viewer_scope = ?", @@ -461,7 +469,8 @@ class UserDB: try: return json.dumps(value) except TypeError as exc: - raise ValueError(f"{field} must be JSON-serializable") from exc + msg = f"{field} must be JSON-serializable" + raise ValueError(msg) from exc @staticmethod def _parse_request_row(row: sqlite3.Row | None) -> dict[str, Any] | None: @@ -543,7 +552,8 @@ class UserDB: ).fetchone() parsed = self._parse_request_row(row) if parsed is None: - raise ValueError(f"Request {request_id} not found after creation") + msg = f"Request {request_id} not found after creation" + raise ValueError(msg) return parsed def create_request( @@ -566,11 +576,14 @@ class UserDB: ) -> dict[str, Any]: """Create a download request row and return the created record.""" if not isinstance(book_data, dict): - raise TypeError("book_data must be an object") + msg = "book_data must be an object" + raise TypeError(msg) if release_data is not None and not isinstance(release_data, dict): - raise TypeError("release_data must be an object when provided") + msg = "release_data must be an object when provided" + raise TypeError(msg) if not content_type: - raise ValueError("content_type is required") + msg = "content_type is required" + raise ValueError(msg) normalized_status = normalize_request_status(status) normalized_delivery_state = normalize_delivery_state(delivery_state) @@ -690,27 +703,46 @@ class UserDB: "last_failure_reason", } ) + _REQUEST_UPDATE_STATEMENTS: ClassVar[dict[str, str]] = { + "status": "UPDATE download_requests SET status = ? WHERE id = ?", + "source_hint": "UPDATE download_requests SET source_hint = ? WHERE id = ?", + "content_type": "UPDATE download_requests SET content_type = ? WHERE id = ?", + "request_level": "UPDATE download_requests SET request_level = ? WHERE id = ?", + "policy_mode": "UPDATE download_requests SET policy_mode = ? WHERE id = ?", + "book_data": "UPDATE download_requests SET book_data = ? WHERE id = ?", + "release_data": "UPDATE download_requests SET release_data = ? WHERE id = ?", + "note": "UPDATE download_requests SET note = ? WHERE id = ?", + "admin_note": "UPDATE download_requests SET admin_note = ? WHERE id = ?", + "reviewed_by": "UPDATE download_requests SET reviewed_by = ? WHERE id = ?", + "reviewed_at": "UPDATE download_requests SET reviewed_at = ? WHERE id = ?", + "delivery_state": "UPDATE download_requests SET delivery_state = ? WHERE id = ?", + "delivery_updated_at": "UPDATE download_requests SET delivery_updated_at = ? WHERE id = ?", + "last_failure_reason": "UPDATE download_requests SET last_failure_reason = ? WHERE id = ?", + } def update_request( self, request_id: int, expected_current_status: str | None = None, - **kwargs, + **kwargs: object, ) -> dict[str, Any]: """Update request fields and return the updated record.""" if not kwargs: request = self.get_request(request_id) if request is None: - raise ValueError(f"Request {request_id} not found") + msg = f"Request {request_id} not found" + raise ValueError(msg) if expected_current_status is not None: normalized_expected_status = normalize_request_status(expected_current_status) if request["status"] != normalized_expected_status: - raise ValueError("Request state changed before update") + msg = "Request state changed before update" + raise ValueError(msg) return request for key in kwargs: if key not in self._ALLOWED_REQUEST_UPDATE_COLUMNS: - raise ValueError(f"Invalid request column: {key}") + msg = f"Invalid request column: {key}" + raise ValueError(msg) with self._lock: conn = self._connect() @@ -721,12 +753,14 @@ class UserDB: ).fetchone() current = self._parse_request_row(row) if current is None: - raise ValueError(f"Request {request_id} not found") + msg = f"Request {request_id} not found" + raise ValueError(msg) if expected_current_status is not None: normalized_expected_status = normalize_request_status(expected_current_status) if current["status"] != normalized_expected_status: - raise ValueError("Request state changed before update") + msg = "Request state changed before update" + raise ValueError(msg) updates = dict(kwargs) @@ -746,35 +780,35 @@ class UserDB: if "delivery_updated_at" in updates: delivery_updated_at = updates["delivery_updated_at"] if delivery_updated_at is not None and not isinstance(delivery_updated_at, str): - raise TypeError("delivery_updated_at must be a string when provided") + msg = "delivery_updated_at must be a string when provided" + raise TypeError(msg) if "content_type" in updates and not updates["content_type"]: - raise ValueError("content_type is required") + msg = "content_type is required" + raise ValueError(msg) if "request_level" in updates: updates["request_level"] = normalize_request_level(updates["request_level"]) if "book_data" in updates: if not isinstance(updates["book_data"], dict): - raise TypeError("book_data must be an object") + msg = "book_data must be an object" + raise TypeError(msg) updates["book_data"] = self._serialize_json(updates["book_data"], "book_data") if "release_data" in updates: if updates["release_data"] is not None and not isinstance( updates["release_data"], dict ): - raise TypeError("release_data must be an object when provided") + msg = "release_data must be an object when provided" + raise TypeError(msg) updates["release_data"] = self._serialize_json( updates["release_data"], "release_data", ) - set_clause = ", ".join(f"{column} = ?" for column in updates) - values = [*list(updates.values()), request_id] - conn.execute( - f"UPDATE download_requests SET {set_clause} WHERE id = ?", - values, - ) + for column, value in updates.items(): + conn.execute(self._REQUEST_UPDATE_STATEMENTS[column], (value, request_id)) conn.commit() updated_row = conn.execute( @@ -783,7 +817,8 @@ class UserDB: ).fetchone() parsed = self._parse_request_row(updated_row) if parsed is None: - raise ValueError(f"Request {request_id} not found after update") + msg = f"Request {request_id} not found after update" + raise ValueError(msg) return parsed finally: conn.close() @@ -865,7 +900,8 @@ class UserDB: ).fetchone() current = self._parse_request_row(row) if current is None: - raise ValueError(f"Request {request_id} not found") + msg = f"Request {request_id} not found" + raise ValueError(msg) conn.execute( """ @@ -893,7 +929,8 @@ class UserDB: conn.commit() parsed = self._parse_request_row(updated_row) if parsed is None: - raise ValueError(f"Request {request_id} not found after rollback") + msg = f"Request {request_id} not found after rollback" + raise ValueError(msg) return parsed finally: conn.close() diff --git a/shelfmark/core/user_settings_overrides.py b/shelfmark/core/user_settings_overrides.py index ee389b34..c1f7325a 100644 --- a/shelfmark/core/user_settings_overrides.py +++ b/shelfmark/core/user_settings_overrides.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: def get_settings_registry() -> ModuleType: + """Load settings modules and return the shared settings registry module.""" # Ensure settings modules are loaded before reading registry metadata. import_module("shelfmark.config.notifications_settings") import_module("shelfmark.config.security") @@ -23,6 +24,7 @@ def get_settings_registry() -> ModuleType: def get_ordered_user_overridable_fields(tab_name: str) -> list[tuple[str, Any]]: + """Return user-overridable fields for a tab in UI display order.""" settings_registry = get_settings_registry() tab = settings_registry.get_settings_tab(tab_name) if not tab: @@ -32,6 +34,7 @@ def get_ordered_user_overridable_fields(tab_name: str) -> list[tuple[str, Any]]: def build_user_preferences_payload(user_db: UserDB, user_id: int, tab_name: str) -> dict[str, Any]: + """Build the effective user-preferences payload for a settings tab.""" from shelfmark.core.config import config as app_config settings_registry = get_settings_registry() diff --git a/shelfmark/core/utils.py b/shelfmark/core/utils.py index 45901a5f..65ec0c48 100644 --- a/shelfmark/core/utils.py +++ b/shelfmark/core/utils.py @@ -4,6 +4,7 @@ import base64 import importlib import os import re +import sqlite3 from pathlib import Path from threading import Lock from typing import TYPE_CHECKING @@ -57,6 +58,7 @@ def normalize_http_url( _xmlrpc_patch_lock = Lock() _xmlrpc_patch_applied = False +_XMLRPC_PATCH_ERRORS = (ImportError, AttributeError, OSError, RuntimeError) def get_hardened_xmlrpc_client() -> ModuleType: @@ -70,7 +72,7 @@ def get_hardened_xmlrpc_client() -> ModuleType: monkey_patch() _xmlrpc_patch_applied = True - except Exception: + except _XMLRPC_PATCH_ERRORS: # Keep runtime behavior unchanged if defusedxml is unavailable. _xmlrpc_patch_applied = False @@ -173,7 +175,7 @@ def _resolve_destination_username( if not user: return "" return str(user.get("username") or "").strip() - except Exception: + except ImportError, OSError, sqlite3.Error: return "" @@ -257,7 +259,7 @@ def get_aa_content_type_dir(content_type: str | None = None) -> Path | None: def get_ingest_dir(content_type: str | None = None) -> Path: - """DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead.""" + """Return the legacy ingest directory for a content type.""" from shelfmark.core.config import config # Check new DESTINATION setting first, then legacy INGEST_DIR diff --git a/shelfmark/download/archive.py b/shelfmark/download/archive.py index db7c5c64..2766cbc9 100644 --- a/shelfmark/download/archive.py +++ b/shelfmark/download/archive.py @@ -134,7 +134,8 @@ def extract_archive( elif suffix == "rar": extracted_files, warnings = _extract_rar(archive_path, output_dir) else: - raise ArchiveExtractionError(f"Unsupported archive format: {suffix}") + msg = f"Unsupported archive format: {suffix}" + raise ArchiveExtractionError(msg) is_audiobook = check_audiobook(content_type) file_type_label = "audiobook" if is_audiobook else "book" @@ -174,7 +175,8 @@ def extract_archive_raw( if suffix == "rar": return _extract_rar(archive_path, output_dir) - raise ArchiveExtractionError(f"Unsupported archive format: {suffix}") + msg = f"Unsupported archive format: {suffix}" + raise ArchiveExtractionError(msg) def _extract_files_from_archive(archive: ArchiveType, output_dir: Path) -> list[Path]: @@ -222,31 +224,37 @@ def _extract_zip(archive_path: Path, output_dir: Path) -> tuple[list[Path], list # Check for password protection for info in zf.infolist(): if info.flag_bits & 0x1: # Encrypted flag - raise PasswordProtectedError("ZIP archive is password protected") + msg = "ZIP archive is password protected" + raise PasswordProtectedError(msg) # Test archive integrity bad_file = zf.testzip() if bad_file: - raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}") + msg = f"Corrupted file in archive: {bad_file}" + raise CorruptedArchiveError(msg) return _extract_files_from_archive(zf, output_dir), [] except zipfile.BadZipFile as e: - raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}") from e + msg = f"Invalid or corrupted ZIP: {e}" + raise CorruptedArchiveError(msg) from e except PermissionError as e: - raise ArchiveExtractionError(f"Permission denied: {e}") from e + msg = f"Permission denied: {e}" + raise ArchiveExtractionError(msg) from e def _extract_rar(archive_path: Path, output_dir: Path) -> tuple[list[Path], list[str]]: """Extract files from a RAR archive.""" if not RAR_AVAILABLE: - raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed") + msg = "RAR extraction not available - rarfile library not installed" + raise ArchiveExtractionError(msg) try: with rarfile.RarFile(archive_path, "r") as rf: # Check for password protection if rf.needs_password(): - raise PasswordProtectedError("RAR archive is password protected") + msg = "RAR archive is password protected" + raise PasswordProtectedError(msg) # Test archive integrity rf.testrar() @@ -254,8 +262,11 @@ def _extract_rar(archive_path: Path, output_dir: Path) -> tuple[list[Path], list return _extract_files_from_archive(rf, output_dir), [] except rarfile.BadRarFile as e: - raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}") from e + msg = f"Invalid or corrupted RAR: {e}" + raise CorruptedArchiveError(msg) from e except rarfile.RarCannotExec as e: - raise ArchiveExtractionError("unrar binary not found - install unrar package") from e + msg = "unrar binary not found - install unrar package" + raise ArchiveExtractionError(msg) from e except PermissionError as e: - raise ArchiveExtractionError(f"Permission denied: {e}") from e + msg = f"Permission denied: {e}" + raise ArchiveExtractionError(msg) from e diff --git a/shelfmark/download/clients/__init__.py b/shelfmark/download/clients/__init__.py index 2c49a9fc..453749cb 100644 --- a/shelfmark/download/clients/__init__.py +++ b/shelfmark/download/clients/__init__.py @@ -17,6 +17,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum from functools import wraps +from importlib import import_module from pathlib import Path from typing import TYPE_CHECKING, TypeVar, cast @@ -39,6 +40,7 @@ RETRYABLE_EXCEPTIONS = ( _MIN_RETRYABLE_STATUS = 500 _MIN_PROGRESS_PERCENT = 0 _MAX_PROGRESS_PERCENT = 100 +_RNG = random.SystemRandom() def with_retry( @@ -47,7 +49,7 @@ def with_retry( max_delay: float = 10.0, jitter: float = 0.5, ) -> Callable[[Callable[..., T]], Callable[..., T]]: - """Decorator for retrying API calls with exponential backoff. + """Retry API calls with exponential backoff. Args: max_attempts: Maximum number of attempts (default 3) @@ -68,7 +70,7 @@ def with_retry( def decorator(func: Callable[..., T]) -> Callable[..., T]: @wraps(func) - def wrapper(*args, **kwargs) -> T: + def wrapper(*args: object, **kwargs: object) -> T: last_exception = None for attempt in range(1, max_attempts + 1): @@ -86,7 +88,7 @@ def with_retry( # Calculate delay with exponential backoff delay = min(base_delay * (2 ** (attempt - 1)), max_delay) # Add jitter to prevent thundering herd - delay += random.uniform(0, delay * jitter) + delay += _RNG.uniform(0, delay * jitter) _logger.debug( "Retry %s/%s for %s after %.1fs (error: %s)", attempt, @@ -232,7 +234,7 @@ class DownloadClient(ABC): # Join and normalize return os.path.normpath(str(Path(valid[0]).joinpath(*valid[1:]))) - def __init_subclass__(cls, **kwargs) -> None: + def __init_subclass__(cls, **kwargs: object) -> None: """Validate that subclasses define required class attributes.""" super().__init_subclass__(**kwargs) @@ -288,6 +290,7 @@ class DownloadClient(ABC): name: Display name for the download category: Category/label for organization (None = client default) expected_hash: Optional info_hash hint (torrents only) + **kwargs: Client-specific options passed through to the implementation. Returns: Client-specific download ID (hash for torrents, ID for NZBGet). @@ -356,12 +359,32 @@ class DownloadClient(ABC): # Client registry: protocol -> list of client classes _CLIENTS: dict[str, list[type[DownloadClient]]] = {} +_BUILTIN_CLIENT_MODULES = ( + "shelfmark.download.clients.deluge", + "shelfmark.download.clients.nzbget", + "shelfmark.download.clients.qbittorrent", + "shelfmark.download.clients.rtorrent", + "shelfmark.download.clients.sabnzbd", + "shelfmark.download.clients.transmission", +) +_builtin_client_state = {"loaded": False} + + +def _ensure_builtin_clients_registered() -> None: + """Import built-in client modules once to populate the registry.""" + if _builtin_client_state["loaded"]: + return + + for module_name in _BUILTIN_CLIENT_MODULES: + import_module(module_name) + + _builtin_client_state["loaded"] = True def register_client( protocol: str, ) -> Callable[[type[DownloadClient]], type[DownloadClient]]: - """Decorator to register a download client for a protocol. + """Register a download client for a protocol. Multiple clients can be registered for the same protocol. The `is_configured()` method determines which one is active. @@ -398,6 +421,8 @@ def get_client(protocol: str) -> DownloadClient | None: Configured client instance, or None if not available/configured. """ + _ensure_builtin_clients_registered() + if protocol not in _CLIENTS: return None @@ -415,6 +440,8 @@ def list_configured_clients() -> list[str]: List of protocol names (e.g., ["torrent", "usenet"]). """ + _ensure_builtin_clients_registered() + result = [] for protocol, client_classes in _CLIENTS.items(): for cls in client_classes: @@ -431,14 +458,8 @@ def get_all_clients() -> dict[str, list[type[DownloadClient]]]: Dict of protocol -> list of client classes. """ + _ensure_builtin_clients_registered() return dict(_CLIENTS) -# Import client implementations to trigger registration -# These imports are at the bottom to avoid circular imports -from shelfmark.download.clients import deluge as deluge -from shelfmark.download.clients import nzbget as nzbget -from shelfmark.download.clients import qbittorrent as qbittorrent -from shelfmark.download.clients import rtorrent as rtorrent -from shelfmark.download.clients import sabnzbd as sabnzbd -from shelfmark.download.clients import transmission as transmission +_ensure_builtin_clients_registered() diff --git a/shelfmark/download/clients/base_handler.py b/shelfmark/download/clients/base_handler.py index f3f903c7..b07c2ebd 100644 --- a/shelfmark/download/clients/base_handler.py +++ b/shelfmark/download/clients/base_handler.py @@ -29,9 +29,13 @@ if TYPE_CHECKING: from shelfmark.core.models import DownloadTask logger = setup_logger(__name__) +_CLIENT_CLEANUP_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError) # How often to poll the download client for status (seconds) POLL_INTERVAL = 2 +WINDOWS_DRIVE_PREFIX_LENGTH = 2 +SECONDS_PER_MINUTE = 60 +SECONDS_PER_HOUR = 3600 # How long to wait for completed files to appear (seconds) COMPLETED_PATH_RETRY_INTERVAL = 5 COMPLETED_PATH_MAX_ATTEMPTS = 12 # 12 attempts * 5s = 60s grace period @@ -60,7 +64,7 @@ def _diagnose_path_issue(path: str) -> str: """ # Detect Windows-style paths (won't work in Linux containers) - if len(path) >= 2 and path[1] == ":": + if len(path) >= WINDOWS_DRIVE_PREFIX_LENGTH and path[1] == ":": return ( f"Path '{path}' appears to be a Windows path. " f"Shelfmark runs in Linux and cannot access Windows paths directly. " @@ -110,6 +114,7 @@ class ExternalClientHandler(DownloadHandler, ABC): """Shared lifecycle handler for sources that hand off to torrent/usenet clients.""" def __init__(self) -> None: + """Initialize cleanup tracking for client-managed downloads.""" # Track downloads that may need client-side cleanup after Shelfmark completes import. # task_id -> (client, download_id, protocol) self._cleanup_refs: dict[str, tuple[DownloadClient, str, str]] = {} @@ -123,7 +128,7 @@ class ExternalClientHandler(DownloadHandler, ABC): """Resolve source-specific task metadata into a client download request.""" def _on_download_complete(self, task: DownloadTask) -> None: - """Hook called after successful completion; override for source cleanup.""" + """Run post-completion source cleanup hooks.""" return def _get_client(self, protocol: str) -> DownloadClient | None: @@ -135,7 +140,7 @@ class ExternalClientHandler(DownloadHandler, ABC): return list_configured_clients() def _poll_interval(self) -> float: - """Polling interval for status checks (seconds).""" + """Return the polling interval for status checks.""" return POLL_INTERVAL def _completed_path_retry_interval(self) -> float: @@ -163,6 +168,7 @@ class ExternalClientHandler(DownloadHandler, ABC): return config.get(audiobook_key, "") or None if audiobook_key else None def post_process_cleanup(self, task: DownloadTask, *, success: bool) -> None: + """Clean up external-client state after post-processing finishes.""" if not success: self._cleanup_refs.pop(task.task_id, None) return @@ -180,7 +186,7 @@ class ExternalClientHandler(DownloadHandler, ABC): try: self._delete_local_download_data(client, download_id) self._remove_usenet_download(client, download_id, delete_files=True, archive=True) - except Exception as e: + except _CLIENT_CLEANUP_ERRORS as e: logger.warning( "Failed to cleanup usenet download %s in %s: %s", download_id, @@ -193,7 +199,7 @@ class ExternalClientHandler(DownloadHandler, ABC): return try: client.remove(download_id, delete_files=False) - except Exception as e: + except _CLIENT_CLEANUP_ERRORS as e: logger.warning( "Failed to remove torrent %s from %s: %s", download_id, @@ -219,7 +225,7 @@ class ExternalClientHandler(DownloadHandler, ABC): """Best-effort local deletion of client download data.""" try: raw_path = client.get_download_path(download_id) - except Exception as e: + except _CLIENT_CLEANUP_ERRORS as e: logger.debug( "Failed to resolve download path for %s %s: %s", client.name, download_id, e ) @@ -268,7 +274,7 @@ class ExternalClientHandler(DownloadHandler, ABC): logger.info( "Deleted local download data for %s %s: %s", client.name, download_id, delete_path ) - except Exception as e: + except _CLIENT_CLEANUP_ERRORS as e: logger.warning( "Failed to delete local download data for %s %s: %s", client.name, download_id, e ) @@ -300,7 +306,7 @@ class ExternalClientHandler(DownloadHandler, ABC): # Permanent delete for failed usenet downloads (SABnzbd archive=0). self._delete_local_download_data(client, download_id) self._remove_usenet_download(client, download_id, delete_files=True, archive=False) - except Exception as e: + except _CLIENT_CLEANUP_ERRORS as e: logger.warning( "Failed to remove download %s from %s after %s: %s", download_id, @@ -321,7 +327,7 @@ class ExternalClientHandler(DownloadHandler, ABC): try: self._delete_local_download_data(client, download_id) self._remove_usenet_download(client, download_id, delete_files=True, archive=True) - except Exception as e: + except _CLIENT_CLEANUP_ERRORS as e: logger.warning( "Failed to remove download %s from %s after cancellation: %s", download_id, @@ -557,12 +563,15 @@ class ExternalClientHandler(DownloadHandler, ABC): msg += f" ({speed_mb:.1f} MB/s)" if status.eta and status.eta > 0: - if status.eta < 60: + if status.eta < SECONDS_PER_MINUTE: msg += f" - {status.eta}s left" - elif status.eta < 3600: - msg += f" - {status.eta // 60}m left" + elif status.eta < SECONDS_PER_HOUR: + msg += f" - {status.eta // SECONDS_PER_MINUTE}m left" else: - msg += f" - {status.eta // 3600}h {(status.eta % 3600) // 60}m left" + msg += ( + f" - {status.eta // SECONDS_PER_HOUR}h " + f"{(status.eta % SECONDS_PER_HOUR) // SECONDS_PER_MINUTE}m left" + ) return msg diff --git a/shelfmark/download/clients/deluge.py b/shelfmark/download/clients/deluge.py index 690992f0..9204765e 100644 --- a/shelfmark/download/clients/deluge.py +++ b/shelfmark/download/clients/deluge.py @@ -40,11 +40,25 @@ ONE_WEEK_IN_SECONDS = 604800 class DelugeRpcError(RuntimeError): + """Raised when Deluge returns a JSON-RPC error response.""" + def __init__(self, message: str, code: int | None = None) -> None: + """Initialize the RPC error with an optional Deluge error code.""" super().__init__(message) self.code = code +_DELUGE_CLIENT_ERRORS = ( + AttributeError, + DelugeRpcError, + OSError, + requests.exceptions.RequestException, + RuntimeError, + TypeError, + ValueError, +) + + def _get_error_message(error: object) -> tuple[str, int | None]: if isinstance(error, dict): return str(error.get("message") or error), error.get("code") @@ -63,6 +77,7 @@ class DelugeClient(DownloadClient): name = "deluge" def __init__(self) -> None: + """Initialize the client from the configured Deluge connection settings.""" raw_host = str(config.get("DELUGE_HOST", "localhost") or "") raw_port = str(config.get("DELUGE_PORT", "8112") or "8112") password = str(config.get("DELUGE_PASSWORD", "") or "") @@ -146,8 +161,7 @@ class DelugeClient(DownloadClient): self._authenticated = True def _select_daemon_host_id(self, hosts: list) -> str: - # Hosts returned by web.get_hosts look like: - # [[host_id, host, port, status], ...] + # Deluge returns entries containing host id, host, port, and status. preferred_hosts = {"127.0.0.1", "localhost"} for entry in hosts: @@ -201,13 +215,10 @@ class DelugeClient(DownloadClient): def _get_daemon_version(self) -> object: """Fetch daemon version, preferring daemon.get_version when available.""" - try: + with suppress(*_DELUGE_CLIENT_ERRORS): methods = self._rpc_call("system.listMethods") if isinstance(methods, list) and "daemon.get_version" in methods: return self._rpc_call("daemon.get_version") - except Exception: - # Fall back to daemon.info to preserve existing behavior. - pass return self._rpc_call("daemon.info") @@ -218,25 +229,27 @@ class DelugeClient(DownloadClient): try: # label.add will error if the plugin is unavailable or the label exists. - with suppress(Exception): + with suppress(*_DELUGE_CLIENT_ERRORS): self._rpc_call("label.add", label) self._rpc_call("label.set_torrent", torrent_id, label) - except Exception as e: + except _DELUGE_CLIENT_ERRORS as e: logger.debug("Could not set Deluge label '%s' for %s: %s", label, torrent_id, e) @staticmethod def is_configured() -> bool: + """Return whether Deluge is the active configured torrent client.""" client = config.get("PROWLARR_TORRENT_CLIENT", "") host = config.get("DELUGE_HOST", "") password = config.get("DELUGE_PASSWORD", "") return client == "deluge" and bool(host) and bool(password) def test_connection(self) -> tuple[bool, str]: + """Test connectivity and authentication against the Deluge server.""" try: self._ensure_connected() version = self._get_daemon_version() - except Exception as e: + except _DELUGE_CLIENT_ERRORS as e: self._authenticated = False self._connected = False return False, f"Connection failed: {e!s}" @@ -249,8 +262,9 @@ class DelugeClient(DownloadClient): name: str, category: str | None = None, expected_hash: str | None = None, - **kwargs, + **kwargs: object, ) -> str: + """Add a torrent to Deluge and return the torrent id.""" try: self._ensure_connected() @@ -298,7 +312,7 @@ class DelugeClient(DownloadClient): logger.info("Added torrent to Deluge: %s", torrent_id) - except Exception: + except _DELUGE_CLIENT_ERRORS: self._authenticated = False self._connected = False logger.exception("Deluge add failed") @@ -307,6 +321,7 @@ class DelugeClient(DownloadClient): return torrent_id def get_status(self, download_id: str) -> DownloadStatus: + """Return the current Deluge status for a torrent.""" try: self._ensure_connected() @@ -352,7 +367,7 @@ class DelugeClient(DownloadClient): if eta is not None: try: eta = int(eta) - except Exception: + except TypeError, ValueError: eta = None if eta is not None and (eta < 0 or eta > ONE_WEEK_IN_SECONDS): @@ -376,10 +391,11 @@ class DelugeClient(DownloadClient): eta=eta, ) - except Exception as e: + except _DELUGE_CLIENT_ERRORS as e: return DownloadStatus.error(self._log_error("get_status", e)) def remove(self, download_id: str, *, delete_files: bool = False) -> bool: + """Remove a torrent from Deluge, optionally deleting its files.""" try: self._ensure_connected() @@ -392,13 +408,14 @@ class DelugeClient(DownloadClient): ) return True - except Exception as e: + except _DELUGE_CLIENT_ERRORS as e: self._log_error("remove", e) return False else: return False def get_download_path(self, download_id: str) -> str | None: + """Return the resolved download path for a Deluge torrent.""" try: self._ensure_connected() @@ -414,7 +431,7 @@ class DelugeClient(DownloadClient): str(status.get("name", "")), ) - except Exception as e: + except _DELUGE_CLIENT_ERRORS as e: self._log_error("get_download_path", e, level="debug") return None else: @@ -423,6 +440,7 @@ class DelugeClient(DownloadClient): def find_existing( self, url: str, category: str | None = None ) -> tuple[str, DownloadStatus] | None: + """Find an existing Deluge torrent matching a release URL.""" try: self._ensure_connected() @@ -440,7 +458,7 @@ class DelugeClient(DownloadClient): full_status = self.get_status(torrent_info.info_hash) return (torrent_info.info_hash, full_status) - except Exception as e: + except _DELUGE_CLIENT_ERRORS as e: self._authenticated = False self._connected = False logger.debug("Error checking for existing torrent: %s", e) diff --git a/shelfmark/download/clients/nzbget.py b/shelfmark/download/clients/nzbget.py index ab56c524..11a49f4e 100644 --- a/shelfmark/download/clients/nzbget.py +++ b/shelfmark/download/clients/nzbget.py @@ -19,6 +19,7 @@ from shelfmark.download.clients import ( from shelfmark.download.network import get_ssl_verify logger = setup_logger(__name__) +_NZBGET_CLIENT_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, ValueError) @register_client("usenet") @@ -59,7 +60,7 @@ class NZBGetClient(DownloadClient): if result: logger.info("Removed NZB from NZBGet (%s): %s", command, download_id) return True, None - except Exception as e: + except _NZBGET_CLIENT_ERRORS as e: return False, e return False, None @@ -115,7 +116,7 @@ class NZBGetClient(DownloadClient): return False, "Could not connect to NZBGet" except requests.exceptions.Timeout: return False, "Connection timed out" - except Exception as e: + except _NZBGET_CLIENT_ERRORS as e: return False, f"Connection failed: {e!s}" else: return True, f"Connected to NZBGet {version}" @@ -126,7 +127,7 @@ class NZBGetClient(DownloadClient): name: str, category: str | None = None, expected_hash: str | None = None, - **kwargs, + **kwargs: object, ) -> str: """Add NZB by URL. @@ -138,6 +139,7 @@ class NZBGetClient(DownloadClient): name: Display name for the download category: Category for organization (uses configured default if not specified) expected_hash: Optional info_hash hint (unused) + **kwargs: Client-specific options passed through to the implementation. Returns: NZBGet download ID (NZBID). @@ -193,7 +195,7 @@ class NZBGetClient(DownloadClient): logger.exception("Failed to fetch NZB from URL") msg = f"Failed to fetch NZB: {e}" raise RuntimeError(msg) from e - except Exception: + except _NZBGET_CLIENT_ERRORS: logger.exception("NZBGet add failed") raise @@ -286,7 +288,7 @@ class NZBGetClient(DownloadClient): # Not found in queue or history return DownloadStatus.error("Download not found") - except Exception as e: + except _NZBGET_CLIENT_ERRORS as e: return DownloadStatus.error(self._log_error("get_status", e)) def remove(self, download_id: str, *, delete_files: bool = False) -> bool: diff --git a/shelfmark/download/clients/qbittorrent.py b/shelfmark/download/clients/qbittorrent.py index 85b9ae14..3915e9c7 100644 --- a/shelfmark/download/clients/qbittorrent.py +++ b/shelfmark/download/clients/qbittorrent.py @@ -1,11 +1,15 @@ """qBittorrent download client for Prowlarr integration.""" +from __future__ import annotations + import time from http import HTTPStatus from pathlib import Path from types import SimpleNamespace from typing import NoReturn +import requests + from shelfmark.core.config import config from shelfmark.core.logger import setup_logger from shelfmark.core.utils import normalize_http_url @@ -19,6 +23,15 @@ from shelfmark.download.clients.torrent_utils import ( ) from shelfmark.download.network import get_ssl_verify +try: + import qbittorrentapi as _qbittorrentapi +except ImportError: + _ImportedQBittorrentApiError = RuntimeError + _ImportedQBittorrentLoginFailed = RuntimeError +else: + _ImportedQBittorrentApiError = getattr(_qbittorrentapi, "APIError", RuntimeError) + _ImportedQBittorrentLoginFailed = getattr(_qbittorrentapi, "LoginFailed", RuntimeError) + logger = setup_logger(__name__) _HASH_LENGTH_40 = 40 @@ -28,6 +41,25 @@ _HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND _ONE_WEEK_IN_SECONDS = 604800 +def _resolve_qbittorrent_exception_type(candidate: object) -> type[Exception]: + if isinstance(candidate, type) and issubclass(candidate, Exception): + return candidate + return RuntimeError + + +_QBittorrentApiError = _resolve_qbittorrent_exception_type(_ImportedQBittorrentApiError) +_QBittorrentLoginFailed = _resolve_qbittorrent_exception_type(_ImportedQBittorrentLoginFailed) +_QBITTORRENT_CLIENT_ERRORS = ( + _QBittorrentLoginFailed, + _QBittorrentApiError, + AttributeError, + OSError, + RuntimeError, + TypeError, + ValueError, +) + + def _hashes_match(hash1: str, hash2: str) -> bool: """Compare hashes, handling Amarr's 40-char zero-padded hashes vs 32-char ed2k hashes.""" h1, h2 = hash1.lower(), hash2.lower() @@ -106,8 +138,6 @@ class QBittorrentClient(DownloadClient): A false result with no error means "not loaded yet". """ - import requests - url = f"{self._base_url}/api/v2/torrents/properties" params = {"hash": torrent_hash} @@ -142,7 +172,13 @@ class QBittorrentClient(DownloadClient): return False, f"Cannot connect to qBittorrent at {self._base_url}" except requests.exceptions.Timeout: return False, f"qBittorrent request timed out at {self._base_url}" - except Exception as e: + except requests.exceptions.InvalidSchema: + return ( + False, + "qBittorrent URL is invalid (missing http:// or https://). " + f"Configured: {self._base_url}", + ) + except _QBITTORRENT_CLIENT_ERRORS as e: return False, f"qBittorrent API error: {type(e).__name__}: {e}" else: return True, None @@ -193,8 +229,6 @@ class QBittorrentClient(DownloadClient): (torrents, error_message) """ - import requests - url = f"{self._base_url}/api/v2/torrents/info" def do_request(params: dict[str, str]) -> requests.Response: @@ -268,15 +302,15 @@ class QBittorrentClient(DownloadClient): except requests.exceptions.Timeout: logger.warning("qBittorrent request timed out at %s", self._base_url) return [], f"qBittorrent request timed out at {self._base_url}" - except Exception as e: + except requests.exceptions.InvalidSchema: + logger.debug("Failed to get torrents info: invalid qBittorrent URL: %s", self._base_url) + return ( + [], + "qBittorrent URL is invalid (missing http:// or https://). " + f"Configured: {self._base_url}", + ) + except _QBITTORRENT_CLIENT_ERRORS as e: logger.debug("Failed to get torrents info: %s", e) - # requests raises InvalidSchema when the base URL doesn't include http(s) - if type(e).__name__ == "InvalidSchema": - return ( - [], - "qBittorrent URL is invalid (missing http:// or https://). " - f"Configured: {self._base_url}", - ) return [], f"qBittorrent API error: {type(e).__name__}: {e}" else: return torrents, None @@ -293,7 +327,7 @@ class QBittorrentClient(DownloadClient): try: self._client.auth_log_in() api_version = self._client.app.web_api_version - except Exception as e: + except _QBITTORRENT_CLIENT_ERRORS as e: return False, f"Connection failed: {e!s}" else: return True, f"Connected to qBittorrent (API v{api_version})" @@ -304,7 +338,7 @@ class QBittorrentClient(DownloadClient): name: str, category: str | None = None, expected_hash: str | None = None, - **kwargs, + **kwargs: object, ) -> str: """Add torrent by URL (magnet or .torrent). @@ -313,6 +347,7 @@ class QBittorrentClient(DownloadClient): name: Display name for the torrent category: Category for organization (uses configured default if not specified) expected_hash: Optional info_hash hint (from Prowlarr) + **kwargs: Client-specific options passed through to the implementation. Returns: Torrent hash (info_hash). @@ -330,7 +365,7 @@ class QBittorrentClient(DownloadClient): if category: try: self._client.torrents_create_category(name=category) - except Exception as e: + except _QBITTORRENT_CLIENT_ERRORS as e: # Conflict409Error means category exists - that's expected # Log other errors but continue since download may still work if "Conflict" not in type(e).__name__ and "409" not in str(e): @@ -402,7 +437,7 @@ class QBittorrentClient(DownloadClient): "Torrent add was not confirmed within the visibility grace period (response=%s), returning expected hash", result_text, ) - except Exception: + except _QBITTORRENT_CLIENT_ERRORS: logger.exception("qBittorrent add failed") raise else: @@ -498,7 +533,7 @@ class QBittorrentClient(DownloadClient): download_speed=torrent_speed, eta=eta, ) - except Exception as e: + except _QBITTORRENT_CLIENT_ERRORS as e: return DownloadStatus.error(self._log_error("get_status", e)) def remove(self, download_id: str, *, delete_files: bool = False) -> bool: @@ -519,7 +554,7 @@ class QBittorrentClient(DownloadClient): download_id, " (with files)" if delete_files else "", ) - except Exception as e: + except _QBITTORRENT_CLIENT_ERRORS as e: self._log_error("remove", e) return False else: @@ -555,7 +590,7 @@ class QBittorrentClient(DownloadClient): return None return self._resolve_completed_download_path(torrent) - except Exception as e: + except _QBITTORRENT_CLIENT_ERRORS as e: self._log_error("get_download_path", e, level="debug") return None @@ -593,8 +628,6 @@ class QBittorrentClient(DownloadClient): """ import os - import requests - def get_with_auth(url: str, params: dict[str, str]) -> requests.Response: self._client.auth_log_in() resp = self._client._session.get(url, params=params, timeout=10) @@ -637,7 +670,7 @@ class QBittorrentClient(DownloadClient): return None return os.path.normpath(str(Path(save_path) / top_level)) - except Exception as e: + except _QBITTORRENT_CLIENT_ERRORS as e: logger.debug( "qBittorrent could not derive path from files: %s: %s", type(e).__name__, @@ -671,7 +704,7 @@ class QBittorrentClient(DownloadClient): if torrent and isinstance(getattr(torrent, "hash", None), str): torrent_hash = torrent.hash return (torrent_hash.lower(), self.get_status(torrent_hash.lower())) - except Exception as e: + except _QBITTORRENT_CLIENT_ERRORS as e: logger.debug("Error checking for existing torrent: %s", e) return None else: diff --git a/shelfmark/download/clients/rtorrent.py b/shelfmark/download/clients/rtorrent.py index 2124d1b0..df9ed39b 100644 --- a/shelfmark/download/clients/rtorrent.py +++ b/shelfmark/download/clients/rtorrent.py @@ -4,6 +4,7 @@ Uses xmlrpc to communicate with rTorrent's RPC interface. """ import ssl +import xmlrpc.client as stdlib_xmlrpc_client from typing import NoReturn from urllib.parse import urlparse @@ -24,6 +25,14 @@ logger = setup_logger(__name__) _ETA_MAX_SECONDS = 604800 +_RTORRENT_CLIENT_ERRORS = ( + AttributeError, + OSError, + RuntimeError, + TypeError, + ValueError, + stdlib_xmlrpc_client.Error, +) def _create_rtorrent_server_proxy(url: str) -> object: @@ -86,7 +95,7 @@ class RTorrentClient(DownloadClient): """Test connection to rTorrent.""" try: version = self._rpc.system.client_version() - except Exception as e: + except _RTORRENT_CLIENT_ERRORS as e: return False, f"Connection failed: {e!s}" else: return True, f"Connected to rTorrent {version}" @@ -97,7 +106,7 @@ class RTorrentClient(DownloadClient): name: str, category: str | None = None, expected_hash: str | None = None, - **kwargs, + **kwargs: object, ) -> str: """Add torrent by URL (magnet or .torrent). @@ -106,6 +115,7 @@ class RTorrentClient(DownloadClient): name: Display name for the torrent category: Category for organization (uses configured label if not specified) expected_hash: Optional info_hash hint (from Prowlarr) + **kwargs: Client-specific options passed through to the implementation. Returns: Torrent hash (info_hash). @@ -153,7 +163,7 @@ class RTorrentClient(DownloadClient): logger.debug("Added torrent to rTorrent: %s", torrent_hash) - except Exception: + except _RTORRENT_CLIENT_ERRORS: logger.exception("rTorrent add failed") raise else: @@ -213,7 +223,7 @@ class RTorrentClient(DownloadClient): try: state = int(state) - except Exception: + except TypeError, ValueError: state = 0 complete = bool(complete) @@ -255,7 +265,7 @@ class RTorrentClient(DownloadClient): eta=eta, ) - except Exception as e: + except _RTORRENT_CLIENT_ERRORS as e: error_type = type(e).__name__ logger.exception("rTorrent get_status failed (%s)", error_type) return DownloadStatus.error(f"{error_type}: {e}") @@ -284,7 +294,7 @@ class RTorrentClient(DownloadClient): download_id, " (with files)" if delete_files else "", ) - except Exception as e: + except _RTORRENT_CLIENT_ERRORS as e: error_type = type(e).__name__ logger.exception("rTorrent remove failed (%s)", error_type) return False @@ -303,7 +313,7 @@ class RTorrentClient(DownloadClient): """ try: return self._get_torrent_path(download_id) - except Exception as e: + except _RTORRENT_CLIENT_ERRORS as e: error_type = type(e).__name__ logger.debug("rTorrent get_download_path failed (%s): %s", error_type, e) return None @@ -321,9 +331,13 @@ class RTorrentClient(DownloadClient): status = self.get_status(torrent_info.info_hash) if status.state != DownloadStatus.error("").state: return (torrent_info.info_hash, status) - except Exception: - pass - except Exception as e: + except _RTORRENT_CLIENT_ERRORS as exc: + logger.debug( + "Could not fetch existing rTorrent status for %s: %s", + torrent_info.info_hash, + exc, + ) + except _RTORRENT_CLIENT_ERRORS as e: logger.debug("Error checking for existing torrent: %s", e) return None else: @@ -333,7 +347,7 @@ class RTorrentClient(DownloadClient): """Get the download directory from rTorrent config.""" try: return self._rpc.directory.default() - except Exception: + except _RTORRENT_CLIENT_ERRORS: return "/downloads" def _get_torrent_path(self, download_id: str) -> str | None: @@ -355,7 +369,7 @@ class RTorrentClient(DownloadClient): if not details: return None path = details[0][0] - except Exception: + except _RTORRENT_CLIENT_ERRORS: return None else: return path or None diff --git a/shelfmark/download/clients/sabnzbd.py b/shelfmark/download/clients/sabnzbd.py index 42d1659b..1a944d02 100644 --- a/shelfmark/download/clients/sabnzbd.py +++ b/shelfmark/download/clients/sabnzbd.py @@ -22,6 +22,13 @@ logger = setup_logger(__name__) _ETA_PART_COUNT = 3 _SPEED_PARTS_MIN = 2 +_SABNZBD_CLIENT_ERRORS = ( + requests.exceptions.RequestException, + AttributeError, + RuntimeError, + TypeError, + ValueError, +) def _parse_eta(eta_str: str) -> int | None: @@ -212,7 +219,7 @@ class SABnzbdClient(DownloadClient): return response.content def _get_prowlarr_headers(self, url: str) -> dict: - # TODO: Move this source-specific Prowlarr auth handling into a source hook. + # TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook. api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip() if not api_key: return {} @@ -283,7 +290,7 @@ class SABnzbdClient(DownloadClient): return False, "Could not connect to SABnzbd" except requests.exceptions.Timeout: return False, "Connection timed out" - except Exception as e: + except _SABNZBD_CLIENT_ERRORS as e: return False, f"Connection failed: {e!s}" else: return True, f"Connected to SABnzbd {version}" @@ -294,7 +301,7 @@ class SABnzbdClient(DownloadClient): name: str, category: str | None = None, expected_hash: str | None = None, - **kwargs, + **kwargs: object, ) -> str: """Add NZB by URL. @@ -303,6 +310,7 @@ class SABnzbdClient(DownloadClient): name: Display name for the download category: Category for organization (uses configured default if not specified) expected_hash: Optional info_hash hint (unused) + **kwargs: Client-specific options passed through to the implementation. Returns: SABnzbd nzo_id. @@ -321,7 +329,7 @@ class SABnzbdClient(DownloadClient): result = self._api_post_file(nzb_content, nzb_filename, name, category) nzo_id = self._extract_nzo_id(result) logger.info("Added NZB to SABnzbd: %s", nzo_id) - except Exception as e: + except _SABNZBD_CLIENT_ERRORS as e: logger.warning("SABnzbd addfile failed, falling back to addurl: %s", e) else: return nzo_id @@ -337,7 +345,7 @@ class SABnzbdClient(DownloadClient): ) nzo_id = self._extract_nzo_id(result) logger.info("Added NZB to SABnzbd via addurl: %s", nzo_id) - except Exception: + except _SABNZBD_CLIENT_ERRORS: logger.exception("SABnzbd add failed") raise else: @@ -447,7 +455,7 @@ class SABnzbdClient(DownloadClient): # Not found logger.warning("SABnzbd: download %s not found in queue or history", download_id) return DownloadStatus.error("Download not found") - except Exception as e: + except _SABNZBD_CLIENT_ERRORS as e: return DownloadStatus.error(self._log_error("get_status", e)) def remove(self, download_id: str, *, delete_files: bool = False, archive: bool = True) -> bool: @@ -477,7 +485,7 @@ class SABnzbdClient(DownloadClient): if result.get("status"): logger.info("Removed NZB from SABnzbd queue: %s", download_id) return True - except Exception as e: + except _SABNZBD_CLIENT_ERRORS as e: logger.debug("SABnzbd queue delete skipped for %s: %s", download_id, e) # If not in queue (or queue delete failed), try to remove from history. @@ -496,7 +504,7 @@ class SABnzbdClient(DownloadClient): action = "archived" if archive else "removed" logger.info("NZB %s from SABnzbd history: %s", action, download_id) return True - except Exception as e: + except _SABNZBD_CLIENT_ERRORS as e: self._log_error("remove", e) return False @@ -583,7 +591,7 @@ class SABnzbdClient(DownloadClient): logger.debug("Found existing NZB in SABnzbd history: %s", nzo_id) return (nzo_id, status) - except Exception as e: + except _SABNZBD_CLIENT_ERRORS as e: logger.debug("Error checking for existing NZB: %s", e) return None else: diff --git a/shelfmark/download/clients/settings.py b/shelfmark/download/clients/settings.py index e51c83fb..6d554d3e 100644 --- a/shelfmark/download/clients/settings.py +++ b/shelfmark/download/clients/settings.py @@ -16,10 +16,53 @@ from shelfmark.core.settings_registry import ( from shelfmark.core.utils import get_hardened_xmlrpc_client, normalize_http_url from shelfmark.download.network import get_ssl_verify +try: + import qbittorrentapi as _qbittorrentapi +except ImportError: + _ImportedQBittorrentApiError = RuntimeError + _ImportedQBittorrentLoginFailed = RuntimeError +else: + _ImportedQBittorrentApiError = getattr(_qbittorrentapi, "APIError", RuntimeError) + _ImportedQBittorrentLoginFailed = getattr(_qbittorrentapi, "LoginFailed", RuntimeError) + +try: + from transmission_rpc import TransmissionError as _ImportedTransmissionError +except ImportError: + _ImportedTransmissionError = RuntimeError + if TYPE_CHECKING: from collections.abc import Iterator # ==================== Test Connection Callbacks ==================== +_DELUGE_HOST_ENTRY_MIN_LENGTH = 2 + + +def _resolve_exception_type(candidate: object) -> type[Exception]: + if isinstance(candidate, type) and issubclass(candidate, Exception): + return candidate + return RuntimeError + + +_QBittorrentApiError = _resolve_exception_type(_ImportedQBittorrentApiError) +_QBittorrentLoginFailed = _resolve_exception_type(_ImportedQBittorrentLoginFailed) +_TransmissionError = _resolve_exception_type(_ImportedTransmissionError) +_QBITTORRENT_SETTINGS_ERRORS = ( + _QBittorrentLoginFailed, + _QBittorrentApiError, + AttributeError, + OSError, + RuntimeError, + TypeError, + ValueError, +) +_TRANSMISSION_SETTINGS_ERRORS = ( + _TransmissionError, + AttributeError, + OSError, + RuntimeError, + TypeError, + ValueError, +) def _raise_runtime_error(message: str) -> NoReturn: @@ -84,7 +127,7 @@ def _test_qbittorrent_connection(current_values: dict[str, Any] | None = None) - api_version = client.app.web_api_version except ImportError: return {"success": False, "message": "qbittorrent-api package not installed"} - except Exception as e: + except _QBITTORRENT_SETTINGS_ERRORS as e: return {"success": False, "message": f"Connection failed: {e!s}"} else: return {"success": True, "message": f"Connected to qBittorrent (API v{api_version})"} @@ -150,7 +193,7 @@ def _test_transmission_connection(current_values: dict[str, Any] | None = None) version = session.version except ImportError: return {"success": False, "message": "transmission-rpc package not installed"} - except Exception as e: + except _TRANSMISSION_SETTINGS_ERRORS as e: return {"success": False, "message": f"Connection failed: {e!s}"} else: return {"success": True, "message": f"Connected to Transmission {version}"} @@ -245,7 +288,7 @@ def _test_deluge_connection(current_values: dict[str, Any] | None = None) -> dic for entry in hosts: if ( isinstance(entry, list) - and len(entry) >= 2 + and len(entry) >= _DELUGE_HOST_ENTRY_MIN_LENGTH and entry[1] in {"127.0.0.1", "localhost"} ): host_id = entry[0] diff --git a/shelfmark/download/clients/torrent_utils.py b/shelfmark/download/clients/torrent_utils.py index 068f03ae..cfb8f6a3 100644 --- a/shelfmark/download/clients/torrent_utils.py +++ b/shelfmark/download/clients/torrent_utils.py @@ -3,6 +3,7 @@ import base64 import hashlib import re +from binascii import Error as BinasciiError from dataclasses import dataclass from urllib.parse import parse_qs, urljoin, urlparse @@ -21,6 +22,14 @@ _BTIH_PREFIX_BYTE = 0x12 _BTIH_DIGEST_LENGTH = 32 _BTIH_HASH_LENGTH_40 = 40 _BTIH_HASH_LENGTH_32 = 32 +_TORRENT_FETCH_ERRORS = ( + requests.exceptions.RequestException, + OSError, + RuntimeError, + TypeError, + ValueError, +) +_TORRENT_PARSE_ERRORS = (IndexError, KeyError, TypeError, ValueError) @dataclass @@ -82,7 +91,7 @@ def extract_torrent_info( return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False) headers: dict[str, str] = {"Accept": "application/x-bittorrent"} - # TODO: Move this source-specific Prowlarr auth handling into a source hook. + # TODO(shelfmark): Move this source-specific Prowlarr auth handling into a source hook. api_key = str(config.get("PROWLARR_API_KEY", "") or "").strip() if api_key: headers["X-Api-Key"] = api_key @@ -135,21 +144,18 @@ def extract_torrent_info( # Check if response is actually a magnet link (text response) # Some indexers return magnet links as plain text instead of redirecting if len(torrent_data) < _MAGNET_RESPONSE_MAX_BYTES: # Magnet links are typically short - try: - text_content = torrent_data.decode("utf-8", errors="ignore").strip() - if text_content.startswith("magnet:"): - logger.debug("Download URL returned magnet link as response body") - info_hash = extract_hash_from_magnet(text_content) - if not info_hash and expected_hash: - info_hash = expected_hash - return TorrentInfo( - info_hash=info_hash, - torrent_data=None, - is_magnet=True, - magnet_url=text_content, - ) - except Exception: - pass # Not text, continue with torrent parsing + text_content = torrent_data.decode("utf-8", errors="ignore").strip() + if text_content.startswith("magnet:"): + logger.debug("Download URL returned magnet link as response body") + info_hash = extract_hash_from_magnet(text_content) + if not info_hash and expected_hash: + info_hash = expected_hash + return TorrentInfo( + info_hash=info_hash, + torrent_data=None, + is_magnet=True, + magnet_url=text_content, + ) info_hash = extract_info_hash_from_torrent(torrent_data) or expected_hash if info_hash: @@ -157,7 +163,7 @@ def extract_torrent_info( else: logger.warning("Could not extract hash from torrent file") return TorrentInfo(info_hash=info_hash, torrent_data=torrent_data, is_magnet=False) - except Exception as e: + except _TORRENT_FETCH_ERRORS as e: logger.debug("Could not fetch torrent file: %s", e) return TorrentInfo(info_hash=expected_hash, torrent_data=None, is_magnet=False) @@ -256,9 +262,10 @@ def extract_info_hash_from_torrent(torrent_data: bytes) -> str | None: info_bencoded = bencode_encode(decoded[b"info"]) info_dict = decoded[b"info"] if isinstance(info_dict, dict) and b"pieces" in info_dict: - return hashlib.sha1(info_bencoded).hexdigest().lower() + # BitTorrent v1 info hashes are defined as SHA-1. + return hashlib.sha1(info_bencoded).hexdigest().lower() # noqa: S324 return hashlib.sha256(info_bencoded).hexdigest().lower() - except Exception as e: + except _TORRENT_PARSE_ERRORS as e: logger.debug("Failed to parse torrent file: %s", e) return None @@ -288,7 +295,7 @@ def extract_hash_from_magnet(magnet_url: str) -> str | None: padded = raw_value.upper() + "=" * (-len(raw_value) % 8) try: data = base64.b32decode(padded, casefold=True) - except Exception: + except BinasciiError, ValueError: return None if not data: @@ -326,8 +333,10 @@ def extract_hash_from_magnet(magnet_url: str) -> str | None: if re.match(r"^[A-Z2-7]{32}$", hash_value.upper()): try: return base64.b32decode(hash_value.upper()).hex().lower() - except Exception: - pass + except BinasciiError, ValueError: + logger.debug( + "Could not decode base32 BTIH hash from magnet URI: %s", hash_value + ) # Fallback: return as-is return hash_value.lower() diff --git a/shelfmark/download/clients/transmission.py b/shelfmark/download/clients/transmission.py index 8676fb47..831da814 100644 --- a/shelfmark/download/clients/transmission.py +++ b/shelfmark/download/clients/transmission.py @@ -20,6 +20,11 @@ from shelfmark.download.clients.torrent_utils import ( ) from shelfmark.download.network import get_ssl_verify +try: + from transmission_rpc import TransmissionError as _ImportedTransmissionError +except ImportError: + _ImportedTransmissionError = RuntimeError + if TYPE_CHECKING: from collections.abc import Iterator @@ -27,6 +32,20 @@ logger = setup_logger(__name__) _SEEDING_PROGRESS_PERCENT = 100 _ETA_MAX_SECONDS = 604800 +_TransmissionError = ( + _ImportedTransmissionError + if isinstance(_ImportedTransmissionError, type) + and issubclass(_ImportedTransmissionError, Exception) + else RuntimeError +) +_TRANSMISSION_CLIENT_ERRORS = ( + _TransmissionError, + AttributeError, + OSError, + RuntimeError, + TypeError, + ValueError, +) @contextmanager @@ -43,13 +62,13 @@ def _transmission_session_verify_override(url: str) -> Iterator[None]: try: import transmission_rpc.client as transmission_rpc_client - except Exception: + + original_session_factory = transmission_rpc_client.requests.Session + except AttributeError, ImportError: # If internals differ, gracefully fall back to default behavior. yield return - original_session_factory = transmission_rpc_client.requests.Session - def _session_factory(*args: object, **kwargs: object) -> object: session = original_session_factory(*args, **kwargs) session.verify = False @@ -69,7 +88,7 @@ def _apply_transmission_ssl_verify(client: object, url: str) -> None: return try: session.verify = get_ssl_verify(url) - except Exception as e: + except (AttributeError, OSError, TypeError, ValueError) as e: logger.debug("Unable to apply Transmission TLS verify setting: %s", e) @@ -138,7 +157,7 @@ class TransmissionClient(DownloadClient): try: session = self._client.get_session() version = session.version - except Exception as e: + except _TRANSMISSION_CLIENT_ERRORS as e: return False, f"Connection failed: {e!s}" else: return True, f"Connected to Transmission {version}" @@ -149,7 +168,7 @@ class TransmissionClient(DownloadClient): name: str, category: str | None = None, expected_hash: str | None = None, - **kwargs, + **kwargs: object, ) -> str: """Add torrent by URL (magnet or .torrent). @@ -158,6 +177,7 @@ class TransmissionClient(DownloadClient): name: Display name for the torrent category: Category for organization (uses configured default if not specified) expected_hash: Optional info_hash hint (from Prowlarr) + **kwargs: Client-specific options passed through to the implementation. Returns: Torrent hash (info_hash). @@ -206,10 +226,10 @@ class TransmissionClient(DownloadClient): if seed_kwargs: try: self._client.change_torrent(ids=torrent_hash, **seed_kwargs) - except Exception as e: + except _TRANSMISSION_CLIENT_ERRORS as e: logger.warning("Failed to set seeding limits for %s: %s", torrent_hash, e) - except Exception: + except _TRANSMISSION_CLIENT_ERRORS: logger.exception("Transmission add failed") raise else: @@ -292,7 +312,7 @@ class TransmissionClient(DownloadClient): except KeyError: return DownloadStatus.error("Torrent not found") - except Exception as e: + except _TRANSMISSION_CLIENT_ERRORS as e: return DownloadStatus.error(self._log_error("get_status", e)) def remove(self, download_id: str, *, delete_files: bool = False) -> bool: @@ -316,7 +336,7 @@ class TransmissionClient(DownloadClient): download_id, " (with files)" if delete_files else "", ) - except Exception as e: + except _TRANSMISSION_CLIENT_ERRORS as e: self._log_error("remove", e) return False else: @@ -341,7 +361,7 @@ class TransmissionClient(DownloadClient): getattr(torrent, "download_dir", ""), torrent_name, ) - except Exception as e: + except _TRANSMISSION_CLIENT_ERRORS as e: self._log_error("get_download_path", e, level="debug") return None @@ -361,6 +381,6 @@ class TransmissionClient(DownloadClient): return None else: return (torrent_info.info_hash, status) - except Exception as e: + except _TRANSMISSION_CLIENT_ERRORS as e: logger.debug("Error checking for existing torrent: %s", e) return None diff --git a/shelfmark/download/fs.py b/shelfmark/download/fs.py index 191b235e..9224307b 100644 --- a/shelfmark/download/fs.py +++ b/shelfmark/download/fs.py @@ -53,7 +53,14 @@ def _call_and_capture[T]( ) -> tuple[bool, T | Exception]: try: return True, func(*args, **kwargs) - except Exception as exc: + except ( + AttributeError, + OSError, + RuntimeError, + TypeError, + ValueError, + subprocess.SubprocessError, + ) as exc: return False, exc @@ -118,10 +125,11 @@ def _verify_transfer_size( actual_size = run_blocking_io(dest.stat).st_size if actual_size != expected_size: - raise OSError( + msg = ( f"File {action} incomplete, data loss may have occurred. " f"'{dest}' was {actual_size} bytes instead of expected {expected_size}." ) + raise OSError(msg) def _is_stale_handle_error(error: Exception) -> bool: @@ -206,7 +214,8 @@ def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path: else: return try_path - raise RuntimeError(f"Could not write file after {max_attempts} attempts: {dest_path}") + msg = f"Could not write file after {max_attempts} attempts: {dest_path}" + raise RuntimeError(msg) def _is_permission_error(e: Exception) -> bool: @@ -521,7 +530,8 @@ def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> else: return try_path - raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}") + msg = f"Could not move file after {max_attempts} attempts: {dest_path}" + raise RuntimeError(msg) def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path: @@ -572,7 +582,8 @@ def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) else: return try_path - raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}") + msg = f"Could not create hardlink after {max_attempts} attempts: {dest_path}" + raise RuntimeError(msg) def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path: @@ -665,4 +676,5 @@ def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> else: return try_path - raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}") + msg = f"Could not copy file after {max_attempts} attempts: {dest_path}" + raise RuntimeError(msg) diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index cfe06f0b..ec24c890 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -11,6 +11,7 @@ from urllib.parse import urljoin, urlparse import requests from tqdm import tqdm +from shelfmark.bypass import BypassCancelledError from shelfmark.core.config import config as app_config from shelfmark.core.logger import setup_logger from shelfmark.download import network @@ -21,6 +22,7 @@ if TYPE_CHECKING: from types import ModuleType logger = setup_logger(__name__) +_RNG = random.SystemRandom() _MAX_REDIRECTS = 5 _HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN @@ -30,6 +32,17 @@ _HTTP_STATUS_OK = HTTPStatus.OK _HTTP_STATUS_RANGE_NOT_SATISFIABLE = HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE _HTTP_STATUS_PARTIAL_CONTENT = HTTPStatus.PARTIAL_CONTENT _HTTP_STATUS_NON_RETRYABLE = (_HTTP_STATUS_FORBIDDEN, _HTTP_STATUS_NOT_FOUND) +_STATUS_CALLBACK_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError) +_BYPASSER_ERRORS = ( + AttributeError, + BypassCancelledError, + KeyError, + OSError, + RuntimeError, + TypeError, + ValueError, + requests.exceptions.RequestException, +) # Bypasser modules are imported lazily to support dynamic selection based on config _internal_bypasser = None @@ -90,7 +103,7 @@ def get_bypassed_page( selector: network.AAMirrorSelector | None = None, cancel_flag: Event | None = None, ) -> str | None: - """Wrapper that delegates to the appropriate bypasser based on config.""" + """Fetch a bypassed page using the active bypasser implementation.""" if _is_using_external_bypasser(): return _get_external_bypasser().get_bypassed_page(url, selector, cancel_flag) return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag) @@ -168,7 +181,7 @@ def parse_size_string(size: str) -> float | None: def _backoff_delay(attempt: int, base: float = 0.25, cap: float = 3.0) -> float: """Exponential backoff with jitter.""" - return min(cap, base * (2 ** (attempt - 1))) + random.random() * base + return min(cap, base * (2 ** (attempt - 1))) + _RNG.random() * base def _get_status_code(e: Exception) -> int | None: @@ -218,11 +231,18 @@ def html_get_page( """Fetch HTML content from a URL with retry mechanism. Args: + url: URL to fetch. + retry: Maximum number of attempts before giving up. + selector: Mirror selector used for AA mirror and DNS rotation. + cancel_flag: Optional event used to abort retries early. + status_callback: Optional callback for UI status updates. allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation instead of switching to the bypasser. Use for search operations. + use_bypasser: Whether to start with the bypasser instead of direct HTTP. include_response_url: If True, return `(html, final_url)` to expose the resolved response URL after redirects. success_delay: Optional delay (seconds) after successful fetch. + session: Optional requests session to reuse across attempts. """ @@ -258,7 +278,7 @@ def html_get_page( return try: status_callback("resolving", "Bypassing protection...") - except Exception: + except _STATUS_CALLBACK_ERRORS: return heartbeat_thread = Thread( @@ -268,7 +288,7 @@ def html_get_page( try: result = get_bypassed_page(current_url, selector, cancel_flag) return _result(result or "", current_url) - except Exception as e: + except _BYPASSER_ERRORS as e: logger.warning("Bypasser error: %s: %s", type(e).__name__, e) return _result("", current_url) finally: @@ -521,7 +541,7 @@ def download_url( time.sleep(0.5) # Retry with fresh cookies (don't increment attempt) continue - except Exception as cookie_err: + except _BYPASSER_ERRORS as cookie_err: logger.warning("Z-Library cookie refresh failed: %s", cookie_err) # Non-retryable errors diff --git a/shelfmark/download/network.py b/shelfmark/download/network.py index a848cda9..1477e380 100644 --- a/shelfmark/download/network.py +++ b/shelfmark/download/network.py @@ -5,12 +5,14 @@ import ipaddress import socket import urllib.parse import urllib.request -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta +from http import HTTPStatus from socket import AddressFamily, SocketKind from typing import TYPE_CHECKING, Any, cast import dns.resolver import requests +from dns.exception import DNSException from shelfmark.core.config import config as app_config from shelfmark.core.logger import setup_logger @@ -92,8 +94,7 @@ def get_proxies(url: str = "") -> dict: def get_ssl_verify(url: str = "") -> bool: - """Return the ``verify`` value for outbound requests based on the - CERTIFICATE_VALIDATION setting. + """Return the ``verify`` value for outbound requests. - ``enabled`` → always ``True`` - ``disabled_local`` → ``False`` for local/private addresses, ``True`` otherwise @@ -120,8 +121,7 @@ _ssl_warnings_suppressed = False def _apply_ssl_warning_suppression() -> None: - """Suppress or restore urllib3 InsecureRequestWarning based on the - CERTIFICATE_VALIDATION setting. + """Suppress or restore urllib3 InsecureRequestWarning. Called once at init and again whenever the setting changes via the UI. Only modifies warning filters when the mode is not 'enabled', so the @@ -161,6 +161,7 @@ except ImportError: _using_gevent_locks = False logger = setup_logger(__name__) +_GETADDRINFO_SOCKADDR_INDEX = 4 def _call_dns_rotation_callback( @@ -173,7 +174,7 @@ def _call_dns_rotation_callback( try: logger.debug("Calling DNS rotation callback: %s", callback.__name__) callback(provider_name, servers, doh_url) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.warning("DNS rotation callback %s failed: %s", callback.__name__, e) @@ -228,7 +229,9 @@ def _load_state() -> dict[str, Any]: """Return current in-memory network state (no disk persistence).""" if state.get("chosen_at"): chosen = datetime.fromisoformat(state["chosen_at"]) - if datetime.now() - chosen > timedelta(days=STATE_TTL_DAYS): + if chosen.tzinfo is None: + chosen = chosen.replace(tzinfo=UTC) + if datetime.now(UTC) - chosen > timedelta(days=STATE_TTL_DAYS): state.clear() return state @@ -239,7 +242,7 @@ def _save_state(aa_url: str | None = None, dns_provider: str | None = None) -> N state["aa_base_url"] = aa_url if dns_provider: state["dns_provider"] = dns_provider - state["chosen_at"] = datetime.now().isoformat() + state["chosen_at"] = datetime.now(UTC).isoformat() # AA URL failover state @@ -451,7 +454,7 @@ class DoHResolver: key = (hostname, record_type) if key in self._cache: ips, timestamp = self._cache[key] - if datetime.now() - timestamp < timedelta(seconds=self.CACHE_TTL): + if datetime.now(UTC) - timestamp < timedelta(seconds=self.CACHE_TTL): logger.debug("DoH cache hit for %s: %s", hostname, ips) return ips # Cache expired, remove it @@ -461,7 +464,7 @@ class DoHResolver: def _set_cached(self, hostname: str, record_type: str, ips: list[str]) -> None: """Cache DNS result.""" if ips: # Only cache non-empty results - self._cache[(hostname, record_type)] = (ips, datetime.now()) + self._cache[(hostname, record_type)] = (ips, datetime.now(UTC)) def resolve(self, hostname: str, record_type: str) -> list[str]: """Resolve a hostname using DoH. @@ -522,7 +525,7 @@ class DoHResolver: self._set_cached(hostname, record_type, answers) # Don't log here - the caller (custom_getaddrinfo) will log the final result - except Exception as e: + except (OSError, ValueError, requests.RequestException) as e: logger.warning("DoH resolution failed for %s: %s", hostname, e) return [] else: @@ -545,7 +548,7 @@ def resolve_with_custom_dns( try: answers = resolver.resolve(hostname, record_type) return [str(answer) for answer in answers] - except Exception: + except DNSException: # Don't log here - let the caller handle it to prevent spam # Don't trigger DNS switch here either - caller handles it return [] @@ -575,7 +578,7 @@ def create_custom_getaddrinfo( host: str | bytes | None, port: str | bytes | int | None, family: int = 0, - type: int = 0, + socket_type: int = 0, proto: int = 0, flags: int = 0, ) -> Sequence[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]]: @@ -601,15 +604,21 @@ def create_custom_getaddrinfo( # Skip logging entirely for localhost to reduce noise if host_str in ("localhost", "127.0.0.1", "::1"): return - try: - ips = [entry[4][0] for entry in res if len(entry) >= 5 and entry[4]] - msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}" - if is_bypass: - logger.debug(msg) - else: - logger.info(msg) - except Exception: - pass # Silently ignore logging failures + ips = [] + for entry in res: + if not isinstance(entry, tuple) or len(entry) <= _GETADDRINFO_SOCKADDR_INDEX: + continue + sockaddr = entry[_GETADDRINFO_SOCKADDR_INDEX] + if not isinstance(sockaddr, tuple) or not sockaddr: + continue + ip = sockaddr[0] + if isinstance(ip, str): + ips.append(ip) + msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}" + if is_bypass: + logger.debug(msg) + else: + logger.info(msg) # Skip custom resolution for IP addresses, local addresses, or if skip check passes if ( @@ -618,7 +627,7 @@ def create_custom_getaddrinfo( or (skip_check and skip_check(host_str)) ): # Quietly bypass custom resolution for IP/local targets - res = original_getaddrinfo(host, port, family, type, proto, flags) + res = original_getaddrinfo(host, port, family, socket_type, proto, flags) _log_results("system resolver (bypass)", "system", res, is_bypass=True) return res @@ -630,7 +639,13 @@ def create_custom_getaddrinfo( ipv4_answers = resolve_ipv4(host_str) results.extend( [ - (socket.AF_INET, cast("SocketKind", type), proto, "", (answer, port_int)) + ( + socket.AF_INET, + cast("SocketKind", socket_type), + proto, + "", + (answer, port_int), + ) for answer in ipv4_answers ] ) @@ -639,7 +654,14 @@ def create_custom_getaddrinfo( _log_results("custom resolver", _current_dns_label(), results) return results - except Exception as e: + except ( + DNSException, + OSError, + RuntimeError, + TypeError, + ValueError, + requests.RequestException, + ) as e: logger.warning( "Custom DNS resolution failed for %s: %s, falling back to system DNS", host_str, e ) @@ -660,14 +682,22 @@ def create_custom_getaddrinfo( "Custom DNS returned no addresses for %s; falling back to system resolver", host_str ) try: - res = original_getaddrinfo(host, port, family, type, proto, flags) + res = original_getaddrinfo(host, port, family, socket_type, proto, flags) _log_results("system resolver (fallback)", "system", res) - except Exception: + except OSError: logger.exception("System DNS resolution also failed for %s", host_str) # Last resort: Try to connect to the hostname directly if family in {0, socket.AF_INET}: logger.warning("Using direct hostname as last resort for %s", host_str) - return [(socket.AF_INET, cast("SocketKind", type), proto, "", (host_str, port_int))] + return [ + ( + socket.AF_INET, + cast("SocketKind", socket_type), + proto, + "", + (host_str, port_int), + ) + ] raise # Re-raise the exception if we can't provide a last resort else: return res @@ -686,14 +716,14 @@ def create_system_failover_getaddrinfo() -> Callable[ host: str | bytes | None, port: str | bytes | int | None, family: int = 0, - type: int = 0, + socket_type: int = 0, proto: int = 0, flags: int = 0, ) -> Sequence[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]]: host_str = _decode_host(host) try: - return original_getaddrinfo(host, port, family, type, proto, flags) - except Exception as e: + return original_getaddrinfo(host, port, family, socket_type, proto, flags) + except OSError as e: if host_str not in _switch_logged: logger.warning("System DNS resolution failed for %s: %s", host_str, e) @@ -708,14 +738,14 @@ def create_system_failover_getaddrinfo() -> Callable[ logger.info("Switching DNS provider after system DNS failure for %s", host_str) _switch_logged.add(host_str) if switch_dns_provider(): - return socket.getaddrinfo(host, port, family, type, proto, flags) + return socket.getaddrinfo(host, port, family, socket_type, proto, flags) raise return system_failover_getaddrinfo def _init_doh_resolver_internal(doh_server: str) -> DoHResolver: - """Internal: Initialize DNS over HTTPS resolver with specified server. + """Initialize a DNS-over-HTTPS resolver for the given server. Args: doh_server: The DoH server URL @@ -739,7 +769,7 @@ def _init_doh_resolver_internal(doh_server: str) -> DoHResolver: # Restore custom getaddrinfo if it was previously set socket.getaddrinfo = temp_getaddrinfo - except Exception: + except OSError: logger.exception("Failed to resolve DoH server %s", server_hostname) # Fall back to a known public DNS if resolution fails server_ip = "1.1.1.1" @@ -773,7 +803,7 @@ def _init_doh_resolver_internal(doh_server: str) -> DoHResolver: def _init_custom_resolver_internal(servers: list[str]) -> dns.resolver.Resolver: - """Internal: Initialize custom DNS resolver with specified servers. + """Initialize a custom DNS resolver for the given servers. Args: servers: List of DNS server IPs to use @@ -856,8 +886,7 @@ def rotate_dns_provider() -> bool: def rotate_dns_and_reset_aa() -> bool: - """Switch DNS provider (auto mode) and reset AA URL list to the first entry. - Returns True if DNS switched; False if no providers left or not in auto mode. + """Switch DNS provider and reset the AA URL list. Note: This function can be called during initialization, so we must NOT call _ensure_initialized() here to avoid recursive init loops. @@ -1090,13 +1119,13 @@ def _initialize_aa_state() -> None: response = requests.get( url, proxies=get_proxies(url), timeout=3, verify=get_ssl_verify(url) ) - if response.status_code == 200: + if response.status_code == HTTPStatus.OK: _current_aa_url_index = i _aa_base_url = url _save_state(aa_url=_aa_base_url) break - except Exception: - pass + except (OSError, requests.RequestException) as exc: + logger.debug("Could not reach AA mirror candidate %s: %s", url, exc) if not _aa_base_url or _aa_base_url == "auto": _aa_base_url = _aa_urls[0] _current_aa_url_index = 0 @@ -1223,11 +1252,13 @@ def set_aa_url_index(new_index: int) -> bool: class AAMirrorSelector: - """Small helper to keep AA mirror switching consistent across call sites. + """Keep AA mirror switching consistent across call sites. + Tracks attempts per DNS cycle and rewrites URLs safely. """ def __init__(self) -> None: + """Initialize mirror state from the current AA configuration.""" self._ensure_fresh_state(reset_attempts=True) def _ensure_fresh_state(self, *, reset_attempts: bool = False) -> None: @@ -1251,7 +1282,8 @@ class AAMirrorSelector: return url def next_mirror_or_rotate_dns(self, *, allow_dns: bool = True) -> tuple[str | None, str]: - """Advance to next mirror; if exhausted and allowed, rotate DNS and reset to first. + """Advance to the next mirror or rotate DNS if needed. + Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'. """ self.attempts_this_dns += 1 diff --git a/shelfmark/download/orchestrator.py b/shelfmark/download/orchestrator.py index 1e2d0168..3caca2d7 100644 --- a/shelfmark/download/orchestrator.py +++ b/shelfmark/download/orchestrator.py @@ -33,6 +33,7 @@ from shelfmark.release_sources import ( ) logger = setup_logger(__name__) +_RNG = random.SystemRandom() # ============================================================================= @@ -263,7 +264,7 @@ def queue_release( error_msg = f"Missing required field in release data: {e}" logger.warning(error_msg) return False, error_msg - except Exception as e: + except (AttributeError, OSError, RuntimeError, TypeError) as e: error_msg = f"Error queueing release: {e}" logger.error_trace(error_msg) return False, error_msg @@ -303,7 +304,7 @@ def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]: with Path(path).open("rb") as f: return f.read(), task - except Exception as e: + except OSError as e: logger.error_trace(f"Error getting book data: {e}") if task: task.download_path = None @@ -545,7 +546,7 @@ def _capture_task_error( task.last_error_type = normalized_type -def _format_download_exception_message(exc: Exception) -> str: +def _format_download_exception_message(exc: BaseException) -> str: if isinstance(exc, PermissionError) and "/cwa-book-ingest" in str(exc): return "Destination misconfigured. Go to Settings → Downloads to update." if isinstance(exc, PermissionError): @@ -555,141 +556,122 @@ def _format_download_exception_message(exc: Exception) -> str: def _download_task(task_id: str, cancel_flag: Event) -> str | None: """Download a task via appropriate handler, then post-process to ingest.""" - try: - # Check for cancellation before starting - if cancel_flag.is_set(): - logger.info("Task %s: cancelled before starting", task_id) - return None - - task = book_queue.get_task(task_id) - if not task: - logger.error("Task not found in queue: %s", task_id) - return None - - title_label = task.title or "Unknown title" - logger.info( - "Task %s: starting download (%s) - %s", - task_id, - get_source_display_name(task.source), - title_label, - ) - - def progress_callback(progress: float) -> None: - update_download_progress(task_id, progress) - - def status_callback(status: str, message: str | None = None) -> None: - status_key = status.lower() - if status_key == "error": - _capture_task_error( - task, - message=message or "Download failed", - exc_type="StatusCallbackError", - ) - return - # Don't propagate terminal statuses to the queue here. Output modules - # call status_callback("complete") before returning the download path, - # but _process_single_download needs to set download_path on the task - # first so the terminal hook captures it for history persistence. - if status_key in ("complete", "cancelled"): - if message is not None: - book_queue.update_status_message(task_id, message) - return - update_download_status(task_id, status, message) - - # Get the download handler based on the task's source - handler = get_handler(task.source) - temp_file: Path | None = None - - if task.staged_path: - staged_file = Path(task.staged_path) - if run_blocking_io(staged_file.exists): - temp_file = staged_file - logger.info("Task %s: reusing staged file for retry: %s", task_id, staged_file) - else: - task.staged_path = None - - if temp_file is None: - temp_path = handler.download( - task, - cancel_flag, - progress_callback, - status_callback, - ) - - # Handler returns temp path - orchestrator handles post-processing - if not temp_path: - return None - - temp_file = Path(temp_path) - if not run_blocking_io(temp_file.exists): - logger.error("Handler returned non-existent path: %s", temp_path) - _capture_task_error( - task, - message=f"Download file missing: {temp_path}", - exc_type="MissingDownloadPath", - ) - return None - - # Check cancellation before post-processing - if cancel_flag.is_set(): - logger.info("Task %s: cancelled before post-processing", task_id) - if not is_torrent_source(temp_file, task): - safe_cleanup_path(temp_file, task) - return None - - logger.info("Task %s: download finished; starting post-processing", task_id) - logger.debug("Task %s: post-processing input path: %s", task_id, temp_file) - task.staged_path = str(temp_file) - preserve_source_on_failure = True - - # Post-processing: output routing + file processing pipeline - result = post_process_download( - temp_file, - task, - cancel_flag, - status_callback, - preserve_source_on_failure=preserve_source_on_failure, - ) - - if cancel_flag.is_set(): - logger.info("Task %s: post-processing cancelled", task_id) - elif result: - logger.info("Task %s: post-processing complete", task_id) - logger.debug("Task %s: post-processing result: %s", task_id, result) - else: - logger.warning("Task %s: post-processing failed", task_id) - if not task.last_error_message: - _capture_task_error( - task, - message="Download failed", - exc_type="UnknownFailure", - ) - - try: - handler.post_process_cleanup(task, success=bool(result)) - except Exception as e: - logger.warning("Post-processing cleanup hook failed for %s: %s", task_id, e) - - if result: - task.staged_path = None - _clear_task_error_state(task) - - except Exception as e: - if cancel_flag.is_set(): - logger.info("Task %s: cancelled during error handling", task_id) - else: - logger.error_trace("Task %s: error downloading: %s", task_id, e) - task = book_queue.get_task(task_id) - if task: - _capture_task_error( - task, - message=_format_download_exception_message(e), - exc_type=type(e).__name__, - ) + # Check for cancellation before starting + if cancel_flag.is_set(): + logger.info("Task %s: cancelled before starting", task_id) return None + task = book_queue.get_task(task_id) + if not task: + logger.error("Task not found in queue: %s", task_id) + return None + + title_label = task.title or "Unknown title" + logger.info( + "Task %s: starting download (%s) - %s", + task_id, + get_source_display_name(task.source), + title_label, + ) + + def progress_callback(progress: float) -> None: + update_download_progress(task_id, progress) + + def status_callback(status: str, message: str | None = None) -> None: + status_key = status.lower() + if status_key == "error": + _capture_task_error( + task, + message=message or "Download failed", + exc_type="StatusCallbackError", + ) + return + # Don't propagate terminal statuses to the queue here. Output modules + # call status_callback("complete") before returning the download path, + # but _process_single_download needs to set download_path on the task + # first so the terminal hook captures it for history persistence. + if status_key in ("complete", "cancelled"): + if message is not None: + book_queue.update_status_message(task_id, message) + return + update_download_status(task_id, status, message) + + # Get the download handler based on the task's source + handler = get_handler(task.source) + temp_file: Path | None = None + + if task.staged_path: + staged_file = Path(task.staged_path) + if run_blocking_io(staged_file.exists): + temp_file = staged_file + logger.info("Task %s: reusing staged file for retry: %s", task_id, staged_file) + else: + task.staged_path = None + + if temp_file is None: + temp_path = handler.download( + task, + cancel_flag, + progress_callback, + status_callback, + ) + + # Handler returns temp path - orchestrator handles post-processing + if not temp_path: + return None + + temp_file = Path(temp_path) + if not run_blocking_io(temp_file.exists): + logger.error("Handler returned non-existent path: %s", temp_path) + _capture_task_error( + task, + message=f"Download file missing: {temp_path}", + exc_type="MissingDownloadPath", + ) + return None + + # Check cancellation before post-processing + if cancel_flag.is_set(): + logger.info("Task %s: cancelled before post-processing", task_id) + if not is_torrent_source(temp_file, task): + safe_cleanup_path(temp_file, task) + return None + + logger.info("Task %s: download finished; starting post-processing", task_id) + logger.debug("Task %s: post-processing input path: %s", task_id, temp_file) + task.staged_path = str(temp_file) + preserve_source_on_failure = True + + # Post-processing: output routing + file processing pipeline + result = post_process_download( + temp_file, + task, + cancel_flag, + status_callback, + preserve_source_on_failure=preserve_source_on_failure, + ) + + if cancel_flag.is_set(): + logger.info("Task %s: post-processing cancelled", task_id) + elif result: + logger.info("Task %s: post-processing complete", task_id) + logger.debug("Task %s: post-processing result: %s", task_id, result) else: - return result + logger.warning("Task %s: post-processing failed", task_id) + if not task.last_error_message: + _capture_task_error( + task, + message="Download failed", + exc_type="UnknownFailure", + ) + + handler.post_process_cleanup(task, success=bool(result)) + + if result: + task.staged_path = None + _clear_task_error_state(task) + + return result def update_download_progress(book_id: str, progress: float) -> None: @@ -854,61 +836,38 @@ def _finalize_download_failure(task_id: str) -> None: def _process_single_download(task_id: str, cancel_flag: Event) -> None: """Process a single download job.""" - try: - # Status will be updated through callbacks during download process - # (resolving -> downloading -> complete) - download_path = _download_task(task_id, cancel_flag) + # Status will be updated through callbacks during download process + # (resolving -> downloading -> complete) + download_path = _download_task(task_id, cancel_flag) - # Clean up progress tracking - _cleanup_progress_tracking(task_id) + # Clean up progress tracking + _cleanup_progress_tracking(task_id) - if cancel_flag.is_set(): - book_queue.update_status(task_id, QueueStatus.CANCELLED) - # Broadcast cancellation - if ws_manager: - ws_manager.broadcast_status_update(queue_status()) - return - - if download_path: - book_queue.update_download_path(task_id, download_path) - book_queue.update_status(task_id, QueueStatus.COMPLETE) - else: - _finalize_download_failure(task_id) - - # Broadcast final status (completed or error) + if cancel_flag.is_set(): + book_queue.update_status(task_id, QueueStatus.CANCELLED) + # Broadcast cancellation if ws_manager: ws_manager.broadcast_status_update(queue_status()) + return - except Exception as e: - # Clean up progress tracking even on error - _cleanup_progress_tracking(task_id) + if download_path: + book_queue.update_download_path(task_id, download_path) + book_queue.update_status(task_id, QueueStatus.COMPLETE) + else: + _finalize_download_failure(task_id) - if not cancel_flag.is_set(): - logger.error_trace(f"Error in download processing: {e}") - task = book_queue.get_task(task_id) - if task: - _capture_task_error( - task, - message=f"Download failed: {type(e).__name__}: {e!s}", - exc_type=type(e).__name__, - ) - _finalize_download_failure(task_id) - else: - logger.info("Download cancelled: %s", task_id) - book_queue.update_status(task_id, QueueStatus.CANCELLED) - - # Broadcast error/cancelled status - if ws_manager: - ws_manager.broadcast_status_update(queue_status()) + # Broadcast final status (completed or error) + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) def concurrent_download_loop() -> None: - """Main download coordinator using ThreadPoolExecutor for concurrent downloads.""" + """Run the main concurrent download coordinator.""" max_workers = config.MAX_CONCURRENT_DOWNLOADS logger.info("Starting concurrent download loop with %s workers", max_workers) with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="Download") as executor: - active_futures: dict[Future, str] = {} # Track active download futures + active_futures: dict[Future, tuple[str, Event]] = {} # Track active download futures stalled_tasks: set[str] = set() # Track tasks already cancelled due to stall while True: @@ -916,17 +875,52 @@ def concurrent_download_loop() -> None: # Clean up completed futures completed_futures = [f for f in active_futures if f.done()] for future in completed_futures: - task_id = active_futures.pop(future) + task_id, cancel_flag = active_futures.pop(future) stalled_tasks.discard(task_id) - try: - future.result() # This will raise any exceptions from the worker - except Exception as e: - logger.error_trace(f"Future exception for {task_id}: {e}") + if future.cancelled(): + _cleanup_progress_tracking(task_id) + if cancel_flag.is_set(): + logger.info("Download cancelled: %s", task_id) + book_queue.update_status(task_id, QueueStatus.CANCELLED) + else: + logger.warning("Future cancelled unexpectedly for %s", task_id) + task = book_queue.get_task(task_id) + if task: + _capture_task_error( + task, + message="Download failed: CancelledError", + exc_type="CancelledError", + ) + _finalize_download_failure(task_id) + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) + continue + + worker_error = future.exception() + if worker_error is None: + continue + + _cleanup_progress_tracking(task_id) + if cancel_flag.is_set(): + logger.info("Download cancelled: %s", task_id) + book_queue.update_status(task_id, QueueStatus.CANCELLED) + else: + logger.error_trace("Future exception for %s: %s", task_id, worker_error) + task = book_queue.get_task(task_id) + if task: + _capture_task_error( + task, + message=_format_download_exception_message(worker_error), + exc_type=type(worker_error).__name__, + ) + _finalize_download_failure(task_id) + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) # Check for stalled downloads (no activity in STALL_TIMEOUT seconds) current_time = time.time() with _progress_lock: - for _future, task_id in list(active_futures.items()): + for _future, (task_id, _cancel_flag) in list(active_futures.items()): if task_id in stalled_tasks: continue last_active = _last_activity.get(task_id, current_time) @@ -948,7 +942,7 @@ def concurrent_download_loop() -> None: # Stagger concurrent downloads to avoid rate limiting on shared download servers # Only delay if other downloads are already active if active_futures: - stagger_delay = random.uniform(2, 5) + stagger_delay = _RNG.uniform(2, 5) logger.debug("Staggering download start by %.1fs", stagger_delay) time.sleep(stagger_delay) @@ -956,11 +950,11 @@ def concurrent_download_loop() -> None: # Submit download job to thread pool future = executor.submit(_process_single_download, task_id, cancel_flag) - active_futures[future] = task_id + active_futures[future] = (task_id, cancel_flag) # Brief sleep to prevent busy waiting time.sleep(config.MAIN_LOOP_SLEEP_TIME) - except Exception as e: + except (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError) as e: logger.error_trace("Download coordinator loop error: %s", e) time.sleep(COORDINATOR_LOOP_ERROR_RETRY_DELAY) diff --git a/shelfmark/download/outputs/__init__.py b/shelfmark/download/outputs/__init__.py index db565648..2b3f4fb9 100644 --- a/shelfmark/download/outputs/__init__.py +++ b/shelfmark/download/outputs/__init__.py @@ -1,3 +1,5 @@ +"""Output registry and shared types for post-download delivery handlers.""" + from __future__ import annotations from collections.abc import Callable @@ -13,6 +15,8 @@ OutputHandler = Callable[[Path, DownloadTask, Event, StatusCallback, bool], str @dataclass(frozen=True) class OutputRegistration: + """Registered output handler with support checks and priority metadata.""" + mode: str supports_task: Callable[[DownloadTask], bool] handler: OutputHandler @@ -28,6 +32,8 @@ def register_output( supports_task: Callable[[DownloadTask], bool], priority: int = 0, ) -> Callable[[OutputHandler], OutputHandler]: + """Register an output handler for a named delivery mode.""" + def decorator(handler: OutputHandler) -> OutputHandler: _OUTPUT_REGISTRY.append( OutputRegistration( @@ -44,6 +50,7 @@ def register_output( def load_output_handlers() -> None: + """Load built-in output handlers exactly once.""" global _OUTPUTS_LOADED if _OUTPUTS_LOADED: return @@ -80,6 +87,7 @@ def _derive_output_mode(task: DownloadTask) -> str: def resolve_output_handler(task: DownloadTask) -> OutputRegistration | None: + """Resolve the best output handler for a download task.""" load_output_handlers() desired_mode = _derive_output_mode(task) diff --git a/shelfmark/download/outputs/booklore.py b/shelfmark/download/outputs/booklore.py index 4869f7fe..4f823d29 100644 --- a/shelfmark/download/outputs/booklore.py +++ b/shelfmark/download/outputs/booklore.py @@ -1,3 +1,5 @@ +"""Booklore output integration for uploading completed downloads.""" + from __future__ import annotations import os @@ -53,6 +55,8 @@ class BookloreError(Exception): @dataclass(frozen=True) class BookloreConfig: + """Configuration required to upload files into Booklore.""" + base_url: str username: str password: str @@ -85,6 +89,7 @@ def build_booklore_config( values: Mapping[str, Any], user_id: int | None = None, ) -> BookloreConfig: + """Build and validate the effective Booklore configuration.""" base_url = str(values.get("BOOKLORE_HOST", "")).strip() username = str(values.get("BOOKLORE_USERNAME", "")).strip() password = values.get("BOOKLORE_PASSWORD", "") or "" @@ -139,6 +144,7 @@ def build_booklore_config( def booklore_login(booklore_config: BookloreConfig) -> str: + """Authenticate with Booklore and return an API token.""" url = f"{booklore_config.base_url}/api/v1/auth/login" payload = { "username": booklore_config.username, @@ -182,6 +188,7 @@ def booklore_login(booklore_config: BookloreConfig) -> str: def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list[dict[str, Any]]: + """Fetch the available Booklore libraries for the current user.""" url = f"{booklore_config.base_url}/api/v1/libraries" headers = {"Authorization": f"Bearer {token}"} @@ -200,6 +207,7 @@ def booklore_list_libraries(booklore_config: BookloreConfig, token: str) -> list def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: Path) -> None: + """Upload a completed file into Booklore.""" if booklore_config.upload_to_bookdrop: url = f"{booklore_config.base_url}/api/v1/files/upload/bookdrop" params = None @@ -244,6 +252,7 @@ def booklore_upload_file(booklore_config: BookloreConfig, token: str, file_path: def booklore_refresh_library(booklore_config: BookloreConfig, token: str) -> None: + """Trigger a Booklore library refresh after upload.""" url = f"{booklore_config.base_url}/api/v1/libraries/{booklore_config.library_id}/refresh" headers = {"Authorization": f"Bearer {token}"} @@ -438,7 +447,7 @@ def _post_process_booklore( logger.warning("Task %s: Booklore upload failed: %s", task.task_id, e) status_callback("error", str(e)) return None - except Exception as e: + except (OSError, TypeError, ValueError) as e: logger.error_trace("Task %s: unexpected error uploading to Booklore: %s", task.task_id, e) status_callback("error", f"{BOOKLORE_DISPLAY_NAME} upload failed: {e}") return None @@ -464,6 +473,7 @@ def process_booklore_output( *, preserve_source_on_failure: bool = False, ) -> str | None: + """Process a completed download through the Booklore output.""" return _post_process_booklore( temp_file, task, diff --git a/shelfmark/download/outputs/email.py b/shelfmark/download/outputs/email.py index 11e7a6a4..7f19f5e1 100644 --- a/shelfmark/download/outputs/email.py +++ b/shelfmark/download/outputs/email.py @@ -1,3 +1,5 @@ +"""Email output integration for delivering completed downloads as attachments.""" + from __future__ import annotations import mimetypes @@ -44,6 +46,8 @@ class EmailOutputError(Exception): @dataclass(frozen=True) class EmailSmtpConfig: + """SMTP connection settings for the email output.""" + host: str port: int security: str @@ -57,25 +61,28 @@ class EmailSmtpConfig: def _parse_int(value: Any, label: str, *, minimum: int = 1) -> int: if value is None or value == "": - raise EmailOutputError(f"{label} is required") + msg = f"{label} is required" + raise EmailOutputError(msg) try: parsed = int(value) except (TypeError, ValueError) as exc: - raise EmailOutputError(f"{label} must be a number") from exc + msg = f"{label} must be a number" + raise EmailOutputError(msg) from exc if parsed < minimum: - raise EmailOutputError(f"{label} must be >= {minimum}") + msg = f"{label} must be >= {minimum}" + raise EmailOutputError(msg) return parsed def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig: + """Build and validate SMTP settings for the email output.""" host = str(values.get("EMAIL_SMTP_HOST", "") or "").strip() port = _parse_int(values.get("EMAIL_SMTP_PORT", 587), "SMTP port", minimum=1) security = str(values.get("EMAIL_SMTP_SECURITY", SECURITY_STARTTLS) or "").strip().lower() if security not in ALLOWED_SECURITY: - raise EmailOutputError( - f"SMTP security must be one of: {', '.join(sorted(ALLOWED_SECURITY))}" - ) + msg = f"SMTP security must be one of: {', '.join(sorted(ALLOWED_SECURITY))}" + raise EmailOutputError(msg) username = str(values.get("EMAIL_SMTP_USERNAME", "") or "").strip() password = values.get("EMAIL_SMTP_PASSWORD", "") or "" @@ -88,9 +95,11 @@ def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig: allow_unverified_tls = bool(values.get("EMAIL_ALLOW_UNVERIFIED_TLS", False)) if not host: - raise EmailOutputError("SMTP host is required") + msg = "SMTP host is required" + raise EmailOutputError(msg) if username and not password: - raise EmailOutputError("SMTP password is required when username is set") + msg = "SMTP password is required when username is set" + raise EmailOutputError(msg) if not from_addr: # If From is not configured, fall back to the SMTP username if it is an email address. @@ -98,9 +107,8 @@ def build_email_smtp_config(values: Mapping[str, Any]) -> EmailSmtpConfig: if username_email and "@" in username_email: from_addr = f"Shelfmark <{username_email}>" else: - raise EmailOutputError( - "From address is required (or set SMTP username to an email address)." - ) + msg = "From address is required (or set SMTP username to an email address)." + raise EmailOutputError(msg) return EmailSmtpConfig( host=host, @@ -161,6 +169,7 @@ def compose_email_message( recipient: str, files: list[Path], ) -> EmailMessage: + """Compose the outbound email message for a completed download.""" message = EmailMessage() message["From"] = smtp_config.from_addr message["To"] = recipient @@ -220,9 +229,11 @@ def test_smtp_connection(smtp_config: EmailSmtpConfig) -> None: if smtp_config.username: smtp.login(smtp_config.username, smtp_config.password) except smtplib.SMTPAuthenticationError as exc: - raise EmailOutputError("SMTP authentication failed") from exc + msg = "SMTP authentication failed" + raise EmailOutputError(msg) from exc except (smtplib.SMTPConnectError, smtplib.SMTPServerDisconnected, TimeoutError, OSError) as exc: - raise EmailOutputError(f"Could not connect to SMTP server: {exc}") from exc + msg = f"Could not connect to SMTP server: {exc}" + raise EmailOutputError(msg) from exc finally: if smtp is not None: with suppress(Exception): @@ -232,6 +243,7 @@ def test_smtp_connection(smtp_config: EmailSmtpConfig) -> None: def send_email_message(smtp_config: EmailSmtpConfig, message: EmailMessage) -> None: + """Send a prepared email message using the configured SMTP transport.""" smtp: smtplib.SMTP | None = None try: if smtp_config.security == SECURITY_SSL: @@ -259,9 +271,11 @@ def send_email_message(smtp_config: EmailSmtpConfig, message: EmailMessage) -> N smtp.send_message(message) except smtplib.SMTPAuthenticationError as exc: - raise EmailOutputError("SMTP authentication failed") from exc + msg = "SMTP authentication failed" + raise EmailOutputError(msg) from exc except (smtplib.SMTPException, TimeoutError, OSError) as exc: - raise EmailOutputError(f"Failed to send email: {exc}") from exc + msg = f"Failed to send email: {exc}" + raise EmailOutputError(msg) from exc finally: if smtp is not None: with suppress(Exception): @@ -428,7 +442,7 @@ def _post_process_email( logger.warning("Task %s: email send failed: %s", task.task_id, exc) status_callback("error", str(exc)) return None - except Exception as exc: + except (OSError, TypeError, ValueError) as exc: logger.error_trace("Task %s: unexpected error sending email: %s", task.task_id, exc) status_callback("error", f"Email send failed: {exc}") return None @@ -454,6 +468,7 @@ def process_email_output( *, preserve_source_on_failure: bool = False, ) -> str | None: + """Process a completed download through the email output.""" return _post_process_email( temp_file, task, diff --git a/shelfmark/download/outputs/folder.py b/shelfmark/download/outputs/folder.py index 72c73ac2..692c3493 100644 --- a/shelfmark/download/outputs/folder.py +++ b/shelfmark/download/outputs/folder.py @@ -1,3 +1,5 @@ +"""Folder output handler for copying or linking files into a target directory.""" + from __future__ import annotations from dataclasses import dataclass diff --git a/shelfmark/download/permissions_debug.py b/shelfmark/download/permissions_debug.py index c49eb8d2..757c6c99 100644 --- a/shelfmark/download/permissions_debug.py +++ b/shelfmark/download/permissions_debug.py @@ -21,6 +21,8 @@ if TYPE_CHECKING: logger = setup_logger(__name__) _T = TypeVar("_T") +_PERMISSION_DEBUG_ERRORS = (LookupError, OSError, RuntimeError, TypeError, ValueError) +_IO_OFFLOAD_FALLBACK_ERRORS = (RuntimeError, TypeError) def _log_path_permissions(probe: Path, label: str) -> None: @@ -39,7 +41,7 @@ def _log_path_permissions(probe: Path, label: str) -> None: _run_io(probe.exists), _run_io(probe.is_dir), ) - except Exception as stat_error: + except _PERMISSION_DEBUG_ERRORS as stat_error: logger.debug("Path permissions (%s): stat failed for %s: %s", label, probe, stat_error) @@ -51,12 +53,12 @@ def _run_io[T](func: Callable[..., _T], *args: Any, **kwargs: Any) -> _T: """ try: from shelfmark.download.fs import run_blocking_io as _run_blocking_io - except Exception: + except ImportError: return func(*args, **kwargs) try: return _run_blocking_io(func, *args, **kwargs) - except Exception: + except _IO_OFFLOAD_FALLBACK_ERRORS: # Fall back to direct call if threadpool offload is unavailable. return func(*args, **kwargs) @@ -66,7 +68,7 @@ def _format_uid(uid: int) -> str: import pwd return pwd.getpwuid(uid).pw_name - except Exception: + except ImportError, KeyError: return str(uid) @@ -75,7 +77,7 @@ def _format_gid(gid: int) -> str: import grp return grp.getgrgid(gid).gr_name - except Exception: + except ImportError, KeyError: return str(gid) @@ -103,7 +105,7 @@ def log_path_permission_context(label: str, path: Path) -> None: for probe in [path, path.parent]: try: resolved = _run_io(probe.resolve) - except Exception: + except OSError, RuntimeError: resolved = probe try: @@ -121,14 +123,14 @@ def log_path_permission_context(label: str, path: Path) -> None: _run_io(probe.is_dir), _run_io(probe.is_symlink), ) - except Exception as stat_error: + except _PERMISSION_DEBUG_ERRORS as stat_error: logger.debug( "Path permissions (%s): stat failed for %s: %s", label, probe, stat_error, ) - except Exception as context_error: + except _PERMISSION_DEBUG_ERRORS as context_error: logger.debug("Permission context (%s): failed to collect: %s", label, context_error) @@ -153,5 +155,5 @@ def log_transfer_permission_context(label: str, source: Path, dest: Path, error: for probe in [source, dest, dest.parent]: _log_path_permissions(probe, label) - except Exception as context_error: + except _PERMISSION_DEBUG_ERRORS as context_error: logger.debug("Permission context (%s): failed to collect: %s", label, context_error) diff --git a/shelfmark/download/postprocess/custom_script.py b/shelfmark/download/postprocess/custom_script.py index 673881f3..3e39899f 100644 --- a/shelfmark/download/postprocess/custom_script.py +++ b/shelfmark/download/postprocess/custom_script.py @@ -1,3 +1,5 @@ +"""Custom script execution helpers for post-processing hooks.""" + from __future__ import annotations import json @@ -48,6 +50,8 @@ def resolve_custom_script_target(target_path: Path, destination: Path, path_mode @dataclass(frozen=True) class CustomScriptExecution: + """Resolved command inputs for a single custom script run.""" + script_path: str target_arg: Path target_abs: Path @@ -59,6 +63,8 @@ class CustomScriptExecution: @dataclass(frozen=True) class CustomScriptTransferSummary: + """Transfer metadata exposed to custom post-process scripts.""" + op_counts: dict[str, int] use_hardlink: bool is_torrent: bool @@ -67,6 +73,8 @@ class CustomScriptTransferSummary: @dataclass(frozen=True) class CustomScriptContext: + """Runtime context exposed to custom post-process scripts.""" + task: DownloadTask phase: str output_mode: str @@ -87,6 +95,7 @@ def prepare_custom_script_execution( phase: str, payload: dict[str, Any] | None = None, ) -> CustomScriptExecution: + """Resolve script arguments and payload for a custom hook invocation.""" mode = (path_mode or "absolute").strip().lower() if mode != "relative": mode = "absolute" @@ -110,6 +119,7 @@ def run_custom_script( status_callback: Callable[[str, str | None], None], timeout_seconds: int = DEFAULT_CUSTOM_SCRIPT_TIMEOUT_SECONDS, ) -> bool: + """Run a prepared custom script and report success.""" cwd: str | None = None if execution.mode == "relative": # Make relative paths unambiguous by running the script from the destination folder. diff --git a/shelfmark/download/postprocess/destination.py b/shelfmark/download/postprocess/destination.py index ed49528f..76c875fc 100644 --- a/shelfmark/download/postprocess/destination.py +++ b/shelfmark/download/postprocess/destination.py @@ -1,3 +1,5 @@ +"""Destination planning helpers for post-processing outputs.""" + from __future__ import annotations import uuid @@ -56,7 +58,7 @@ def validate_destination( ) run_blocking_io(test_path.write_text, test_content) run_blocking_io(test_path.unlink, missing_ok=True) - except Exception as exc: + except OSError as exc: logger.debug("Destination write probe path: %s", test_path) log_path_permission_context("destination_write_probe", destination) logger.warning("Destination not writable: %s (%s)", destination, exc) diff --git a/shelfmark/download/postprocess/policy.py b/shelfmark/download/postprocess/policy.py index 75716ccd..7c0d79cb 100644 --- a/shelfmark/download/postprocess/policy.py +++ b/shelfmark/download/postprocess/policy.py @@ -44,7 +44,7 @@ def get_supported_audiobook_formats() -> list[str]: return [fmt.lower() for fmt in formats] -def get_file_organization(is_audiobook: bool) -> str: +def get_file_organization(*, is_audiobook: bool) -> str: """Get the file organization mode for the content type.""" key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION" mode = core_config.config.get(key, "rename") @@ -62,7 +62,7 @@ def get_file_organization(is_audiobook: bool) -> str: return mode -def get_template(is_audiobook: bool, organization_mode: str) -> str: +def get_template(*, is_audiobook: bool, organization_mode: str) -> str: """Get the template for the content type and organization mode.""" # Determine the correct key based on content type and organization mode if is_audiobook: diff --git a/shelfmark/download/postprocess/prepare.py b/shelfmark/download/postprocess/prepare.py index 67b403f4..69aa2912 100644 --- a/shelfmark/download/postprocess/prepare.py +++ b/shelfmark/download/postprocess/prepare.py @@ -1,3 +1,5 @@ +"""Preparation helpers for staging files before final output handling.""" + from __future__ import annotations from typing import TYPE_CHECKING @@ -54,6 +56,7 @@ def prepare_output_files( *, preserve_source_on_failure: bool = False, ) -> PreparedFiles | None: + """Prepare staged files and output metadata for final processing.""" if output_plan is None: output_plan = build_output_plan( temp_file, diff --git a/shelfmark/download/postprocess/scan.py b/shelfmark/download/postprocess/scan.py index 8724f2ea..5b732fdd 100644 --- a/shelfmark/download/postprocess/scan.py +++ b/shelfmark/download/postprocess/scan.py @@ -1,3 +1,5 @@ +"""Scanning helpers for discovering candidate files after download completion.""" + from __future__ import annotations import os @@ -26,6 +28,7 @@ logger = setup_logger("shelfmark.download.postprocess.pipeline") def get_supported_formats(content_type: str | None = None) -> list[str]: + """Return supported file extensions for the requested content type.""" if check_audiobook(content_type): return get_supported_audiobook_formats() return get_book_formats() @@ -60,6 +63,7 @@ def extract_archive_files( *, cleanup_archive: bool, ) -> tuple[list[Path], list[Path], list[Path], str | None]: + """Extract an archive and classify the resulting files.""" content_type = task.content_type try: @@ -213,6 +217,7 @@ def collect_directory_files( status_callback: Callable[[str, str | None], None] | None = None, cleanup_archives: bool = False, ) -> tuple[list[Path], list[Path], list[Path], str | None]: + """Collect supported files from a directory, extracting archives when allowed.""" content_type = task.content_type book_files, rejected_files, archive_files, scan_error = scan_directory_tree( directory, content_type @@ -313,6 +318,7 @@ def collect_staged_files( status_callback: Callable[[str, str | None], None] | None, cleanup_archives: bool, ) -> tuple[list[Path], list[Path], list[Path], str | None]: + """Collect supported files from a staged file or directory path.""" if run_blocking_io(working_path.is_dir): if status_callback: status_callback("resolving", "Processing download folder") diff --git a/shelfmark/download/postprocess/steps.py b/shelfmark/download/postprocess/steps.py index 99f54d64..b792bbf4 100644 --- a/shelfmark/download/postprocess/steps.py +++ b/shelfmark/download/postprocess/steps.py @@ -1,3 +1,5 @@ +"""Helpers for recording debug steps in the post-processing pipeline.""" + from __future__ import annotations from shelfmark.core.logger import setup_logger @@ -8,10 +10,12 @@ logger = setup_logger("shelfmark.download.postprocess.pipeline") def record_step(steps: list[PlanStep], name: str, **details: object) -> None: + """Append a named debug step to the processing plan.""" steps.append(PlanStep(name=name, details=details)) def log_plan_steps(task_id: str, steps: list[PlanStep]) -> None: + """Log a compact summary of recorded post-processing steps.""" if not steps: return summary = " -> ".join(step.name for step in steps) diff --git a/shelfmark/download/postprocess/transfer.py b/shelfmark/download/postprocess/transfer.py index 7aaf90a7..dbc04dd6 100644 --- a/shelfmark/download/postprocess/transfer.py +++ b/shelfmark/download/postprocess/transfer.py @@ -1,3 +1,5 @@ +"""File transfer helpers for post-processing output delivery.""" + from __future__ import annotations import os @@ -32,6 +34,7 @@ if TYPE_CHECKING: from shelfmark.core.models import DownloadTask logger = setup_logger("shelfmark.download.postprocess.pipeline") +_TRANSFER_PROCESS_ERRORS = (AttributeError, KeyError, OSError, RuntimeError, TypeError, ValueError) def should_hardlink(task: DownloadTask) -> bool: @@ -53,6 +56,7 @@ def should_hardlink(task: DownloadTask) -> bool: def build_metadata_dict(task: DownloadTask) -> dict: + """Build template metadata from a download task.""" return { "Author": task.author, "Title": task.title, @@ -67,6 +71,7 @@ def build_metadata_dict(task: DownloadTask) -> dict: def build_file_metadata( task: DownloadTask, source_file: Path, part_number: str | None = None ) -> dict: + """Build template metadata for a specific source file.""" metadata = build_metadata_dict(task) metadata["OriginalName"] = source_file.stem if part_number is not None: @@ -121,10 +126,7 @@ def is_torrent_source(source_path: Path, task: DownloadTask) -> bool: try: return run_blocking_io(source_path.resolve) == run_blocking_io(original_path.resolve) except OSError, ValueError: - try: - return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path)) - except Exception: - return False + return os.path.normpath(str(source_path)) == os.path.normpath(str(original_path)) def _max_attempts_for_batch(file_count: int, default: int = 100) -> int: @@ -167,6 +169,7 @@ def transfer_book_files( preserve_source: bool = False, organization_mode: str | None = None, ) -> tuple[list[Path], str | None, dict[str, int]]: + """Transfer discovered book files into their final destination layout.""" if not book_files: return [], "No book files found", {"hardlink": 0, "copy": 0, "move": 0} @@ -178,7 +181,7 @@ def transfer_book_files( op_counts: dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0} if organization_mode == "organize": - template = get_template(is_audiobook, "organize") + template = get_template(is_audiobook=is_audiobook, organization_mode="organize") if len(book_files) == 1: source_file = book_files[0] @@ -239,7 +242,7 @@ def transfer_book_files( if not task.format: task.format = book_file.suffix.lower().lstrip(".") - template = get_template(is_audiobook, "rename") + template = get_template(is_audiobook=is_audiobook, organization_mode="rename") metadata = build_file_metadata(task, book_file) extension = book_file.suffix.lstrip(".") or task.format or "" @@ -315,7 +318,7 @@ def process_directory( processed_paths = final_paths - except Exception as exc: + except _TRANSFER_PROCESS_ERRORS as exc: logger.error_trace( "Task %s: error processing directory %s: %s", task.task_id, directory, exc ) @@ -337,6 +340,7 @@ def transfer_file_to_library( *, use_hardlink: bool, ) -> str | None: + """Transfer a single file into a library path derived from metadata.""" extension = source_path.suffix.lstrip(".") or task.format template_metadata = dict(metadata) template_metadata.setdefault("OriginalName", source_path.stem) @@ -379,6 +383,7 @@ def transfer_directory_to_library( *, use_hardlink: bool, ) -> str | None: + """Transfer a directory tree into a library path derived from metadata.""" content_type = task.content_type.lower() if task.content_type else None source_files, _, _, scan_error = scan_directory_tree(source_dir, content_type) if scan_error: diff --git a/shelfmark/download/postprocess/types.py b/shelfmark/download/postprocess/types.py index a764475c..ea844b97 100644 --- a/shelfmark/download/postprocess/types.py +++ b/shelfmark/download/postprocess/types.py @@ -1,3 +1,5 @@ +"""Typed data containers used by the post-processing pipeline.""" + from __future__ import annotations from dataclasses import dataclass @@ -11,6 +13,8 @@ if TYPE_CHECKING: @dataclass(frozen=True) class TransferPlan: + """Plan describing how files should move from source to output.""" + source_path: Path use_hardlink: bool allow_archive_extraction: bool @@ -19,6 +23,8 @@ class TransferPlan: @dataclass(frozen=True) class OutputPlan: + """Resolved output mode, staging strategy, and transfer settings.""" + mode: str stage_action: StageAction staging_dir: Path @@ -28,6 +34,8 @@ class OutputPlan: @dataclass(frozen=True) class PreparedFiles: + """Prepared file set ready for transfer or output handling.""" + output_plan: OutputPlan working_path: Path files: list[Path] @@ -37,5 +45,7 @@ class PreparedFiles: @dataclass(frozen=True) class PlanStep: + """Recorded post-processing step and its debug metadata.""" + name: str details: dict[str, Any] diff --git a/shelfmark/download/postprocess/workspace.py b/shelfmark/download/postprocess/workspace.py index bb73d935..f8cafd80 100644 --- a/shelfmark/download/postprocess/workspace.py +++ b/shelfmark/download/postprocess/workspace.py @@ -1,6 +1,9 @@ +"""Workspace helpers for managing mutable post-processing directories.""" + from __future__ import annotations import shutil +from contextlib import suppress from pathlib import Path from typing import TYPE_CHECKING @@ -27,7 +30,7 @@ def is_within_tmp_dir(path: Path) -> bool: # This is a *negative* check only; for potential TMP paths we still resolve to # prevent symlink escapes from being treated as managed. tmp_dir = _tmp_dir() - try: + with suppress(Exception): if ( path.is_absolute() and tmp_dir.is_absolute() @@ -35,9 +38,6 @@ def is_within_tmp_dir(path: Path) -> bool: and tmp_dir not in path.parents ): return False - except Exception: - # Fall back to the slower resolve-based check below. - pass try: run_blocking_io(path.resolve).relative_to(run_blocking_io(tmp_dir.resolve)) @@ -48,7 +48,7 @@ def is_within_tmp_dir(path: Path) -> bool: def is_managed_workspace_path(path: Path) -> bool: - """True if Shelfmark should treat this path as mutable. + """Return whether Shelfmark should treat this path as mutable. The managed workspace is `TMP_DIR`. Anything outside it should be treated as read-only for safety (e.g. torrent seeding directories). @@ -90,6 +90,7 @@ def cleanup_output_staging( task: DownloadTask, cleanup_paths: list[Path] | None = None, ) -> None: + """Clean up staging paths created for output processing.""" if output_plan.stage_action != STAGE_NONE: cleanup_target = output_plan.staging_dir if output_plan.staging_dir == _tmp_dir(): diff --git a/shelfmark/download/staging.py b/shelfmark/download/staging.py index e45a93fd..c5e838e1 100644 --- a/shelfmark/download/staging.py +++ b/shelfmark/download/staging.py @@ -1,3 +1,5 @@ +"""Helpers for staging downloaded files before post-processing.""" + from __future__ import annotations import hashlib @@ -29,7 +31,7 @@ def get_staging_dir() -> Path: def get_staging_path(task_id: str, extension: str) -> Path: """Get a staging path for a download.""" staging_dir = get_staging_dir() - safe_id = hashlib.md5(task_id.encode()).hexdigest()[:16] + safe_id = hashlib.blake2b(task_id.encode(), digest_size=8).hexdigest() return staging_dir / f"{safe_id}.{extension.lstrip('.')}" @@ -39,7 +41,7 @@ def build_staging_dir(prefix: str | None, task_id: str) -> Path: if not prefix: return base_dir - safe_id = hashlib.md5(task_id.encode()).hexdigest()[:8] + safe_id = hashlib.blake2b(task_id.encode(), digest_size=4).hexdigest() staging_dir = base_dir / f"{prefix}_{safe_id}" counter = 1 diff --git a/shelfmark/main.py b/shelfmark/main.py index e34ed097..75f7a09a 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -1,5 +1,6 @@ """Flask app - routes, WebSocket handlers, and middleware.""" +import binascii import io import logging import os @@ -7,7 +8,7 @@ import re import sqlite3 import time from contextlib import suppress -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from functools import wraps from importlib import import_module from pathlib import Path @@ -31,8 +32,12 @@ from shelfmark.config.env import ( HIDE_LOCAL_AUTH, OIDC_AUTO_REDIRECT, RELEASE_VERSION, + SESSION_COOKIE_NAME, + SESSION_COOKIE_SECURE_ENV, _is_config_dir_writable, + string_to_bool, ) +from shelfmark.config.security import _migrate_security_settings from shelfmark.config.settings import _SUPPORTED_BOOK_LANGUAGE from shelfmark.core.activity_view_state_service import ActivityViewStateService from shelfmark.core.auth_modes import ( @@ -74,6 +79,7 @@ from shelfmark.core.requests_service import ( reopen_failed_request, sync_delivery_states_from_queue_status, ) +from shelfmark.core.user_db import UserDB from shelfmark.core.utils import normalize_base_path from shelfmark.download import orchestrator as backend from shelfmark.release_sources import ( @@ -87,6 +93,9 @@ if TYPE_CHECKING: from shelfmark.metadata_providers import BookMetadata, MetadataProvider logger = setup_logger(__name__) +FLASK_SECRET_KEY_MIN_BYTES = 32 +_OPERATIONAL_ERRORS = (OSError, RuntimeError, TypeError, ValueError, sqlite3.Error) +_IMPORT_OPERATIONAL_ERRORS = (ImportError, *_OPERATIONAL_ERRORS) def _raise_runtime_error(message: str) -> NoReturn: @@ -102,7 +111,7 @@ BASE_PATH = normalize_base_path(app_config.get("URL_BASE", "")) app = Flask(__name__) app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 0 # Disable caching app.config["APPLICATION_ROOT"] = BASE_PATH or "/" -app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore +app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore[assignment] if BASE_PATH: app.wsgi_app = PrefixMiddleware(app.wsgi_app, BASE_PATH, bypass_paths={"/api/health"}) @@ -149,17 +158,11 @@ except ImportError as e: logger.warning("Failed to import plugin modules: %s", e) # Migrate legacy security settings if needed -from shelfmark.config.security import _migrate_security_settings - _migrate_security_settings() # Initialize user database and register multi-user routes # If CONFIG_DIR doesn't exist or is read-only, multi-user features will be disabled -import os as _os - -from shelfmark.core.user_db import UserDB - -_user_db_path = str(Path(_os.environ.get("CONFIG_DIR", "/config")) / "users.db") +_user_db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db") user_db: UserDB | None = None download_history_service: DownloadHistoryService | None = None activity_view_state_service: ActivityViewStateService | None = None @@ -180,7 +183,7 @@ except (sqlite3.OperationalError, OSError) as e: logger.warning( "User database initialization failed: %s. Multi-user authentication features will be disabled. Ensure CONFIG_DIR (%s) exists and is writable.", e, - _os.environ.get("CONFIG_DIR", "/config"), + os.environ.get("CONFIG_DIR", "/config"), ) user_db = None download_history_service = None @@ -190,15 +193,16 @@ except (sqlite3.OperationalError, OSError) as e: backend.start() # Rate limiting for login attempts -# Structure: {username: {'count': int, 'lockout_until': datetime}} +# Map usernames to their failed-attempt counters and lockout timestamps. failed_login_attempts: dict[str, dict[str, Any]] = {} MAX_LOGIN_ATTEMPTS = 10 LOCKOUT_DURATION_MINUTES = 30 +LOGIN_ATTEMPT_WARNING_THRESHOLD = 5 def cleanup_old_lockouts() -> None: """Remove expired lockout entries to prevent memory buildup.""" - current_time = datetime.now() + current_time = datetime.now(UTC) expired_users = [ username for username, data in failed_login_attempts.items() @@ -217,7 +221,7 @@ def is_account_locked(username: str) -> bool: return False lockout_until = failed_login_attempts[username].get("lockout_until") - return lockout_until is not None and datetime.now() < lockout_until + return lockout_until is not None and datetime.now(UTC) < lockout_until def record_failed_login(username: str, ip_address: str) -> bool: @@ -240,7 +244,7 @@ def record_failed_login(username: str, ip_address: str) -> bool: ) if count >= MAX_LOGIN_ATTEMPTS: - lockout_until = datetime.now() + timedelta(minutes=LOCKOUT_DURATION_MINUTES) + lockout_until = datetime.now(UTC) + timedelta(minutes=LOCKOUT_DURATION_MINUTES) failed_login_attempts[username]["lockout_until"] = lockout_until logger.warning( "Account locked for user '%s' until %s due to %s failed login attempts", @@ -497,7 +501,7 @@ if user_db is not None: emit_request_updates=_emit_request_updates, ws_manager=ws_manager, ) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.warning("Failed to register request routes: %s", e) @@ -525,6 +529,7 @@ class LogNoiseFilter(logging.Filter): """ def filter(self, record: logging.LogRecord) -> bool: + """Return whether a log record should be emitted.""" message = record.getMessage() if hasattr(record, "getMessage") else str(record.msg) # Exclude GET /api/status requests (polling noise) @@ -564,8 +569,6 @@ werkzeug_logger.setLevel(logger.level) werkzeug_logger.addFilter(LogNoiseFilter()) # Set up authentication defaults -from shelfmark.config.env import SESSION_COOKIE_NAME, SESSION_COOKIE_SECURE_ENV, string_to_bool - SESSION_COOKIE_SECURE = string_to_bool(SESSION_COOKIE_SECURE_ENV) @@ -576,7 +579,7 @@ def _load_or_create_secret_key() -> bytes: try: if secret_path.exists(): secret_key = secret_path.read_bytes() - if len(secret_key) >= 32: + if len(secret_key) >= FLASK_SECRET_KEY_MIN_BYTES: return secret_key logger.warning( "Invalid persisted Flask secret key at %s (length=%s). Regenerating.", @@ -730,7 +733,7 @@ def proxy_auth_middleware() -> Response | tuple[Response, int] | None: session["db_user_id"] = db_user["id"] session.permanent = False - except Exception: + except _OPERATIONAL_ERRORS: logger.exception("Proxy auth middleware error") return jsonify({"error": "Authentication error"}), 500 else: @@ -753,8 +756,10 @@ def set_security_headers(response: Response) -> Response: def login_required( f: Callable[..., Response | tuple[Response, int]], ) -> Callable[..., Response | tuple[Response, int]]: + """Require authentication for a Flask route.""" + @wraps(f) - def decorated_function(*args, **kwargs) -> Response | tuple[Response, int]: + def decorated_function(*args: object, **kwargs: object) -> Response | tuple[Response, int]: auth_mode = get_auth_mode() # If no authentication is configured, allow access @@ -778,7 +783,7 @@ def login_required( ): return jsonify({"error": "Admin access required"}), 403 - except Exception: + except RuntimeError, TypeError, ValueError: logger.exception("Admin access check error") return jsonify({"error": "Internal Server Error"}), 500 @@ -821,6 +826,7 @@ def serve_frontend_assets(filename: str) -> Response: @app.route("/") def index() -> Response: """Serve the React frontend application. + Authentication is handled by the React app itself. """ return _serve_index_html() @@ -859,9 +865,9 @@ if DEBUG: @app.route("/api/debug", methods=["GET"]) @login_required def debug() -> Response | tuple[Response, int]: - """This will run the /app/genDebug.sh script, which will generate a debug zip with all the logs - The file will be named /tmp/shelfmark-debug.zip - And then return it to the user + """Run `/app/genDebug.sh`, generate a debug zip, and return it. + + The file is written to `/tmp/shelfmark-debug.zip` before being returned. """ try: logger.info("Debug endpoint called, stopping GUI and generating debug info...") @@ -888,14 +894,14 @@ if DEBUG: except subprocess.CalledProcessError as e: logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}") return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Debug endpoint error: {e}") return jsonify({"error": str(e)}), 500 @app.route("/api/restart", methods=["GET"]) @login_required def restart() -> Response | tuple[Response, int]: - """Restart the application""" + """Restart the application.""" os._exit(0) @@ -991,7 +997,7 @@ def api_download_release() -> Response | tuple[Response, int]: """ try: - data = request.get_json() + data = request.get_json(silent=True) if not data: return jsonify({"error": "No data provided"}), 400 @@ -1035,7 +1041,7 @@ def api_download_release() -> Response | tuple[Response, int]: if success: return jsonify({"status": "queued", "priority": priority}) return jsonify({"error": error_msg or "Failed to queue release"}), 500 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Release download error: {e}") return jsonify({"error": str(e)}), 500 @@ -1124,7 +1130,7 @@ def api_config() -> Response | tuple[Response, int]: ), # For universal mode } return jsonify(config) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Config error: {e}") return jsonify({"error": str(e)}), 500 @@ -1132,6 +1138,7 @@ def api_config() -> Response | tuple[Response, int]: @app.route("/api/health", methods=["GET"]) def api_health() -> Response | tuple[Response, int]: """Health check endpoint for container orchestration. + No authentication required. Returns: @@ -1215,7 +1222,7 @@ def _notify_admin_for_terminal_download_status( ) try: notify_admin(event, context) - except Exception as exc: + except (RuntimeError, TypeError, ValueError) as exc: logger.warning( "Failed to trigger admin notification for download %s (%s): %s", task_id, @@ -1226,7 +1233,7 @@ def _notify_admin_for_terminal_download_status( return try: notify_user(owner_user_id, event, context) - except Exception as exc: + except (RuntimeError, TypeError, ValueError) as exc: logger.warning( "Failed to trigger user notification for download %s (%s, user_id=%s): %s", task_id, @@ -1264,10 +1271,7 @@ def _record_download_queued(task_id: str, task: Any) -> None: origin = "requested" if request_id else "direct" source_name = normalize_source(getattr(task, "source", None)) - try: - source_display = get_source_display_name(source_name) - except Exception: - source_display = None + source_display = get_source_display_name(source_name) try: download_history_service.record_download( @@ -1279,14 +1283,14 @@ def _record_download_queued(task_id: str, task: Any) -> None: source_display_name=source_display, title=str(getattr(task, "title", "Unknown title") or "Unknown title"), author=normalize_optional_text(getattr(task, "author", None)), - format=normalize_optional_text(getattr(task, "format", None)), + file_format=normalize_optional_text(getattr(task, "format", None)), size=normalize_optional_text(getattr(task, "size", None)), preview=normalize_optional_text(getattr(task, "preview", None)), content_type=normalize_optional_text(getattr(task, "content_type", None)), origin=origin, retry_payload=backend.serialize_task_for_retry(task), ) - except Exception as exc: + except _OPERATIONAL_ERRORS as exc: logger.warning("Failed to record download at queue time for task %s: %s", task_id, exc) return @@ -1312,7 +1316,7 @@ def _record_download_queued(task_id: str, task: Any) -> None: "task_id": task_id, }, ) - except Exception as exc: + except _OPERATIONAL_ERRORS as exc: logger.warning("Failed to reset activity viewer state for task %s: %s", task_id, exc) @@ -1334,7 +1338,7 @@ def _record_download_terminal_snapshot(task_id: str, status: QueueStatus, task: retry_payload=backend.serialize_task_for_retry(task), ) finalized_download = True - except Exception as exc: + except _OPERATIONAL_ERRORS as exc: logger.warning("Failed to finalize download history for task %s: %s", task_id, exc) if finalized_download: @@ -1375,7 +1379,7 @@ def _record_download_terminal_snapshot(task_id: str, status: QueueStatus, task: item_key=f"request:{request_id}", ) _emit_request_update_events([reopened_request]) - except Exception as exc: + except _OPERATIONAL_ERRORS as exc: logger.warning( "Failed to reopen request %s after terminal download error %s: %s", request_id, @@ -1429,23 +1433,25 @@ def _emit_request_update_events(updated_requests: list[dict[str, Any]]) -> None: if not updated_requests or ws_manager is None: return - try: - socketio_ref = getattr(ws_manager, "socketio", None) - is_enabled = getattr(ws_manager, "is_enabled", None) - if socketio_ref is None or not callable(is_enabled) or not is_enabled(): - return - - for updated in updated_requests: - payload = { - "request_id": updated["id"], - "status": updated["status"], - "delivery_state": updated.get("delivery_state"), - "title": (updated.get("book_data") or {}).get("title") or "Unknown title", - } - socketio_ref.emit("request_update", payload, to=f"user_{updated['user_id']}") - socketio_ref.emit("request_update", payload, to="admins") - except Exception as exc: - logger.warning("Failed to emit delivery request_update events: %s", exc) + for updated in updated_requests: + payload = { + "request_id": updated["id"], + "status": updated["status"], + "delivery_state": updated.get("delivery_state"), + "title": (updated.get("book_data") or {}).get("title") or "Unknown title", + } + emit_ws_event( + ws_manager, + event_name="request_update", + room=f"user_{updated['user_id']}", + payload=payload, + ) + emit_ws_event( + ws_manager, + event_name="request_update", + room="admins", + payload=payload, + ) @app.route("/api/status", methods=["GET"]) @@ -1472,7 +1478,7 @@ def api_status() -> Response | tuple[Response, int]: ) _emit_request_update_events(updated_requests) return jsonify(status) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Status error: {e}") return jsonify({"error": str(e)}), 500 @@ -1521,7 +1527,7 @@ def api_local_download() -> Response | tuple[Response, int]: data = io.BytesIO(file_data) return send_file(data, download_name=file_name, as_attachment=True) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Local download error: {e}") return jsonify({"error": str(e)}), 500 @@ -1571,7 +1577,7 @@ def api_cover(cover_id: str) -> Response | tuple[Response, int]: try: original_url = base64.urlsafe_b64decode(encoded_url).decode() - except Exception as e: + except (binascii.Error, UnicodeDecodeError) as e: logger.warning("Failed to decode cover URL: %s", e) return jsonify({"error": "Invalid cover URL encoding"}), 400 @@ -1584,7 +1590,7 @@ def api_cover(cover_id: str) -> Response | tuple[Response, int]: response = app.response_class(response=image_data, status=200, mimetype=content_type) response.headers["Cache-Control"] = "public, max-age=86400" response.headers["X-Cache"] = "MISS" - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Cover fetch error: {e}") return jsonify({"error": str(e)}), 500 else: @@ -1633,7 +1639,7 @@ def api_cancel_download(book_id: str) -> Response | tuple[Response, int]: if success: return jsonify({"status": "cancelled", "book_id": book_id}) return jsonify({"error": "Failed to cancel download or book not found"}), 404 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Cancel download error: {e}") return jsonify({"error": str(e)}), 500 @@ -1683,7 +1689,9 @@ def api_retry_download(book_id: str) -> Response | tuple[Response, int]: ), 403 success, error = backend.retry_download(book_id) else: - assert history_row is not None + if history_row is None: + logger.error("Download history row disappeared while retrying task %s", book_id) + return jsonify({"error": "Download history not found"}), 404 request_id = normalize_positive_int(history_row.get("request_id")) retry_payload = history_row.get("retry_payload") final_status = history_row.get("final_status") @@ -1705,7 +1713,7 @@ def api_retry_download(book_id: str) -> Response | tuple[Response, int]: return jsonify({"error": error}), 404 return jsonify({"error": error or "Download cannot be retried"}), 409 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Retry download error: {e}") return jsonify({"error": str(e)}), 500 @@ -1726,7 +1734,7 @@ def api_set_priority(book_id: str) -> Response | tuple[Response, int]: """ try: - data = request.get_json() + data = request.get_json(silent=True) if not data or "priority" not in data: return jsonify({"error": "Priority not provided"}), 400 @@ -1738,7 +1746,7 @@ def api_set_priority(book_id: str) -> Response | tuple[Response, int]: return jsonify({"error": "Failed to update priority or book not found"}), 404 except ValueError: return jsonify({"error": "Invalid priority value"}), 400 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Set priority error: {e}") return jsonify({"error": str(e)}), 500 @@ -1756,7 +1764,7 @@ def api_reorder_queue() -> Response | tuple[Response, int]: """ try: - data = request.get_json() + data = request.get_json(silent=True) if not data or "book_priorities" not in data: return jsonify({"error": "book_priorities not provided"}), 400 @@ -1774,7 +1782,7 @@ def api_reorder_queue() -> Response | tuple[Response, int]: if success: return jsonify({"status": "reordered", "updated_count": len(book_priorities)}) return jsonify({"error": "Failed to reorder queue"}), 500 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Reorder queue error: {e}") return jsonify({"error": str(e)}), 500 @@ -1791,7 +1799,7 @@ def api_queue_order() -> Response | tuple[Response, int]: try: queue_order = backend.get_queue_order() return jsonify({"queue": queue_order}) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Queue order error: {e}") return jsonify({"error": str(e)}), 500 @@ -1808,7 +1816,7 @@ def api_active_downloads() -> Response | tuple[Response, int]: try: active_downloads = backend.get_active_downloads() return jsonify({"active_downloads": active_downloads}) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Active downloads error: {e}") return jsonify({"error": str(e)}), 500 @@ -1855,7 +1863,7 @@ def _failed_login_response(username: str, ip_address: str) -> tuple[Response, in ), 429 attempts_remaining = MAX_LOGIN_ATTEMPTS - failed_login_attempts[username]["count"] - if attempts_remaining <= 5: + if attempts_remaining <= LOGIN_ATTEMPT_WARNING_THRESHOLD: return jsonify( {"error": f"Invalid username or password. {attempts_remaining} attempts remaining."} ), 401 @@ -1866,6 +1874,7 @@ def _failed_login_response(username: str, ip_address: str) -> tuple[Response, in @app.route("/api/auth/login", methods=["POST"]) def api_login() -> Response | tuple[Response, int]: """Login endpoint that validates credentials and creates a session. + Supports both built-in credentials and CWA database authentication. Includes rate limiting: 10 failed attempts = 30 minute lockout. @@ -1880,7 +1889,7 @@ def api_login() -> Response | tuple[Response, int]: """ try: ip_address = get_client_ip() - data = request.get_json() + data = request.get_json(silent=True) if not data: return jsonify({"error": "No data provided"}), 400 @@ -1901,7 +1910,7 @@ def api_login() -> Response | tuple[Response, int]: # Check if account is locked due to failed login attempts if is_account_locked(username): lockout_until = failed_login_attempts[username].get("lockout_until") - remaining_time = (lockout_until - datetime.now()).total_seconds() / 60 + remaining_time = (lockout_until - datetime.now(UTC)).total_seconds() / 60 logger.warning( "Login attempt blocked for locked account '%s' from IP %s", username, ip_address ) @@ -1960,7 +1969,7 @@ def api_login() -> Response | tuple[Response, int]: return _failed_login_response(username, ip_address) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Built-in auth error: {e}") return jsonify({"error": "Authentication system error"}), 500 @@ -2017,14 +2026,14 @@ def api_login() -> Response | tuple[Response, int]: ) return jsonify({"success": True}) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"CWA database error during login: {e}") return jsonify({"error": "Authentication system error"}), 500 # Should not reach here, but handle gracefully return jsonify({"error": "Unknown authentication mode"}), 500 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Login error: {e}") return jsonify({"error": "Login failed"}), 500 @@ -2032,6 +2041,7 @@ def api_login() -> Response | tuple[Response, int]: @app.route("/api/auth/logout", methods=["POST"]) def api_logout() -> Response | tuple[Response, int]: """Logout endpoint that clears the session. + For proxy auth, returns the logout URL if configured. Returns: @@ -2052,7 +2062,7 @@ def api_logout() -> Response | tuple[Response, int]: return jsonify({"success": True, "logout_url": logout_url}) return jsonify({"success": True}) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Logout error: {e}") return jsonify({"error": "Logout failed"}), 500 @@ -2091,8 +2101,8 @@ def api_auth_check() -> Response | tuple[Response, int]: db_user = user_db.get_user(user_id=session["db_user_id"]) if db_user: display_name = db_user.get("display_name") or None - except Exception: - pass + except (sqlite3.Error, TypeError, ValueError) as exc: + logger.debug("Could not load display name for session user: %s", exc) response_data = { "authenticated": is_authenticated, @@ -2120,7 +2130,7 @@ def api_auth_check() -> Response | tuple[Response, int]: response_data["oidc_auto_redirect"] = True return jsonify(response_data) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Auth check error: {e}") return jsonify( { @@ -2182,8 +2192,12 @@ def api_metadata_providers() -> Response | tuple[Response, int]: kwargs = get_provider_kwargs(info["name"]) provider = get_provider(info["name"], **kwargs) provider_info["available"] = provider.is_available() - except Exception: - pass + except _OPERATIONAL_ERRORS as exc: + logger.debug( + "Metadata provider %s availability check failed: %s", + info["name"], + exc, + ) providers.append(provider_info) @@ -2195,7 +2209,7 @@ def api_metadata_providers() -> Response | tuple[Response, int]: "configured_provider_combined": configured_combined_metadata_provider or None, } ) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Metadata providers error: {e}") return jsonify({"error": str(e)}), 500 @@ -2264,7 +2278,7 @@ def api_metadata_config() -> Response | tuple[Response, int]: "default_sort": get_provider_default_sort(provider_name, user_id=db_user_id), } ) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Metadata config error: {e}") return jsonify({"error": str(e)}), 500 @@ -2409,7 +2423,7 @@ def api_metadata_search() -> Response | tuple[Response, int]: if search_result.source_title: response_data["source_title"] = search_result.source_title return jsonify(response_data) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Metadata search error: {e}") return jsonify({"error": str(e)}), 500 @@ -2450,7 +2464,7 @@ def api_metadata_field_options() -> Response: options = provider.get_search_field_options(field_key, query=query_text or None) return jsonify({"options": options}) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.warning("Metadata field options endpoint error: %s", e) return jsonify({"options": []}) @@ -2467,13 +2481,15 @@ def _resolve_metadata_provider(provider_name: str) -> MetadataProvider: ) if not is_provider_registered(provider_name): - raise ValueError(f"Unknown metadata provider: {provider_name}") + msg = f"Unknown metadata provider: {provider_name}" + raise ValueError(msg) kwargs = get_provider_kwargs(provider_name) prov = get_provider(provider_name, **kwargs) if not prov.is_available(): - raise RuntimeError(f"Provider '{provider_name}' is not available") + msg = f"Provider '{provider_name}' is not available" + raise RuntimeError(msg) return prov @@ -2514,7 +2530,7 @@ def api_metadata_book(provider: str, book_id: str) -> Response | tuple[Response, return jsonify({"error": str(e)}), 400 except RuntimeError as e: return jsonify({"error": str(e)}), 503 - except Exception as e: + except (OSError, TypeError, sqlite3.Error) as e: logger.error_trace(f"Metadata book error: {e}") return jsonify({"error": str(e)}), 500 @@ -2524,20 +2540,20 @@ def _handle_target_errors( ) -> Callable[ [Callable[..., Response | tuple[Response, int]]], Callable[..., Response | tuple[Response, int]] ]: - """Decorator that wraps a metadata-target route with standard error handling.""" + """Wrap a metadata-target route with standard error handling.""" def decorator( fn: Callable[..., Response | tuple[Response, int]], ) -> Callable[..., Response | tuple[Response, int]]: @wraps(fn) - def wrapper(*args, **kwargs) -> Response | tuple[Response, int]: + def wrapper(*args: object, **kwargs: object) -> Response | tuple[Response, int]: try: return fn(*args, **kwargs) except (NotImplementedError, ValueError) as e: return jsonify({"error": str(e)}), 400 except RuntimeError as e: return jsonify({"error": str(e)}), 502 - except Exception as e: + except (OSError, TypeError, sqlite3.Error) as e: logger.error_trace(f"{fallback_message}: {e}") return jsonify({"error": fallback_message}), 500 @@ -2589,7 +2605,7 @@ def api_metadata_book_targets_update( if not isinstance(selected, bool): return jsonify({"error": "selected must be a boolean"}), 400 - result = prov.set_book_target_state(book_id, target, selected) + result = prov.set_book_target_state(book_id, target, selected=selected) response: dict = { "success": True, "changed": bool(result.get("changed", True)), @@ -2680,7 +2696,7 @@ def api_releases() -> Response | tuple[Response, int]: ) except ValueError: return None, [], f"Unknown source: {source_name}" - except Exception as e: + except (SourceUnavailableError, *_OPERATIONAL_ERRORS) as e: logger.warning("Release search failed for source %s: %s", source_name, e) return None, [], f"{source_name}: {e!s}" else: @@ -2809,7 +2825,7 @@ def api_releases() -> Response | tuple[Response, int]: try: first_source = source_instances[sources_to_search[0]] column_config = serialize_column_config(first_source.get_column_config()) - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.warning("Failed to get column config: %s", e) # Convert book to dict and transform cover_url @@ -2851,7 +2867,7 @@ def api_releases() -> Response | tuple[Response, int]: except SourceUnavailableError as e: logger.warning("Release search unavailable: %s", e) return jsonify({"error": str(e)}), 503 - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Releases search error: {e}") return jsonify({"error": str(e)}), 500 @@ -2870,7 +2886,7 @@ def api_release_sources() -> Response | tuple[Response, int]: sources = list_available_sources() return jsonify(sources) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Release sources error: {e}") return jsonify({"error": str(e)}), 500 @@ -2892,7 +2908,7 @@ def api_release_source_record(source_name: str, record_id: str) -> Response | tu except SourceUnavailableError as e: logger.warning("Release source record unavailable: %s", e) return jsonify({"error": str(e)}), 503 - except Exception as e: + except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Release source record error: {e}") return jsonify({"error": str(e)}), 500 @@ -2918,7 +2934,7 @@ def api_settings_get_all() -> Response | tuple[Response, int]: data = serialize_all_settings(include_values=True) return jsonify(data) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Settings get error: {e}") return jsonify({"error": str(e)}), 500 @@ -2952,7 +2968,7 @@ def api_settings_get_tab(tab_name: str) -> Response | tuple[Response, int]: return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404 return jsonify(serialize_tab(tab, include_values=True)) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Settings get tab error: {e}") return jsonify({"error": str(e)}), 500 @@ -2988,7 +3004,7 @@ def api_settings_update_tab(tab_name: str) -> Response | tuple[Response, int]: if not tab: return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404 - values = request.get_json() + values = request.get_json(silent=True) if values is None or not isinstance(values, dict): return jsonify({"error": "Request body must be a JSON object"}), 400 @@ -3001,7 +3017,7 @@ def api_settings_update_tab(tab_name: str) -> Response | tuple[Response, int]: if result["success"]: return jsonify(result) return jsonify(result), 400 - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Settings update error: {e}") return jsonify({"error": str(e)}), 500 @@ -3039,7 +3055,7 @@ def api_settings_execute_action(tab_name: str, action_key: str) -> Response | tu if result["success"]: return jsonify(result) return jsonify(result), 400 - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Settings action error: {e}") return jsonify({"error": str(e)}), 500 @@ -3065,7 +3081,7 @@ def api_onboarding_get() -> Response | tuple[Response, int]: config = get_onboarding_config() return jsonify(config) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Onboarding get error: {e}") return jsonify({"error": str(e)}), 500 @@ -3087,7 +3103,7 @@ def api_onboarding_save() -> Response | tuple[Response, int]: import_module("shelfmark.config.settings") from shelfmark.core.onboarding import save_onboarding_settings - data = request.get_json() + data = request.get_json(silent=True) if not data: return jsonify({"success": False, "message": "No data provided"}), 400 @@ -3096,7 +3112,7 @@ def api_onboarding_save() -> Response | tuple[Response, int]: if result["success"]: return jsonify(result) return jsonify(result), 400 - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Onboarding save error: {e}") return jsonify({"error": str(e)}), 500 @@ -3115,7 +3131,7 @@ def api_onboarding_skip() -> Response | tuple[Response, int]: mark_onboarding_complete() return jsonify({"success": True, "message": "Onboarding skipped"}) - except Exception as e: + except _IMPORT_OPERATIONAL_ERRORS as e: logger.error_trace(f"Onboarding skip error: {e}") return jsonify({"error": str(e)}), 500 @@ -3125,6 +3141,7 @@ def api_onboarding_skip() -> Response | tuple[Response, int]: @app.route("/") def catch_all(path: str) -> Response: """Serve the React app for any route not matched by API endpoints. + This allows React Router to handle client-side routing. Authentication is handled by the React app itself. """ @@ -3146,7 +3163,7 @@ def handle_connect() -> None: # Join appropriate room based on authenticated user session is_admin, db_user_id, can_access_status = _resolve_status_scope() - ws_manager.join_user_room(request.sid, is_admin, db_user_id) + ws_manager.join_user_room(request.sid, is_admin=is_admin, db_user_id=db_user_id) # Send initial status to the newly connected client (filtered) try: @@ -3157,7 +3174,7 @@ def handle_connect() -> None: user_id = None if is_admin else db_user_id status = backend.queue_status(user_id=user_id) emit("status_update", status) - except Exception: + except _OPERATIONAL_ERRORS: logger.exception("Error sending initial status") @@ -3178,7 +3195,7 @@ def handle_status_request() -> None: """Handle manual status request from client.""" try: is_admin, db_user_id, can_access_status = _resolve_status_scope() - ws_manager.sync_user_room(request.sid, is_admin, db_user_id) + ws_manager.sync_user_room(request.sid, is_admin=is_admin, db_user_id=db_user_id) if not can_access_status: emit("status_update", {}) @@ -3187,7 +3204,7 @@ def handle_status_request() -> None: user_id = None if is_admin else db_user_id status = backend.queue_status(user_id=user_id) emit("status_update", status) - except Exception: + except _OPERATIONAL_ERRORS: logger.exception("Error handling status request") emit("error", {"message": "Failed to get status"}) diff --git a/shelfmark/metadata_providers/__init__.py b/shelfmark/metadata_providers/__init__.py index 3de5ec6a..3d1f05e1 100644 --- a/shelfmark/metadata_providers/__init__.py +++ b/shelfmark/metadata_providers/__init__.py @@ -418,6 +418,7 @@ class MetadataProvider(ABC): self, book_id: str, target: str, + *, selected: bool, ) -> dict[str, Any]: """Set whether a book belongs to a provider-managed list or shelf. @@ -436,7 +437,7 @@ _PROVIDER_KWARGS_FACTORIES: dict[str, Any] = {} # Callable[[], Dict] def register_provider( name: str, ) -> Callable[[type[MetadataProvider]], type[MetadataProvider]]: - """Decorator to register a metadata provider.""" + """Register a metadata provider.""" def decorator(cls: type[MetadataProvider]) -> type[MetadataProvider]: _PROVIDERS[name] = cls @@ -448,7 +449,7 @@ def register_provider( def register_provider_kwargs( name: str, ) -> Callable[[Callable[[], dict[str, Any]]], Callable[[], dict[str, Any]]]: - """Decorator to register a provider's kwargs factory. + """Register a provider kwargs factory. The decorated function should return a Dict of kwargs to pass to the provider constructor. This allows each provider to define its own @@ -469,8 +470,8 @@ def register_provider_kwargs( return decorator -def get_provider(name: str, **kwargs) -> MetadataProvider: - """Factory - instantiate any registered provider.""" +def get_provider(name: str, **kwargs: object) -> MetadataProvider: + """Instantiate a registered metadata provider.""" if name not in _PROVIDERS: msg = f"Unknown metadata provider: {name}" raise ValueError(msg) diff --git a/shelfmark/metadata_providers/googlebooks.py b/shelfmark/metadata_providers/googlebooks.py index 2049f7ac..4c9e14da 100644 --- a/shelfmark/metadata_providers/googlebooks.py +++ b/shelfmark/metadata_providers/googlebooks.py @@ -117,7 +117,7 @@ class GoogleBooksProvider(MetadataProvider): key_prefix="googlebooks:search", ) def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> list[BookMetadata]: - """Cached search implementation.""" + """Return cached search results for Google Books.""" # Build query string with Google Books operators author_value = options.fields.get("author", "").strip() title_value = options.fields.get("title", "").strip() @@ -275,7 +275,6 @@ class GoogleBooksProvider(MetadataProvider): if not volume_id or not title: return None - # Authors (list) authors = volume_info.get("authors", []) # ISBNs - extract from industryIdentifiers @@ -353,7 +352,7 @@ class GoogleBooksProvider(MetadataProvider): display_fields=display_fields, ) - except Exception as e: + except (AttributeError, TypeError, ValueError) as e: logger.debug("Failed to parse Google Books volume: %s", e) return None diff --git a/shelfmark/metadata_providers/hardcover.py b/shelfmark/metadata_providers/hardcover.py index f8d1abf1..a4cb1302 100644 --- a/shelfmark/metadata_providers/hardcover.py +++ b/shelfmark/metadata_providers/hardcover.py @@ -3,7 +3,8 @@ import re from contextlib import suppress from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime +from http import HTTPStatus from typing import Any, ClassVar from urllib.parse import urlparse @@ -42,6 +43,10 @@ logger = setup_logger(__name__) HARDCOVER_API_URL = "https://api.hardcover.app/v1/graphql" HARDCOVER_PAGE_SIZE = 25 # Hardcover API returns max 25 results per page +HARDCOVER_MIN_AUTHOR_PARTS = 2 +HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH = 2 +HARDCOVER_MAX_SERIES_OPTIONS = 7 +HARDCOVER_API_KEY_MIN_LENGTH = 100 HARDCOVER_LIST_URL_PATTERN = re.compile( r"^/(?:@([\w.-]+)/)?lists?/([\w-]+)/?$", re.IGNORECASE, @@ -693,11 +698,11 @@ def _simplify_author_for_search(author: str) -> str | None: # Handle "Last, First ..." -> "First ... Last" if "," in normalized: parts = [p.strip() for p in normalized.split(",") if p.strip()] - if len(parts) >= 2: + if len(parts) >= HARDCOVER_MIN_AUTHOR_PARTS: normalized = " ".join([*parts[1:], parts[0]]).strip() tokens = normalized.split(" ") - if len(tokens) < 2: + if len(tokens) < HARDCOVER_MIN_AUTHOR_PARTS: return None keep_suffixes = {"jr", "jr.", "sr", "sr.", "ii", "iii", "iv", "v"} @@ -1117,7 +1122,7 @@ class HardcoverProvider(MetadataProvider): ) -> list[dict[str, Any]]: """Run a Hardcover search request for field-level typeahead options.""" normalized_query = _normalize_search_text(query) - if not self.api_key or len(normalized_query) < 2: + if not self.api_key or len(normalized_query) < HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH: return [] result = self._execute_query( @@ -1226,7 +1231,7 @@ class HardcoverProvider(MetadataProvider): exclude_compilations = app_config.get("HARDCOVER_EXCLUDE_COMPILATIONS", False) exclude_unreleased = app_config.get("HARDCOVER_EXCLUDE_UNRELEASED", False) - current_year = datetime.now().year + current_year = datetime.now(UTC).year options: list[dict[str, str]] = [] seen_labels: set[str] = set() @@ -1327,7 +1332,7 @@ class HardcoverProvider(MetadataProvider): if description: option["description"] = description options.append(option) - if len(options) >= 7: + if len(options) >= HARDCOVER_MAX_SERIES_OPTIONS: break return options @@ -1423,7 +1428,7 @@ class HardcoverProvider(MetadataProvider): str(series_data.get("name") or "").strip() if isinstance(series_data, dict) else "" ) allow_split_parts = _series_allows_split_parts(series_name) - today = datetime.now().date() + today = datetime.now(UTC).date() book_series_rows = ( series_data.get("book_series", []) if isinstance(series_data, dict) else [] @@ -1516,7 +1521,7 @@ class HardcoverProvider(MetadataProvider): @cacheable(ttl=120, key_prefix="hardcover:user_lists") def _get_user_lists_cached(self, _cache_user_id: str) -> list[dict[str, str]]: - """Cached wrapper keyed by Hardcover user id to avoid cross-user cache leakage.""" + """Return cached user lists keyed by Hardcover user id.""" return self._fetch_user_lists() def _fetch_current_user_books_by_status( @@ -1544,7 +1549,7 @@ class HardcoverProvider(MetadataProvider): page: int, limit: int, ) -> SearchResult: - """Cached wrapper keyed by Hardcover user id and status shelf.""" + """Return cached status-shelf books keyed by user id and shelf.""" return self._fetch_user_books_by_status(status_id, page, limit) def _fetch_user_books_by_status(self, status_id: int, page: int, limit: int) -> SearchResult: @@ -1706,7 +1711,8 @@ class HardcoverProvider(MetadataProvider): book_id_int = coerce_int(book_id, 0) if book_id_int < 1: - raise ValueError("book_id must be a valid Hardcover book id") + msg = "book_id must be a valid Hardcover book id" + raise ValueError(msg) state = self._fetch_book_target_state(book_id_int) options = [ @@ -1722,21 +1728,31 @@ class HardcoverProvider(MetadataProvider): return options - def set_book_target_state(self, book_id: str, target: str, selected: bool) -> dict[str, Any]: + def set_book_target_state( + self, + book_id: str, + target: str, + *, + selected: bool, + ) -> dict[str, Any]: """Set whether a Hardcover book belongs to a status shelf or user list.""" if not self.api_key: - raise ValueError("Hardcover is not configured") + msg = "Hardcover is not configured" + raise ValueError(msg) book_id_int = coerce_int(book_id, 0) if book_id_int < 1: - raise ValueError("book_id must be a valid Hardcover book id") + msg = "book_id must be a valid Hardcover book id" + raise ValueError(msg) selected_target = str(target or "").strip() if not selected_target: - raise ValueError("target is required") + msg = "target is required" + raise ValueError(msg) if selected_target not in self._get_writable_targets(): - raise ValueError("Unsupported Hardcover target") + msg = "Unsupported Hardcover target" + raise ValueError(msg) state = self._fetch_book_target_state(book_id_int) status_ids_to_invalidate: set[int] = set() @@ -1769,7 +1785,8 @@ class HardcoverProvider(MetadataProvider): if changed: list_ids_to_invalidate.add(list_id) else: - raise ValueError("Unsupported Hardcover target") + msg = "Unsupported Hardcover target" + raise ValueError(msg) if changed: self._invalidate_book_target_caches( @@ -1787,13 +1804,15 @@ class HardcoverProvider(MetadataProvider): def _unwrap_me_data(result: dict | None) -> dict: """Extract and validate the ``me`` payload from a GraphQL result.""" if not isinstance(result, dict): - raise HardcoverTargetPayloadError("Hardcover could not load book targets") + msg = "Hardcover could not load book targets" + raise HardcoverTargetPayloadError(msg) me_data = result.get("me", {}) if isinstance(me_data, list) and me_data: me_data = me_data[0] if not isinstance(me_data, dict): - raise HardcoverTargetPayloadError("Hardcover returned an invalid target payload") + msg = "Hardcover returned an invalid target payload" + raise HardcoverTargetPayloadError(msg) return me_data def _fetch_book_target_state(self, book_id: int) -> HardcoverBookTargetState: @@ -2052,7 +2071,8 @@ class HardcoverProvider(MetadataProvider): try: return int(value.split(":", 1)[1]) except (IndexError, ValueError) as exc: - raise ValueError(f"Invalid Hardcover {label}") from exc + msg = f"Invalid Hardcover {label}" + raise ValueError(msg) from exc @staticmethod def _check_mutation_result(result: Any, key: str, *, check_error: bool = True) -> None: @@ -2071,7 +2091,8 @@ class HardcoverProvider(MetadataProvider): raise ValueError(error_text) if payload.get("id") is not None: return - raise RuntimeError("Hardcover could not complete this action") + msg = "Hardcover could not complete this action" + raise RuntimeError(msg) def search(self, options: MetadataSearchOptions) -> list[BookMetadata]: """Search for books using Hardcover's search API.""" @@ -2140,7 +2161,7 @@ class HardcoverProvider(MetadataProvider): @cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="hardcover:search") def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> SearchResult: - """Cached search implementation.""" + """Return cached Hardcover search results.""" # Determine query and fields based on custom search fields # Note: Hardcover API requires 'weights' when using 'fields' parameter author_value = options.fields.get("author", "").strip() @@ -2195,7 +2216,7 @@ class HardcoverProvider(MetadataProvider): # Parse hits, filtering compilations and unreleased books if enabled exclude_compilations = app_config.get("HARDCOVER_EXCLUDE_COMPILATIONS", False) exclude_unreleased = app_config.get("HARDCOVER_EXCLUDE_UNRELEASED", False) - current_year = datetime.now().year + current_year = datetime.now(UTC).year books = [] for hit in hits: item = _unwrap_hit_document(hit) @@ -2416,29 +2437,34 @@ class HardcoverProvider(MetadataProvider): except requests.Timeout as e: logger.warning("Hardcover API request timed out") if raise_on_error: - raise RuntimeError("Hardcover API request timed out") from e + msg = "Hardcover API request timed out" + raise RuntimeError(msg) from e return None except requests.HTTPError as e: - if e.response.status_code == 401: + if e.response.status_code == HTTPStatus.UNAUTHORIZED: logger.exception("Hardcover API key is invalid") if raise_on_error: - raise RuntimeError("Hardcover API key is invalid") from e + msg = "Hardcover API key is invalid" + raise RuntimeError(msg) from e else: logger.exception("Hardcover API HTTP error") if raise_on_error: - raise RuntimeError(f"Hardcover API HTTP error: {e}") from e + msg = f"Hardcover API HTTP error: {e}" + raise RuntimeError(msg) from e return None except HardcoverGraphQLError: raise except ValueError as e: logger.exception("Hardcover API returned invalid JSON") if raise_on_error: - raise RuntimeError("Hardcover API returned an invalid response") from e + msg = "Hardcover API returned an invalid response" + raise RuntimeError(msg) from e return None except (TypeError, requests.RequestException) as e: logger.exception("Hardcover API request failed") if raise_on_error: - raise RuntimeError("Hardcover API request failed") from e + msg = "Hardcover API request failed" + raise RuntimeError(msg) from e return None def _parse_search_result(self, item: dict) -> BookMetadata | None: @@ -2705,10 +2731,13 @@ def _test_hardcover_connection(current_values: dict[str, Any] | None = None) -> _save_connected_user(None, None) return {"success": False, "message": "API key is required"} - if key_len < 100: + if key_len < HARDCOVER_API_KEY_MIN_LENGTH: return { "success": False, - "message": f"API key seems too short ({key_len} chars). Expected 500+ chars.", + "message": ( + f"API key seems too short ({key_len} chars). " + f"Expected {HARDCOVER_API_KEY_MIN_LENGTH}+ chars." + ), } connection_result = {"success": False, "message": "API request failed - check your API key"} diff --git a/shelfmark/metadata_providers/openlibrary.py b/shelfmark/metadata_providers/openlibrary.py index 189c59ac..5c8303c3 100644 --- a/shelfmark/metadata_providers/openlibrary.py +++ b/shelfmark/metadata_providers/openlibrary.py @@ -4,6 +4,7 @@ import re import threading import time from collections import deque +from http import HTTPStatus from typing import Any, ClassVar import requests @@ -39,6 +40,8 @@ COVERS_BASE_URL = "https://covers.openlibrary.org" # We use a sliding window with 90 requests per 60 seconds for safety margin RATE_LIMIT_REQUESTS = 90 RATE_LIMIT_WINDOW_SECONDS = 60 +ISBN_10_LENGTH = 10 +ISBN_13_LENGTH = 13 class RateLimiter: @@ -148,7 +151,7 @@ class OpenLibraryProvider(MetadataProvider): ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="openlibrary:search" ) def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> list[BookMetadata]: - """Cached search implementation.""" + """Return cached Open Library search results.""" _rate_limiter.wait_if_needed() # Build query params @@ -210,7 +213,7 @@ class OpenLibraryProvider(MetadataProvider): logger.warning("Open Library search timed out") return [] except requests.HTTPError as e: - if e.response.status_code == 503: + if e.response.status_code == HTTPStatus.SERVICE_UNAVAILABLE: logger.warning("Open Library service unavailable (503)") else: logger.exception("Open Library HTTP error") @@ -249,7 +252,7 @@ class OpenLibraryProvider(MetadataProvider): logger.warning("Open Library get_book timed out") return None except requests.HTTPError as e: - if e.response.status_code == 404: + if e.response.status_code == HTTPStatus.NOT_FOUND: logger.debug("Open Library work not found: %s", book_id) else: logger.exception("Open Library HTTP error") @@ -310,7 +313,7 @@ class OpenLibraryProvider(MetadataProvider): return self._parse_edition(edition, clean_isbn) except requests.HTTPError as e: - if e.response.status_code == 404: + if e.response.status_code == HTTPStatus.NOT_FOUND: logger.debug("Open Library ISBN not found: %s", isbn) else: logger.exception("Open Library ISBN search HTTP error") @@ -339,8 +342,8 @@ class OpenLibraryProvider(MetadataProvider): # Get ISBNs - find first ISBN-10 and ISBN-13 isbns = doc.get("isbn", []) - isbn_10 = next((i for i in isbns if len(i) == 10), None) - isbn_13 = next((i for i in isbns if len(i) == 13), None) + isbn_10 = next((i for i in isbns if len(i) == ISBN_10_LENGTH), None) + isbn_13 = next((i for i in isbns if len(i) == ISBN_13_LENGTH), None) # Get cover URL cover_id = doc.get("cover_i") diff --git a/shelfmark/release_sources/__init__.py b/shelfmark/release_sources/__init__.py index 58c56a80..379cc003 100644 --- a/shelfmark/release_sources/__init__.py +++ b/shelfmark/release_sources/__init__.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from enum import StrEnum +from importlib import import_module from typing import TYPE_CHECKING, Any, ClassVar, Literal if TYPE_CHECKING: @@ -382,7 +383,7 @@ class DownloadHandler(ABC): """Execute download and return a path to the downloaded payload.""" def post_process_cleanup(self, task: DownloadTask, *, success: bool) -> None: - """Optional hook called after orchestrator post-processing. + """Run optional cleanup after orchestrator post-processing. This is primarily used for external download clients, where the handler may need to trigger client-side cleanup only after Shelfmark has safely imported the files. @@ -398,12 +399,30 @@ class DownloadHandler(ABC): _SOURCES: dict[str, type[ReleaseSource]] = {} _HANDLERS: dict[str, type[DownloadHandler]] = {} +_BUILTIN_SOURCE_MODULES = ( + "shelfmark.release_sources.audiobookbay", + "shelfmark.release_sources.direct_download", + "shelfmark.release_sources.irc", + "shelfmark.release_sources.prowlarr", +) +_builtin_source_state = {"loaded": False} + + +def _ensure_builtin_sources_registered() -> None: + """Import built-in source modules once to populate source registries.""" + if _builtin_source_state["loaded"]: + return + + for module_name in _BUILTIN_SOURCE_MODULES: + import_module(module_name) + + _builtin_source_state["loaded"] = True def register_source( name: str, ) -> Callable[[type[ReleaseSource]], type[ReleaseSource]]: - """Decorator to register a release source.""" + """Register a release source.""" def decorator(cls: type[ReleaseSource]) -> type[ReleaseSource]: _SOURCES[name] = cls @@ -415,7 +434,7 @@ def register_source( def register_handler( name: str, ) -> Callable[[type[DownloadHandler]], type[DownloadHandler]]: - """Decorator to register a download handler.""" + """Register a download handler.""" def decorator(cls: type[DownloadHandler]) -> type[DownloadHandler]: _HANDLERS[name] = cls @@ -426,6 +445,7 @@ def register_handler( def get_source(name: str) -> ReleaseSource: """Get a release source instance by name.""" + _ensure_builtin_sources_registered() if name not in _SOURCES: msg = f"Unknown release source: {name}" raise ValueError(msg) @@ -434,6 +454,7 @@ def get_source(name: str) -> ReleaseSource: def get_handler(name: str) -> DownloadHandler: """Get a download handler instance by name.""" + _ensure_builtin_sources_registered() if name not in _HANDLERS: msg = f"Unknown download handler: {name}" raise ValueError(msg) @@ -442,6 +463,7 @@ def get_handler(name: str) -> DownloadHandler: def list_available_sources() -> list[dict]: """List all registered sources with their availability status.""" + _ensure_builtin_sources_registered() result = [] for name, src_class in _SOURCES.items(): instance = src_class() @@ -462,6 +484,7 @@ def list_available_sources() -> list[dict]: def get_source_display_name(name: str) -> str: """Get display name for a source by its identifier.""" + _ensure_builtin_sources_registered() if name in _SOURCES: return _SOURCES[name]().display_name return name.replace("_", " ").title() @@ -505,14 +528,10 @@ def browse_record_to_book_metadata( def source_results_are_releases(name: str) -> bool: """Whether a source's browse/search results already map to concrete releases.""" + _ensure_builtin_sources_registered() if name not in _SOURCES: return False return _SOURCES[name]().search_results_are_releases() -# Import source implementations to trigger registration -# These must be imported AFTER the base classes and registry are defined -from shelfmark.release_sources import audiobookbay as audiobookbay -from shelfmark.release_sources import direct_download as direct_download -from shelfmark.release_sources import irc as irc -from shelfmark.release_sources import prowlarr as prowlarr +_ensure_builtin_sources_registered() diff --git a/shelfmark/release_sources/audiobookbay/source.py b/shelfmark/release_sources/audiobookbay/source.py index a78d0395..a1612b62 100644 --- a/shelfmark/release_sources/audiobookbay/source.py +++ b/shelfmark/release_sources/audiobookbay/source.py @@ -25,6 +25,7 @@ from shelfmark.release_sources.audiobookbay import scraper from shelfmark.release_sources.audiobookbay.utils import normalize_hostname, parse_size logger = setup_logger(__name__) +MIN_RELEVANCE_QUERY_WORD_LENGTH = 2 # Map language names to ISO 639-1 codes (matching frontend color maps) @@ -133,7 +134,7 @@ def _parse_bitrate_to_kbps(bitrate: str | None) -> int | None: def _generate_source_id(detail_url: str) -> str: """Generate a unique source ID from detail URL.""" - return hashlib.md5(detail_url.encode()).hexdigest() + return hashlib.blake2b(detail_url.encode(), digest_size=16).hexdigest() @register_source("audiobookbay") @@ -246,7 +247,11 @@ class AudiobookBaySource(ReleaseSource): ) # Extract query words for relevance checking - query_words = {word.lower() for word in query_lower.split() if len(word) > 2} + query_words = { + word.lower() + for word in query_lower.split() + if len(word) > MIN_RELEVANCE_QUERY_WORD_LENGTH + } releases = [] for result in results: diff --git a/shelfmark/release_sources/direct_download.py b/shelfmark/release_sources/direct_download.py index 04c361d2..a4fe9744 100644 --- a/shelfmark/release_sources/direct_download.py +++ b/shelfmark/release_sources/direct_download.py @@ -5,6 +5,7 @@ import json import re import time from dataclasses import replace +from http import HTTPStatus from typing import TYPE_CHECKING, ClassVar, NoReturn from urllib.parse import quote @@ -68,6 +69,7 @@ _DOWNLOAD_SOURCES = [ _SOURCE_FAILURE_THRESHOLD = 4 _MIN_VALID_FILE_SIZE = 10 * 1024 +_AA_COUNTDOWN_MAX_SECONDS = 300 # Sources that require Cloudflare bypass _CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "welib"}) @@ -163,7 +165,7 @@ def _normalize_size(size_str: str) -> str: return _SIZE_UNIT_PATTERN.sub(lambda m: m.group(1).upper(), size_str.strip()) -class SearchUnavailable(SourceUnavailableError): +class SearchUnavailableError(SourceUnavailableError): """Raised when Anna's Archive cannot be reached via any mirror/DNS.""" @@ -178,7 +180,7 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]: List[BrowseRecord]: List of matching books Raises: - SearchUnavailable: If Anna's Archive cannot be reached + SearchUnavailableError: If Anna's Archive cannot be reached Exception: If parsing fails """ @@ -224,9 +226,8 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]: html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False) if not html: # Network/mirror exhaustion path bubbles up so API can notify clients - raise SearchUnavailable( - "Unable to reach download source. Network restricted or mirrors are blocked." - ) + msg = "Unable to reach download source. Network restricted or mirrors are blocked." + raise SearchUnavailableError(msg) if "No files found." in html: logger.info("No books found for query: %s", query) @@ -237,7 +238,8 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]: if not tbody: logger.warning("No results table found for query: %s", query) - raise RuntimeError("No books found. Please try another query.") + msg = "No books found. Please try another query." + raise RuntimeError(msg) books = [] if isinstance(tbody, Tag): @@ -274,9 +276,8 @@ def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseR html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False) if not html: - raise SearchUnavailable( - "Unable to reach download source. Network restricted or mirrors are blocked." - ) + msg = "Unable to reach download source. Network restricted or mirrors are blocked." + raise SearchUnavailableError(msg) soup = BeautifulSoup(html, "html.parser") @@ -320,7 +321,8 @@ def _parse_book_info_page( data = soup.select_one("body > main > div:nth-of-type(1)") if not data: - raise RuntimeError(f"Failed to parse book info for ID: {book_id}") + msg = f"Failed to parse book info for ID: {book_id}" + raise RuntimeError(msg) preview: str = "" @@ -381,15 +383,15 @@ def _parse_book_info_page( divs = [div for div in divs if div.text.strip() != ""] all_details = _find_in_divs(divs, " · ") - format = "" + file_format = "" size = "" content = "" for _details in all_details: _details = _details.split(" · ") for f in _details: - if format == "" and f.strip().lower() in config.SUPPORTED_FORMATS: - format = f.strip().lower() + if file_format == "" and f.strip().lower() in config.SUPPORTED_FORMATS: + file_format = f.strip().lower() if size == "" and any(u in f.strip().lower() for u in ("mb", "kb", "gb")): size = _normalize_size(f) if content == "": @@ -397,11 +399,11 @@ def _parse_book_info_page( if ct in f.strip().lower(): content = ct break - if format == "" or size == "": + if file_format == "" or size == "": for f in _details: stripped = f.strip().lower() - if format == "" and stripped and " " not in stripped: - format = stripped + if file_format == "" and stripped and " " not in stripped: + file_format = stripped if size == "" and "." in stripped: size = _normalize_size(f) @@ -418,7 +420,7 @@ def _parse_book_info_page( content=content, publisher=(_find_in_divs(divs, "icon-[mdi--company]", is_class=True) or [""])[0], author=(_find_in_divs(divs, "icon-[mdi--user-edit]", is_class=True) or [""])[0], - format=format, + format=file_format, size=size, description=description, download_urls=urls, @@ -438,7 +440,7 @@ def _parse_book_info_page( if "downloads_total" in summary_data: info["Downloads"] = [str(summary_data["downloads_total"])] except ( - SearchUnavailable, + SearchUnavailableError, RuntimeError, json.JSONDecodeError, TypeError, @@ -592,7 +594,7 @@ def _fetch_aa_page_urls(book_info: BrowseRecord, urls_by_source: dict[str, list[ try: fresh_book_info = get_book_info(book_info.id, fetch_download_count=False) _group_urls_by_source(fresh_book_info.download_urls, urls_by_source) - except (SearchUnavailable, RuntimeError, TypeError, AttributeError) as e: + except (SearchUnavailableError, RuntimeError, TypeError, AttributeError) as e: logger.warning("Failed to fetch AA page: %s", e) @@ -740,7 +742,7 @@ def _get_download_urls_from_welib( status_callback=status_callback, ) except ( - SearchUnavailable, + SearchUnavailableError, requests.exceptions.RequestException, RuntimeError, ValueError, @@ -780,7 +782,7 @@ def _extract_libgen_download_url(link: str, cancel_flag: Event | None = None) -> verify=network.get_ssl_verify(link), ) - if response.status_code != 200: + if response.status_code != HTTPStatus.OK: logger.debug("Libgen fast: %s returned %s", link, response.status_code) return "" @@ -1058,13 +1060,13 @@ def _extract_slow_download_url( countdown_seconds = _extract_countdown_seconds(soup, html_str) if countdown_seconds > 0: - MAX_COUNTDOWN_SECONDS = 600 - sleep_time = min(countdown_seconds, MAX_COUNTDOWN_SECONDS) - if countdown_seconds > MAX_COUNTDOWN_SECONDS: + max_countdown_seconds = 600 + sleep_time = min(countdown_seconds, max_countdown_seconds) + if countdown_seconds > max_countdown_seconds: logger.warning( "Countdown %ss exceeds max, capping at %ss", countdown_seconds, - MAX_COUNTDOWN_SECONDS, + max_countdown_seconds, ) logger.info("AA waitlist: %ss for %s", sleep_time, title) @@ -1114,36 +1116,36 @@ def _extract_countdown_seconds(soup: BeautifulSoup, html_str: str) -> int: countdown_attr = re.search(r'data-countdown=["\'](\d+)["\']', html_str) if countdown_attr: seconds = int(countdown_attr.group(1)) - if 0 < seconds < 300: + if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS: return seconds js_countdown = re.search(r"countdown:\s*(\d+)", html_str) if js_countdown: seconds = int(js_countdown.group(1)) - if 0 < seconds < 300: + if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS: return seconds js_var = re.search(r"(?:var|let|const)\s+countdown\s*=\s*(\d+)", html_str) if js_var: seconds = int(js_var.group(1)) - if 0 < seconds < 300: + if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS: return seconds countdown_secs = re.search(r"countdownSeconds\s*=\s*(\d+)", html_str) if countdown_secs: seconds = int(countdown_secs.group(1)) - if 0 < seconds < 300: + if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS: return seconds json_countdown = re.search(r'["\']countdown[_-]?seconds["\']\s*:\s*(\d+)', html_str) if json_countdown: seconds = int(json_countdown.group(1)) - if 0 < seconds < 300: + if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS: return seconds wait_text = re.search(r"wait\s+(\d+)\s+seconds", html_str, re.IGNORECASE) if wait_text: seconds = int(wait_text.group(1)) - if 0 < seconds < 300: + if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS: return seconds return 0 @@ -1156,7 +1158,7 @@ def _parse_countdown_seconds_from_element(element: Tag) -> int | None: except ValueError, TypeError: return None - if 0 < seconds < 300: + if 0 < seconds < _AA_COUNTDOWN_MAX_SECONDS: return seconds return None @@ -1204,6 +1206,7 @@ class DirectDownloadSource(ReleaseSource): supported_content_types: ClassVar[list[str]] = ["ebook"] # Direct downloads only support ebooks def __init__(self) -> None: + """Initialize per-instance search state for direct downloads.""" # Tracks which search method was used in the last search() call # "isbn" = ISBN search returned results, "title_author" = title+author was used self._last_search_type: str = "title_author" @@ -1307,6 +1310,7 @@ class DirectDownloadSource(ReleaseSource): Args: book: Book metadata from provider + plan: Precomputed search plan with normalized queries and filters. expand_search: If True, skip ISBN and use title+author directly languages: Language codes to filter by (overrides book.language/config) content_type: Ignored - Direct download uses format filtering instead @@ -1347,7 +1351,7 @@ class DirectDownloadSource(ReleaseSource): self._last_search_type = "isbn" return [_browse_record_to_release(record) for record in results] logger.debug("No ISBN results, falling back to title+author") - except SearchUnavailable: + except SearchUnavailableError: raise except (ValueError, TypeError, AttributeError, RuntimeError) as e: logger.warning("ISBN search failed: %s", e) @@ -1372,7 +1376,7 @@ class DirectDownloadSource(ReleaseSource): if bi.id not in seen_ids: seen_ids.add(bi.id) all_results.append(bi) - except SearchUnavailable: + except SearchUnavailableError: raise except Exception: logger.exception("Search error") @@ -1392,7 +1396,7 @@ class DirectDownloadSource(ReleaseSource): if bi.id not in seen_ids: seen_ids.add(bi.id) all_results.append(bi) - except SearchUnavailable: + except SearchUnavailableError: raise except Exception: logger.exception("Search error") @@ -1477,7 +1481,7 @@ class DirectDownloadHandler(DownloadHandler): progress_callback: Callable[[float], None], status_callback: Callable[[str, str | None], None], ) -> str | None: - """Internal method to execute the download with fetched browse record. + """Execute the direct-download flow with a fetched browse record. This contains the core download logic: cascade through sources, handle bypass, move to final location. diff --git a/shelfmark/release_sources/irc/client.py b/shelfmark/release_sources/irc/client.py index a85f8804..d48b59c9 100644 --- a/shelfmark/release_sources/irc/client.py +++ b/shelfmark/release_sources/irc/client.py @@ -10,7 +10,7 @@ import time from contextlib import suppress from dataclasses import dataclass, field from enum import Enum, auto -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Self from shelfmark.core.logger import setup_logger @@ -78,6 +78,7 @@ class IRCClient: use_tls: bool = True, version: str = "Shelfmark 1.0", ) -> None: + """Initialize the IRC client with connection settings and defaults.""" if not nick: msg = "IRC nickname is required" raise IRCError(msg) @@ -456,9 +457,11 @@ class IRCClient: """Check if currently connected.""" return self._connected and self._socket is not None - def __enter__(self) -> IRCClient: + def __enter__(self) -> Self: + """Connect and return the IRC client for context-manager usage.""" self.connect() return self - def __exit__(self, *args) -> None: + def __exit__(self, *args: object) -> None: + """Disconnect the IRC client when leaving a context manager.""" self.disconnect() diff --git a/shelfmark/release_sources/irc/connection_manager.py b/shelfmark/release_sources/irc/connection_manager.py index 3a5d199a..0ba9de84 100644 --- a/shelfmark/release_sources/irc/connection_manager.py +++ b/shelfmark/release_sources/irc/connection_manager.py @@ -6,15 +6,17 @@ Maintains persistent IRC connections to avoid reconnecting between search and do import threading import time from contextlib import suppress +from typing import Self from shelfmark.core.logger import setup_logger -from .client import IRCClient +from .client import IRCClient, IRCError logger = setup_logger(__name__) # How long to keep an idle connection before closing it IDLE_TIMEOUT = 300.0 # 5 minutes +_IRC_CONNECTION_ERRORS = (IRCError, OSError, RuntimeError) class IRCConnectionManager: @@ -28,7 +30,7 @@ class IRCConnectionManager: _instance: IRCConnectionManager | None = None _lock = threading.Lock() - def __new__(cls) -> IRCConnectionManager: + def __new__(cls) -> Self: """Singleton pattern - only one connection manager.""" if cls._instance is None: with cls._lock: @@ -38,6 +40,7 @@ class IRCConnectionManager: return cls._instance def __init__(self) -> None: + """Initialize connection caches for the singleton manager.""" if self._initialized: return @@ -87,7 +90,7 @@ class IRCConnectionManager: logger.info("Closing idle IRC connection: %s", key) try: client.disconnect() - except Exception as e: + except _IRC_CONNECTION_ERRORS as e: logger.debug("Error closing idle connection: %s", e) def get_connection( @@ -183,7 +186,7 @@ class IRCConnectionManager: self._last_used[key] = time.time() self._channels[key] = channel self._connecting.pop(key, None) - except Exception: + except _IRC_CONNECTION_ERRORS: # Clear connecting flag on failure with self._conn_lock: self._connecting.pop(key, None) @@ -220,7 +223,7 @@ class IRCConnectionManager: try: client.disconnect() - except Exception as e: + except _IRC_CONNECTION_ERRORS as e: logger.debug("Error closing connection: %s", e) logger.debug("Closed IRC connection: %s", key) @@ -242,7 +245,7 @@ class IRCConnectionManager: """Disconnect one IRC client and log failures.""" try: client.disconnect() - except Exception as e: + except _IRC_CONNECTION_ERRORS as e: logger.debug("Error closing connection %s: %s", key, e) diff --git a/shelfmark/release_sources/irc/source.py b/shelfmark/release_sources/irc/source.py index 0d492dc3..26e1cbb7 100644 --- a/shelfmark/release_sources/irc/source.py +++ b/shelfmark/release_sources/irc/source.py @@ -75,6 +75,7 @@ class IRCReleaseSource(ReleaseSource): can_be_default = False # Exclude from default source options (requires deliberate selection) def __init__(self) -> None: + """Initialize per-search IRC source state.""" # Track online servers from most recent search self._online_servers: set[str] | None = None diff --git a/shelfmark/release_sources/prowlarr/api.py b/shelfmark/release_sources/prowlarr/api.py index f1e7bc36..1423d36b 100644 --- a/shelfmark/release_sources/prowlarr/api.py +++ b/shelfmark/release_sources/prowlarr/api.py @@ -1,5 +1,6 @@ """Prowlarr API client for connection testing, indexer listing, and search.""" +from contextlib import suppress from http import HTTPStatus from typing import Any @@ -15,12 +16,20 @@ logger = setup_logger(__name__) _HTTP_STATUS_UNAUTHORIZED = HTTPStatus.UNAUTHORIZED _BOOK_CATEGORY_RANGE_START = 7000 _BOOK_CATEGORY_RANGE_END = 8000 +_PROWLARR_CLIENT_ERRORS = ( + requests.exceptions.RequestException, + OSError, + RuntimeError, + TypeError, + ValueError, +) class ProwlarrClient: """Client for interacting with the Prowlarr API.""" def __init__(self, url: str, api_key: str, timeout: int = 30) -> None: + """Initialize the API client with base URL, key, and timeout.""" self.base_url = normalize_http_url(url) self.api_key = api_key self.timeout = timeout @@ -54,11 +63,9 @@ class ProwlarrClient: ) if not response.ok: - try: + with suppress(Exception): error_body = response.text[:500] logger.error("Prowlarr API error response: %s", error_body) - except Exception: - pass response.raise_for_status() return response.json() @@ -91,7 +98,7 @@ class ProwlarrClient: if e.response is not None and e.response.status_code == _HTTP_STATUS_UNAUTHORIZED: return False, "Invalid API key" return False, f"HTTP error {status}" - except Exception as e: + except _PROWLARR_CLIENT_ERRORS as e: return False, f"Connection failed: {e!s}" else: logger.info("Prowlarr connection successful: version %s", version) @@ -101,7 +108,7 @@ class ProwlarrClient: """Get all configured indexers.""" try: return self._request("GET", "/api/v1/indexer") - except Exception: + except _PROWLARR_CLIENT_ERRORS: logger.exception("Failed to get indexers") return [] @@ -215,11 +222,9 @@ class ProwlarrClient: verify=get_ssl_verify(url), ) if not response.ok: - try: + with suppress(Exception): error_body = response.text[:500] logger.error("Prowlarr Torznab error response: %s", error_body) - except Exception: - pass response.raise_for_status() results = parse_torznab_xml(response.text) diff --git a/shelfmark/release_sources/prowlarr/handler.py b/shelfmark/release_sources/prowlarr/handler.py index f67c58d3..76082b88 100644 --- a/shelfmark/release_sources/prowlarr/handler.py +++ b/shelfmark/release_sources/prowlarr/handler.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from shelfmark.core.config import config as config +from shelfmark.core.config import config from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import normalize_optional_text from shelfmark.download.clients import ( @@ -36,6 +36,13 @@ if TYPE_CHECKING: from shelfmark.core.models import DownloadTask logger = setup_logger(__name__) +__all__ = [ + "ProwlarrHandler", + "POLL_INTERVAL", + "COMPLETED_PATH_RETRY_INTERVAL", + "COMPLETED_PATH_MAX_ATTEMPTS", + "config", +] # Backwards-compat constants for tests patching this module. POLL_INTERVAL = _DEFAULT_POLL_INTERVAL diff --git a/shelfmark/release_sources/prowlarr/settings.py b/shelfmark/release_sources/prowlarr/settings.py index ef2cdad4..0e4437cd 100644 --- a/shelfmark/release_sources/prowlarr/settings.py +++ b/shelfmark/release_sources/prowlarr/settings.py @@ -2,6 +2,8 @@ from typing import Any +import requests + from shelfmark.core.settings_registry import ( ActionButton, CheckboxField, @@ -16,6 +18,15 @@ from shelfmark.core.utils import normalize_http_url # ==================== Dynamic Options Loaders ==================== +_PROWLARR_SETTINGS_ERRORS = ( + requests.exceptions.RequestException, + AttributeError, + OSError, + RuntimeError, + TypeError, + ValueError, +) + def _get_indexer_options() -> list[dict[str, str]]: """Fetch available indexers from Prowlarr for the multi-select field. @@ -62,7 +73,7 @@ def _get_indexer_options() -> list[dict[str, str]]: } ) - except Exception: + except _PROWLARR_SETTINGS_ERRORS: logger.exception("Failed to fetch Prowlarr indexers") return [] @@ -95,7 +106,7 @@ def _test_prowlarr_connection(current_values: dict[str, Any] | None = None) -> d try: client = ProwlarrClient(url, api_key) success, message = client.test_connection() - except Exception as e: + except _PROWLARR_SETTINGS_ERRORS as e: return {"success": False, "message": f"Connection failed: {e!s}"} else: return {"success": success, "message": message} diff --git a/shelfmark/release_sources/prowlarr/source.py b/shelfmark/release_sources/prowlarr/source.py index cf5a50ce..c7d4849b 100644 --- a/shelfmark/release_sources/prowlarr/source.py +++ b/shelfmark/release_sources/prowlarr/source.py @@ -38,6 +38,7 @@ logger = setup_logger(__name__) _SIZE_UNIT_BASE = 1024 _TWO_FORMATS = 2 +_PROWLARR_SOURCE_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, ValueError) def _raise_timeout_error(message: str) -> NoReturn: @@ -443,6 +444,7 @@ class ProwlarrSource(ReleaseSource): ] # Explicitly declare support for both def __init__(self) -> None: + """Initialize per-instance search state for Prowlarr.""" self.last_search_type: str | None = None def get_column_config(self) -> ReleaseColumnConfig: @@ -482,7 +484,7 @@ class ProwlarrSource(ReleaseSource): default_indexers = ( sorted(selected_indexer_names) if selected_indexer_names else None ) - except Exception as e: + except _PROWLARR_SOURCE_ERRORS as e: logger.warning("Failed to fetch indexer list for column config: %s", e) return ReleaseColumnConfig( @@ -617,7 +619,7 @@ class ProwlarrSource(ReleaseSource): if idx_id is not None: with suppress(TypeError, ValueError): ids.append(int(idx_id)) - except Exception as e: + except _PROWLARR_SOURCE_ERRORS as e: logger.warning("Failed to resolve indexer names to IDs: %s", e) return None else: @@ -635,7 +637,7 @@ class ProwlarrSource(ReleaseSource): try: enabled_indexers = client.get_enabled_indexers_detailed() - except Exception as e: + except _PROWLARR_SOURCE_ERRORS as e: logger.warning("Failed to load enabled Prowlarr indexers: %s", e) return [] diff --git a/shelfmark/release_sources/prowlarr/torznab.py b/shelfmark/release_sources/prowlarr/torznab.py index f96a370f..9d9931eb 100644 --- a/shelfmark/release_sources/prowlarr/torznab.py +++ b/shelfmark/release_sources/prowlarr/torznab.py @@ -44,7 +44,9 @@ def _coerce_float(value: str | None) -> float | None: def _strip_author_from_title(title: str, author: str | None) -> str: - """Prowlarr's MyAnonamouse parser appends " by {author}" into the title while + """Strip duplicate trailing author text from a Torznab title. + + Prowlarr's MyAnonamouse parser appends " by {author}" into the title while also emitting author/booktitle fields. Shelfmark's UI shows author separately, so strip the duplicated " by author" segment when present. """ @@ -59,8 +61,9 @@ def _strip_author_from_title(title: str, author: str | None) -> str: def parse_torznab_xml(xml_text: str) -> list[dict[str, Any]]: - """Parse a Torznab/Newznab XML response into a list of dicts that roughly match - Prowlarr's JSON search results shape. + """Parse a Torznab/Newznab XML response into Prowlarr-like result dicts. + + This keeps the parsed shape close to Prowlarr's JSON search results. """ if not xml_text or not xml_text.strip(): return [] diff --git a/tests/bypass/test_internal_bypasser.py b/tests/bypass/test_internal_bypasser.py index eb39d769..b21ac30f 100644 --- a/tests/bypass/test_internal_bypasser.py +++ b/tests/bypass/test_internal_bypasser.py @@ -38,6 +38,7 @@ def test_bypass_tries_all_methods_before_abort(monkeypatch): def test_extract_cookies_from_cdp_filters_and_stores_ua(): import time + import shelfmark.bypass.internal_bypasser as internal_bypasser class FakeCookie: @@ -80,6 +81,7 @@ def test_extract_cookies_from_cdp_filters_and_stores_ua(): def test_extract_cookies_from_cdp_normalizes_session_expiry(): import time + import shelfmark.bypass.internal_bypasser as internal_bypasser class FakeCookie: @@ -121,3 +123,90 @@ def test_extract_cookies_from_cdp_normalizes_session_expiry(): # Verify fallback to "expires" key for expiry checks internal_bypasser._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10 assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {} + + +def test_get_page_info_returns_safe_defaults_on_cdp_errors(): + from seleniumbase.undetected.cdp_driver.connection import ProtocolException + + import shelfmark.bypass.internal_bypasser as internal_bypasser + + class FakePage: + async def get_title(self): + raise ProtocolException("no title") + + async def evaluate(self, _expr): + raise ProtocolException("no body") + + async def get_current_url(self): + raise ProtocolException("no url") + + title, body, current_url = asyncio.run(internal_bypasser._get_page_info(FakePage())) + + assert title == "" + assert body == "" + assert current_url == "" + + +def test_try_with_cached_cookies_returns_none_on_request_exception(monkeypatch): + import time + + import requests + + import shelfmark.bypass.internal_bypasser as internal_bypasser + + internal_bypasser.clear_cf_cookies() + internal_bypasser._cf_cookies["example.com"] = { + "cf_clearance": { + "value": "abc", + "domain": "example.com", + "path": "/", + "expiry": int(time.time()) + 3600, + "secure": True, + "httpOnly": True, + } + } + + def _raise(*_args, **_kwargs): + raise requests.RequestException("boom") + + monkeypatch.setattr(internal_bypasser.requests, "get", _raise) + + assert internal_bypasser._try_with_cached_cookies("https://example.com", "example.com") is None + + +def test_get_bypassed_page_retries_next_mirror_after_runtime_error(monkeypatch): + import shelfmark.bypass.internal_bypasser as internal_bypasser + + class FakeSelector: + def __init__(self): + self.urls = ["https://mirror-one.example/book", "https://mirror-two.example/book"] + self.index = 0 + + def rewrite(self, _url): + return self.urls[self.index] + + def next_mirror_or_rotate_dns(self, *, allow_dns=True): + del allow_dns + self.index = 1 + return "https://mirror-two.example", "mirror" + + calls: list[str] = [] + + def _fake_get(url, retry=None, cancel_flag=None): + del retry, cancel_flag + calls.append(url) + if len(calls) == 1: + raise RuntimeError("browser hiccup") + return "ok" + + monkeypatch.setattr(internal_bypasser, "_try_with_cached_cookies", lambda *_args, **_kwargs: None) + monkeypatch.setattr(internal_bypasser, "get", _fake_get) + + selector = FakeSelector() + result = internal_bypasser.get_bypassed_page("https://orig.example/book", selector=selector) + + assert result == "ok" + assert calls == [ + "https://mirror-one.example/book", + "https://mirror-two.example/book", + ] diff --git a/tests/config/test_dns_settings_live_apply.py b/tests/config/test_dns_settings_live_apply.py new file mode 100644 index 00000000..a1b0414a --- /dev/null +++ b/tests/config/test_dns_settings_live_apply.py @@ -0,0 +1,38 @@ +def test_update_settings_network_logs_dns_apply_failure(monkeypatch): + import shelfmark.config.settings # noqa: F401 + import shelfmark.core.settings_registry as registry + from shelfmark.core.config import config as config_obj + from shelfmark.core.settings_registry import update_settings + + monkeypatch.setattr("shelfmark.core.settings_registry.save_config_file", lambda _tab, _values: True) + monkeypatch.setattr(config_obj, "refresh", lambda: None) + + import shelfmark.download.network as network + + def failing_set_dns_provider(*args, **kwargs) -> None: + raise RuntimeError("dns apply failed") + + monkeypatch.setattr(network, "set_dns_provider", failing_set_dns_provider) + + warnings: list[tuple[str, tuple[object, ...]]] = [] + monkeypatch.setattr( + registry.logger, + "warning", + lambda message, *args: warnings.append((str(message), args)), + ) + + result = update_settings( + "network", + { + "CUSTOM_DNS": "manual", + "CUSTOM_DNS_MANUAL": "1.1.1.1,8.8.8.8", + }, + ) + + assert result["success"] is True + assert len(warnings) == 1 + message, args = warnings[0] + assert message == "Failed to apply DNS settings: %s" + assert len(args) == 1 + assert isinstance(args[0], RuntimeError) + assert str(args[0]) == "dns apply failed" diff --git a/tests/config/test_mirror_settings_live_apply.py b/tests/config/test_mirror_settings_live_apply.py index 9e710d22..4cd1364c 100644 --- a/tests/config/test_mirror_settings_live_apply.py +++ b/tests/config/test_mirror_settings_live_apply.py @@ -1,7 +1,6 @@ def test_update_settings_mirrors_applies_aa_changes_live(monkeypatch): # Ensure settings tabs are registered (mirrors tab lives here). import shelfmark.config.settings # noqa: F401 - from shelfmark.core.config import config as config_obj from shelfmark.core.settings_registry import update_settings @@ -26,3 +25,37 @@ def test_update_settings_mirrors_applies_aa_changes_live(monkeypatch): assert result["success"] is True assert called["force"] is True + +def test_update_settings_mirrors_logs_live_apply_failure(monkeypatch): + import shelfmark.config.settings # noqa: F401 + import shelfmark.core.settings_registry as registry + from shelfmark.core.config import config as config_obj + from shelfmark.core.settings_registry import update_settings + + monkeypatch.setattr("shelfmark.core.settings_registry.save_config_file", lambda _tab, _values: True) + monkeypatch.setattr(config_obj, "refresh", lambda: None) + + import shelfmark.download.network as network + + def failing_init_aa(*, force: bool = False) -> None: + del force + raise RuntimeError("mirror apply failed") + + monkeypatch.setattr(network, "init_aa", failing_init_aa) + + warnings: list[tuple[str, tuple[object, ...]]] = [] + monkeypatch.setattr( + registry.logger, + "warning", + lambda message, *args: warnings.append((str(message), args)), + ) + + result = update_settings("mirrors", {"AA_BASE_URL": "https://annas-archive.li"}) + + assert result["success"] is True + assert len(warnings) == 1 + message, args = warnings[0] + assert message == "Failed to apply AA mirror settings: %s" + assert len(args) == 1 + assert isinstance(args[0], RuntimeError) + assert str(args[0]) == "mirror apply failed" diff --git a/tests/core/test_activity_routes_api.py b/tests/core/test_activity_routes_api.py index 87cf4192..41ea4bd0 100644 --- a/tests/core/test_activity_routes_api.py +++ b/tests/core/test_activity_routes_api.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib +import sqlite3 import uuid from types import SimpleNamespace from unittest.mock import ANY, patch @@ -66,7 +67,7 @@ def _record_terminal_download( source_display_name=source_display_name, title=title, author=author, - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -390,7 +391,7 @@ class TestActivityRoutes: source_display_name="Direct Download", title="Active Dismiss Task", author="Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -554,7 +555,7 @@ class TestActivityRoutes: source_display_name="Direct Download", title="Stale Active Download", author="Stale Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -605,7 +606,7 @@ class TestActivityRoutes: source_display_name="Prowlarr", title="Interrupted Requested Download", author="Stale Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -782,6 +783,34 @@ class TestActivityRoutes: assert response.status_code == 403 assert response.json["code"] == "user_identity_unavailable" + def test_dismiss_many_with_user_db_lookup_failure_returns_identity_unavailable( + self, main_module, client + ): + user = _create_user(main_module, prefix="reader") + _set_session(client, user_id=user["username"], db_user_id=user["id"], is_admin=False) + + with ( + patch.object(main_module, "get_auth_mode", return_value="builtin"), + patch.object( + main_module.user_db, + "get_user", + side_effect=sqlite3.OperationalError("database is locked"), + ), + patch("shelfmark.core.activity_routes.logger.warning") as mock_warning, + ): + response = client.post( + "/api/activity/dismiss-many", + json={"items": [{"item_type": "download", "item_key": "download:test-db-error"}]}, + ) + + assert response.status_code == 403 + assert response.json["code"] == "user_identity_unavailable" + mock_warning.assert_any_call( + "Failed to validate activity db identity %s: %s", + user["id"], + ANY, + ) + def test_clear_history_logs_identity_failure(self, main_module, client): admin = _create_user(main_module, prefix="admin", role="admin") _set_session(client, user_id=admin["username"], db_user_id=None, is_admin=True) @@ -896,7 +925,7 @@ class TestActivityRoutes: source_display_name="Direct Download", title="Stale Active Task", author="Stale Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -938,7 +967,7 @@ class TestActivityRoutes: source_display_name="Prowlarr", title="Stale Active Requested Task", author="Stale Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -1027,7 +1056,7 @@ class TestActivityRoutes: source_display_name="Prowlarr", title="Retry Gone Request", author="Retry Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -1068,7 +1097,7 @@ class TestActivityRoutes: source_display_name="Direct Download", title="Active Downloading Task", author="Active Author", - format="epub", + file_format="epub", size="2 MB", preview=None, content_type="ebook", diff --git a/tests/core/test_admin_users_api.py b/tests/core/test_admin_users_api.py index e1ce34c9..7e854daf 100644 --- a/tests/core/test_admin_users_api.py +++ b/tests/core/test_admin_users_api.py @@ -700,6 +700,24 @@ class TestAdminUserUpdateEndpoint: assert resp.json["error"] == "Invalid settings payload" assert any("Unknown setting: destination" in msg for msg in resp.json["details"]) + def test_update_user_settings_warns_when_runtime_refresh_fails(self, admin_client, user_db): + user = user_db.create_user(username="alice") + + with ( + patch("shelfmark.core.admin_routes.app_config.refresh", side_effect=RuntimeError("boom")), + patch("shelfmark.core.admin_routes.logger.warning") as mock_warning, + ): + resp = admin_client.put( + f"/api/admin/users/{user['id']}", + json={"settings": {"DESTINATION": "/books/alice"}}, + ) + + assert resp.status_code == 200 + settings = user_db.get_user_settings(user["id"]) + assert settings["DESTINATION"] == "/books/alice" + mock_warning.assert_called_once() + assert "failed to refresh runtime config" in mock_warning.call_args[0][0] + def test_update_response_excludes_password_hash(self, admin_client, user_db): user = user_db.create_user(username="alice", password_hash="secret") diff --git a/tests/core/test_download_api_guardrails.py b/tests/core/test_download_api_guardrails.py index debb30cf..14c8a07c 100644 --- a/tests/core/test_download_api_guardrails.py +++ b/tests/core/test_download_api_guardrails.py @@ -151,7 +151,7 @@ class TestReleaseDownloadEndpointGuardrails: assert captured["release_data"] == {**payload, "content_type": "audiobook"} assert captured["priority"] == 1 - def test_non_json_payload_returns_500_current_behavior(self, main_module, client): + def test_non_json_payload_returns_400(self, main_module, client): with patch.object(main_module, "get_auth_mode", return_value="none"): with patch.object(main_module.backend, "queue_release") as mock_queue_release: resp = client.post( @@ -161,8 +161,8 @@ class TestReleaseDownloadEndpointGuardrails: ) body = resp.get_json() - assert resp.status_code == 500 - assert "Unsupported Media Type" in body["error"] + assert resp.status_code == 400 + assert body == {"error": "No data provided"} mock_queue_release.assert_not_called() def test_admin_can_queue_release_on_behalf_of_another_user(self, main_module, client): @@ -520,7 +520,7 @@ class TestRetryDownloadEndpointGuardrails: source_display_name="Direct Download", title="Persisted Direct Task", author="Direct Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", @@ -705,7 +705,7 @@ class TestRetryDownloadEndpointGuardrails: source_display_name="Prowlarr", title="Persisted Requested Book", author="Request Author", - format="epub", + file_format="epub", size="1 MB", preview=None, content_type="ebook", diff --git a/tests/core/test_download_history_service.py b/tests/core/test_download_history_service.py index e1f8a1ed..ba0e4eef 100644 --- a/tests/core/test_download_history_service.py +++ b/tests/core/test_download_history_service.py @@ -30,7 +30,7 @@ def test_record_download_stores_utc_iso_timestamps(): source_display_name="Direct Download", title="Example", author=None, - format=None, + file_format=None, size=None, preview=None, content_type="ebook", diff --git a/tests/core/test_download_processing.py b/tests/core/test_download_processing.py index a0a7f1fc..95f952d6 100644 --- a/tests/core/test_download_processing.py +++ b/tests/core/test_download_processing.py @@ -506,7 +506,7 @@ class TestProcessDirectory: with patch('shelfmark.core.config.config') as mock_config, \ patch('shelfmark.config.env.TMP_DIR', temp_dirs["staging"]), \ - patch('shelfmark.download.postprocess.transfer.atomic_move', side_effect=Exception("Move failed")): + patch('shelfmark.download.postprocess.transfer.atomic_move', side_effect=RuntimeError("Move failed")): mock_config.USE_BOOK_TITLE = False mock_config.get = MagicMock(side_effect=lambda key, default=None, **_kwargs: { diff --git a/tests/core/test_hardlink.py b/tests/core/test_hardlink.py index 4dfa5916..79837e54 100644 --- a/tests/core/test_hardlink.py +++ b/tests/core/test_hardlink.py @@ -1345,6 +1345,34 @@ class TestTorrentSourceCleanupProtection: assert is_torrent_source(torrent_path, task) is True assert is_torrent_source(staging_path, task) is False + def testis_torrent_source_falls_back_to_normalized_paths(self, tmp_path, monkeypatch): + """If resolve() fails, path comparison should still fall back safely.""" + import shelfmark.download.postprocess.transfer as transfer_module + + from shelfmark.download.postprocess.pipeline import is_torrent_source + from shelfmark.core.models import DownloadTask, SearchMode + + torrent_path = tmp_path / "downloads" / "book.epub" + fallback_path = tmp_path / "downloads" / ".." / "downloads" / "book.epub" + + task = DownloadTask( + task_id="test", + source="prowlarr", + title="Test", + author="Author", + format="epub", + search_mode=SearchMode.UNIVERSAL, + original_download_path=str(torrent_path), + ) + + monkeypatch.setattr( + transfer_module, + "run_blocking_io", + lambda _func, *_args, **_kwargs: (_ for _ in ()).throw(OSError("resolve failed")), + ) + + assert is_torrent_source(fallback_path, task) is True + class TestEdgeCases: """Edge cases and error handling.""" diff --git a/tests/core/test_image_cache.py b/tests/core/test_image_cache.py new file mode 100644 index 00000000..27783f08 --- /dev/null +++ b/tests/core/test_image_cache.py @@ -0,0 +1,22 @@ +"""Tests for targeted image cache safety and fetch fallbacks.""" + +import requests + +from shelfmark.core.image_cache import ImageCacheService + + +def test_is_safe_url_rejects_invalid_ipv6_url() -> None: + assert ImageCacheService._is_safe_url("http://[") is False + + +def test_fetch_and_cache_returns_none_on_request_exception(tmp_path, monkeypatch) -> None: + cache = ImageCacheService(tmp_path) + monkeypatch.setattr(cache, "_is_safe_url", lambda _url: True) + + def fake_get(*args, **kwargs): + raise requests.exceptions.TooManyRedirects("too many redirects") + + monkeypatch.setattr("shelfmark.core.image_cache.requests.get", fake_get) + + assert cache.fetch_and_cache("cover-1", "https://example.com/cover.jpg") is None + assert "cover-1" not in cache._index diff --git a/tests/core/test_oidc_routes.py b/tests/core/test_oidc_routes.py index 1cb815fc..f56c834c 100644 --- a/tests/core/test_oidc_routes.py +++ b/tests/core/test_oidc_routes.py @@ -167,6 +167,15 @@ class TestOIDCLoginEndpoint: class TestOIDCCallbackEndpoint: + def test_normalize_claims_returns_empty_dict_for_invalid_mapping(self): + from shelfmark.core.oidc_routes import _normalize_claims + + class BadClaims: + def __iter__(self): + raise TypeError("bad claims") + + assert _normalize_claims(BadClaims()) == {} + @patch("shelfmark.core.oidc_routes._get_oidc_client") def test_callback_creates_session(self, mock_get_client, client): fake_client = Mock() @@ -339,6 +348,20 @@ class TestOIDCCallbackEndpoint: assert error is not None assert "issuer validation failed" in error + @patch("shelfmark.core.oidc_routes._get_oidc_client") + def test_callback_tolerates_metadata_lookup_failure_during_claim_diagnostics( + self, mock_get_client, client + ): + fake_client = Mock() + fake_client.authorize_access_token.side_effect = InvalidClaimError("iss") + fake_client.load_server_metadata.side_effect = RuntimeError("metadata failed") + mock_get_client.return_value = (fake_client, MOCK_OIDC_CONFIG) + + resp = client.get("/api/auth/oidc/callback?code=abc123&state=test-state") + error = _get_oidc_error(resp) + assert error is not None + assert "issuer validation failed" in error + @patch("shelfmark.core.oidc_routes._get_oidc_client") def test_callback_redirects_when_auto_provision_disabled_and_no_email_match( self, mock_get_client, client diff --git a/tests/core/test_original_name_template_variable.py b/tests/core/test_original_name_template_variable.py index 05ed18ba..29e10505 100644 --- a/tests/core/test_original_name_template_variable.py +++ b/tests/core/test_original_name_template_variable.py @@ -32,7 +32,9 @@ class TestOriginalNameTransferTemplates: monkeypatch.setattr( "shelfmark.download.postprocess.transfer.get_template", - lambda _is_audiobook, mode: "{OriginalName}" if mode == "rename" else "{Author}/{Title}", + lambda *, is_audiobook, organization_mode: ( + "{OriginalName}" if organization_mode == "rename" else "{Author}/{Title}" + ), ) task = DownloadTask( @@ -70,7 +72,7 @@ class TestOriginalNameTransferTemplates: monkeypatch.setattr( "shelfmark.download.postprocess.transfer.get_template", - lambda _is_audiobook, mode: "{Author}/{Title}/{OriginalName}", + lambda *, is_audiobook, organization_mode: "{Author}/{Title}/{OriginalName}", ) task = DownloadTask( diff --git a/tests/core/test_per_user_downloads.py b/tests/core/test_per_user_downloads.py index 695681b5..dc8fb8e1 100644 --- a/tests/core/test_per_user_downloads.py +++ b/tests/core/test_per_user_downloads.py @@ -5,6 +5,8 @@ Tests that DownloadTask has a user_id field and that the queue can be filtered by user. """ +import sqlite3 + from shelfmark.core.models import DownloadTask, QueueStatus from shelfmark.core.queue import BookQueue @@ -277,6 +279,52 @@ class TestUserDestinationTemplate: result = get_destination(is_audiobook=True, user_id=42, username="alice") assert result == Path("/audiobooks/alice") + def test_get_destination_looks_up_username_from_user_db(self, monkeypatch, tmp_path): + from pathlib import Path + + from shelfmark.core.config import config + from shelfmark.core.user_db import UserDB + from shelfmark.core.utils import get_destination + + monkeypatch.setenv("CONFIG_DIR", str(tmp_path)) + + user_db = UserDB(str(tmp_path / "users.db")) + user_db.initialize() + user = user_db.create_user(username="alice") + + def fake_config_get(key, default=None, user_id=None): + if key == "DESTINATION": + return "/books/{User}" + if key == "INGEST_DIR": + return "/books" + return default + + monkeypatch.setattr(config, "get", fake_config_get) + result = get_destination(is_audiobook=False, user_id=user["id"], username=None) + assert result == Path("/books/alice") + + def test_get_destination_falls_back_when_user_db_lookup_fails(self, monkeypatch): + from pathlib import Path + + from shelfmark.core.config import config + from shelfmark.core.utils import get_destination + + def fake_config_get(key, default=None, user_id=None): + if key == "DESTINATION": + return "/books/{User}" + if key == "INGEST_DIR": + return "/books" + return default + + monkeypatch.setattr(config, "get", fake_config_get) + monkeypatch.setattr( + "shelfmark.core.user_db.UserDB.get_user", + lambda self, **kwargs: (_ for _ in ()).throw(sqlite3.OperationalError("locked")), + ) + + result = get_destination(is_audiobook=False, user_id=42, username=None) + assert result == Path("/books") + class TestTaskToDictUsername: """Tests that _task_to_dict includes username for frontend display.""" diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py new file mode 100644 index 00000000..8869a7e2 --- /dev/null +++ b/tests/core/test_utils.py @@ -0,0 +1,27 @@ +"""Tests for shared utility helpers.""" + +import sys +import types +import xmlrpc.client as stdlib_xmlrpc_client + +from shelfmark.core import utils + + +def test_get_hardened_xmlrpc_client_tolerates_patch_runtime_error(monkeypatch) -> None: + fake_package = types.ModuleType("defusedxml") + fake_module = types.ModuleType("defusedxml.xmlrpc") + + def failing_monkey_patch() -> None: + raise RuntimeError("patch failed") + + fake_module.monkey_patch = failing_monkey_patch + fake_package.xmlrpc = fake_module + + monkeypatch.setitem(sys.modules, "defusedxml", fake_package) + monkeypatch.setitem(sys.modules, "defusedxml.xmlrpc", fake_module) + monkeypatch.setattr(utils, "_xmlrpc_patch_applied", False) + + client_module = utils.get_hardened_xmlrpc_client() + + assert client_module is stdlib_xmlrpc_client + assert utils._xmlrpc_patch_applied is False diff --git a/tests/download/test_fs_run_blocking_io.py b/tests/download/test_fs_run_blocking_io.py index 5dd691f3..50e89ef0 100644 --- a/tests/download/test_fs_run_blocking_io.py +++ b/tests/download/test_fs_run_blocking_io.py @@ -49,3 +49,20 @@ def test_run_blocking_io_handles_gevent_patched_subprocess_run() -> None: f"stdout:\n{completed.stdout}\n" f"stderr:\n{completed.stderr}" ) + + +def test_run_blocking_io_reraises_captured_operational_errors(monkeypatch) -> None: + from shelfmark.download import fs + + class _FakePool: + def apply(self, func, args): + return func(*args) + + monkeypatch.setattr(fs, "_use_gevent_threadpool", lambda: True) + monkeypatch.setattr(fs, "_get_io_threadpool", lambda: _FakePool()) + + def _boom() -> None: + raise RuntimeError("boom") + + with pytest.raises(RuntimeError, match="boom"): + fs.run_blocking_io(_boom) diff --git a/tests/download/test_http_bypasser_fallbacks.py b/tests/download/test_http_bypasser_fallbacks.py new file mode 100644 index 00000000..c40d02a3 --- /dev/null +++ b/tests/download/test_http_bypasser_fallbacks.py @@ -0,0 +1,91 @@ +"""Tests for HTTP bypasser fallback handling.""" + +import requests + +from shelfmark.bypass import BypassCancelledError + + +class _FakeResponse: + def __init__(self, status_code: int, *, url: str = "") -> None: + self.status_code = status_code + self.url = url + + +class _ImmediateThread: + def __init__(self, *args, **kwargs) -> None: + self._target = kwargs["target"] + + def start(self) -> None: + self._target() + + def join(self, timeout: float | None = None) -> None: + del timeout + + +def test_html_get_page_ignores_heartbeat_callback_failure(monkeypatch): + import shelfmark.download.http as http + + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + monkeypatch.setattr(http, "Thread", _ImmediateThread) + monkeypatch.setattr(http, "get_bypassed_page", lambda *_args, **_kwargs: "OK") + + calls: list[tuple[str, str | None]] = [] + + def status_callback(status: str, message: str | None) -> None: + calls.append((status, message)) + if len(calls) > 1: + raise RuntimeError("callback failed") + + html = http.html_get_page( + "https://example.com", + retry=1, + use_bypasser=True, + status_callback=status_callback, + ) + + assert html == "OK" + assert calls == [ + ("resolving", "Bypassing protection..."), + ("resolving", "Bypassing protection..."), + ] + + +def test_html_get_page_returns_empty_on_bypass_cancellation(monkeypatch): + import shelfmark.download.http as http + + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + + def failing_bypasser(*_args, **_kwargs): + raise BypassCancelledError("Bypass cancelled") + + monkeypatch.setattr(http, "get_bypassed_page", failing_bypasser) + + html = http.html_get_page("https://example.com", retry=1, use_bypasser=True) + + assert html == "" + + +def test_download_url_ignores_zlib_cookie_refresh_failure(monkeypatch): + import shelfmark.download.http as http + + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http.time, "sleep", lambda _seconds: None) + + def fake_get(_url: str, **_kwargs): + error = requests.exceptions.HTTPError("forbidden") + error.response = _FakeResponse(403, url=_url) + raise error + + def failing_bypasser(*_args, **_kwargs): + raise RuntimeError("refresh failed") + + monkeypatch.setattr(http.requests, "get", fake_get) + monkeypatch.setattr(http, "get_bypassed_page", failing_bypasser) + + result = http.download_url( + "https://z-lib.fm/download/book", + referer="https://z-lib.fm/books/example", + ) + + assert result is None diff --git a/tests/download/test_orchestrator_retry.py b/tests/download/test_orchestrator_retry.py index c7dfb27e..3f95bd3d 100644 --- a/tests/download/test_orchestrator_retry.py +++ b/tests/download/test_orchestrator_retry.py @@ -1,11 +1,9 @@ from __future__ import annotations -from threading import Event from pathlib import Path +from threading import Event from unittest.mock import MagicMock -import pytest - from shelfmark.core.models import DownloadTask, QueueStatus from shelfmark.core.queue import BookQueue @@ -249,3 +247,25 @@ def test_output_stage_retry_falls_back_to_download_when_staged_file_missing(monk handler.download.assert_called_once() assert seen_temp_files == [downloaded_file] assert task.staged_path is None + + +def test_get_book_data_clears_download_path_when_file_read_fails(monkeypatch, tmp_path): + import shelfmark.download.orchestrator as orchestrator + + missing_file = tmp_path / "missing.epub" + task = DownloadTask( + task_id="task-book-data-1", + source="direct_download", + title="Missing File", + download_path=str(missing_file), + ) + + mock_queue = MagicMock() + mock_queue.get_task.return_value = task + monkeypatch.setattr(orchestrator, "book_queue", mock_queue) + + file_data, returned_task = orchestrator.get_book_data(task.task_id) + + assert file_data is None + assert returned_task is task + assert task.download_path is None diff --git a/tests/download/test_orchestrator_user_output_mode.py b/tests/download/test_orchestrator_user_output_mode.py index bd6233f3..f220de99 100644 --- a/tests/download/test_orchestrator_user_output_mode.py +++ b/tests/download/test_orchestrator_user_output_mode.py @@ -1,3 +1,5 @@ +from unittest.mock import MagicMock + from shelfmark.core.models import SearchMode @@ -152,3 +154,26 @@ def test_queue_release_persists_generic_retry_resolution_fields(monkeypatch): assert task.retry_ratio_limit == 1.25 assert task.retry_seeding_time_limit_minutes == 90 assert task.can_retry_without_staged_source is True + + +def test_queue_release_returns_error_for_operational_queue_failure(monkeypatch): + import shelfmark.download.orchestrator as orchestrator + + monkeypatch.setattr( + orchestrator.book_queue, + "add", + MagicMock(side_effect=RuntimeError("queue offline")), + ) + monkeypatch.setattr(orchestrator, "ws_manager", None) + + success, error = orchestrator.queue_release( + { + "source": "direct_download", + "source_id": "release-broken-1", + "title": "Broken Queue", + "content_type": "ebook", + } + ) + + assert success is False + assert error == "Error queueing release: queue offline" diff --git a/tests/download/test_ssl_verify.py b/tests/download/test_ssl_verify.py index 05904261..a629c06e 100644 --- a/tests/download/test_ssl_verify.py +++ b/tests/download/test_ssl_verify.py @@ -4,7 +4,6 @@ import warnings import pytest - # --------------------------------------------------------------------------- # get_ssl_verify() # --------------------------------------------------------------------------- @@ -158,6 +157,7 @@ class TestApplySslWarningSuppression: def test_disabled_mode_suppresses_warnings(self, monkeypatch): import urllib3 + import shelfmark.download.network as network monkeypatch.setattr(network.app_config, "get", lambda k, d="": "disabled" if k == "CERTIFICATE_VALIDATION" else d) @@ -176,6 +176,7 @@ class TestApplySslWarningSuppression: def test_disabled_local_mode_suppresses_warnings(self, monkeypatch): import urllib3 + import shelfmark.download.network as network monkeypatch.setattr(network.app_config, "get", lambda k, d="": "disabled_local" if k == "CERTIFICATE_VALIDATION" else d) @@ -186,6 +187,7 @@ class TestApplySslWarningSuppression: def test_enabled_mode_restores_warnings(self, monkeypatch): import urllib3 + import shelfmark.download.network as network # First suppress @@ -213,7 +215,6 @@ class TestCertificateValidationSetting: def _get_network_fields(self): import shelfmark.config.settings # noqa: F401 — ensure settings tabs are registered - from shelfmark.core.settings_registry import get_settings_tab tab = get_settings_tab("network") @@ -252,7 +253,6 @@ class TestCertificateValidationSetting: def test_update_settings_certificate_validation_triggers_suppression(monkeypatch): """Changing CERTIFICATE_VALIDATION via update_settings calls _apply_ssl_warning_suppression.""" import shelfmark.config.settings # noqa: F401 — ensure settings tabs are registered - from shelfmark.core.config import config as config_obj from shelfmark.core.settings_registry import update_settings @@ -272,3 +272,39 @@ def test_update_settings_certificate_validation_triggers_suppression(monkeypatch assert result["success"] is True assert called["count"] == 1 + + +def test_update_settings_certificate_validation_logs_live_apply_failure(monkeypatch): + """Runtime live-apply failures should warn without failing the save.""" + import shelfmark.config.settings # noqa: F401 + import shelfmark.core.settings_registry as registry + from shelfmark.core.config import config as config_obj + from shelfmark.core.settings_registry import update_settings + + monkeypatch.setattr("shelfmark.core.settings_registry.save_config_file", lambda _tab, _values: True) + monkeypatch.setattr(config_obj, "refresh", lambda: None) + + import shelfmark.download.network as network + + monkeypatch.setattr( + network, + "_apply_ssl_warning_suppression", + lambda: (_ for _ in ()).throw(RuntimeError("ssl apply failed")), + ) + + warnings_logged: list[tuple[str, tuple[object, ...]]] = [] + monkeypatch.setattr( + registry.logger, + "warning", + lambda message, *args: warnings_logged.append((str(message), args)), + ) + + result = update_settings("network", {"CERTIFICATE_VALIDATION": "disabled"}) + + assert result["success"] is True + assert len(warnings_logged) == 1 + message, args = warnings_logged[0] + assert message == "Failed to apply certificate validation setting: %s" + assert len(args) == 1 + assert isinstance(args[0], RuntimeError) + assert str(args[0]) == "ssl apply failed" diff --git a/tests/e2e/test_auth_endpoints.py b/tests/e2e/test_auth_endpoints.py index 36ab48d1..af2970af 100644 --- a/tests/e2e/test_auth_endpoints.py +++ b/tests/e2e/test_auth_endpoints.py @@ -8,7 +8,7 @@ from __future__ import annotations import importlib import sqlite3 -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from typing import Any, Tuple from unittest.mock import Mock, patch @@ -71,7 +71,7 @@ class TestGetAuthMode: assert main_module.get_auth_mode() == "cwa" def test_get_auth_mode_default_on_error(self, main_module): - with patch.object(main_module.app_config, "get", side_effect=Exception("boom")): + with patch.object(main_module.app_config, "get", side_effect=RuntimeError("boom")): assert main_module.get_auth_mode() == "none" @@ -335,7 +335,7 @@ class TestRateLimiting: main_module.failed_login_attempts.clear() main_module.failed_login_attempts["testuser"] = { "count": 10, - "lockout_until": datetime.now() + timedelta(hours=1), + "lockout_until": datetime.now(UTC) + timedelta(hours=1), } assert main_module.is_account_locked("testuser") is True diff --git a/tests/irc/test_connection_manager.py b/tests/irc/test_connection_manager.py new file mode 100644 index 00000000..7b494c53 --- /dev/null +++ b/tests/irc/test_connection_manager.py @@ -0,0 +1,75 @@ +from unittest.mock import MagicMock + +import pytest + +from shelfmark.release_sources.irc.connection_manager import IRCConnectionManager + + +@pytest.fixture +def manager(): + IRCConnectionManager._instance = None + connection_manager = IRCConnectionManager() + yield connection_manager + connection_manager._running = False + IRCConnectionManager._instance = None + + +def test_cleanup_idle_connections_tolerates_disconnect_error(manager): + client = MagicMock() + client.disconnect.side_effect = OSError("socket closed") + key = manager._connection_key("irc.example.com", 6697, "reader") + + manager._connections[key] = client + manager._last_used[key] = 0 + manager._channels[key] = "#books" + + manager._cleanup_idle_connections() + + assert key not in manager._connections + client.disconnect.assert_called_once() + + +def test_get_connection_clears_connecting_flag_on_failure(manager, monkeypatch): + class FailingIRCClient: + def __init__(self, nick, server, port, *, use_tls): + self.nick = nick + self.server = server + self.port = port + + def connect(self): + raise RuntimeError("connect failed") + + monkeypatch.setattr( + "shelfmark.release_sources.irc.connection_manager.IRCClient", + FailingIRCClient, + ) + + with pytest.raises(RuntimeError, match="connect failed"): + manager.get_connection( + server="irc.example.com", + port=6697, + nick="reader", + use_tls=True, + channel="books", + ) + + key = manager._connection_key("irc.example.com", 6697, "reader") + assert key not in manager._connecting + + +def test_close_connection_tolerates_disconnect_error(manager): + client = MagicMock() + client.server = "irc.example.com" + client.port = 6697 + client.nick = "reader" + client.disconnect.side_effect = RuntimeError("disconnect failed") + key = manager._connection_key(client.server, client.port, client.nick) + + manager._connections[key] = client + manager._last_used[key] = 1.0 + manager._channels[key] = "#books" + + manager.close_connection(client) + + assert key not in manager._connections + client.disconnect.assert_called_once() diff --git a/tests/metadata/test_googlebooks_parse.py b/tests/metadata/test_googlebooks_parse.py new file mode 100644 index 00000000..6708c35f --- /dev/null +++ b/tests/metadata/test_googlebooks_parse.py @@ -0,0 +1,56 @@ +from shelfmark.metadata_providers.googlebooks import GoogleBooksProvider + + +class TestGoogleBooksParseVolume: + def test_parse_volume_returns_metadata_for_valid_payload(self): + provider = GoogleBooksProvider(api_key="test-key") + + result = provider._parse_volume( + { + "id": "volume-1", + "volumeInfo": { + "title": "Test Book", + "authors": ["Alice Author"], + "industryIdentifiers": [ + {"type": "ISBN_10", "identifier": "1234567890"}, + {"type": "ISBN_13", "identifier": "9781234567897"}, + ], + "imageLinks": { + "thumbnail": "http://example.com/cover.jpg&edge=curl", + }, + "publisher": "Test Publisher", + "publishedDate": "2024-03-01", + "language": "en", + "categories": ["Fiction", "Fantasy"], + "description": "A book.", + "infoLink": "https://example.com/books/volume-1", + "averageRating": 4.2, + "ratingsCount": 1200, + }, + } + ) + + assert result is not None + assert result.provider_id == "volume-1" + assert result.title == "Test Book" + assert result.authors == ["Alice Author"] + assert result.isbn_10 == "1234567890" + assert result.isbn_13 == "9781234567897" + assert result.cover_url == "https://example.com/cover.jpg" + assert result.publish_year == 2024 + assert result.display_fields[0].value == "4.2 (1,200)" + + def test_parse_volume_returns_none_for_malformed_rating_payload(self): + provider = GoogleBooksProvider(api_key="test-key") + + result = provider._parse_volume( + { + "id": "volume-2", + "volumeInfo": { + "title": "Broken Book", + "averageRating": "not-a-number", + }, + } + ) + + assert result is None diff --git a/tests/metadata/test_hardcover_lists.py b/tests/metadata/test_hardcover_lists.py index 6e2cd827..ebbcaded 100644 --- a/tests/metadata/test_hardcover_lists.py +++ b/tests/metadata/test_hardcover_lists.py @@ -283,7 +283,7 @@ class TestHardcoverLists: result = provider.set_book_target_state( "123", "status:1", - True, + selected=True, ) assert result == {"changed": True, "deselected_target": "status:2"} @@ -336,7 +336,7 @@ class TestHardcoverLists: monkeypatch.setattr(provider, "_execute_query", fake_execute) - result = provider.set_book_target_state("123", "id:42", False) + result = provider.set_book_target_state("123", "id:42", selected=False) assert result == {"changed": True} assert cache_stub.invalidated == [ @@ -362,7 +362,7 @@ class TestHardcoverLists: ) try: - provider.set_book_target_state("123", "id:99", True) + provider.set_book_target_state("123", "id:99", selected=True) except ValueError as exc: assert str(exc) == "Unsupported Hardcover target" else: diff --git a/tests/prowlarr/test_deluge_client.py b/tests/prowlarr/test_deluge_client.py index ff1ea809..6ec4643d 100644 --- a/tests/prowlarr/test_deluge_client.py +++ b/tests/prowlarr/test_deluge_client.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock, patch +from shelfmark.download.clients import DownloadStatus from shelfmark.download.clients.torrent_utils import TorrentInfo @@ -97,3 +98,108 @@ class TestDelugeClientAddDownload: magnet, {}, ) + + +class TestDelugeClientErrors: + """Tests for Deluge error handling fallbacks.""" + + def test_test_connection_failure_returns_false(self, monkeypatch): + """Operational client errors should return a failure tuple.""" + config_values = { + "DELUGE_HOST": "http://localhost", + "DELUGE_PORT": "8112", + "DELUGE_PASSWORD": "password", + } + monkeypatch.setattr( + "shelfmark.download.clients.deluge.config.get", + make_config_getter(config_values), + ) + + from shelfmark.download.clients.deluge import DelugeClient + + client = DelugeClient() + monkeypatch.setattr(client, "_ensure_connected", MagicMock(side_effect=RuntimeError("offline"))) + + success, message = client.test_connection() + + assert success is False + assert "offline" in message + assert client._authenticated is False + assert client._connected is False + + def test_get_status_failure_returns_error_status(self, monkeypatch): + """Status lookup failures should degrade to DownloadStatus.error().""" + config_values = { + "DELUGE_HOST": "http://localhost", + "DELUGE_PORT": "8112", + "DELUGE_PASSWORD": "password", + } + monkeypatch.setattr( + "shelfmark.download.clients.deluge.config.get", + make_config_getter(config_values), + ) + + from shelfmark.download.clients.deluge import DelugeClient + + client = DelugeClient() + monkeypatch.setattr(client, "_ensure_connected", lambda: None) + monkeypatch.setattr(client, "_rpc_call", MagicMock(side_effect=RuntimeError("status failed"))) + + status = client.get_status("torrent-id") + + assert isinstance(status, DownloadStatus) + assert status.state_value == "error" + assert "status failed" in status.message + + def test_remove_failure_returns_false(self, monkeypatch): + """Removal failures should return False rather than raising.""" + config_values = { + "DELUGE_HOST": "http://localhost", + "DELUGE_PORT": "8112", + "DELUGE_PASSWORD": "password", + } + monkeypatch.setattr( + "shelfmark.download.clients.deluge.config.get", + make_config_getter(config_values), + ) + + from shelfmark.download.clients.deluge import DelugeClient + + client = DelugeClient() + monkeypatch.setattr(client, "_ensure_connected", lambda: None) + monkeypatch.setattr(client, "_rpc_call", MagicMock(side_effect=RuntimeError("remove failed"))) + + assert client.remove("torrent-id") is False + + def test_find_existing_failure_returns_none_and_resets_connection(self, monkeypatch): + """Lookup failures should clear client state and return no match.""" + config_values = { + "DELUGE_HOST": "http://localhost", + "DELUGE_PORT": "8112", + "DELUGE_PASSWORD": "password", + } + monkeypatch.setattr( + "shelfmark.download.clients.deluge.config.get", + make_config_getter(config_values), + ) + + from shelfmark.download.clients.deluge import DelugeClient + + client = DelugeClient() + client._authenticated = True + client._connected = True + monkeypatch.setattr(client, "_ensure_connected", lambda: None) + monkeypatch.setattr(client, "_rpc_call", MagicMock(side_effect=RuntimeError("lookup failed"))) + + magnet = "magnet:?xt=urn:btih:ABCDEF1234567890ABCDEF1234567890ABCDEF12&dn=test" + with patch("shelfmark.download.clients.deluge.extract_torrent_info", autospec=True) as mock_extract: + mock_extract.return_value = TorrentInfo( + info_hash="abcdef1234567890abcdef1234567890abcdef12", + torrent_data=None, + is_magnet=True, + magnet_url=magnet, + ) + assert client.find_existing(magnet) is None + + assert client._authenticated is False + assert client._connected is False diff --git a/tests/prowlarr/test_handler.py b/tests/prowlarr/test_handler.py index 90467f59..bb0a0790 100644 --- a/tests/prowlarr/test_handler.py +++ b/tests/prowlarr/test_handler.py @@ -1007,7 +1007,7 @@ class TestProwlarrHandlerPostProcessCleanup: mock_client.name = "nzbget" handler._cleanup_refs[task.task_id] = (mock_client, "123", "usenet") - with patch("shelfmark.release_sources.prowlarr.handler.config.get", return_value="move"): + with patch("shelfmark.download.clients.base_handler.config.get", return_value="move"): handler.post_process_cleanup(task, success=True) mock_client.remove.assert_called_once_with("123", delete_files=True) @@ -1020,7 +1020,110 @@ class TestProwlarrHandlerPostProcessCleanup: mock_client.name = "nzbget" handler._cleanup_refs[task.task_id] = (mock_client, "123", "usenet") - with patch("shelfmark.release_sources.prowlarr.handler.config.get", return_value="copy"): + with patch("shelfmark.download.clients.base_handler.config.get", return_value="copy"): handler.post_process_cleanup(task, success=True) mock_client.remove.assert_not_called() + + def test_usenet_move_logs_cleanup_failure(self): + handler = ProwlarrHandler() + task = DownloadTask(task_id="cleanup-failure", source="prowlarr", title="Test") + + mock_client = MagicMock() + mock_client.name = "nzbget" + handler._cleanup_refs[task.task_id] = (mock_client, "123", "usenet") + handler._delete_local_download_data = MagicMock(side_effect=ConnectionError("offline")) + + with ( + patch("shelfmark.download.clients.base_handler.config.get", return_value="move"), + patch("shelfmark.download.clients.base_handler.logger.warning") as mock_warning, + ): + handler.post_process_cleanup(task, success=True) + + mock_warning.assert_called_once() + args = mock_warning.call_args.args + assert args[0] == "Failed to cleanup usenet download %s in %s: %s" + assert args[1] == "123" + assert args[2] == "nzbget" + assert str(args[3]) == "offline" + + def test_torrent_remove_logs_cleanup_failure(self): + handler = ProwlarrHandler() + task = DownloadTask(task_id="torrent-cleanup-failure", source="prowlarr", title="Test") + + mock_client = MagicMock() + mock_client.name = "qbittorrent" + mock_client.remove.side_effect = ConnectionError("offline") + handler._cleanup_refs[task.task_id] = (mock_client, "123", "torrent") + + with ( + patch("shelfmark.download.clients.base_handler.config.get", return_value="remove"), + patch("shelfmark.download.clients.base_handler.logger.warning") as mock_warning, + ): + handler.post_process_cleanup(task, success=True) + + mock_warning.assert_called_once() + args = mock_warning.call_args.args + assert args[0] == "Failed to remove torrent %s from %s: %s" + assert args[1] == "123" + assert args[2] == "qbittorrent" + assert str(args[3]) == "offline" + + def test_delete_local_download_data_ignores_path_lookup_failure(self): + handler = ProwlarrHandler() + mock_client = MagicMock() + mock_client.name = "nzbget" + mock_client.get_download_path.side_effect = ConnectionError("path lookup failed") + + with patch("shelfmark.download.clients.base_handler.logger.debug") as mock_debug: + handler._delete_local_download_data(mock_client, "123") + + mock_debug.assert_called_once() + args = mock_debug.call_args.args + assert args[0] == "Failed to resolve download path for %s %s: %s" + assert args[1] == "nzbget" + assert args[2] == "123" + assert str(args[3]) == "path lookup failed" + + def test_delete_local_download_data_logs_delete_failure(self, tmp_path, monkeypatch): + import shelfmark.download.clients.base_handler as base_handler + import shelfmark.core.path_mappings as path_mappings + + handler = ProwlarrHandler() + download_file = tmp_path / "downloads" / "book.epub" + download_file.parent.mkdir(parents=True) + download_file.write_text("content") + + mock_client = MagicMock() + mock_client.name = "nzbget" + mock_client.get_download_path.return_value = str(download_file) + + monkeypatch.setattr(path_mappings, "get_client_host_identifier", lambda _client: "") + monkeypatch.setattr(path_mappings, "parse_remote_path_mappings", lambda _value: []) + monkeypatch.setattr( + path_mappings, + "remap_remote_to_local_with_match", + lambda *, mappings, host, remote_path: (remote_path, None), + ) + + def fake_run_blocking_io(func, *args, **kwargs): + name = getattr(func, "__name__", "") + if name == "exists": + return True + if name == "is_dir": + return False + if name == "unlink": + raise ConnectionError("delete failed") + return func(*args, **kwargs) + + monkeypatch.setattr(base_handler, "run_blocking_io", fake_run_blocking_io) + + with patch("shelfmark.download.clients.base_handler.logger.warning") as mock_warning: + handler._delete_local_download_data(mock_client, "123") + + mock_warning.assert_called_once() + args = mock_warning.call_args.args + assert args[0] == "Failed to delete local download data for %s %s: %s" + assert args[1] == "nzbget" + assert args[2] == "123" + assert str(args[3]) == "delete failed" diff --git a/tests/prowlarr/test_nzbget_client.py b/tests/prowlarr/test_nzbget_client.py index d02e7da4..2bb9b6df 100644 --- a/tests/prowlarr/test_nzbget_client.py +++ b/tests/prowlarr/test_nzbget_client.py @@ -218,7 +218,7 @@ class TestNZBGetClientRPCCall: ) client = NZBGetClient() - with pytest.raises(Exception) as exc_info: + with pytest.raises(RuntimeError) as exc_info: client._rpc_call("invalid_method") assert "Invalid method" in str(exc_info.value) @@ -510,7 +510,7 @@ class TestNZBGetClientAddDownload: ) client = NZBGetClient() - with pytest.raises(Exception) as exc_info: + with pytest.raises(RuntimeError) as exc_info: client.add_download("https://example.com/download.nzb", "Test") assert "fetch" in str(exc_info.value).lower() diff --git a/tests/prowlarr/test_qbittorrent_client.py b/tests/prowlarr/test_qbittorrent_client.py index e89df1e9..b2799493 100644 --- a/tests/prowlarr/test_qbittorrent_client.py +++ b/tests/prowlarr/test_qbittorrent_client.py @@ -158,7 +158,7 @@ class TestQBittorrentClientTestConnection: ) mock_client_instance = MagicMock() - mock_client_instance.auth_log_in.side_effect = Exception("401 Unauthorized") + mock_client_instance.auth_log_in.side_effect = RuntimeError("401 Unauthorized") mock_client_class = MagicMock(return_value=mock_client_instance) with patch.dict('sys.modules', {'qbittorrentapi': MagicMock(Client=mock_client_class)}): @@ -679,7 +679,7 @@ class TestQBittorrentClientAddDownload: client = qb_module.QBittorrentClient() magnet = f"magnet:?xt=urn:btih:{valid_hash}&dn=test" - with pytest.raises(Exception, match="Failed to add torrent: Fails\\."): + with pytest.raises(RuntimeError, match="Failed to add torrent: Fails\\."): client.add_download(magnet, "Test") def test_add_download_omits_empty_category(self, monkeypatch): @@ -803,7 +803,7 @@ class TestQBittorrentClientRemove: ) mock_client_instance = MagicMock() - mock_client_instance.torrents_delete.side_effect = Exception("Not found") + mock_client_instance.torrents_delete.side_effect = RuntimeError("Not found") mock_client_class = MagicMock(return_value=mock_client_instance) with patch.dict('sys.modules', {'qbittorrentapi': MagicMock(Client=mock_client_class)}): diff --git a/tests/prowlarr/test_rtorrent_client.py b/tests/prowlarr/test_rtorrent_client.py index 165a1255..b9391bf0 100644 --- a/tests/prowlarr/test_rtorrent_client.py +++ b/tests/prowlarr/test_rtorrent_client.py @@ -159,7 +159,7 @@ class TestRTorrentClientTestConnection: ) mock_rpc = MagicMock() - mock_rpc.system.client_version.side_effect = Exception("Connection refused") + mock_rpc.system.client_version.side_effect = RuntimeError("Connection refused") mock_xmlrpc = create_mock_xmlrpc_module() mock_xmlrpc.ServerProxy.return_value = mock_rpc @@ -328,7 +328,7 @@ class TestRTorrentClientAddDownload: ) mock_rpc = MagicMock() - mock_rpc.load.start.side_effect = Exception("RPC Error") + mock_rpc.load.start.side_effect = RuntimeError("RPC Error") mock_xmlrpc = create_mock_xmlrpc_module() mock_xmlrpc.ServerProxy.return_value = mock_rpc @@ -352,7 +352,7 @@ class TestRTorrentClientAddDownload: ) client = RTorrentClient() - with pytest.raises(Exception) as excinfo: + with pytest.raises(RuntimeError) as excinfo: client.add_download("magnet:...", "Test") assert "RPC Error" in str(excinfo.value) @@ -626,7 +626,7 @@ class TestRTorrentClientRemove: ) mock_rpc = MagicMock() - mock_rpc.d.stop.side_effect = Exception("Connection lost") + mock_rpc.d.stop.side_effect = RuntimeError("Connection lost") mock_xmlrpc = create_mock_xmlrpc_module() mock_xmlrpc.ServerProxy.return_value = mock_rpc diff --git a/tests/prowlarr/test_source.py b/tests/prowlarr/test_source.py index 92c05eb1..7e95a7b8 100644 --- a/tests/prowlarr/test_source.py +++ b/tests/prowlarr/test_source.py @@ -425,3 +425,36 @@ class TestProwlarrLocalizedQueries: (query,), ) ] + + def test_get_column_config_ignores_indexer_lookup_failure(self, monkeypatch): + source = ProwlarrSource() + + class FailingClient: + def get_enabled_indexers_detailed(self): + raise RuntimeError("indexers unavailable") + + monkeypatch.setattr(source, "_get_client", lambda: FailingClient()) + monkeypatch.setattr(source, "_get_selected_indexer_ids", lambda: None) + + config = source.get_column_config() + + assert config.available_indexers is None + assert config.default_indexers is None + + def test_resolve_indexer_ids_from_names_returns_none_on_lookup_failure(self): + source = ProwlarrSource() + + class FailingClient: + def get_enabled_indexers_detailed(self): + raise RuntimeError("indexers unavailable") + + assert source._resolve_indexer_ids_from_names(FailingClient(), ["Alpha"]) is None + + def test_get_search_indexer_ids_returns_empty_on_lookup_failure(self): + source = ProwlarrSource() + + class FailingClient: + def get_enabled_indexers_detailed(self): + raise RuntimeError("indexers unavailable") + + assert source._get_search_indexer_ids(FailingClient(), None, [7000]) == [] diff --git a/tests/prowlarr/test_transmission_client.py b/tests/prowlarr/test_transmission_client.py index 6d342348..2790c85e 100644 --- a/tests/prowlarr/test_transmission_client.py +++ b/tests/prowlarr/test_transmission_client.py @@ -5,11 +5,10 @@ These tests mock the transmission-rpc library to test the client logic without requiring a running Transmission instance. """ -from unittest.mock import MagicMock, patch -from datetime import timedelta -import pytest import sys import types +from datetime import timedelta +from unittest.mock import MagicMock, patch from shelfmark.download.clients import DownloadStatus @@ -284,7 +283,7 @@ class TestTransmissionClientTestConnection: ) mock_client_instance = MagicMock() - mock_client_instance.get_session.side_effect = Exception("Connection refused") + mock_client_instance.get_session.side_effect = RuntimeError("Connection refused") mock_transmission_rpc = create_mock_transmission_rpc_module() mock_transmission_rpc.Client.return_value = mock_client_instance @@ -645,7 +644,7 @@ class TestTransmissionClientRemove: ) mock_client_instance = MagicMock() - mock_client_instance.remove_torrent.side_effect = Exception("Not found") + mock_client_instance.remove_torrent.side_effect = RuntimeError("Not found") mock_transmission_rpc = create_mock_transmission_rpc_module() mock_transmission_rpc.Client.return_value = mock_client_instance