starred/shelfmark
1
0
Fork 0
mirror of https://github.com/calibrain/shelfmark.git synced 2026-09-24 13:40:21 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Final tweaks and code cleanup (#392)

Browse Source
This commit is contained in:
Alex
2026-01-04 14:12:36 +00:00
committed by GitHub
parent b0206f76f8
commit 3d84c5b42f
53 changed files with 1190 additions and 2603 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cwa_book_downloader/api/websocket.py
+4 -24
View File
@@ -28,29 +28,17 @@ class WebSocketManager:
logger.info("WebSocket manager initialized")
def register_on_first_connect(self, callback: Callable[[], None]):
"""Register a callback to be called when the first client connects.
This is useful for warming up resources (like the Cloudflare bypasser)
when a user starts using the web UI.
"""
"""Register a callback for when the first client connects."""
self._on_first_connect_callbacks.append(callback)
logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
def register_on_all_disconnect(self, callback: Callable[[], None]):
"""Register a callback to be called when all clients disconnect.
This can be used to trigger cleanup or resource release.
"""
"""Register a callback for when all clients disconnect."""
self._on_all_disconnect_callbacks.append(callback)
logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
def request_warmup_on_next_connect(self):
"""Request that warmup callbacks be triggered on the next client connect.
This is used when resources (like the Cloudflare bypasser) shut down due to
inactivity while clients are still connected. The next connect event should
trigger warmup even though it's not technically the "first" connection.
"""
"""Request warmup callbacks on the next client connect (e.g., after idle shutdown)."""
with self._connection_lock:
self._needs_rewarm = True
logger.debug("Warmup requested for next client connect")
@@ -165,15 +153,7 @@ class WebSocketManager:
message: str,
phase: str = 'searching'
):
"""Broadcast search status update for a release source search.
Args:
source: Release source name (e.g., 'irc', 'direct_download')
provider: Metadata provider name (e.g., 'hardcover')
book_id: Book ID from the metadata provider
message: Human-readable status message
phase: Search phase ('connecting', 'searching', 'downloading', 'parsing', 'complete', 'error')
"""
"""Broadcast search status update for a release source search."""
if not self.is_enabled():
return
cwa_book_downloader/bypass/external_bypasser.py
+2 -23
View File
@@ -28,14 +28,7 @@ BACKOFF_CAP = 10.0
def _fetch_via_bypasser(target_url: str) -> Optional[str]:
"""Make a single request to the external bypasser service.
Args:
target_url: The URL to fetch through the bypasser
Returns:
HTML content if successful, None otherwise
"""
"""Make a single request to the external bypasser service. Returns HTML or None."""
bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191")
bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1")
bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000)
@@ -105,21 +98,7 @@ def get_bypassed_page(
selector: Optional["network.AAMirrorSelector"] = None,
cancel_flag: Optional[Event] = None
) -> Optional[str]:
"""Fetch HTML content from a URL using an external Cloudflare bypasser service.
Retries with exponential backoff and mirror/DNS rotation on failure.
Args:
url: Target URL to fetch
selector: Mirror selector for AA URL rewriting and rotation
cancel_flag: Optional threading Event to signal cancellation
Returns:
HTML content if successful, None otherwise
Raises:
BypassCancelledException: If cancel_flag is set during operation
"""
"""Fetch HTML via external bypasser with retries and mirror rotation."""
from cwa_book_downloader.download import network as network_module
sel = selector or network_module.AAMirrorSelector()
cwa_book_downloader/bypass/internal_bypasser.py
+1 -5
View File
@@ -268,11 +268,7 @@ def _has_cloudflare_patterns(body: str, url: str) -> bool:
return "cf-" in body or "cloudflare" in url.lower() or "/cdn-cgi/" in url
def _detect_challenge_type(sb) -> str:
"""Detect what type of challenge we're facing.
Returns:
str: 'cloudflare', 'ddos_guard', or 'none' if no challenge detected
"""
"""Detect challenge type: 'cloudflare', 'ddos_guard', or 'none'."""
try:
title, body, current_url = _get_page_info(sb)
cwa_book_downloader/config/env.py
+86 -125
View File
@@ -1,4 +1,4 @@
"""Environment variable parsing. No local dependencies - import first."""
"""Bootstrap environment variables. No local dependencies - import first."""
import json
import os
@@ -7,17 +7,12 @@ from pathlib import Path
def string_to_bool(s: str) -> bool:
"""Convert string to boolean."""
return s.lower() in ["true", "yes", "1", "y"]
def _read_debug_from_config() -> bool:
"""
Read DEBUG setting directly from config JSON file.
This is called at import time before the config singleton is available.
Priority: ENV var > config file > default (False)
"""
# Check env var first (takes priority)
"""Read DEBUG from env var or config file (import-time safe)."""
env_debug = os.environ.get("DEBUG")
if env_debug is not None:
return string_to_bool(env_debug)
@@ -38,20 +33,18 @@ def _read_debug_from_config() -> bool:
return False
# Authentication and session settings
SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
def _is_sqlite_file(path: Path) -> bool:
"""Check if a file is a valid SQLite database by reading magic bytes."""
try:
with open(path, "rb") as f:
header = f.read(16)
return header[:16] == b"SQLite format 3\x00"
except (OSError, PermissionError):
return False
def _resolve_cwa_db_path() -> Path | None:
"""
Resolve the Calibre-Web database path.
Priority:
1. CWA_DB_PATH env var (backwards compatibility)
2. Default path /auth/app.db if it exists and is a valid SQLite file
Returns None if no valid database is found.
"""
# Check env var first (backwards compatibility)
"""Resolve CWA database path from env var or default location."""
env_path = os.getenv("CWA_DB_PATH")
if env_path:
path = Path(env_path)
@@ -66,99 +59,6 @@ def _resolve_cwa_db_path() -> Path | None:
return None
def _is_sqlite_file(path: Path) -> bool:
"""Check if a file is a valid SQLite database by reading magic bytes."""
try:
with open(path, "rb") as f:
header = f.read(16)
return header[:16] == b"SQLite format 3\x00"
except (OSError, PermissionError):
return False
CWA_DB_PATH = _resolve_cwa_db_path()
CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
LOG_DIR = LOG_ROOT / "cwa-book-downloader"
TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
_SUPPORTED_AUDIOBOOK_FORMATS = os.getenv("SUPPORTED_AUDIOBOOK_FORMATS", "m4b,mp3").lower()
_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
DEBUG = _read_debug_from_config()
# Debug: skip specific download sources for testing fallback chains
# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))
# Version information from Docker build
BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
# Log level is derived from DEBUG - no separate LOG_LEVEL setting
LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "1"))
DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "auto").strip()
USE_DOH = string_to_bool(os.getenv("USE_DOH", "true"))
BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
BYPASS_WARMUP_ON_CONNECT = string_to_bool(os.getenv("BYPASS_WARMUP_ON_CONNECT", "true"))
# Logging settings
LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
USING_EXTERNAL_BYPASSER = string_to_bool(os.getenv("USING_EXTERNAL_BYPASSER", "false"))
if USING_EXTERNAL_BYPASSER:
EXT_BYPASSER_URL = os.getenv("EXT_BYPASSER_URL", "http://flaresolverr:8191").strip()
EXT_BYPASSER_PATH = os.getenv("EXT_BYPASSER_PATH", "/v1").strip()
EXT_BYPASSER_TIMEOUT = int(os.getenv("EXT_BYPASSER_TIMEOUT", "60000"))
USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
# If using Tor, we don't need to set custom DNS, use DOH, or proxy
if USING_TOR:
_CUSTOM_DNS = ""
USE_DOH = False
HTTP_PROXY = ""
HTTPS_PROXY = ""
# Detect Tor variant (has tor binary installed)
TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
# Calibre-Web URL for navigation button
CALIBRE_WEB_URL = os.getenv("CALIBRE_WEB_URL", "").strip()
# Metadata provider settings (Stage 2)
# Set to "hardcover" or "openlibrary" to enable metadata-first search mode
METADATA_PROVIDER = os.getenv("METADATA_PROVIDER", "").strip().lower()
HARDCOVER_API_KEY = os.getenv("HARDCOVER_API_KEY", "").strip()
# Cache TTL settings (in seconds)
METADATA_CACHE_SEARCH_TTL = int(os.getenv("METADATA_CACHE_SEARCH_TTL", "300")) # 5 minutes
METADATA_CACHE_BOOK_TTL = int(os.getenv("METADATA_CACHE_BOOK_TTL", "600")) # 10 minutes
# Cover image cache settings
def _is_config_dir_writable() -> bool:
"""Check if the config directory exists and is writable."""
try:
@@ -173,20 +73,81 @@ def _is_config_dir_writable() -> bool:
def is_covers_cache_enabled() -> bool:
"""Check if cover caching is enabled (dynamic, respects settings changes).
Cache is only enabled if:
1. The COVERS_CACHE_ENABLED setting is true
2. The config directory is writable
"""
"""Check if cover caching is enabled (requires setting + writable config dir)."""
from cwa_book_downloader.core.config import config
setting_enabled = config.get("COVERS_CACHE_ENABLED", True)
return setting_enabled and _is_config_dir_writable()
# Legacy static value - use is_covers_cache_enabled() for dynamic checks
_COVERS_CACHE_ENABLED_ENV = string_to_bool(os.getenv("COVERS_CACHE_ENABLED", "true"))
COVERS_CACHE_ENABLED = _COVERS_CACHE_ENABLED_ENV and _is_config_dir_writable()
COVERS_CACHE_DIR = CONFIG_DIR / "covers"
COVERS_CACHE_TTL = int(os.getenv("COVERS_CACHE_TTL", "0")) # 0 = forever (covers are static)
COVERS_CACHE_MAX_SIZE_MB = int(os.getenv("COVERS_CACHE_MAX_SIZE_MB", "500"))
# =============================================================================
# Bootstrap paths - needed before settings registry is available
# =============================================================================
CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config"))
LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
LOG_DIR = LOG_ROOT / "cwa-book-downloader"
LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
INGEST_DIR = Path(os.getenv("INGEST_DIR", "/books"))
# =============================================================================
# Logger configuration - needed before settings registry is available
# =============================================================================
DEBUG = _read_debug_from_config()
LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
# =============================================================================
# Flask configuration - needed before app starts
# =============================================================================
FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
# =============================================================================
# Authentication
# =============================================================================
SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false")
CWA_DB_PATH = _resolve_cwa_db_path()
# =============================================================================
# Version information from Docker build
# =============================================================================
BUILD_VERSION = os.getenv("BUILD_VERSION", "N/A")
RELEASE_VERSION = os.getenv("RELEASE_VERSION", "N/A")
# =============================================================================
# Capability detection - runtime checks, not user-configurable
# =============================================================================
DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None
USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
# =============================================================================
# Debug/development settings
# =============================================================================
# Debug: skip specific download sources for testing fallback chains
# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
# =============================================================================
# Legacy migration support - will be removed in future version
# =============================================================================
# Legacy welib settings - replaced by SOURCE_PRIORITY OrderableListField
# Kept for migration: if set, used to build initial SOURCE_PRIORITY config
_LEGACY_PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
_LEGACY_ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))
cwa_book_downloader/config/security.py
+4 -6
View File
@@ -20,15 +20,14 @@ logger = setup_logger(__name__)
def _clear_builtin_credentials() -> Dict[str, Any]:
"""Clear built-in credentials to allow public access."""
import json
from cwa_book_downloader.core.settings_registry import _get_config_file_path, _ensure_config_dir
try:
config = load_config_file("security")
config.pop("BUILTIN_USERNAME", None)
config.pop("BUILTIN_PASSWORD_HASH", None)
# Save the cleared config
from cwa_book_downloader.core.settings_registry import _get_config_file_path, _ensure_config_dir
import json
_ensure_config_dir("security")
config_path = _get_config_file_path("security")
with open(config_path, 'w') as f:
@@ -102,9 +101,8 @@ def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]:
def security_settings():
"""Security and authentication settings."""
from cwa_book_downloader.config.env import CWA_DB_PATH
import os
cwa_db_available = CWA_DB_PATH and os.path.exists(CWA_DB_PATH)
cwa_db_available = CWA_DB_PATH is not None and CWA_DB_PATH.exists()
fields = [
TextField(
cwa_book_downloader/config/settings.py
+27 -68
View File
@@ -9,23 +9,11 @@ from cwa_book_downloader.core.logger import setup_logger
logger = setup_logger(__name__)
# Log configuration values at DEBUG level, filtering out module imports and functions
logger.debug("Environment configuration:")
for key, value in env.__dict__.items():
# Skip private attributes, modules, types, and callables (functions)
if key.startswith('_'):
continue
if isinstance(value, type) or callable(value):
continue
# Don't log module objects (they have __name__ attribute)
if hasattr(value, '__name__') and hasattr(value, '__file__'):
continue
# Redact sensitive values
if key == "AA_DONATOR_KEY" and isinstance(value, str) and value.strip():
value = "REDACTED"
if key == "HARDCOVER_API_KEY" and isinstance(value, str) and value.strip():
value = "REDACTED"
logger.debug(f" {key}: {value}")
# Log bootstrap configuration values at DEBUG level
logger.debug("Bootstrap configuration:")
for key in ['CONFIG_DIR', 'LOG_DIR', 'TMP_DIR', 'INGEST_DIR', 'DEBUG', 'DOCKERMODE']:
if hasattr(env, key):
logger.debug(f" {key}: {getattr(env, key)}")
# Load supported book languages from data file
# Path is relative to the package root, not this file
@@ -52,47 +40,20 @@ logger.debug(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
CUSTOM_DNS: list[str] = []
DOH_SERVER: str = ""
# Warn about external bypasser DNS limitations
if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS:
logger.warning(
"Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
"not this application's custom DNS settings. If you experience DNS-related blocks, "
"configure DNS at the Docker/system level for your FlareSolverr container, "
"or consider using the internal bypasser which integrates with the app's DNS system."
)
# Recording directory for debugging internal cloudflare bypasser
RECORDING_DIR = env.LOG_DIR / "recording"
# Anna's Archive settings
AA_BASE_URL = env._AA_BASE_URL
AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
# File format settings
SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
logger.debug(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
SUPPORTED_AUDIOBOOK_FORMATS = env._SUPPORTED_AUDIOBOOK_FORMATS.split(",")
logger.debug(f"SUPPORTED_AUDIOBOOK_FORMATS: {SUPPORTED_AUDIOBOOK_FORMATS}")
# Complex language processing logic kept in config.py
BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
if len(BOOK_LANGUAGE) == 0:
BOOK_LANGUAGE = ['en']
# Custom script settings with validation logic
CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
if CUSTOM_SCRIPT:
if not os.path.exists(CUSTOM_SCRIPT):
logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
CUSTOM_SCRIPT = ""
elif not os.access(CUSTOM_SCRIPT, os.X_OK):
logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
CUSTOM_SCRIPT = ""
# Debugging settings
if not env.USING_EXTERNAL_BYPASSER:
# Recording directory for debugging internal cloudflare bypasser
RECORDING_DIR = env.LOG_DIR / "recording"
def _log_external_bypasser_warning() -> None:
"""Log warning about external bypasser DNS limitations (called after config is available)."""
from cwa_book_downloader.core.config import config
if config.get("USING_EXTERNAL_BYPASSER", False) and config.get("USE_CF_BYPASS", True):
logger.warning(
"Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
"not this application's custom DNS settings. If you experience DNS-related blocks, "
"configure DNS at the Docker/system level for your FlareSolverr container, "
"or consider using the internal bypasser which integrates with the app's DNS system."
)
from cwa_book_downloader.core.settings_registry import (
@@ -182,9 +143,7 @@ def _get_metadata_provider_options():
def _get_metadata_provider_options_with_none():
"""Build metadata provider options with a 'Use main provider' option first."""
options = [{"value": "", "label": "Use book provider"}]
options.extend(_get_metadata_provider_options())
return options
return [{"value": "", "label": "Use book provider"}] + _get_metadata_provider_options()
def _get_release_source_options():
@@ -492,7 +451,7 @@ def download_settings():
key="DESTINATION",
label="Destination",
description="Directory where downloaded files are saved.",
default="/cwa-book-ingest",
default="/books",
required=True,
),
SelectField(
@@ -789,49 +748,49 @@ def download_source_settings():
TextField(
key="AA_CONTENT_TYPE_DIR_FICTION",
label="Fiction Books",
placeholder="/cwa-book-ingest/fiction",
placeholder="/books/fiction",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
TextField(
key="AA_CONTENT_TYPE_DIR_NON_FICTION",
label="Non-Fiction Books",
placeholder="/cwa-book-ingest/non-fiction",
placeholder="/books/non-fiction",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
TextField(
key="AA_CONTENT_TYPE_DIR_UNKNOWN",
label="Unknown Books",
placeholder="/cwa-book-ingest/unknown",
placeholder="/books/unknown",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
TextField(
key="AA_CONTENT_TYPE_DIR_MAGAZINE",
label="Magazines",
placeholder="/cwa-book-ingest/magazines",
placeholder="/books/magazines",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
TextField(
key="AA_CONTENT_TYPE_DIR_COMIC",
label="Comic Books",
placeholder="/cwa-book-ingest/comics",
placeholder="/books/comics",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
TextField(
key="AA_CONTENT_TYPE_DIR_STANDARDS",
label="Standards Documents",
placeholder="/cwa-book-ingest/standards",
placeholder="/books/standards",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
TextField(
key="AA_CONTENT_TYPE_DIR_MUSICAL_SCORE",
label="Musical Scores",
placeholder="/cwa-book-ingest/scores",
placeholder="/books/scores",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
TextField(
key="AA_CONTENT_TYPE_DIR_OTHER",
label="Other",
placeholder="/cwa-book-ingest/other",
placeholder="/books/other",
show_when={"field": "AA_CONTENT_TYPE_ROUTING", "value": True},
),
]
cwa_book_downloader/core/cache.py
+9 -63
View File
@@ -24,24 +24,13 @@ class CacheService:
"""Thread-safe in-memory cache with TTL support."""
def __init__(self, max_size: int = 1000):
"""Initialize cache service.
Args:
max_size: Maximum number of entries before oldest are evicted.
"""
"""Initialize cache with max_size entries before eviction."""
self._cache: Dict[str, CacheEntry] = {}
self._lock = threading.Lock()
self._max_size = max_size
def get(self, key: str) -> Optional[Any]:
"""Get cached value if not expired.
Args:
key: Cache key to retrieve.
Returns:
Cached value or None if not found/expired.
"""
"""Get cached value if not expired."""
with self._lock:
entry = self._cache.get(key)
if entry is None:
@@ -54,13 +43,7 @@ class CacheService:
return entry.value
def set(self, key: str, value: Any, ttl: int) -> None:
"""Cache value with TTL.
Args:
key: Cache key.
value: Value to cache.
ttl: Time to live in seconds.
"""
"""Cache value with TTL in seconds."""
with self._lock:
# Evict oldest entries if at capacity
if len(self._cache) >= self._max_size:
@@ -72,14 +55,7 @@ class CacheService:
)
def invalidate(self, key: str) -> bool:
"""Remove specific cache entry.
Args:
key: Cache key to remove.
Returns:
True if entry was removed, False if not found.
"""
"""Remove specific cache entry. Returns True if found."""
with self._lock:
if key in self._cache:
del self._cache[key]
@@ -92,11 +68,7 @@ class CacheService:
self._cache.clear()
def cleanup_expired(self) -> int:
"""Remove all expired entries.
Returns:
Number of entries removed.
"""
"""Remove all expired entries. Returns count removed."""
with self._lock:
now = time.time()
expired_keys = [
@@ -108,10 +80,7 @@ class CacheService:
return len(expired_keys)
def _evict_oldest(self) -> None:
"""Evict oldest entries (by expiration time) to make room.
Called with lock held.
"""
"""Evict ~10% of oldest entries. Called with lock held."""
if not self._cache:
return
@@ -126,11 +95,7 @@ class CacheService:
del self._cache[key]
def stats(self) -> Dict[str, int]:
"""Get cache statistics.
Returns:
Dict with size and max_size.
"""
"""Get cache statistics (size, max_size)."""
with self._lock:
return {
"size": len(self._cache),
@@ -148,15 +113,7 @@ def get_metadata_cache() -> CacheService:
def cache_key(*args, **kwargs) -> str:
"""Generate cache key from arguments.
Args:
*args: Positional arguments to include in key.
**kwargs: Keyword arguments to include in key.
Returns:
String cache key.
"""
"""Generate cache key from arguments."""
parts = [str(arg) for arg in args]
parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
return ":".join(parts)
@@ -168,18 +125,7 @@ def cacheable(
ttl_default: int = 300,
key_prefix: str = ""
):
"""Decorator for caching function results.
Args:
ttl: Static time to live in seconds (use this OR ttl_key, not both).
ttl_key: Config key to read TTL from (e.g., "METADATA_CACHE_SEARCH_TTL").
ttl_default: Default TTL if ttl_key not found in config.
key_prefix: Optional prefix for cache keys.
Examples:
@cacheable(ttl=300, key_prefix="hardcover:search") # Static TTL
@cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", key_prefix="hardcover:search") # Dynamic TTL
"""
"""Decorator for caching function results. Use ttl (static) or ttl_key (from config)."""
def decorator(func: Callable[..., T]) -> Callable[..., T]:
@wraps(func)
def wrapper(*args, **kwargs) -> T:
cwa_book_downloader/core/config.py
+5 -4
View File
@@ -65,11 +65,12 @@ class Config:
def _load_settings(self) -> None:
"""Load all settings from the registry."""
# Ensure all plugin settings are registered before loading
# This handles cases where config is accessed before plugins are imported
# Ensure all settings modules are imported before loading
# This handles cases where config is accessed before settings are registered
try:
import cwa_book_downloader.release_sources # noqa: F401
import cwa_book_downloader.metadata_providers # noqa: F401
import cwa_book_downloader.config.settings # noqa: F401 - main app settings
import cwa_book_downloader.release_sources # noqa: F401 - plugin settings
import cwa_book_downloader.metadata_providers # noqa: F401 - plugin settings
except ImportError:
pass
cwa_book_downloader/core/image_cache.py
+15 -21
View File
@@ -92,10 +92,13 @@ class ImageCacheService:
def _load_index(self) -> None:
"""Load cache index from disk."""
if not self.index_path.exists():
self._index = {}
return
try:
if self.index_path.exists():
with open(self.index_path, 'r') as f:
self._index = json.load(f)
with open(self.index_path, 'r') as f:
self._index = json.load(f)
except (json.JSONDecodeError, IOError):
self._index = {}
@@ -179,9 +182,7 @@ class ImageCacheService:
"""Check if a cache entry is expired."""
if self.ttl_seconds == 0:
return False
cached_at = entry.get('cached_at', 0)
return (time.time() - cached_at) > self.ttl_seconds
return (time.time() - entry.get('cached_at', 0)) > self.ttl_seconds
def _is_negative_expired(self, entry: Dict[str, Any]) -> bool:
"""Check if a negative cache entry is expired.
@@ -193,12 +194,8 @@ class ImageCacheService:
return False
cached_at = entry.get('cached_at', 0)
# Transient failures (timeouts, connection errors) use shorter TTL
if entry.get('transient', False):
return (time.time() - cached_at) > TRANSIENT_CACHE_TTL
return (time.time() - cached_at) > NEGATIVE_CACHE_TTL
ttl = TRANSIENT_CACHE_TTL if entry.get('transient', False) else NEGATIVE_CACHE_TTL
return (time.time() - cached_at) > ttl
def _calculate_total_size(self) -> int:
"""Calculate total size of cached images."""
@@ -255,14 +252,13 @@ class ImageCacheService:
with self._lock:
entry = self._index.get(cache_id)
# Try reloading from disk if not found (handles multiprocess case)
if not entry:
# Try reloading from disk (handles multiprocess case)
self._load_index()
entry = self._index.get(cache_id)
if not entry:
self._misses += 1
return None
if not entry:
self._misses += 1
return None
# Check for negative cache (failed fetch)
if entry.get('negative', False):
@@ -526,10 +522,8 @@ class ImageCacheService:
self.put_negative(cache_id, transient=True)
return None
except requests.exceptions.HTTPError as e:
if e.response is not None and e.response.status_code == 404:
self.put_negative(cache_id)
else:
self.put_negative(cache_id, transient=True)
is_404 = e.response is not None and e.response.status_code == 404
self.put_negative(cache_id, transient=not is_404)
return None
except Exception:
return None
cwa_book_downloader/core/logger.py
+1 -11
View File
@@ -72,17 +72,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger:
# Create logger as CustomLogger instance
logger = CustomLogger(name)
log_level = logging.INFO
if LOG_LEVEL == "DEBUG":
log_level = logging.DEBUG
elif LOG_LEVEL == "INFO":
log_level = logging.INFO
elif LOG_LEVEL == "WARNING":
log_level = logging.WARNING
elif LOG_LEVEL == "ERROR":
log_level = logging.ERROR
elif LOG_LEVEL == "CRITICAL":
log_level = logging.CRITICAL
log_level = getattr(logging, LOG_LEVEL, logging.INFO)
logger.setLevel(log_level)
formatter = logging.Formatter(
cwa_book_downloader/core/models.py
+8 -6
View File
@@ -146,12 +146,14 @@ class BookInfo:
"""
# Resolve format if needed
if not self.format:
for url in (self.download_urls[0] if self.download_urls else None, fallback_url):
if url:
ext = url.split(".")[-1].lower()
if ext and len(ext) <= 5 and ext.isalnum():
self.format = ext
break
urls = [self.download_urls[0]] if self.download_urls else []
if fallback_url:
urls.append(fallback_url)
for url in urls:
ext = url.split(".")[-1].lower()
if ext and len(ext) <= 5 and ext.isalnum():
self.format = ext
break
return build_filename(self.title, self.author, self.year, self.format)
cwa_book_downloader/core/naming.py
+5 -10
View File
@@ -32,20 +32,19 @@ def _sanitize(name: str, max_length: int = 245) -> str:
def sanitize_filename(name: str, max_length: int = 245) -> str:
"""Sanitize a string for use as a filename."""
"""Sanitize a string for use as a filename or path component."""
return _sanitize(name, max_length)
def sanitize_path_component(name: str, max_length: int = 245) -> str:
"""Sanitize a string for use as a path component."""
return _sanitize(name, max_length)
# Alias for backwards compatibility
sanitize_path_component = sanitize_filename
def format_series_position(position: Optional[Union[int, float]]) -> str:
if position is None:
return ""
# Check if it's effectively an integer
# Display as integer if whole number
if isinstance(position, float) and position.is_integer():
return str(int(position))
@@ -110,11 +109,7 @@ def parse_naming_template(
return ""
# Sanitize the value
# If suffix contains a slash, this is meant to be a folder component
if '/' in suffix:
value = sanitize_path_component(value)
else:
value = sanitize_filename(value)
value = sanitize_filename(value)
return f"{prefix}{value}{suffix}"
cwa_book_downloader/core/queue.py
+23 -92
View File
@@ -12,11 +12,7 @@ from cwa_book_downloader.core.models import QueueStatus, QueueItem, DownloadTask
class BookQueue:
"""Thread-safe download queue manager with priority support and cancellation.
Stores DownloadTask objects which are source-agnostic download descriptors.
Works with both Direct Download and Universal modes.
"""
"""Thread-safe download queue manager with priority support and cancellation."""
def __init__(self) -> None:
self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue()
@@ -33,14 +29,7 @@ class BookQueue:
return timedelta(seconds=app_config.get("STATUS_TIMEOUT", 3600))
def add(self, task: DownloadTask) -> bool:
"""Add a download task to the queue.
Args:
task: The download task to queue (includes task_id, priority, etc.)
Returns:
True if added successfully, False if already exists
"""
"""Add a download task to the queue. Returns False if already exists."""
with self._lock:
task_id = task.task_id
@@ -59,11 +48,7 @@ class BookQueue:
return True
def get_next(self) -> Optional[Tuple[str, Event]]:
"""Get next task ID from queue with cancellation flag.
Returns:
Tuple of (task_id, cancel_flag) or None if queue is empty
"""
"""Get next task ID from queue with cancellation flag."""
# Use iterative approach to avoid stack overflow if many items are cancelled
while True:
try:
@@ -85,14 +70,7 @@ class BookQueue:
return None
def get_task(self, task_id: str) -> Optional[DownloadTask]:
"""Get a task by its ID.
Args:
task_id: The task identifier
Returns:
The DownloadTask if found, None otherwise
"""
"""Get a task by its ID."""
with self._lock:
return self._task_data.get(task_id)
@@ -171,14 +149,7 @@ class BookQueue:
return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
def cancel_download(self, task_id: str) -> bool:
"""Cancel a download or clear a completed/errored item.
Args:
task_id: Task identifier to cancel or clear
Returns:
bool: True if cancellation/clearing was successful
"""
"""Cancel a download or clear a completed/errored item."""
with self._lock:
current_status = self._status.get(task_id)
@@ -205,15 +176,7 @@ class BookQueue:
return False
def set_priority(self, task_id: str, new_priority: int) -> bool:
"""Change the priority of a queued task.
Args:
task_id: Task identifier
new_priority: New priority level (lower = higher priority)
Returns:
bool: True if priority was successfully changed
"""
"""Change the priority of a queued task (lower = higher priority)."""
with self._lock:
if task_id not in self._status or self._status[task_id] != QueueStatus.QUEUED:
return False
@@ -245,14 +208,7 @@ class BookQueue:
return found
def reorder_queue(self, task_priorities: Dict[str, int]) -> bool:
"""Bulk reorder queue by setting new priorities.
Args:
task_priorities: Dict mapping task_id to new priority
Returns:
bool: True if reordering was successful
"""
"""Bulk reorder queue by mapping task_id to new priority."""
with self._lock:
# Extract all items from queue
all_items = []
@@ -283,39 +239,18 @@ class BookQueue:
return list(self._active_downloads.keys())
def has_pending_work(self) -> bool:
"""Check if there are any active downloads or queued items.
This is useful for determining if the bypasser should stay active
even when the UI is closed.
Returns:
bool: True if there are active downloads or queued items
"""
"""Check if there are any active downloads or queued items."""
with self._lock:
# Check for active downloads
if self._active_downloads:
return True
# Check for queued items (excluding cancelled ones)
for task_id, status in self._status.items():
if status == QueueStatus.QUEUED:
return True
return False
return any(status == QueueStatus.QUEUED for status in self._status.values())
def clear_completed(self) -> int:
"""Remove all completed, errored, or cancelled tasks from tracking.
Returns:
int: Number of tasks removed
"""
"""Remove all completed, errored, or cancelled tasks from tracking."""
terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED}
with self._lock:
to_remove = []
for task_id, status in self._status.items():
if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
to_remove.append(task_id)
to_remove = [task_id for task_id, status in self._status.items() if status in terminal_statuses]
removed_count = len(to_remove)
for task_id in to_remove:
self._status.pop(task_id, None)
self._status_timestamps.pop(task_id, None)
@@ -323,14 +258,13 @@ class BookQueue:
self._cancel_flags.pop(task_id, None)
self._active_downloads.pop(task_id, None)
return removed_count
return len(to_remove)
def refresh(self) -> None:
"""Remove any tasks that are done downloading or have stale status."""
terminal_statuses = {QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED}
with self._lock:
current_time = datetime.now()
# Create a list of items to remove to avoid modifying dict during iteration
to_remove = []
for task_id, status in self._status.items():
@@ -338,28 +272,25 @@ class BookQueue:
if not task:
continue
path = task.download_path
if path and not Path(path).exists():
# Clear stale download paths
if task.download_path and not Path(task.download_path).exists():
task.download_path = None
path = None
# Check for completed downloads
if status == QueueStatus.AVAILABLE:
if not path:
self._update_status(task_id, QueueStatus.DONE)
# Mark available downloads as done if file is gone
if status == QueueStatus.AVAILABLE and not task.download_path:
self._update_status(task_id, QueueStatus.DONE)
# Check for stale status entries
last_update = self._status_timestamps.get(task_id)
if last_update and (current_time - last_update) > self._status_timeout:
if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]:
if status in terminal_statuses:
to_remove.append(task_id)
# Remove stale entries
for task_id in to_remove:
del self._status[task_id]
del self._status_timestamps[task_id]
if task_id in self._task_data:
del self._task_data[task_id]
self._status.pop(task_id, None)
self._status_timestamps.pop(task_id, None)
self._task_data.pop(task_id, None)
# Global instance of BookQueue
book_queue = BookQueue()
cwa_book_downloader/core/settings_registry.py
+9 -21
View File
@@ -240,9 +240,7 @@ def _get_config_file_path(tab_name: str) -> Path:
# Core settings tabs share the main settings.json file
if tab_name in ("general", "search_mode"):
return config_dir / "settings.json"
else:
plugins_dir = config_dir / "plugins"
return plugins_dir / f"{tab_name}.json"
return config_dir / "plugins" / f"{tab_name}.json"
def _ensure_config_dir(tab_name: str) -> None:
@@ -463,12 +461,9 @@ def is_value_from_env(field: SettingsField) -> bool:
if isinstance(field, (ActionButton, HeadingField)):
return False
# UI-only settings never come from ENV (env_supported=False)
# Default to True for backwards compatibility
env_supported = getattr(field, 'env_supported', True)
if env_supported is False:
if not getattr(field, 'env_supported', True):
return False
env_var_name = field.get_env_var_name()
return env_var_name in os.environ
return field.get_env_var_name() in os.environ
def serialize_field(field: SettingsField, tab_name: str, include_value: bool = True) -> Dict[str, Any]:
@@ -511,19 +506,12 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T
"requiresRestart": getattr(field, 'requires_restart', False),
}
# Add conditional visibility if specified
show_when = getattr(field, 'show_when', None)
if show_when:
result["showWhen"] = show_when
# Add conditional disable if specified
disabled_when = getattr(field, 'disabled_when', None)
if disabled_when:
result["disabledWhen"] = disabled_when
# Add universal_only flag if set
universal_only = getattr(field, 'universal_only', False)
if universal_only:
# Add optional properties if set
if getattr(field, 'show_when', None):
result["showWhen"] = field.show_when
if getattr(field, 'disabled_when', None):
result["disabledWhen"] = field.disabled_when
if getattr(field, 'universal_only', False):
result["universalOnly"] = True
# Add type-specific properties
cwa_book_downloader/core/utils.py
+22 -64
View File
@@ -1,14 +1,15 @@
"""
Shared utility functions for the CWA Book Downloader.
Provides common helper functions used across the application.
"""
"""Shared utility functions for the CWA Book Downloader."""
import base64
from pathlib import Path
from typing import Optional
def is_audiobook(content_type: Optional[str]) -> bool:
"""Check if content type indicates an audiobook."""
return bool(content_type and "audiobook" in content_type.lower())
CONTENT_TYPES = [
"book (fiction)",
"book (non-fiction)",
@@ -50,14 +51,7 @@ _LEGACY_CONTENT_TYPE_TO_CONFIG_KEY = {
def get_destination(is_audiobook: bool = False) -> Path:
"""Get the base destination directory.
Args:
is_audiobook: If True, returns audiobook destination (with fallback to books destination)
Returns:
Path to the destination directory
"""
"""Get base destination directory. Audiobooks fall back to main destination."""
from cwa_book_downloader.core.config import config
if is_audiobook:
@@ -68,63 +62,40 @@ def get_destination(is_audiobook: bool = False) -> Path:
# Main destination (also fallback for audiobooks)
# Check new setting first, then legacy INGEST_DIR
destination = config.get("DESTINATION", "") or config.get("INGEST_DIR", "/cwa-book-ingest")
destination = config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books")
return Path(destination)
def get_aa_content_type_dir(content_type: Optional[str] = None) -> Optional[Path]:
"""Get override directory for Anna's Archive content-type routing.
Only returns a path if AA_CONTENT_TYPE_ROUTING is enabled AND
a custom directory is configured for the given content type.
Args:
content_type: The AA content type (e.g., "book (fiction)", "magazine")
Returns:
Path to the override directory if configured, None otherwise
"""
"""Get override directory for AA content-type routing if configured."""
from cwa_book_downloader.core.config import config
# Check if content-type routing is enabled
if not config.get("AA_CONTENT_TYPE_ROUTING", False):
# Also check legacy setting for backwards compatibility
if not config.get("USE_CONTENT_TYPE_DIRECTORIES", False):
return None
# Check if content-type routing is enabled (new or legacy setting)
if not config.get("AA_CONTENT_TYPE_ROUTING", False) and not config.get("USE_CONTENT_TYPE_DIRECTORIES", False):
return None
if not content_type:
return None
# Normalize content type for lookup
content_type_lower = content_type.lower().strip()
# Try new AA-specific config keys first
config_key = _AA_CONTENT_TYPE_TO_CONFIG_KEY.get(content_type_lower)
if config_key:
custom_dir = config.get(config_key, "")
if custom_dir:
return Path(custom_dir)
# Fall back to legacy config keys for backwards compatibility
legacy_key = _LEGACY_CONTENT_TYPE_TO_CONFIG_KEY.get(content_type_lower)
if legacy_key:
custom_dir = config.get(legacy_key, "")
if custom_dir:
return Path(custom_dir)
# Try new AA-specific config keys first, then legacy keys
for mapping in (_AA_CONTENT_TYPE_TO_CONFIG_KEY, _LEGACY_CONTENT_TYPE_TO_CONFIG_KEY):
config_key = mapping.get(content_type_lower)
if config_key:
custom_dir = config.get(config_key, "")
if custom_dir:
return Path(custom_dir)
return None
def get_ingest_dir(content_type: Optional[str] = None) -> Path:
"""Get the ingest directory for a content type, falling back to default.
DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead.
Kept for backwards compatibility during migration.
"""
"""DEPRECATED: Use get_destination() and get_aa_content_type_dir() instead."""
from cwa_book_downloader.core.config import config
# Check new DESTINATION setting first, then legacy INGEST_DIR
default_ingest_dir = Path(config.get("DESTINATION", "") or config.get("INGEST_DIR", "/cwa-book-ingest"))
default_ingest_dir = Path(config.get("DESTINATION", "") or config.get("INGEST_DIR", "/books"))
if not content_type:
return default_ingest_dir
@@ -138,20 +109,7 @@ def get_ingest_dir(content_type: Optional[str] = None) -> Path:
def transform_cover_url(cover_url: Optional[str], cache_id: str) -> Optional[str]:
"""
Transform an external cover URL to a local proxy URL when caching is enabled.
When cover caching is enabled, external cover image URLs are transformed
to local proxy URLs that cache the images on first access. This reduces
external requests and provides a consistent caching layer.
Args:
cover_url: Original cover URL (external or already local)
cache_id: Unique identifier for the cache entry (e.g., "provider_bookid")
Returns:
Transformed URL if caching enabled and URL is external, otherwise original URL
"""
"""Transform external cover URL to local proxy URL when caching is enabled."""
if not cover_url:
return cover_url
cwa_book_downloader/download/archive.py
+56 -136
View File
@@ -10,6 +10,8 @@ from typing import List, Optional, Tuple
from cwa_book_downloader.core.logger import setup_logger
from cwa_book_downloader.core.config import config
from cwa_book_downloader.core.naming import parse_naming_template, sanitize_filename
from cwa_book_downloader.core.utils import is_audiobook as check_audiobook
from cwa_book_downloader.download.fs import atomic_write, atomic_move
logger = setup_logger(__name__)
@@ -76,6 +78,8 @@ def _get_template(is_audiobook: bool, organization_mode: str) -> str:
template = config.get(legacy_key, "")
if not template:
if organization_mode == "organize":
return "{Author}/{Title} ({Year})"
return "{Author} - {Title} ({Year})"
return template
@@ -83,8 +87,7 @@ def _get_template(is_audiobook: bool, organization_mode: str) -> str:
def _build_filename_from_task(task, extension: str, organization_mode: str) -> str:
"""Build a filename from task metadata using the configured template."""
content_type = task.content_type.lower() if task.content_type else ""
is_audiobook = "audiobook" in content_type
is_audiobook = check_audiobook(task.content_type)
template = _get_template(is_audiobook, organization_mode)
metadata = {
@@ -138,7 +141,7 @@ def is_archive(file_path: Path) -> bool:
def _is_supported_file(file_path: Path, content_type: Optional[str] = None) -> bool:
"""Check if file matches user's supported formats setting based on content type."""
ext = file_path.suffix.lower().lstrip(".")
if content_type and content_type.lower() == "audiobook":
if check_audiobook(content_type):
supported_formats = _get_supported_audiobook_formats()
else:
supported_formats = _get_supported_formats()
@@ -156,19 +159,8 @@ def _filter_files(
extracted_files: List[Path],
content_type: Optional[str] = None,
) -> Tuple[List[Path], List[Path], List[Path]]:
"""
Filter extracted files based on content type.
For audiobooks: filters to audio formats using SUPPORTED_AUDIOBOOK_FORMATS
For books: filters to book formats using SUPPORTED_FORMATS
Returns:
Tuple of (matched_files, rejected_format_files, other_files)
- matched_files: Match user's supported formats for this content type
- rejected_format_files: Valid formats for this type but not enabled by user
- other_files: Unrelated files (images, html, etc)
"""
is_audiobook = content_type and content_type.lower() == "audiobook"
"""Filter files by content type. Returns (matched, rejected_format, other)."""
is_audiobook = check_audiobook(content_type)
known_extensions = ALL_AUDIO_EXTENSIONS if is_audiobook else ALL_EBOOK_EXTENSIONS
matched_files = []
@@ -191,30 +183,7 @@ def extract_archive(
output_dir: Path,
content_type: Optional[str] = None,
) -> Tuple[List[Path], List[str], List[Path]]:
"""
Extract files from an archive based on content type.
Extracts all files, then filters based on content type:
- Audiobooks: keeps files matching SUPPORTED_AUDIOBOOK_FORMATS
- Books: keeps files matching SUPPORTED_FORMATS
Non-matching files (HTML, images, etc.) are deleted.
Args:
archive_path: Path to the archive file
output_dir: Directory to extract files to
content_type: Content type (e.g., "audiobook") to determine which formats to keep
Returns:
Tuple of (matched_files, warnings, rejected_files)
- matched_files: Paths to extracted files matching supported formats
- warnings: List of warning messages
- rejected_files: Files that were rejected (format not enabled)
Raises:
ArchiveExtractionError: If extraction fails
PasswordProtectedError: If archive requires password
CorruptedArchiveError: If archive is corrupted
"""
"""Extract archive and filter by content type. Returns (matched, warnings, rejected)."""
suffix = archive_path.suffix.lower().lstrip(".")
if suffix == "zip":
@@ -224,7 +193,7 @@ def extract_archive(
else:
raise ArchiveExtractionError(f"Unsupported archive format: {suffix}")
is_audiobook = content_type and content_type.lower() == "audiobook"
is_audiobook = check_audiobook(content_type)
file_type_label = "audiobook" if is_audiobook else "book"
# Filter files based on content type
@@ -256,14 +225,46 @@ def extract_archive(
return matched_files, warnings, rejected_files
def _extract_zip(
archive_path: Path,
output_dir: Path,
) -> Tuple[List[Path], List[str]]:
"""Extract files from a ZIP archive."""
def _extract_files_from_archive(archive, output_dir: Path) -> List[Path]:
"""Extract files from ZipFile or RarFile to output_dir with security checks."""
extracted_files = []
warnings = []
for info in archive.infolist():
if info.is_dir():
continue
# Use only filename, strip directory path (security: prevent path traversal)
filename = Path(info.filename).name
if not filename:
continue
# Security: reject filenames with null bytes or path separators
# Check both / and \ since archives may be created on different OSes
if "\x00" in filename or "/" in filename or "\\" in filename:
logger.warning(f"Skipping suspicious filename in archive: {info.filename!r}")
continue
# Extract to output_dir with flat structure
target_path = output_dir / filename
# Security: verify resolved path stays within output directory (defense-in-depth)
try:
target_path.resolve().relative_to(output_dir.resolve())
except ValueError:
logger.warning(f"Path traversal attempt blocked: {info.filename!r}")
continue
with archive.open(info) as src:
data = src.read()
final_path = atomic_write(target_path, data)
extracted_files.append(final_path)
logger.debug(f"Extracted: {filename}")
return extracted_files
def _extract_zip(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
"""Extract files from a ZIP archive."""
try:
with zipfile.ZipFile(archive_path, "r") as zf:
# Check for password protection
@@ -276,45 +277,19 @@ def _extract_zip(
if bad_file:
raise CorruptedArchiveError(f"Corrupted file in archive: {bad_file}")
# Extract all files
for info in zf.infolist():
if info.is_dir():
continue
# Use only filename, strip directory path (security: prevent path traversal)
filename = Path(info.filename).name
if not filename:
continue
# Extract to output_dir with flat structure
target_path = output_dir / filename
target_path = _handle_duplicate_filename(target_path)
with zf.open(info) as src, open(target_path, "wb") as dst:
dst.write(src.read())
extracted_files.append(target_path)
logger.debug(f"Extracted: {filename}")
return _extract_files_from_archive(zf, output_dir), []
except zipfile.BadZipFile as e:
raise CorruptedArchiveError(f"Invalid or corrupted ZIP: {e}")
except PermissionError as e:
raise ArchiveExtractionError(f"Permission denied: {e}")
return extracted_files, warnings
def _extract_rar(
archive_path: Path,
output_dir: Path,
) -> Tuple[List[Path], List[str]]:
def _extract_rar(archive_path: Path, output_dir: Path) -> Tuple[List[Path], List[str]]:
"""Extract files from a RAR archive."""
if not RAR_AVAILABLE:
raise ArchiveExtractionError("RAR extraction not available - rarfile library not installed")
extracted_files = []
warnings = []
try:
with rarfile.RarFile(archive_path, "r") as rf:
# Check for password protection
@@ -324,25 +299,7 @@ def _extract_rar(
# Test archive integrity
rf.testrar()
# Extract all files
for info in rf.infolist():
if info.is_dir():
continue
# Use only filename, strip directory path (security: prevent path traversal)
filename = Path(info.filename).name
if not filename:
continue
# Extract to output_dir with flat structure
target_path = output_dir / filename
target_path = _handle_duplicate_filename(target_path)
with rf.open(info) as src, open(target_path, "wb") as dst:
dst.write(src.read())
extracted_files.append(target_path)
logger.debug(f"Extracted: {filename}")
return _extract_files_from_archive(rf, output_dir), []
except rarfile.BadRarFile as e:
raise CorruptedArchiveError(f"Invalid or corrupted RAR: {e}")
@@ -351,25 +308,6 @@ def _extract_rar(
except PermissionError as e:
raise ArchiveExtractionError(f"Permission denied: {e}")
return extracted_files, warnings
def _handle_duplicate_filename(target_path: Path) -> Path:
"""Handle duplicate filenames by appending counter."""
if not target_path.exists():
return target_path
base = target_path.stem
ext = target_path.suffix
parent = target_path.parent
counter = 1
while target_path.exists():
target_path = parent / f"{base}_{counter}{ext}"
counter += 1
return target_path
@dataclass
class ArchiveResult:
@@ -388,27 +326,10 @@ def process_archive(
archive_id: str,
task: Optional["DownloadTask"] = None,
) -> ArchiveResult:
"""
Process an archive file: extract, filter to supported files, move to ingest.
This is the main entry point for archive handling, usable by any download handler.
Filters files based on content type:
- Audiobooks: keeps files matching SUPPORTED_AUDIOBOOK_FORMATS
- Books: keeps files matching SUPPORTED_FORMATS
Args:
archive_path: Path to the downloaded archive file
temp_dir: Base temp directory for extraction (e.g., TMP_DIR)
ingest_dir: Final destination directory for files
archive_id: Unique identifier for temp directory naming
task: Optional download task for filename generation and content type
Returns:
ArchiveResult with success status, final paths, and status message
"""
"""Extract archive, filter to supported formats, move to ingest directory."""
extract_dir = temp_dir / f"extract_{archive_id}"
content_type = task.content_type if task else None
is_audiobook = content_type and content_type.lower() == "audiobook"
is_audiobook = check_audiobook(content_type)
file_type_label = "audiobook" if is_audiobook else "book"
try:
@@ -456,7 +377,7 @@ def process_archive(
final_paths = []
# Determine file organization mode
is_audiobook = task and task.content_type and "audiobook" in task.content_type.lower()
is_audiobook = check_audiobook(task.content_type) if task else False
organization_mode = _get_file_organization(is_audiobook) if task else "none"
for extracted_file in extracted_files:
@@ -472,9 +393,8 @@ def process_archive(
else:
filename = extracted_file.name
final_path = ingest_dir / filename
final_path = _handle_duplicate_filename(final_path)
shutil.move(str(extracted_file), str(final_path))
dest_path = ingest_dir / filename
final_path = atomic_move(extracted_file, dest_path)
final_paths.append(final_path)
logger.debug(f"Moved to ingest: {final_path.name}")
cwa_book_downloader/download/fs.py
+191
View File
@@ -0,0 +1,191 @@
"""Atomic filesystem operations for concurrent-safe file handling.
These utilities handle file collisions atomically, avoiding TOCTOU race conditions
when multiple workers may try to write to the same path simultaneously.
"""
import errno
import os
import shutil
from pathlib import Path
from cwa_book_downloader.core.logger import setup_logger
logger = setup_logger(__name__)
def atomic_write(dest_path: Path, data: bytes, max_attempts: int = 100) -> Path:
"""Write data to a file with atomic collision detection.
If the destination already exists, retries with counter suffix (_1, _2, etc.)
until a unique path is found.
Args:
dest_path: Desired destination path
data: Bytes to write
max_attempts: Maximum collision retries before raising error
Returns:
Path where file was actually written (may differ from dest_path)
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
parent = dest_path.parent
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
try:
# O_CREAT | O_EXCL fails atomically if file exists
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
try:
os.write(fd, data)
finally:
os.close(fd)
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except FileExistsError:
continue
raise RuntimeError(f"Could not write file after {max_attempts} attempts: {dest_path}")
def atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
"""Move a file with collision detection.
Uses os.rename() for same-filesystem moves (atomic, triggers inotify events),
falls back to exclusive create + shutil.move for cross-filesystem moves.
Note: We use os.rename() instead of hardlink+unlink because os.rename()
triggers proper inotify IN_MOVED_TO events that file watchers (like Calibre's
auto-add) rely on to detect new files.
Args:
source_path: Source file to move
dest_path: Desired destination path
max_attempts: Maximum collision retries before raising error
Returns:
Path where file was actually moved (may differ from dest_path)
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
parent = dest_path.parent
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
# Check for existing file (os.rename would overwrite on Unix)
if try_path.exists():
continue
try:
# os.rename is atomic on same filesystem and triggers inotify events
os.rename(str(source_path), str(try_path))
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except FileExistsError:
# Race condition: file created between exists() check and rename()
continue
except OSError as e:
# Cross-filesystem - fall back to exclusive create + move
if e.errno != errno.EXDEV:
raise
try:
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.close(fd)
try:
shutil.move(str(source_path), str(try_path))
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except Exception:
try_path.unlink(missing_ok=True)
raise
except FileExistsError:
continue
raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}")
def atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
"""Create a hardlink with atomic collision detection.
Args:
source_path: Source file to link from
dest_path: Desired destination path for the link
max_attempts: Maximum collision retries before raising error
Returns:
Path where link was actually created (may differ from dest_path)
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
parent = dest_path.parent
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
try:
os.link(str(source_path), str(try_path))
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except FileExistsError:
continue
raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
def atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
"""Copy a file with atomic collision detection.
Uses exclusive create to claim destination, then copies via temp file
to avoid partial files on failure.
Args:
source_path: Source file to copy
dest_path: Desired destination path
max_attempts: Maximum collision retries before raising error
Returns:
Path where file was actually copied (may differ from dest_path)
Raises:
RuntimeError: If no unique path found after max_attempts
"""
base = dest_path.stem
ext = dest_path.suffix
parent = dest_path.parent
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else parent / f"{base}_{attempt}{ext}"
try:
# Atomically claim the destination by creating an exclusive file
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.close(fd)
# Copy to temp file first, then replace to avoid partial files
temp_path = try_path.parent / f".{try_path.name}.tmp"
try:
shutil.copy2(str(source_path), str(temp_path))
temp_path.replace(try_path)
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except Exception:
try_path.unlink(missing_ok=True)
temp_path.unlink(missing_ok=True)
raise
except FileExistsError:
continue
raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}")
cwa_book_downloader/download/http.py
+25 -34
View File
@@ -15,6 +15,8 @@ from cwa_book_downloader.download.network import get_proxies
from cwa_book_downloader.core.config import config as app_config
from cwa_book_downloader.core.logger import setup_logger
logger = setup_logger(__name__)
# Bypasser modules are imported lazily to support dynamic selection based on config
_internal_bypasser = None
_external_bypasser = None
@@ -65,8 +67,7 @@ def get_bypassed_page(url, selector=None, cancel_flag=None):
"""Wrapper that delegates to the appropriate bypasser based on config."""
if _is_using_external_bypasser():
return _get_external_bypasser().get_bypassed_page(url, selector, cancel_flag)
else:
return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag)
return _get_internal_bypasser().get_bypassed_page(url, selector, cancel_flag)
def get_cf_cookies_for_domain(domain):
@@ -84,7 +85,24 @@ def get_cf_user_agent_for_domain(domain):
return None
return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
logger = setup_logger(__name__)
def _apply_cf_bypass(url: str, headers: dict) -> dict:
"""Apply CF bypass cookies and user agent if available.
Modifies headers in-place with the stored user agent (if available).
Returns cookies dict to use with the request.
"""
if not _is_cf_bypass_enabled():
return {}
parsed = urlparse(url)
hostname = parsed.hostname or ""
cookies = get_cf_cookies_for_domain(hostname)
stored_ua = get_cf_user_agent_for_domain(hostname)
if stored_ua:
headers['User-Agent'] = stored_ua
return cookies
# Network settings
REQUEST_TIMEOUT = (5, 10) # (connect, read)
@@ -131,7 +149,7 @@ def _is_retryable_error(e: Exception) -> bool:
if isinstance(e, CONNECTION_ERRORS):
return True
status = _get_status_code(e)
return status in RETRYABLE_CODES if status else False
return status is not None and status in RETRYABLE_CODES
def _try_rotation(original_url: str, current_url: str, selector: network.AAMirrorSelector) -> Optional[str]:
@@ -180,15 +198,8 @@ def html_get_page(
logger.debug(f"GET: {current_url}")
# Try with CF cookies/UA if available (from previous bypass)
cookies = {}
headers = {}
if _is_cf_bypass_enabled():
parsed = urlparse(current_url)
hostname = parsed.hostname or ""
cookies = get_cf_cookies_for_domain(hostname)
stored_ua = get_cf_user_agent_for_domain(hostname)
if stored_ua:
headers['User-Agent'] = stored_ua
cookies = _apply_cf_bypass(current_url, headers)
response = requests.get(current_url, proxies=get_proxies(), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
response.raise_for_status()
time.sleep(1)
@@ -271,20 +282,7 @@ def download_url(
logger.info(f"Downloading: {current_url} (attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
# Try with CF cookies/UA if available
cookies = {}
if _is_cf_bypass_enabled():
parsed = urlparse(current_url)
hostname = parsed.hostname or ""
cookies = get_cf_cookies_for_domain(hostname)
# Use stored UA - Cloudflare ties cf_clearance to the UA that solved the challenge
stored_ua = get_cf_user_agent_for_domain(hostname)
if stored_ua:
headers['User-Agent'] = stored_ua
logger.debug(f"Using stored UA for {hostname}")
else:
logger.debug(f"No stored UA available for {hostname}")
if cookies:
logger.debug(f"Using {len(cookies)} cookies for {hostname}: {list(cookies.keys())}")
cookies = _apply_cf_bypass(current_url, headers)
response = requests.get(current_url, stream=True, proxies=get_proxies(), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
response.raise_for_status()
@@ -392,15 +390,8 @@ def _try_resume(
try:
# Try with CF cookies/UA if available
cookies = {}
resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'}
if _is_cf_bypass_enabled():
parsed = urlparse(url)
hostname = parsed.hostname or ""
cookies = get_cf_cookies_for_domain(hostname)
stored_ua = get_cf_user_agent_for_domain(hostname)
if stored_ua:
resume_headers['User-Agent'] = stored_ua
cookies = _apply_cf_bypass(url, resume_headers)
response = requests.get(
url, stream=True, proxies=get_proxies(), timeout=REQUEST_TIMEOUT,
headers=resume_headers, cookies=cookies
cwa_book_downloader/download/network.py
+64 -75
View File
@@ -1,6 +1,5 @@
"""DNS rotation, mirror selection, and network utilities."""
import os
import requests
import urllib.request
from typing import Sequence, Tuple, Any, Union, cast, List, Optional, Callable
@@ -8,12 +7,9 @@ import socket
import dns.resolver
from socket import AddressFamily, SocketKind
import urllib.parse
import ssl
import ipaddress
from cwa_book_downloader.core.logger import setup_logger
from cwa_book_downloader.config.settings import AA_BASE_URL, AA_AVAILABLE_URLS
from cwa_book_downloader.config import settings as config
from cwa_book_downloader.core.config import config as app_config
from datetime import datetime, timedelta
@@ -107,13 +103,6 @@ def _notify_dns_rotation(provider_name: str, servers: List[str], doh_url: str) -
except Exception as e:
logger.warning(f"DNS rotation callback {callback.__name__} failed: {e}")
def _agent_debug_log(code: str, source: str, reason: str, meta: Optional[dict] = None) -> None:
"""Lightweight debug hook for automated runs; safe no-op on failure."""
try:
logger.debug(f"[agent] code={code} source={source} reason={reason} meta={meta or {}}")
except Exception as exc:
# Avoid raising inside debug logger
logger.debug(f"[agent] log failed: {exc}")
def _load_state():
"""Return current in-memory network state (no disk persistence)."""
@@ -133,7 +122,8 @@ def _save_state(aa_url=None, dns_provider=None):
# AA URL failover state
_current_aa_url_index = 0
_aa_urls = AA_AVAILABLE_URLS.copy()
_aa_urls: List[str] = [] # Initialized lazily in _initialize_aa_state()
_aa_base_url: str = "" # Current active AA URL
def _ensure_initialized() -> None:
"""Lazy guard so runtime setup happens once and late calls still work."""
@@ -234,34 +224,17 @@ def _decode_port(port: Union[str, bytes, int, None]) -> int:
"""Convert port to integer, handling various input types."""
if port is None:
return 0
if isinstance(port, (str, bytes)):
return int(port)
return int(port)
def _is_local_address(host_str: str) -> bool:
"""Check if an address is local or private and should bypass custom DNS."""
# Localhost checks
if (host_str == 'localhost' or
host_str.startswith('127.') or
host_str == '::1' or
host_str == '0.0.0.0'):
if host_str == 'localhost':
return True
# IPv4 private ranges (RFC 1918)
if (host_str.startswith('10.') or
(host_str.startswith('172.') and
len(host_str.split('.')) > 1 and
16 <= int(host_str.split('.')[1]) <= 31) or
host_str.startswith('192.168.')):
return True
# IPv6 private ranges
if (host_str.startswith('fc') or
host_str.startswith('fd') or # Unique local addresses (fc00::/7)
host_str.startswith('fe80:')): # Link-local addresses (fe80::/10)
return True
return False
try:
addr = ipaddress.ip_address(host_str)
return addr.is_private or addr.is_loopback or addr.is_link_local
except ValueError:
return False
def _is_ip_address(host_str: str) -> bool:
"""Check if a string is a valid IP address (IPv4 or IPv6)."""
@@ -642,8 +615,8 @@ def switch_dns_provider() -> bool:
name, servers, doh = DNS_PROVIDERS[_current_dns_index]
CUSTOM_DNS = servers
DOH_SERVER = doh
config.CUSTOM_DNS = servers
config.DOH_SERVER = doh
app_config.CUSTOM_DNS = servers
app_config.DOH_SERVER = doh
logger.warning(f"Switched DNS provider to: {name} (using DoH)")
_save_state(dns_provider=name)
@@ -672,20 +645,20 @@ def rotate_dns_and_reset_aa() -> bool:
"""
Switch DNS provider (auto mode) and reset AA URL list to the first entry.
Returns True if DNS switched; False if no providers left or not in auto mode.
Note: This function can be called during initialization, so we must NOT call
_ensure_initialized() here to avoid recursive init loops.
"""
if not rotate_dns_provider():
return False
# Reset AA URL to first available auto option if using auto AA
global AA_BASE_URL, _current_aa_url_index
if AA_BASE_URL == "auto" or AA_BASE_URL in _aa_urls:
global _aa_base_url, _current_aa_url_index
configured_url = app_config.get("AA_BASE_URL", "auto")
if configured_url == "auto" or _aa_base_url in _aa_urls:
_current_aa_url_index = 0
AA_BASE_URL = _aa_urls[0]
config.AA_BASE_URL = AA_BASE_URL
logger.info(f"After DNS switch, resetting AA URL to: {AA_BASE_URL}")
_save_state(aa_url=AA_BASE_URL)
_aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.org"
logger.info(f"After DNS switch, resetting AA URL to: {_aa_base_url}")
_save_state(aa_url=_aa_base_url)
return True
def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use_doh: bool | None = None) -> bool:
@@ -715,8 +688,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
_dns_exhausted_logged = False
CUSTOM_DNS = []
DOH_SERVER = ""
config.CUSTOM_DNS = []
config.DOH_SERVER = ""
app_config.CUSTOM_DNS = []
app_config.DOH_SERVER = ""
# Restore original system getaddrinfo
socket.getaddrinfo = original_getaddrinfo
logger.info("DNS set to system mode (using OS default resolver)")
@@ -730,8 +703,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
_dns_exhausted_logged = False
CUSTOM_DNS = []
DOH_SERVER = ""
config.CUSTOM_DNS = []
config.DOH_SERVER = ""
app_config.CUSTOM_DNS = []
app_config.DOH_SERVER = ""
logger.info("DNS set to auto mode (system DNS, will rotate on failure with DoH)")
init_dns_resolvers()
_notify_dns_rotation("auto", [], "")
@@ -744,8 +717,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
_current_dns_index = -1 # Not using preset providers
CUSTOM_DNS = manual_servers
DOH_SERVER = "" # No DoH for manual servers
config.CUSTOM_DNS = manual_servers
config.DOH_SERVER = ""
app_config.CUSTOM_DNS = manual_servers
app_config.DOH_SERVER = ""
logger.info(f"DNS set to manual servers: {manual_servers}")
init_dns_resolvers()
_notify_dns_rotation("manual", manual_servers, "")
@@ -759,8 +732,8 @@ def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use
CUSTOM_DNS = servers
# Only set DoH server if DoH is enabled
DOH_SERVER = doh if doh_enabled else ""
config.CUSTOM_DNS = servers
config.DOH_SERVER = DOH_SERVER
app_config.CUSTOM_DNS = servers
app_config.DOH_SERVER = DOH_SERVER
doh_status = "DoH enabled" if doh_enabled else "standard DNS"
logger.info(f"DNS set to: {name} ({doh_status})")
_save_state(dns_provider=name)
@@ -781,14 +754,14 @@ def init_dns_resolvers():
name, servers, doh = DNS_PROVIDERS[_current_dns_index]
CUSTOM_DNS = servers
DOH_SERVER = doh
config.CUSTOM_DNS = servers
config.DOH_SERVER = doh
app_config.CUSTOM_DNS = servers
app_config.DOH_SERVER = doh
logger.info(f"Using DNS provider: {name} (DoH enabled)")
else:
CUSTOM_DNS = []
DOH_SERVER = ""
config.CUSTOM_DNS = []
config.DOH_SERVER = ""
app_config.CUSTOM_DNS = []
app_config.DOH_SERVER = ""
logger.debug("Using system DNS (auto mode - will switch on failure)")
socket.getaddrinfo = cast(Any, create_system_failover_getaddrinfo())
return
@@ -837,13 +810,29 @@ def _looks_like_ip(s: str) -> bool:
# Simple heuristic: contains only digits, dots, and colons
return s.replace(".", "").replace(":", "").isdigit()
def _build_aa_urls() -> List[str]:
"""Build list of available AA URLs from config."""
urls = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
additional = app_config.get("AA_ADDITIONAL_URLS", "")
if additional:
urls.extend(u.strip() for u in additional.split(",") if u.strip())
return urls
def _initialize_aa_state() -> None:
"""Restore or probe AA URL state."""
global AA_BASE_URL, _current_aa_url_index
if AA_BASE_URL == "auto":
global _aa_base_url, _current_aa_url_index, _aa_urls
# Build URL list from config
_aa_urls = _build_aa_urls()
# Get configured base URL from config
configured_url = app_config.get("AA_BASE_URL", "auto")
if configured_url == "auto":
if state.get('aa_base_url') and state['aa_base_url'] in _aa_urls:
_current_aa_url_index = _aa_urls.index(state['aa_base_url'])
AA_BASE_URL = state['aa_base_url']
_aa_base_url = state['aa_base_url']
else:
logger.debug(f"AA_BASE_URL: auto, checking available urls {_aa_urls}")
for i, url in enumerate(_aa_urls):
@@ -851,21 +840,22 @@ def _initialize_aa_state() -> None:
response = requests.get(url, proxies=get_proxies(), timeout=3)
if response.status_code == 200:
_current_aa_url_index = i
AA_BASE_URL = url
_save_state(aa_url=AA_BASE_URL)
_aa_base_url = url
_save_state(aa_url=_aa_base_url)
break
except Exception:
pass
if AA_BASE_URL == "auto":
AA_BASE_URL = _aa_urls[0]
if not _aa_base_url or _aa_base_url == "auto":
_aa_base_url = _aa_urls[0]
_current_aa_url_index = 0
elif AA_BASE_URL not in _aa_urls:
logger.info(f"AA_BASE_URL set to custom value {AA_BASE_URL}; skipping auto-switch")
elif configured_url not in _aa_urls:
logger.info(f"AA_BASE_URL set to custom value {configured_url}; skipping auto-switch")
_aa_base_url = configured_url
else:
_current_aa_url_index = _aa_urls.index(AA_BASE_URL)
_current_aa_url_index = _aa_urls.index(configured_url)
_aa_base_url = configured_url
config.AA_BASE_URL = AA_BASE_URL
logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
logger.info(f"AA_BASE_URL: {_aa_base_url}")
def init_dns(force: bool = False) -> None:
"""Initialize DNS state and resolvers using set_dns_provider() for consistency."""
@@ -939,7 +929,7 @@ def init(force: bool = False) -> None:
if _initialized and not force:
return
# Do the work first, then set flag to prevent race conditions
# where another thread sees _initialized=True but AA_BASE_URL is still "auto"
# where another thread sees _initialized=True but _aa_base_url is still empty
try:
init_dns(force=force)
init_aa(force=force)
@@ -952,7 +942,7 @@ def init(force: bool = False) -> None:
def get_aa_base_url():
"""Get current AA base URL."""
_ensure_initialized()
return AA_BASE_URL
return _aa_base_url
def get_available_aa_urls():
"""Get list of configured AA URLs (copy)."""
@@ -962,14 +952,13 @@ def get_available_aa_urls():
def set_aa_url_index(new_index: int) -> bool:
"""Set AA base URL by index in available list; returns True if applied."""
_ensure_initialized()
global AA_BASE_URL, _current_aa_url_index
global _aa_base_url, _current_aa_url_index
if new_index < 0 or new_index >= len(_aa_urls):
return False
_current_aa_url_index = new_index
AA_BASE_URL = _aa_urls[_current_aa_url_index]
config.AA_BASE_URL = AA_BASE_URL
logger.info(f"Set AA URL to: {AA_BASE_URL}")
_save_state(aa_url=AA_BASE_URL)
_aa_base_url = _aa_urls[_current_aa_url_index]
logger.info(f"Set AA URL to: {_aa_base_url}")
_save_state(aa_url=_aa_base_url)
return True
class AAMirrorSelector:
cwa_book_downloader/download/orchestrator.py
+133 -509
View File
@@ -1,23 +1,7 @@
"""Download queue orchestration and worker management.
## Download Architecture
All downloads follow a two-stage process:
1. **Staging (TMP_DIR)**: Handlers download/copy files to a temp staging area.
- Direct downloads: Downloaded directly to staging
- Torrent downloads: Copied from torrent client's completed folder to staging
- NZB downloads: Moved from NZB client's completed folder to staging
2. **Ingest (INGEST_DIR)**: Orchestrator moves staged files to the final location.
- Archive extraction (RAR/ZIP) happens here
- Custom scripts run here
- Final move to ingest folder
This ensures:
- Handlers don't need to know about ingest folder logic
- Archive handling works uniformly for all sources
- Single point of control for what enters the ingest folder
Two-stage architecture: handlers stage to TMP_DIR, orchestrator moves to INGEST_DIR
with archive extraction and custom script support.
"""
import hashlib
@@ -36,9 +20,16 @@ from cwa_book_downloader.release_sources import direct_download
from cwa_book_downloader.release_sources.direct_download import SearchUnavailable
from cwa_book_downloader.core.config import config
from cwa_book_downloader.config.env import TMP_DIR
from cwa_book_downloader.core.utils import get_ingest_dir, get_destination, get_aa_content_type_dir
from cwa_book_downloader.core.utils import get_ingest_dir, get_destination, get_aa_content_type_dir, is_audiobook as check_audiobook, transform_cover_url
from cwa_book_downloader.core.naming import build_library_path, same_filesystem, assign_part_numbers, parse_naming_template, sanitize_filename
from cwa_book_downloader.download.archive import is_archive, process_archive
from cwa_book_downloader.download.archive import (
is_archive,
process_archive,
_get_file_organization,
_get_template,
_get_supported_formats as _get_book_formats,
_get_supported_audiobook_formats,
)
from cwa_book_downloader.release_sources import get_handler, get_source_display_name
from cwa_book_downloader.core.logger import setup_logger
from cwa_book_downloader.core.models import BookInfo, DownloadTask, QueueStatus, SearchFilters, SearchMode
@@ -54,25 +45,13 @@ logger = setup_logger(__name__)
# The orchestrator handles moving staged files to the ingest folder.
def get_staging_dir() -> Path:
"""Get the staging directory for downloads.
All handlers should stage their downloads here. The orchestrator
handles moving staged files to the final ingest location.
"""
"""Get the staging directory for downloads."""
TMP_DIR.mkdir(parents=True, exist_ok=True)
return TMP_DIR
def get_staging_path(task_id: str, extension: str) -> Path:
"""Get a staging path for a download.
Args:
task_id: Unique task identifier
extension: File extension (e.g., 'epub', 'zip')
Returns:
Path in staging directory for this download
"""
"""Get a staging path for a download."""
staging_dir = get_staging_dir()
# Hash task_id in case it contains invalid filename chars (e.g., Prowlarr URLs)
safe_id = hashlib.md5(task_id.encode()).hexdigest()[:16]
@@ -80,19 +59,7 @@ def get_staging_path(task_id: str, extension: str) -> Path:
def stage_file(source_path: Path, task_id: str, copy: bool = False) -> Path:
"""Stage a file for ingest processing.
Use this when a download client has completed a download and the file
needs to be staged for orchestrator processing.
Args:
source_path: Path to the completed download
task_id: Unique task identifier
copy: If True, copy the file (for torrents). If False, move it.
Returns:
Path to the staged file
"""
"""Stage a file for ingest processing. Use copy=True for torrents to preserve seeding."""
staging_dir = get_staging_dir()
# Stage with original filename, add counter suffix if collision
staged_path = staging_dir / source_path.name
@@ -112,85 +79,8 @@ def stage_file(source_path: Path, task_id: str, copy: bool = False) -> Path:
return staged_path
# =============================================================================
# File Organization Helpers
# =============================================================================
def _get_file_organization(is_audiobook: bool) -> str:
"""Get the file organization mode for the content type.
Returns:
One of: "none", "rename", "organize"
"""
key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
mode = config.get(key, "rename")
# Handle legacy settings migration
if mode not in ("none", "rename", "organize"):
# Check legacy PROCESSING_MODE
legacy_key = "PROCESSING_MODE_AUDIOBOOK" if is_audiobook else "PROCESSING_MODE"
legacy_mode = config.get(legacy_key, "ingest")
if legacy_mode == "library":
return "organize"
# Check legacy USE_BOOK_TITLE for ingest mode
if config.get("USE_BOOK_TITLE", True):
return "rename"
return "none"
return mode
def _get_template(is_audiobook: bool, organization_mode: str) -> str:
"""Get the template for the content type and organization mode.
Returns:
Template string
"""
# Build the key based on content type and organization mode
if is_audiobook:
if organization_mode == "organize":
key = "TEMPLATE_AUDIOBOOK_ORGANIZE"
else:
key = "TEMPLATE_AUDIOBOOK_RENAME"
else:
if organization_mode == "organize":
key = "TEMPLATE_ORGANIZE"
else:
key = "TEMPLATE_RENAME"
template = config.get(key, "")
# Try legacy keys if new setting is empty
if not template:
# Try old unified TEMPLATE key
legacy_key = "TEMPLATE_AUDIOBOOK" if is_audiobook else "TEMPLATE"
template = config.get(legacy_key, "")
if not template:
# Try even older LIBRARY_TEMPLATE key
legacy_key = "LIBRARY_TEMPLATE_AUDIOBOOK" if is_audiobook else "LIBRARY_TEMPLATE"
template = config.get(legacy_key, "")
# Use sensible default if still empty
if not template:
if organization_mode == "organize":
return "{Author}/{Title} ({Year})"
else:
return "{Author} - {Title} ({Year})"
return template
def _should_hardlink(task: DownloadTask) -> bool:
"""Determine if a download should be hardlinked instead of copied.
Hardlinking only applies to torrent downloads (Prowlarr source).
Uses per-content-type settings.
Returns:
True if hardlinking should be used
"""
"""Check if download should be hardlinked (Prowlarr torrents only)."""
# Only Prowlarr downloads (torrents) can be hardlinked
if task.source != "prowlarr":
return False
@@ -200,7 +90,7 @@ def _should_hardlink(task: DownloadTask) -> bool:
return False
# Check per-content-type setting
is_audiobook = task.content_type and "audiobook" in task.content_type.lower()
is_audiobook = check_audiobook(task.content_type)
key = "HARDLINK_TORRENTS_AUDIOBOOK" if is_audiobook else "HARDLINK_TORRENTS"
# Check new setting first, then legacy TORRENT_HARDLINK
@@ -213,30 +103,13 @@ def _should_hardlink(task: DownloadTask) -> bool:
def _should_extract_archives(task: DownloadTask) -> bool:
"""Determine if archives should be extracted for this download.
Archives are NOT extracted when hardlinking is enabled (to preserve torrent seeding).
"""
if _should_hardlink(task):
return False
return True
"""Check if archives should be extracted (disabled when hardlinking)."""
return not _should_hardlink(task)
def _get_final_destination(task: DownloadTask) -> Path:
"""Get the final destination directory for a download.
Handles:
- Per-content-type destinations (books vs audiobooks)
- AA content-type routing override (for Direct mode downloads)
Returns:
Path to the destination directory
"""
content_type = task.content_type.lower() if task.content_type else ""
is_audiobook = "audiobook" in content_type
# Get base destination
base_dest = get_destination(is_audiobook)
"""Get final destination directory, with content-type routing support."""
is_audiobook = check_audiobook(task.content_type)
# For Anna's Archive (direct_download), check for content-type routing override
if task.source == "direct_download" and not is_audiobook:
@@ -244,7 +117,7 @@ def _get_final_destination(task: DownloadTask) -> Path:
if override:
return override
return base_dest
return get_destination(is_audiobook)
def _build_metadata_dict(task: DownloadTask) -> dict:
@@ -261,32 +134,19 @@ def _build_metadata_dict(task: DownloadTask) -> dict:
def _get_supported_formats(content_type: str = None) -> List[str]:
"""Get current supported formats from config singleton based on content type."""
if content_type and content_type.lower() == "audiobook":
formats = config.get("SUPPORTED_AUDIOBOOK_FORMATS", ["m4b", "mp3"])
else:
formats = config.get("SUPPORTED_FORMATS", ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"])
# Handle both list (from MultiSelectField) and comma-separated string (legacy/env)
if isinstance(formats, str):
return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
return [fmt.lower() for fmt in formats]
if check_audiobook(content_type):
return _get_supported_audiobook_formats()
return _get_book_formats()
def _find_book_files_in_directory(directory: Path, content_type: str = None) -> Tuple[List[Path], List[Path]]:
"""Find all book files in a directory matching supported formats.
Args:
directory: Directory to search recursively
content_type: Content type to determine format list (e.g., "audiobook")
Returns:
Tuple of (matching book files, rejected files with unsupported extensions)
"""
"""Find book files matching supported formats. Returns (matches, rejected)."""
book_files = []
rejected_files = []
supported_formats = _get_supported_formats(content_type)
supported_exts = {f".{fmt}" for fmt in supported_formats}
is_audiobook = content_type and content_type.lower() == "audiobook"
is_audiobook = check_audiobook(content_type)
if is_audiobook:
trackable_exts = {'.m4b', '.mp3', '.m4a', '.flac', '.ogg', '.wma', '.aac', '.wav'}
else:
@@ -307,19 +167,7 @@ def process_directory(
ingest_dir: Path,
task: DownloadTask,
) -> Tuple[List[Path], Optional[str]]:
"""Process a staged directory: find book files, handle archives, move to ingest.
For multi-file torrent/usenet downloads. If book files exist, moves them directly.
If only archives exist, extracts them to find book files inside.
Args:
directory: Staged directory containing downloaded files
ingest_dir: Final destination directory for book files
task: Download task for filename generation
Returns:
Tuple of (list of final paths, error message if failed)
"""
"""Process staged directory: find book files, extract archives, move to ingest."""
try:
content_type = task.content_type
book_files, rejected_files = _find_book_files_in_directory(directory, content_type)
@@ -385,8 +233,7 @@ def process_directory(
# Move each book file to destination
final_paths = []
content_type = task.content_type.lower() if task.content_type else ""
is_audiobook = "audiobook" in content_type
is_audiobook = check_audiobook(task.content_type)
organization_mode = _get_file_organization(is_audiobook)
for book_file in book_files:
@@ -428,11 +275,14 @@ def process_directory(
# WebSocket manager (initialized by app.py)
# Track whether WebSocket is available for status reporting
WEBSOCKET_AVAILABLE = True
try:
from cwa_book_downloader.api.websocket import ws_manager
except ImportError:
logger.warning("WebSocket unavailable - real-time updates disabled")
logger.error("WebSocket unavailable - real-time updates disabled")
ws_manager = None
WEBSOCKET_AVAILABLE = False
# Progress update throttling - track last broadcast time per book
_progress_last_broadcast: Dict[str, float] = {}
@@ -443,15 +293,7 @@ _last_activity: Dict[str, float] = {}
STALL_TIMEOUT = 300 # 5 minutes without progress/status update = stalled
def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
"""Search for books matching the query.
Args:
query: Search term
filters: Search filters object
Returns:
List[Dict]: List of book information dictionaries
"""
"""Search for books matching the query."""
try:
books = direct_download.search_books(query, filters)
return [_book_info_to_dict(book) for book in books]
@@ -462,17 +304,7 @@ def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
raise
def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
"""Get detailed information for a specific book.
Args:
book_id: Book identifier
Returns:
Optional[Dict]: Book information dictionary if found, None if not found
Raises:
Exception: If there's an error fetching the book info
"""
"""Get detailed information for a specific book."""
try:
book = direct_download.get_book_info(book_id)
return _book_info_to_dict(book)
@@ -480,25 +312,14 @@ def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
logger.error_trace(f"Error getting book info: {e}")
raise
def queue_book(book_id: str, priority: int = 0, source: str = "direct_download") -> bool:
"""Add a book to the download queue with specified priority.
Fetches display info and creates a DownloadTask. The handler will fetch
the full book details (including download URLs) when processing.
Args:
book_id: Book identifier (e.g., AA MD5 hash)
priority: Priority level (lower number = higher priority)
source: Release source handler to use (default: direct_download)
Returns:
bool: True if book was successfully queued
"""
def queue_book(book_id: str, priority: int = 0, source: str = "direct_download") -> Tuple[bool, Optional[str]]:
"""Add a book to the download queue. Returns (success, error_message)."""
try:
book_info = direct_download.get_book_info(book_id, fetch_download_count=False)
if not book_info:
logger.warning(f"Could not fetch book info for {book_id}")
return False
error_msg = f"Could not fetch book info for {book_id}"
logger.warning(error_msg)
return False, error_msg
# Create a source-agnostic download task
task = DownloadTask(
@@ -516,7 +337,7 @@ def queue_book(book_id: str, priority: int = 0, source: str = "direct_download")
if not book_queue.add(task):
logger.info(f"Book already in queue: {book_info.title}")
return False
return False, "Book is already in the download queue"
logger.info(f"Book queued with priority {priority}: {book_info.title}")
@@ -524,28 +345,19 @@ def queue_book(book_id: str, priority: int = 0, source: str = "direct_download")
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
return True
return True, None
except SearchUnavailable as e:
error_msg = f"Search service unavailable: {e}"
logger.warning(error_msg)
return False, error_msg
except Exception as e:
logger.error_trace(f"Error queueing book: {e}")
return False
error_msg = f"Error queueing book: {e}"
logger.error_trace(error_msg)
return False, error_msg
def queue_release(release_data: dict, priority: int = 0) -> bool:
"""Add a release to the download queue.
This is used when downloading from the ReleaseModal where we already have
all the release data from the search - no need to re-fetch.
Creates a DownloadTask directly from the release data. The handler will
fetch full details when processing.
Args:
release_data: Release dictionary with source, source_id, title, format, etc.
priority: Priority level (lower number = higher priority)
Returns:
bool: True if release was successfully queued
"""
def queue_release(release_data: dict, priority: int = 0) -> Tuple[bool, Optional[str]]:
"""Add a release to the download queue. Returns (success, error_message)."""
try:
source = release_data.get('source', 'direct_download')
extra = release_data.get('extra', {})
@@ -581,7 +393,7 @@ def queue_release(release_data: dict, priority: int = 0) -> bool:
if not book_queue.add(task):
logger.info(f"Release already in queue: {task.title}")
return False
return False, "Release is already in the download queue"
logger.info(f"Release queued with priority {priority}: {task.title}")
@@ -589,28 +401,29 @@ def queue_release(release_data: dict, priority: int = 0) -> bool:
if ws_manager:
ws_manager.broadcast_status_update(queue_status())
return True
return True, None
except ValueError as e:
# Handler not found for this source
logger.warning(f"Unknown release source: {e}")
return False
error_msg = f"Unknown release source: {e}"
logger.warning(error_msg)
return False, error_msg
except KeyError as e:
error_msg = f"Missing required field in release data: {e}"
logger.warning(error_msg)
return False, error_msg
except Exception as e:
logger.error_trace(f"Error queueing release: {e}")
return False
error_msg = f"Error queueing release: {e}"
logger.error_trace(error_msg)
return False, error_msg
def queue_status() -> Dict[str, Dict[str, Any]]:
"""Get current status of the download queue.
Returns:
Dict: Queue status organized by status type with serialized task data
"""
"""Get current status of the download queue."""
status = book_queue.get_status()
for _, tasks in status.items():
for _, task in tasks.items():
if task.download_path:
if not os.path.exists(task.download_path):
task.download_path = None
if task.download_path and not os.path.exists(task.download_path):
task.download_path = None
# Convert Enum keys to strings and DownloadTask objects to dicts for JSON serialization
return {
@@ -622,14 +435,7 @@ def queue_status() -> Dict[str, Dict[str, Any]]:
}
def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]]:
"""Get downloaded file data for a specific task.
Args:
task_id: Task identifier
Returns:
Tuple[Optional[bytes], Optional[DownloadTask]]: File data if available, and the task
"""
"""Get downloaded file data for a specific task."""
task = None
try:
task = book_queue.get_task(task_id)
@@ -649,12 +455,7 @@ def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]
return None, task
def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
"""Convert BookInfo object to dictionary representation.
Transforms external preview URLs to local proxy URLs when cover caching is enabled.
"""
from cwa_book_downloader.core.utils import transform_cover_url
"""Convert BookInfo to dict, transforming cover URLs for caching."""
result = {
key: value for key, value in book.__dict__.items()
if value is not None
@@ -668,14 +469,7 @@ def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
def _task_to_dict(task: DownloadTask) -> Dict[str, Any]:
"""Convert DownloadTask object to dictionary representation.
Maps DownloadTask fields to the format expected by the frontend,
maintaining compatibility with the previous BookInfo-based format.
Transforms external preview URLs to local proxy URLs when cover caching is enabled.
"""
from cwa_book_downloader.core.utils import transform_cover_url
"""Convert DownloadTask to dict for frontend, transforming cover URLs."""
# Transform external preview URLs to local proxy URLs
preview = transform_cover_url(task.preview, task.task_id)
@@ -699,19 +493,7 @@ def _task_to_dict(task: DownloadTask) -> Dict[str, Any]:
def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
"""Download a task with cancellation support.
Delegates to the appropriate handler based on the task's source.
Handlers return a temp file path, orchestrator handles post-processing
(archive extraction, moving to ingest) uniformly for all sources.
Args:
task_id: Task identifier
cancel_flag: Threading event to signal cancellation
Returns:
str: Path to the downloaded file if successful, None otherwise
"""
"""Download a task via appropriate handler, then post-process to ingest."""
try:
# Check for cancellation before starting
if cancel_flag.is_set():
@@ -766,6 +548,11 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]:
logger.info(f"Download cancelled during error handling: {task_id}")
else:
logger.error_trace(f"Error downloading: {e}")
# Update task status so user sees the failure
task = book_queue.get_task(task_id)
if task:
book_queue.update_status(task_id, QueueStatus.ERROR)
book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}")
return None
@@ -774,16 +561,8 @@ def _process_organize_mode(
task: DownloadTask,
status_callback,
) -> Optional[str]:
"""Process a download with file organization (organize mode with folders).
Organizes files into folders based on template (e.g., "{Author}/{Series/}{Title}").
Supports hardlinking for torrent downloads when enabled.
Returns:
Path to organized file if successful, None if failed
"""
content_type = task.content_type.lower() if task.content_type else ""
is_audiobook = "audiobook" in content_type
"""Organize files into library folders using template. Supports hardlinking."""
is_audiobook = check_audiobook(task.content_type)
# Get destination and template
destination = _get_final_destination(task)
@@ -810,32 +589,25 @@ def _process_organize_mode(
# Determine if we should use hardlinking
use_hardlink = False
hardlink_source = None
source = temp_file
if _should_hardlink(task):
hardlink_source = Path(task.original_download_path)
if hardlink_source.exists():
# Check same filesystem (required for hardlinks)
if same_filesystem(hardlink_source, destination):
use_hardlink = True
else:
logger.warning(
f"Cannot hardlink: {hardlink_source} and {destination} are on different filesystems. "
"Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination."
)
status_callback("resolving", "Cannot hardlink (different filesystems), using copy")
if hardlink_source.exists() and same_filesystem(hardlink_source, destination):
use_hardlink = True
source = hardlink_source
elif hardlink_source.exists():
logger.warning(
f"Cannot hardlink: {hardlink_source} and {destination} are on different filesystems. "
"Falling back to copy. To fix: ensure torrent client downloads to same filesystem as destination."
)
status_callback("resolving", "Cannot hardlink (different filesystems), using copy")
# Build metadata dict for template
metadata = _build_metadata_dict(task)
try:
if use_hardlink:
status_callback("resolving", "Creating hardlinks")
else:
status_callback("resolving", "Organizing files")
# Use torrent client path for hardlinks, staging path for moves
source = hardlink_source if use_hardlink else temp_file
status_callback("resolving", "Creating hardlinks" if use_hardlink else "Organizing files")
if source.is_dir():
return _transfer_directory_to_library(
@@ -865,108 +637,34 @@ def _is_torrent_source(source_path: Path, task: DownloadTask) -> bool:
return False
def _get_unique_path(dest_path: Path) -> Path:
"""Return a unique path by appending counter if file already exists.
Note: Has TOCTOU race. Use _atomic_hardlink/_atomic_move for concurrent safety.
"""
if not dest_path.exists():
return dest_path
base = dest_path.stem
ext = dest_path.suffix
def _stage_torrent_path(source: Path) -> Path:
"""Copy torrent source to staging directory to preserve seeding."""
staging_dir = get_staging_dir()
staged_path = staging_dir / source.name
counter = 1
while dest_path.exists():
dest_path = dest_path.parent / f"{base}_{counter}{ext}"
counter += 1
logger.info(f"File already exists, saving as: {dest_path.name}")
return dest_path
if source.is_dir():
while staged_path.exists():
staged_path = staging_dir / f"{source.name}_{counter}"
counter += 1
shutil.copytree(str(source), str(staged_path))
else:
while staged_path.exists():
staged_path = staging_dir / f"{source.stem}_{counter}{source.suffix}"
counter += 1
shutil.copy2(str(source), str(staged_path))
logger.debug(f"Staged torrent {'directory' if source.is_dir() else 'file'}: {staged_path.name}")
return staged_path
def _atomic_hardlink(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
"""Create a hardlink with atomic collision detection. Retries with counter suffix on collision."""
base = dest_path.stem
ext = dest_path.suffix
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else dest_path.parent / f"{base}_{attempt}{ext}"
try:
os.link(str(source_path), str(try_path))
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except FileExistsError:
continue
raise RuntimeError(f"Could not create hardlink after {max_attempts} attempts: {dest_path}")
def _atomic_copy(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
"""Copy a file with atomic collision detection. Retries with counter suffix on collision."""
base = dest_path.stem
ext = dest_path.suffix
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else dest_path.parent / f"{base}_{attempt}{ext}"
try:
# Atomically claim the destination by creating an exclusive file
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.close(fd)
try:
# Copy to temp file first, then replace to avoid partial files
temp_path = try_path.parent / f".{try_path.name}.tmp"
shutil.copy2(str(source_path), str(temp_path))
temp_path.replace(try_path)
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except Exception:
try_path.unlink(missing_ok=True)
temp_path.unlink(missing_ok=True) if 'temp_path' in locals() else None
raise
except FileExistsError:
continue
raise RuntimeError(f"Could not copy file after {max_attempts} attempts: {dest_path}")
def _atomic_move(source_path: Path, dest_path: Path, max_attempts: int = 100) -> Path:
"""Move a file with atomic collision detection. Retries with counter suffix on collision."""
import errno
base = dest_path.stem
ext = dest_path.suffix
for attempt in range(max_attempts):
try_path = dest_path if attempt == 0 else dest_path.parent / f"{base}_{attempt}{ext}"
try:
os.link(str(source_path), str(try_path))
source_path.unlink()
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except FileExistsError:
continue
except OSError as e:
# Cross-filesystem - fall back to exclusive create
if e.errno not in (errno.EXDEV, errno.EMLINK):
raise
try:
fd = os.open(str(try_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.close(fd)
try:
shutil.move(str(source_path), str(try_path))
if attempt > 0:
logger.info(f"File collision resolved: {try_path.name}")
return try_path
except Exception:
try_path.unlink(missing_ok=True)
raise
except FileExistsError:
continue
raise RuntimeError(f"Could not move file after {max_attempts} attempts: {dest_path}")
# Import atomic file operations from shared module
# Re-exported here for backwards compatibility with existing tests/imports
from cwa_book_downloader.download.fs import (
atomic_hardlink as _atomic_hardlink,
atomic_copy as _atomic_copy,
atomic_move as _atomic_move,
)
def _cleanup_staged_files(temp_file: Path, source_dir: Optional[Path] = None) -> None:
@@ -1023,7 +721,7 @@ def _transfer_file_to_library(
if use_hardlink:
_cleanup_staged_files(temp_file)
status_callback("complete", "Complete (library mode)")
status_callback("complete", "Complete")
return str(final_path)
@@ -1100,8 +798,8 @@ def _transfer_directory_to_library(
elif not is_torrent:
_cleanup_staged_files(temp_file, source_dir)
count_msg = f" ({len(transferred_paths)} files," if len(transferred_paths) > 1 else " ("
status_callback("complete", f"Complete{count_msg} library mode)")
message = f"Complete ({len(transferred_paths)} files)" if len(transferred_paths) > 1 else "Complete"
status_callback("complete", message)
return str(transferred_paths[0])
@@ -1112,25 +810,8 @@ def _post_process_download(
cancel_flag: Event,
status_callback,
) -> Optional[str]:
"""Post-process a downloaded file based on file organization settings.
This runs uniformly for all download sources, ensuring consistent behavior.
Handles three organization modes:
- "none": Keep original filename, move to destination
- "rename": Apply template to filename, move to destination
- "organize": Apply template with folders, move to destination
Args:
temp_file: Path to downloaded file in temp directory
task: Download task with metadata
cancel_flag: Cancellation event
status_callback: Callback for status updates
Returns:
Final path in destination directory, or None on failure
"""
content_type = task.content_type.lower() if task.content_type else ""
is_audiobook = "audiobook" in content_type
"""Post-process download: extract archives, apply naming template, move to destination."""
is_audiobook = check_audiobook(task.content_type)
# Validate search_mode
if task.search_mode is None:
@@ -1150,7 +831,11 @@ def _post_process_download(
if result is not None:
return result
# If organize mode fails, fall through to flat mode
logger.warning("Organize mode failed, falling back to flat destination")
logger.warning(
f"Organize mode failed for '{task.title}', falling back to flat destination. "
"Check destination folder permissions and ensure the path is writable."
)
status_callback("resolving", "Organization failed, using flat destination")
# Ensure destination exists
os.makedirs(destination, exist_ok=True)
@@ -1159,26 +844,7 @@ def _post_process_download(
# (Torrent handler returns original path, not staged copy)
if _is_torrent_source(temp_file, task) and not _should_hardlink(task):
status_callback("resolving", "Staging torrent files")
staging_dir = get_staging_dir()
if temp_file.is_dir():
staged_path = staging_dir / temp_file.name
counter = 1
while staged_path.exists():
staged_path = staging_dir / f"{temp_file.name}_{counter}"
counter += 1
shutil.copytree(str(temp_file), str(staged_path))
logger.debug(f"Staged torrent directory: {staged_path.name}")
else:
staged_path = staging_dir / temp_file.name
counter = 1
while staged_path.exists():
staged_path = staging_dir / f"{temp_file.stem}_{counter}{temp_file.suffix}"
counter += 1
shutil.copy2(str(temp_file), str(staged_path))
logger.debug(f"Staged torrent file: {staged_path.name}")
temp_file = staged_path
temp_file = _stage_torrent_path(temp_file)
# Handle archive extraction (RAR/ZIP) - only if not hardlinking
if is_archive(temp_file) and _should_extract_archives(task):
@@ -1215,17 +881,14 @@ def _post_process_download(
status_callback("error", error)
return None
if final_paths:
if len(final_paths) == 1:
message = "Complete"
else:
message = f"Complete ({len(final_paths)} files)"
status_callback("complete", message)
return str(final_paths[0])
else:
if not final_paths:
status_callback("error", "No book files found")
return None
message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
status_callback("complete", message)
return str(final_paths[0])
# Non-archive: run custom script if configured, then move to destination
if config.CUSTOM_SCRIPT:
logger.info(f"Running custom script: {config.CUSTOM_SCRIPT}")
@@ -1297,14 +960,7 @@ def _post_process_download(
return str(final_path)
def update_download_progress(book_id: str, progress: float) -> None:
"""Update download progress with throttled WebSocket broadcasts.
Progress is always stored in the queue, but WebSocket broadcasts are
throttled to avoid flooding clients with updates. Broadcasts occur:
- At most once per DOWNLOAD_PROGRESS_UPDATE_INTERVAL seconds
- Always at 0% (start) and 100% (complete)
- On significant progress jumps (>10%)
"""
"""Update download progress with throttled WebSocket broadcasts."""
book_queue.update_progress(book_id, progress)
# Track activity for stall detection
@@ -1339,13 +995,7 @@ def update_download_progress(book_id: str, progress: float) -> None:
ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None:
"""Update download status with optional detailed message.
Args:
book_id: Book identifier
status: Status string (e.g., 'resolving', 'downloading')
message: Optional detailed status message for UI display
"""
"""Update download status with optional message for UI display."""
# Map string status to QueueStatus enum
status_map = {
'queued': QueueStatus.QUEUED,
@@ -1375,14 +1025,7 @@ def update_download_status(book_id: str, status: str, message: Optional[str] = N
ws_manager.broadcast_status_update(queue_status())
def cancel_download(book_id: str) -> bool:
"""Cancel a download.
Args:
book_id: Book identifier to cancel
Returns:
bool: True if cancellation was successful
"""
"""Cancel a download."""
result = book_queue.cancel_download(book_id)
# Broadcast status update via WebSocket
@@ -1392,26 +1035,11 @@ def cancel_download(book_id: str) -> bool:
return result
def set_book_priority(book_id: str, priority: int) -> bool:
"""Set priority for a queued book.
Args:
book_id: Book identifier
priority: New priority level (lower = higher priority)
Returns:
bool: True if priority was successfully changed
"""
"""Set priority for a queued book (lower = higher priority)."""
return book_queue.set_priority(book_id, priority)
def reorder_queue(book_priorities: Dict[str, int]) -> bool:
"""Bulk reorder queue.
Args:
book_priorities: Dict mapping book_id to new priority
Returns:
bool: True if reordering was successful
"""
"""Bulk reorder queue by mapping book_id to new priority."""
return book_queue.reorder_queue(book_priorities)
def get_queue_order() -> List[Dict[str, Any]]:
@@ -1539,11 +1167,7 @@ _started = False
def start() -> None:
"""Start the download coordinator thread.
This should be called once during application startup.
Calling multiple times is safe - subsequent calls are no-ops.
"""
"""Start the download coordinator thread. Safe to call multiple times."""
global _coordinator_thread, _started
if _started:
cwa_book_downloader/main.py
+55 -67
View File
@@ -135,6 +135,15 @@ def clear_failed_logins(username: str) -> None:
logger.debug(f"Cleared failed login attempts for user: {username}")
def get_client_ip() -> str:
"""Extract client IP address from request, handling reverse proxy forwarding."""
ip_address = request.headers.get('X-Forwarded-For', request.remote_addr) or 'unknown'
# X-Forwarded-For can contain multiple IPs, take the first one
if ',' in ip_address:
ip_address = ip_address.split(',')[0].strip()
return ip_address
def get_auth_mode() -> str:
"""Determine which authentication mode is active.
@@ -171,44 +180,34 @@ if DEBUG:
})
# Custom log filter to exclude routine status endpoint polling and WebSocket noise
class StatusEndpointFilter(logging.Filter):
"""Filter out routine status endpoint requests and WebSocket upgrade errors to reduce log noise."""
def filter(self, record):
if hasattr(record, 'getMessage'):
message = record.getMessage()
# Exclude GET /api/status requests (polling noise)
if 'GET /api/status' in message:
return False
# Exclude WebSocket upgrade errors (benign - falls back to polling)
if 'write() before start_response' in message:
return False
# Exclude the Error on request line that precedes WebSocket errors
if 'Error on request:' in message and record.levelno == logging.ERROR:
return False
return True
class LogNoiseFilter(logging.Filter):
"""Filter out routine status endpoint requests and WebSocket upgrade errors to reduce log noise.
class WebSocketErrorFilter(logging.Filter):
"""Filter out WebSocket upgrade errors that occur in Werkzeug dev server.
These errors are benign - Flask-SocketIO automatically falls back to polling transport.
WebSocket upgrade errors are benign - Flask-SocketIO automatically falls back to polling transport.
The error occurs because Werkzeug's built-in server doesn't fully support WebSocket upgrades.
"""
def filter(self, record):
# Filter out the AssertionError traceback for WebSocket upgrades
message = record.getMessage() if hasattr(record, 'getMessage') else str(record.msg)
# Exclude GET /api/status requests (polling noise)
if 'GET /api/status' in message:
return False
# Exclude WebSocket upgrade errors (benign - falls back to polling)
if 'write() before start_response' in message:
return False
# Exclude the Error on request line that precedes WebSocket errors
if record.levelno == logging.ERROR:
message = record.getMessage() if hasattr(record, 'getMessage') else str(record.msg)
# Filter out the full traceback that includes the WebSocket assertion error
if 'write() before start_response' in message:
if 'Error on request:' in message:
return False
# Also filter the "Error on request" header that precedes it
# Filter WebSocket-related AssertionError tracebacks
if hasattr(record, 'exc_info') and record.exc_info:
exc_type = record.exc_info[0]
exc_type, exc_value = record.exc_info[0], record.exc_info[1]
if exc_type and exc_type.__name__ == 'AssertionError':
# Check if it's the WebSocket-related assertion
exc_value = record.exc_info[1]
if exc_value and 'write() before start_response' in str(exc_value):
return False
return True
# Flask logger
@@ -218,17 +217,15 @@ app.logger.setLevel(logger.level)
werkzeug_logger = logging.getLogger('werkzeug')
werkzeug_logger.handlers = logger.handlers
werkzeug_logger.setLevel(logger.level)
# Add filters to suppress routine status endpoint polling logs and WebSocket upgrade errors
werkzeug_logger.addFilter(StatusEndpointFilter())
werkzeug_logger.addFilter(WebSocketErrorFilter())
# Add filter to suppress routine status endpoint polling logs and WebSocket upgrade errors
werkzeug_logger.addFilter(LogNoiseFilter())
# Set up authentication defaults
# The secret key will reset every time we restart, which will
# require users to authenticate again
from cwa_book_downloader.config.env import SESSION_COOKIE_SECURE_ENV, string_to_bool
# Session cookie security - set to 'true' if exclusively using HTTPS
session_cookie_secure_env = os.getenv('SESSION_COOKIE_SECURE', 'false').lower()
SESSION_COOKIE_SECURE = session_cookie_secure_env in ['true', 'yes', '1']
SESSION_COOKIE_SECURE = string_to_bool(SESSION_COOKIE_SECURE_ENV)
app.config.update(
SECRET_KEY = os.urandom(64),
@@ -238,7 +235,7 @@ app.config.update(
PERMANENT_SESSION_LIFETIME = 604800 # 7 days in seconds
)
logger.info(f"Session cookie secure setting: {SESSION_COOKIE_SECURE} (from env: {session_cookie_secure_env})")
logger.info(f"Session cookie secure setting: {SESSION_COOKIE_SECURE} (from env: {SESSION_COOKIE_SECURE_ENV})")
def login_required(f):
@wraps(f)
@@ -304,10 +301,12 @@ if not app_config.get("USING_EXTERNAL_BYPASSER", False):
if DEBUG:
import subprocess
if app_config.get("USING_EXTERNAL_BYPASSER", False):
STOP_GUI = lambda: None
_stop_gui = lambda: None
else:
from cwa_book_downloader.bypass.internal_bypasser import _reset_driver as STOP_GUI
from cwa_book_downloader.bypass.internal_bypasser import _reset_driver as _stop_gui
@app.route('/api/debug', methods=['GET'])
@login_required
def debug() -> Union[Response, Tuple[Response, int]]:
@@ -317,9 +316,8 @@ if DEBUG:
And then return it to the user
"""
try:
# Run the debug script
logger.info("Debug endpoint called, stopping GUI and generating debug info...")
STOP_GUI()
_stop_gui()
time.sleep(1)
result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True)
if result.returncode != 0:
@@ -329,9 +327,8 @@ if DEBUG:
if not os.path.exists(debug_file_path):
logger.error(f"Debug zip file not found at: {debug_file_path}")
return jsonify({"error": "Failed to generate debug information"}), 500
logger.info(f"Sending debug file: {debug_file_path}")
# Return the file to the user
return send_file(
debug_file_path,
mimetype='application/zip',
@@ -345,7 +342,6 @@ if DEBUG:
logger.error_trace(f"Debug endpoint error: {e}")
return jsonify({"error": str(e)}), 500
if DEBUG:
@app.route('/api/restart', methods=['GET'])
@login_required
def restart() -> Union[Response, Tuple[Response, int]]:
@@ -441,10 +437,10 @@ def api_download() -> Union[Response, Tuple[Response, int]]:
try:
priority = int(request.args.get('priority', 0))
success = backend.queue_book(book_id, priority)
success, error_msg = backend.queue_book(book_id, priority)
if success:
return jsonify({"status": "queued", "priority": priority})
return jsonify({"error": "Failed to queue book"}), 500
return jsonify({"error": error_msg or "Failed to queue book"}), 500
except Exception as e:
logger.error_trace(f"Download error: {e}")
return jsonify({"error": str(e)}), 500
@@ -479,22 +475,16 @@ def api_download_release() -> Union[Response, Tuple[Response, int]]:
return jsonify({"error": "source_id is required"}), 400
priority = data.get('priority', 0)
success = backend.queue_release(data, priority)
success, error_msg = backend.queue_release(data, priority)
if success:
return jsonify({"status": "queued", "priority": priority})
return jsonify({"error": "Failed to queue release"}), 500
return jsonify({"error": error_msg or "Failed to queue release"}), 500
except Exception as e:
logger.error_trace(f"Release download error: {e}")
return jsonify({"error": str(e)}), 500
def _is_settings_enabled() -> bool:
"""Check if the config directory is mounted and writable."""
from cwa_book_downloader.config.env import _is_config_dir_writable
return _is_config_dir_writable()
@app.route('/api/config', methods=['GET'])
@login_required
def api_config() -> Union[Response, Tuple[Response, int]]:
@@ -510,6 +500,7 @@ def api_config() -> Union[Response, Tuple[Response, int]]:
get_provider_search_fields,
get_provider_default_sort,
)
from cwa_book_downloader.config.env import _is_config_dir_writable
config = {
"calibre_web_url": app_config.get("CALIBRE_WEB_URL", ""),
@@ -526,7 +517,7 @@ def api_config() -> Union[Response, Tuple[Response, int]]:
"default_release_source": app_config.get("DEFAULT_RELEASE_SOURCE", "direct_download"),
"auto_open_downloads_sidebar": app_config.get("AUTO_OPEN_DOWNLOADS_SIDEBAR", True),
"download_to_browser": app_config.get("DOWNLOAD_TO_BROWSER", False),
"settings_enabled": _is_settings_enabled(),
"settings_enabled": _is_config_dir_writable(),
# Default sort orders
"default_sort": app_config.get("AA_DEFAULT_SORT", "relevance"), # For direct mode (Anna's Archive)
"metadata_default_sort": get_provider_default_sort(), # For universal mode
@@ -541,11 +532,17 @@ def api_health() -> Union[Response, Tuple[Response, int]]:
"""
Health check endpoint for container orchestration.
No authentication required.
Returns:
flask.Response: JSON with status "ok".
flask.Response: JSON with status "ok" and optional degraded features.
"""
return jsonify({"status": "ok"})
response = {"status": "ok"}
# Report degraded features
if not backend.WEBSOCKET_AVAILABLE:
response["degraded"] = {"websocket": "WebSocket unavailable - real-time updates disabled"}
return jsonify(response)
@app.route('/api/status', methods=['GET'])
@login_required
@@ -874,12 +871,7 @@ def api_login() -> Union[Response, Tuple[Response, int]]:
from cwa_book_downloader.core.settings_registry import load_config_file
try:
# Get client IP address (handles reverse proxy forwarding)
ip_address = request.headers.get('X-Forwarded-For', request.remote_addr)
if ip_address and ',' in ip_address:
# X-Forwarded-For can contain multiple IPs, take the first one
ip_address = ip_address.split(',')[0].strip()
ip_address = get_client_ip()
data = request.get_json()
if not data:
return jsonify({"error": "No data provided"}), 400
@@ -983,11 +975,7 @@ def api_logout() -> Union[Response, Tuple[Response, int]]:
flask.Response: JSON with success status.
"""
try:
# Get client IP address (handles reverse proxy forwarding)
ip_address = request.headers.get('X-Forwarded-For', request.remote_addr)
if ip_address and ',' in ip_address:
ip_address = ip_address.split(',')[0].strip()
ip_address = get_client_ip()
username = session.get('user_id', 'unknown')
session.clear()
logger.info(f"Logout successful for user '{username}' from IP {ip_address}")
cwa_book_downloader/metadata_providers/__init__.py
+17 -133
View File
@@ -1,9 +1,9 @@
"""Metadata provider plugin system - base classes and registry."""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field, asdict
from dataclasses import dataclass, field
from enum import Enum
from typing import List, Optional, Dict, Type, Literal, Any, Union
from typing import Any, Dict, List, Optional, Type, Union
class SearchType(str, Enum):
@@ -79,24 +79,12 @@ class CheckboxSearchField:
SearchField = Union[TextSearchField, NumberSearchField, SelectSearchField, CheckboxSearchField]
def _get_field_type_name(search_field: SearchField) -> str:
"""Get the type name for a search field."""
return search_field.__class__.__name__
def serialize_search_field(search_field: SearchField) -> Dict[str, Any]:
"""Serialize a search field for API response.
Args:
search_field: The search field definition.
Returns:
Dict representation for frontend.
"""
"""Serialize a search field to dict for API response."""
result: Dict[str, Any] = {
"key": search_field.key,
"label": search_field.label,
"type": _get_field_type_name(search_field),
"type": search_field.__class__.__name__,
"placeholder": getattr(search_field, 'placeholder', ''),
"description": getattr(search_field, 'description', ''),
}
@@ -116,11 +104,7 @@ def serialize_search_field(search_field: SearchField) -> Dict[str, Any]:
@dataclass
class MetadataSearchOptions:
"""Options for metadata search queries.
Provides an abstracted interface that works across all metadata providers.
Providers map these options to their specific API parameters.
"""
"""Options for metadata search queries across all providers."""
query: str
search_type: SearchType = SearchType.GENERAL
language: Optional[str] = None # ISO 639-1 code (e.g., "en", "fr")
@@ -132,11 +116,7 @@ class MetadataSearchOptions:
@dataclass
class DisplayField:
"""A display field for metadata cards.
Providers can populate these to show provider-specific metadata
like ratings, page counts, reader counts, etc.
"""
"""A display field for metadata cards (ratings, page counts, etc.)."""
label: str # e.g., "Rating", "Pages", "Readers"
value: str # e.g., "4.5", "496", "8,041"
icon: Optional[str] = None # Icon name: "star", "book", "users", "editions"
@@ -208,19 +188,7 @@ class MetadataProvider(ABC):
@abstractmethod
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
"""Search for books using the provided options.
Args:
options: Search options including query, type, language, sort, pagination.
Returns:
List of BookMetadata matching the search criteria.
Note:
- If search_type is ISBN, this delegates to search_by_isbn()
- Unsupported sort orders fall back to RELEVANCE
- Language filtering is best-effort (not all providers support it)
"""
"""Search for books using the provided options."""
pass
@abstractmethod
@@ -239,17 +207,7 @@ class MetadataProvider(ABC):
pass
def search_paginated(self, options: MetadataSearchOptions) -> SearchResult:
"""Search for books and return results with pagination info.
Default implementation calls search() and estimates has_more.
Providers should override this to return accurate pagination info.
Args:
options: Search options including query, type, language, sort, pagination.
Returns:
SearchResult with books and pagination info.
"""
"""Search with pagination info. Override for accurate pagination."""
books = self.search(options)
# Heuristic: if we got exactly limit results, there might be more
has_more = len(books) >= options.limit
@@ -309,18 +267,7 @@ def list_providers() -> List[dict]:
def get_provider_kwargs(provider_name: str) -> Dict:
"""Get provider-specific initialization kwargs based on configuration.
Looks up the provider's registered kwargs factory and calls it to get
the configuration. Each provider registers its own factory via
@register_provider_kwargs decorator.
Args:
provider_name: Name of the provider.
Returns:
Dict of kwargs to pass to provider constructor.
"""
"""Get provider-specific initialization kwargs from registered factory."""
factory = _PROVIDER_KWARGS_FACTORIES.get(provider_name)
if factory:
return factory()
@@ -328,29 +275,12 @@ def get_provider_kwargs(provider_name: str) -> Dict:
def is_provider_registered(provider_name: str) -> bool:
"""Check if a provider is registered.
Args:
provider_name: Name of the provider.
Returns:
True if provider is registered, False otherwise.
"""
"""Check if a provider is registered."""
return provider_name in _PROVIDERS
def is_provider_enabled(provider_name: str) -> bool:
"""Check if a provider is enabled in settings.
Each provider has an enabled flag (e.g., HARDCOVER_ENABLED, OPENLIBRARY_ENABLED)
that must be explicitly set to True for the provider to be used.
Args:
provider_name: Name of the provider.
Returns:
True if provider is enabled, False otherwise.
"""
"""Check if a provider is enabled in settings."""
from cwa_book_downloader.core.config import config as app_config
# Refresh config to get latest settings
@@ -362,31 +292,12 @@ def is_provider_enabled(provider_name: str) -> bool:
def get_enabled_providers() -> List[str]:
"""Get list of all enabled provider names.
Returns:
List of enabled provider names.
"""
enabled = []
for name in _PROVIDERS:
if is_provider_enabled(name):
enabled.append(name)
return enabled
"""Get list of all enabled provider names."""
return [name for name in _PROVIDERS if is_provider_enabled(name)]
def get_configured_provider(content_type: str = "ebook") -> Optional[MetadataProvider]:
"""Get the currently configured metadata provider, if any.
Uses the METADATA_PROVIDER config setting to determine which provider
to instantiate. For audiobook content type, uses METADATA_PROVIDER_AUDIOBOOK
if configured, otherwise falls back to METADATA_PROVIDER.
Args:
content_type: Content type - "ebook" or "audiobook" (default: "ebook")
Returns:
MetadataProvider instance or None.
"""
"""Get the currently configured metadata provider for the content type."""
from cwa_book_downloader.core.config import config as app_config
# Refresh config to ensure we have the latest saved settings
@@ -422,16 +333,7 @@ def _get_configured_provider_name() -> str:
def get_provider_sort_options(provider_name: Optional[str] = None) -> List[Dict[str, str]]:
"""Get sort options for a metadata provider.
Returns a list of {value, label} dicts suitable for frontend dropdowns.
Args:
provider_name: Provider name. If None, uses configured provider.
Returns:
List of sort option dicts, or default [relevance] if provider not found.
"""
"""Get sort options for a metadata provider as {value, label} dicts."""
if provider_name is None:
provider_name = _get_configured_provider_name()
@@ -448,16 +350,7 @@ def get_provider_sort_options(provider_name: Optional[str] = None) -> List[Dict[
def get_provider_search_fields(provider_name: Optional[str] = None) -> List[Dict[str, Any]]:
"""Get search fields for a metadata provider.
Returns a list of serialized search field dicts suitable for frontend rendering.
Args:
provider_name: Provider name. If None, uses configured provider.
Returns:
List of search field dicts, or empty list if provider not found.
"""
"""Get search fields for a metadata provider as serialized dicts."""
if provider_name is None:
provider_name = _get_configured_provider_name()
@@ -471,16 +364,7 @@ def get_provider_search_fields(provider_name: Optional[str] = None) -> List[Dict
def get_provider_default_sort(provider_name: Optional[str] = None) -> str:
"""Get the default sort order for a metadata provider.
Reads from the provider-specific config setting (e.g., HARDCOVER_DEFAULT_SORT).
Args:
provider_name: Provider name. If None, uses configured provider.
Returns:
Default sort value string, or "relevance" if not configured.
"""
"""Get the default sort order for a metadata provider."""
from cwa_book_downloader.core.config import config as app_config
if provider_name is None:
cwa_book_downloader/metadata_providers/googlebooks.py
+18 -84
View File
@@ -53,15 +53,7 @@ def _googlebooks_kwargs() -> Dict[str, Any]:
@register_provider("googlebooks")
class GoogleBooksProvider(MetadataProvider):
"""Google Books metadata provider using REST API.
Attributes:
name: Internal provider identifier.
display_name: Human-readable name for UI.
requires_auth: True - requires API key.
supported_sorts: Only RELEVANCE and NEWEST supported.
search_fields: Author and title search fields.
"""
"""Google Books metadata provider using REST API."""
name = "googlebooks"
display_name = "Google Books"
@@ -81,11 +73,7 @@ class GoogleBooksProvider(MetadataProvider):
]
def __init__(self, api_key: Optional[str] = None):
"""Initialize provider with API key.
Args:
api_key: Google Books API key. If not provided, reads from config.
"""
"""Initialize provider with optional API key (falls back to config)."""
self.api_key = api_key or app_config.get("GOOGLEBOOKS_API_KEY", "")
self.session = requests.Session()
@@ -94,14 +82,7 @@ class GoogleBooksProvider(MetadataProvider):
return bool(self.api_key)
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
"""Search for books using Google Books API.
Args:
options: Search options (query, type, sort, pagination, fields).
Returns:
List of BookMetadata objects matching the search.
"""
"""Search for books using Google Books API."""
if not self.api_key:
logger.warning("Google Books API key not configured")
return []
@@ -127,40 +108,29 @@ class GoogleBooksProvider(MetadataProvider):
def _search_cached(
self, cache_key: str, options: MetadataSearchOptions
) -> List[BookMetadata]:
"""Cached search implementation.
Args:
cache_key: Cache key for this search (includes all options).
options: Search options.
Returns:
List of BookMetadata objects.
"""
"""Cached search implementation."""
# Build query string with Google Books operators
author_value = options.fields.get("author", "").strip()
title_value = options.fields.get("title", "").strip()
query_parts = []
if author_value and not title_value:
# Author-only search
query_parts.append(f"inauthor:{author_value}")
elif title_value and not author_value:
# Title-only search
# Add field-specific operators
if title_value:
query_parts.append(f"intitle:{title_value}")
elif author_value and title_value:
# Both provided - combine
query_parts.append(f"intitle:{title_value}")
query_parts.append(f"inauthor:{author_value}")
elif options.search_type == SearchType.TITLE:
query_parts.append(f"intitle:{options.query}")
if author_value:
query_parts.append(f"inauthor:{author_value}")
elif options.search_type == SearchType.AUTHOR:
query_parts.append(f"inauthor:{options.query}")
else:
# General search
# Fall back to general search if no specific fields
if not query_parts:
query_parts.append(options.query)
query = "+".join(query_parts) if query_parts else options.query
query = "+".join(query_parts)
# Build request params
params: Dict[str, Any] = {
@@ -205,14 +175,7 @@ class GoogleBooksProvider(MetadataProvider):
key_prefix="googlebooks:book",
)
def get_book(self, book_id: str) -> Optional[BookMetadata]:
"""Get book details by Google Books volume ID.
Args:
book_id: Google Books volume ID.
Returns:
BookMetadata or None if not found.
"""
"""Get book details by Google Books volume ID."""
try:
result = self._make_request(f"/volumes/{book_id}", {})
if not result:
@@ -230,14 +193,7 @@ class GoogleBooksProvider(MetadataProvider):
key_prefix="googlebooks:isbn",
)
def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
"""Search for a book by ISBN.
Args:
isbn: ISBN-10 or ISBN-13.
Returns:
BookMetadata or None if not found.
"""
"""Search for a book by ISBN-10 or ISBN-13."""
# Clean ISBN (remove hyphens and spaces)
clean_isbn = isbn.replace("-", "").replace(" ", "").strip()
@@ -266,15 +222,7 @@ class GoogleBooksProvider(MetadataProvider):
def _make_request(
self, endpoint: str, params: Dict[str, Any]
) -> Optional[Dict[str, Any]]:
"""Make authenticated API request.
Args:
endpoint: API endpoint path (e.g., "/volumes").
params: Query parameters.
Returns:
Response JSON or None on error.
"""
"""Make authenticated API request to endpoint."""
if not self.api_key:
logger.warning("Google Books API key not configured")
return None
@@ -313,14 +261,7 @@ class GoogleBooksProvider(MetadataProvider):
return None
def _parse_volume(self, volume: Dict[str, Any]) -> Optional[BookMetadata]:
"""Parse a volume object into BookMetadata.
Args:
volume: Volume data from Google Books API.
Returns:
BookMetadata or None if parsing fails.
"""
"""Parse a volume object into BookMetadata."""
try:
volume_id = volume.get("id")
volume_info = volume.get("volumeInfo", {})
@@ -419,14 +360,7 @@ class GoogleBooksProvider(MetadataProvider):
def _test_googlebooks_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]:
"""Test the Google Books API connection using current form values.
Args:
current_values: Current unsaved form values from the UI.
Returns:
Dict with 'success' bool and 'message' string.
"""
"""Test the Google Books API connection using current form values."""
current_values = current_values or {}
# Use current form values first, fall back to saved config
cwa_book_downloader/metadata_providers/hardcover.py
+33 -95
View File
@@ -135,11 +135,7 @@ class HardcoverProvider(MetadataProvider):
]
def __init__(self, api_key: Optional[str] = None):
"""Initialize provider with API key.
Args:
api_key: Hardcover API key. If not provided, uses config singleton.
"""
"""Initialize provider with optional API key (falls back to config)."""
raw_key = api_key or app_config.get("HARDCOVER_API_KEY", "")
# Strip "Bearer " prefix if user pasted the full auth header from Hardcover
self.api_key = raw_key.removeprefix("Bearer ").strip() if raw_key else ""
@@ -154,26 +150,32 @@ class HardcoverProvider(MetadataProvider):
"""Check if provider is configured with an API key."""
return bool(self.api_key)
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
"""Search for books using Hardcover's search API.
def _build_search_params(
self, default_query: str, author: str, title: str, series: str
) -> tuple[str, Optional[str], Optional[str]]:
"""Build search query, fields, and weights based on provided values.
Args:
options: Search options (query, type, sort, pagination, fields).
Returns:
List of BookMetadata objects.
Returns (query, fields, weights) tuple. Fields/weights are None for general search.
"""
if series and not author and not title:
return series, "series_names", "1"
if author and not title and not series:
return author, "author_names", "1"
if title and not author and not series:
return title, "title,alternative_titles", "5,1"
if author and title and not series:
return f"{title} {author}", "title,alternative_titles,author_names", "5,1,3"
if series:
query = " ".join(p for p in [series, title, author] if p)
return query, "series_names,title,alternative_titles,author_names", "5,3,1,2"
return default_query, None, None
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
"""Search for books using Hardcover's search API."""
return self.search_paginated(options).books
def search_paginated(self, options: MetadataSearchOptions) -> SearchResult:
"""Search for books with pagination info.
Args:
options: Search options (query, type, sort, pagination, fields).
Returns:
SearchResult with books and pagination info.
"""
"""Search for books with pagination info."""
if not self.api_key:
logger.warning("Hardcover API key not configured")
return SearchResult(books=[], page=options.page, total_found=0, has_more=False)
@@ -193,36 +195,17 @@ class HardcoverProvider(MetadataProvider):
@cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="hardcover:search")
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> SearchResult:
"""Cached search implementation.
Args:
cache_key: Cache key (used by decorator).
options: Search options.
Returns:
SearchResult with books and pagination info.
"""
"""Cached search implementation."""
# Determine query and fields based on custom search fields
# Note: Hardcover API requires 'weights' when using 'fields' parameter
author_value = options.fields.get("author", "").strip()
title_value = options.fields.get("title", "").strip()
series_value = options.fields.get("series", "").strip()
if series_value and not author_value and not title_value:
query, search_fields, search_weights = series_value, "series_names", "1"
elif author_value and not title_value and not series_value:
query, search_fields, search_weights = author_value, "author_names", "1"
elif title_value and not author_value and not series_value:
query, search_fields, search_weights = title_value, "title,alternative_titles", "5,1"
elif author_value and title_value and not series_value:
query = f"{title_value} {author_value}"
search_fields, search_weights = "title,alternative_titles,author_names", "5,1,3"
elif series_value:
query = " ".join(p for p in [series_value, title_value, author_value] if p)
search_fields, search_weights = "series_names,title,alternative_titles,author_names", "5,3,1,2"
else:
query, search_fields, search_weights = options.query, None, None
# Build query and field configuration based on which fields are provided
query, search_fields, search_weights = self._build_search_params(
options.query, author_value, title_value, series_value
)
# Build GraphQL query - include fields/weights parameters only when needed
if search_fields:
@@ -256,7 +239,6 @@ class HardcoverProvider(MetadataProvider):
variables["fields"] = search_fields
variables["weights"] = search_weights
try:
result = self._execute_query(graphql_query, variables)
if not result:
@@ -314,15 +296,7 @@ class HardcoverProvider(MetadataProvider):
return SearchResult(books=[], page=options.page, total_found=0, has_more=False)
def _apply_series_ordering(self, books: List[BookMetadata], series_name: str) -> List[BookMetadata]:
"""Filter books to exact series match and sort by series position.
Args:
books: List of books from search results.
series_name: The series name to match.
Returns:
Filtered and sorted list of books.
"""
"""Filter books to exact series match and sort by series position."""
series_name_lower = series_name.lower()
books_with_position = []
@@ -353,14 +327,7 @@ class HardcoverProvider(MetadataProvider):
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="hardcover:book")
def get_book(self, book_id: str) -> Optional[BookMetadata]:
"""Get book details by Hardcover ID.
Args:
book_id: Hardcover book ID.
Returns:
BookMetadata or None if not found.
"""
"""Get book details by Hardcover ID."""
if not self.api_key:
logger.warning("Hardcover API key not configured")
return None
@@ -433,14 +400,7 @@ class HardcoverProvider(MetadataProvider):
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="hardcover:isbn")
def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
"""Search for a book by ISBN.
Args:
isbn: ISBN-10 or ISBN-13.
Returns:
BookMetadata or None if not found.
"""
"""Search for a book by ISBN-10 or ISBN-13."""
if not self.api_key:
logger.warning("Hardcover API key not configured")
return None
@@ -510,15 +470,7 @@ class HardcoverProvider(MetadataProvider):
return None
def _execute_query(self, query: str, variables: Dict[str, Any]) -> Optional[Dict]:
"""Execute a GraphQL query.
Args:
query: GraphQL query string.
variables: Query variables.
Returns:
Response data dict or None on error.
"""
"""Execute a GraphQL query and return data or None on error."""
try:
response = self.session.post(
HARDCOVER_API_URL,
@@ -549,14 +501,7 @@ class HardcoverProvider(MetadataProvider):
return None
def _parse_search_result(self, item: Dict) -> Optional[BookMetadata]:
"""Parse a search result item into BookMetadata.
Args:
item: Search result item dict.
Returns:
BookMetadata or None if parsing fails.
"""
"""Parse a search result item into BookMetadata."""
try:
book_id = item.get("id") or item.get("document", {}).get("id")
title = item.get("title") or item.get("document", {}).get("title")
@@ -633,14 +578,7 @@ class HardcoverProvider(MetadataProvider):
return None
def _parse_book(self, book: Dict) -> BookMetadata:
"""Parse a book object into BookMetadata.
Args:
book: Book data dict from GraphQL response.
Returns:
BookMetadata object.
"""
"""Parse a book object into BookMetadata."""
# Extract authors - try contributions first (filtered), fall back to cached_contributors
authors = []
contributions = book.get("contributions") or []
cwa_book_downloader/metadata_providers/openlibrary.py
+15 -90
View File
@@ -1,9 +1,10 @@
"""Open Library metadata provider. No API key required, rate limited."""
import re
import time
import threading
from collections import deque
from typing import Any, Deque, Dict, List, Optional, Union
from typing import Any, Deque, Dict, List, Optional
import requests
@@ -42,23 +43,14 @@ class RateLimiter:
"""Simple sliding window rate limiter."""
def __init__(self, max_requests: int, window_seconds: int):
"""Initialize rate limiter.
Args:
max_requests: Maximum requests allowed in the window.
window_seconds: Time window in seconds.
"""
"""Initialize rate limiter with max requests per time window."""
self.max_requests = max_requests
self.window_seconds = window_seconds
self.timestamps: Deque[float] = deque()
self.lock = threading.Lock()
def wait_if_needed(self) -> None:
"""Block until a request is allowed.
Thread-safe implementation that calculates wait time with lock held,
then sleeps without holding the lock to avoid blocking other threads.
"""
"""Block until a request is allowed (thread-safe)."""
wait_time = 0
# Calculate wait time with lock held
@@ -139,14 +131,7 @@ class OpenLibraryProvider(MetadataProvider):
return True
def search(self, options: MetadataSearchOptions) -> List[BookMetadata]:
"""Search for books using Open Library's search API.
Args:
options: Search options (query, type, sort, language, pagination, fields).
Returns:
List of BookMetadata objects.
"""
"""Search for books using Open Library's search API."""
# Handle ISBN search separately
if options.search_type == SearchType.ISBN:
result = self.search_by_isbn(options.query)
@@ -159,15 +144,7 @@ class OpenLibraryProvider(MetadataProvider):
@cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="openlibrary:search")
def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> List[BookMetadata]:
"""Cached search implementation.
Args:
cache_key: Cache key (used by decorator).
options: Search options.
Returns:
List of BookMetadata objects.
"""
"""Cached search implementation."""
_rate_limiter.wait_if_needed()
# Build query params
@@ -240,14 +217,7 @@ class OpenLibraryProvider(MetadataProvider):
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="openlibrary:book")
def get_book(self, book_id: str) -> Optional[BookMetadata]:
"""Get book details by Open Library work ID.
Args:
book_id: Open Library work ID (e.g., "OL12345W").
Returns:
BookMetadata or None if not found.
"""
"""Get book details by Open Library work ID (e.g., 'OL12345W')."""
_rate_limiter.wait_if_needed()
# Normalize the book_id format
@@ -281,14 +251,7 @@ class OpenLibraryProvider(MetadataProvider):
@cacheable(ttl_key="METADATA_CACHE_BOOK_TTL", ttl_default=600, key_prefix="openlibrary:isbn")
def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]:
"""Search for a book by ISBN.
Args:
isbn: ISBN-10 or ISBN-13.
Returns:
BookMetadata or None if not found.
"""
"""Search for a book by ISBN-10 or ISBN-13."""
# Clean ISBN
clean_isbn = isbn.replace("-", "").strip()
@@ -343,14 +306,7 @@ class OpenLibraryProvider(MetadataProvider):
return None
def _parse_search_doc(self, doc: dict) -> Optional[BookMetadata]:
"""Parse a search document into BookMetadata.
Args:
doc: Search result document from Open Library.
Returns:
BookMetadata or None if parsing fails.
"""
"""Parse a search document into BookMetadata."""
try:
# Extract work ID from key
key = doc.get("key", "")
@@ -364,17 +320,10 @@ class OpenLibraryProvider(MetadataProvider):
if not isinstance(authors, list):
authors = [authors] if authors else []
# Get ISBNs
# Get ISBNs - find first ISBN-10 and ISBN-13
isbns = doc.get("isbn", [])
isbn_10 = None
isbn_13 = None
for isbn in isbns:
if len(isbn) == 10 and not isbn_10:
isbn_10 = isbn
elif len(isbn) == 13 and not isbn_13:
isbn_13 = isbn
if isbn_10 and isbn_13:
break
isbn_10 = next((i for i in isbns if len(i) == 10), None)
isbn_13 = next((i for i in isbns if len(i) == 13), None)
# Get cover URL
cover_id = doc.get("cover_i")
@@ -426,15 +375,7 @@ class OpenLibraryProvider(MetadataProvider):
return None
def _parse_work(self, work: dict, work_id: str) -> Optional[BookMetadata]:
"""Parse a work object into BookMetadata.
Args:
work: Work data from Open Library API.
work_id: The work ID.
Returns:
BookMetadata or None if parsing fails.
"""
"""Parse a work object into BookMetadata."""
try:
title = work.get("title")
if not title:
@@ -484,15 +425,7 @@ class OpenLibraryProvider(MetadataProvider):
return None
def _parse_edition(self, edition: dict, isbn: str) -> Optional[BookMetadata]:
"""Parse an edition object into BookMetadata (fallback for ISBN lookup).
Args:
edition: Edition data from Open Library API.
isbn: The ISBN used for lookup.
Returns:
BookMetadata or None if parsing fails.
"""
"""Parse an edition object into BookMetadata (fallback for ISBN lookup)."""
try:
title = edition.get("title")
if not title:
@@ -524,7 +457,6 @@ class OpenLibraryProvider(MetadataProvider):
publish_date = edition.get("publish_date", "")
if publish_date:
# Try to extract year from various formats
import re
year_match = re.search(r'\b(19|20)\d{2}\b', publish_date)
if year_match:
publish_year = int(year_match.group())
@@ -547,14 +479,7 @@ class OpenLibraryProvider(MetadataProvider):
return None
def _get_author_name(self, author_key: str) -> Optional[str]:
"""Get author name from author key.
Args:
author_key: Open Library author key (e.g., "/authors/OL123A").
Returns:
Author name or None.
"""
"""Get author name from author key (e.g., '/authors/OL123A')."""
_rate_limiter.wait_if_needed()
try:
cwa_book_downloader/release_sources/__init__.py
+7 -71
View File
@@ -40,13 +40,7 @@ class Release:
@dataclass
class DownloadProgress:
"""Progress update structure.
DEPRECATED: This class is deprecated and will be removed.
The new DownloadHandler.download() uses simpler callbacks:
- progress_callback(float) for progress percentage
- status_callback(str, Optional[str]) for status and message
"""
"""DEPRECATED: Use progress_callback and status_callback instead."""
status: str # "queued", "resolving", "downloading", "complete", "failed"
progress: float # 0-100
status_message: Optional[str] = None
@@ -227,18 +221,7 @@ class ReleaseSource(ABC):
languages: Optional[List[str]] = None,
content_type: str = "ebook"
) -> List[Release]:
"""Search for releases of a book.
Args:
book: Book metadata from provider
expand_search: If True, use broader search (e.g., title+author instead of ISBN).
Not all sources support this - they may ignore it.
languages: Optional list of language codes to filter by.
If provided, overrides book.language and default settings.
Not all sources support this - they may ignore it.
content_type: Content type - "ebook" or "audiobook" (default: "ebook").
Sources may use this to adjust search categories/filters.
"""
"""Search for releases of a book."""
pass
@abstractmethod
@@ -248,41 +231,13 @@ class ReleaseSource(ABC):
@classmethod
def get_column_config(cls) -> ReleaseColumnConfig:
"""Get the column configuration for this source's release list UI.
Override this method in subclasses to provide custom columns.
Default implementation returns standard columns (language, format, size).
"""
"""Get column configuration for release list UI. Override for custom columns."""
return _default_column_config()
class DownloadHandler(ABC):
"""Interface for executing downloads from a source.
## Staging Architecture
Handlers are responsible for getting files into the STAGING directory (TMP_DIR).
The orchestrator handles all post-processing and moving to the INGEST directory.
This means handlers should:
1. Download/retrieve the file to the staging directory
2. Return the path to the staged file
3. NOT move files to the ingest folder (orchestrator does this)
Examples by source type:
- **Direct downloads**: Download directly to staging dir
- **Torrents**: Copy completed file from torrent client to staging (keep seeding)
- **Usenet**: Move completed file from NZB client to staging
Use the staging helpers from orchestrator:
- `get_staging_dir()` - Get the staging directory path
- `get_staging_path(task_id, ext)` - Get a staging path for a task
- `stage_file(source, task_id, copy=False)` - Stage a file (copy or move)
The orchestrator then handles:
- Archive extraction (RAR/ZIP)
- Custom script execution
- Moving to the final ingest folder
"""Interface for executing downloads. Handlers stage files to TMP_DIR;
orchestrator handles post-processing and move to INGEST_DIR.
"""
@abstractmethod
@@ -293,22 +248,7 @@ class DownloadHandler(ABC):
progress_callback: Callable[[float], None],
status_callback: Callable[[str, Optional[str]], None]
) -> Optional[str]:
"""
Execute download and return path to STAGED file.
Handlers should download/copy files to the staging directory (TMP_DIR),
NOT directly to the ingest folder. The orchestrator handles post-processing
(archive extraction, custom scripts) and final move to ingest.
Args:
task: The download task with task_id and display info
cancel_flag: Event to check for cancellation
progress_callback: Called with progress percentage (0-100)
status_callback: Called with (status, message) for status updates
Returns:
Path to staged file (in TMP_DIR) if successful, None otherwise
"""
"""Execute download and return path to staged file in TMP_DIR."""
pass
@abstractmethod
@@ -354,11 +294,7 @@ def get_handler(name: str) -> DownloadHandler:
def list_available_sources() -> List[dict]:
"""For frontend - list all registered sources with their status.
Returns all sources (not just available ones) so the frontend can show
appropriate UI for disabled/unconfigured sources instead of hiding them.
"""
"""List all registered sources with their availability status."""
result = []
for name, src_class in _SOURCES.items():
instance = src_class()
cwa_book_downloader/release_sources/direct_download.py
+41 -98
View File
@@ -2,12 +2,11 @@
import itertools
import json
import os
import re
import time
from pathlib import Path
from threading import Event
from typing import Callable, Dict, List, Optional, Tuple
from typing import Callable, Dict, List, Optional
from urllib.parse import quote
from bs4 import BeautifulSoup, NavigableString, Tag
@@ -84,29 +83,16 @@ def _is_source_enabled(source_id: str) -> bool:
return False
def _get_enabled_source_order() -> List[str]:
"""Get ordered list of enabled source IDs."""
return [
item["id"]
for item in _get_source_priority()
if item.get("enabled", True)
]
_SIZE_UNIT_PATTERN = re.compile(r'(kb|mb|gb|tb)', re.IGNORECASE)
def _get_source_position(source_id: str) -> int:
"""Get the position of a source in the priority list (lower = higher priority).
Returns 999 if source not found or disabled.
"""
for i, item in enumerate(_get_source_priority()):
if item["id"] == source_id and item.get("enabled", True):
return i
return 999
def _normalize_size(size_str: str) -> str:
"""Normalize size string by uppercasing units (e.g., '5.2 mb' -> '5.2 MB')."""
return _SIZE_UNIT_PATTERN.sub(lambda m: m.group(1).upper(), size_str.strip())
class SearchUnavailable(Exception):
"""Raised when Anna's Archive cannot be reached via any mirror/DNS."""
pass
def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
@@ -148,11 +134,9 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
index = 1
for filter_type, filter_values in vars(filters).items():
if (filter_type == "author" or filter_type == "title") and filter_values:
if filter_type in ("author", "title") and filter_values:
for value in filter_values:
filters_query += (
f"&termtype_{index}={filter_type}&termval_{index}={quote(value)}"
)
filters_query += f"&termtype_{index}={filter_type}&termval_{index}={quote(value)}"
index += 1
selector = network.AAMirrorSelector()
@@ -272,26 +256,25 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str, fetch_download_coun
data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div")
divs = list(data.children)
slow_urls_no_waitlist: List[str] = []
slow_urls_with_waitlist: List[str] = []
def _append_unique(lst: List[str], href: str) -> None:
if href and href not in lst:
lst.append(href)
slow_urls_no_waitlist: set[str] = set()
slow_urls_with_waitlist: set[str] = set()
for anchor in soup.find_all("a"):
try:
text = anchor.text.strip().lower()
href = anchor.get("href", "")
if not href:
continue
next_text = ""
if anchor.next and anchor.next.next:
next_text = getattr(anchor.next.next, 'text', str(anchor.next.next)).strip().lower()
if text.startswith("slow partner server") and "waitlist" in next_text:
if "no waitlist" in next_text:
_append_unique(slow_urls_no_waitlist, href)
slow_urls_no_waitlist.add(href)
else:
_append_unique(slow_urls_with_waitlist, href)
slow_urls_with_waitlist.add(href)
except Exception:
pass
@@ -331,9 +314,8 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str, fetch_download_coun
for f in _details:
if format == "" and f.strip().lower() in config.SUPPORTED_FORMATS:
format = f.strip().lower()
if size == "" and any(u in f.strip().lower() for u in ["mb", "kb", "gb"]):
# Preserve original case but uppercase the unit (e.g., "5.2 mb" -> "5.2 MB")
size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE)
if size == "" and any(u in f.strip().lower() for u in ("mb", "kb", "gb")):
size = _normalize_size(f)
if content == "":
for ct in CONTENT_TYPES:
if ct in f.strip().lower():
@@ -345,8 +327,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str, fetch_download_coun
if format == "" and stripped and " " not in stripped:
format = stripped
if size == "" and "." in stripped:
# Uppercase any size units
size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE)
size = _normalize_size(f)
book_title = _find_in_divs(divs, "🔍")[0].strip("🔍").strip()
@@ -456,43 +437,28 @@ def _extract_book_description(soup: BeautifulSoup) -> Optional[str]:
def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]:
"""Extract metadata from book info divs."""
info: Dict[str, List[str]] = {}
info: Dict[str, set[str]] = {}
# Process the first set of metadata
sub_datas = metadata_divs.find_all("div")[0]
sub_datas = list(sub_datas.children)
for sub_data in sub_datas:
for sub_data in sub_datas.children:
if sub_data.text.strip() == "":
continue
sub_data = list(sub_data.children)
key = sub_data[0].text.strip()
value = sub_data[1].text.strip()
children = list(sub_data.children)
key = children[0].text.strip()
value = children[1].text.strip()
if key not in info:
info[key] = set()
info[key].add(value)
# make set into list
for key, value in info.items():
info[key] = list(value)
# Filter relevant metadata
relevant_prefixes = [
"ISBN-",
"ALTERNATIVE",
"ASIN",
"Goodreads",
"Language",
"Year",
]
relevant_prefixes = ("isbn-", "alternative", "asin", "goodreads", "language", "year")
return {
k.strip(): v
k.strip(): list(v)
for k, v in info.items()
if any(k.lower().startswith(prefix.lower()) for prefix in relevant_prefixes)
and "filename" not in k.lower()
if k.lower().startswith(relevant_prefixes) and "filename" not in k.lower()
}
def _get_source_info(link: str) -> Tuple[str, str]:
def _get_source_info(link: str) -> tuple[str, str]:
"""Get source label and friendly name for a download link.
Args:
@@ -514,41 +480,32 @@ def _get_source_info(link: str) -> Tuple[str, str]:
return "unknown", "Mirror"
def _label_source(link: str) -> str:
"""Get lightweight source tag for logging/metrics."""
return _get_source_info(link)[0]
def _friendly_source_name(link: str) -> str:
"""Get user-friendly name for a download source."""
return _get_source_info(link)[1]
def _group_urls_by_source(urls: List[str], urls_by_source: Dict[str, List[str]]) -> None:
"""Group URLs into urls_by_source dict by their source type."""
for url in urls:
source_type = _url_source_types.get(url)
if source_type:
urls_by_source.setdefault(source_type, []).append(url)
def _fetch_aa_page_urls(book_info: BookInfo, urls_by_source: Dict[str, List[str]]) -> None:
"""Fetch and parse AA page, populating urls_by_source dict.
Groups existing book_info.download_urls by source type. If book_info
has no URLs, fetches the AA page fresh.
"""
# If book_info already has URLs, group them by source type
if book_info.download_urls:
for url in book_info.download_urls:
source_type = _url_source_types.get(url)
if source_type:
if source_type not in urls_by_source:
urls_by_source[source_type] = []
urls_by_source[source_type].append(url)
_group_urls_by_source(book_info.download_urls, urls_by_source)
return
# Otherwise fetch the page fresh
try:
fresh_book_info = get_book_info(book_info.id, fetch_download_count=False)
for url in fresh_book_info.download_urls:
source_type = _url_source_types.get(url)
if source_type:
if source_type not in urls_by_source:
urls_by_source[source_type] = []
urls_by_source[source_type].append(url)
_group_urls_by_source(fresh_book_info.download_urls, urls_by_source)
except Exception as e:
logger.warning(f"Failed to fetch AA page: {e}")
@@ -560,7 +517,6 @@ def _get_urls_for_source(
cancel_flag: Optional[Event],
status_callback: Optional[Callable[[str, Optional[str]], None]],
urls_by_source: Dict[str, List[str]],
aa_page_fetched: bool
) -> List[str]:
"""Get URLs for a specific source, fetching lazily if needed."""
# AA Fast - generate URL dynamically
@@ -585,7 +541,7 @@ def _get_urls_for_source(
# AA page sources - fetch AA page if not already done
if source_id in _AA_PAGE_SOURCES:
if not aa_page_fetched and not urls_by_source:
if not urls_by_source:
if status_callback:
status_callback("resolving", "Fetching download sources")
_fetch_aa_page_urls(book_info, urls_by_source)
@@ -686,7 +642,6 @@ def _download_book(
selector = network.AAMirrorSelector()
source_failures: dict[str, int] = {}
urls_by_source: dict[str, list[str]] = {}
aa_page_fetched = False
url_attempt_counter = 0
# Get enabled sources in priority order
@@ -716,13 +671,9 @@ def _download_book(
# Get URLs for this source (lazy-loads as needed)
urls_to_try = _get_urls_for_source(
source_id, book_info, selector, cancel_flag, status_callback,
urls_by_source, aa_page_fetched
urls_by_source,
)
# Track if we fetched AA page
if source_id in _AA_PAGE_SOURCES and not aa_page_fetched:
aa_page_fetched = bool(urls_by_source)
if not urls_to_try:
continue
@@ -841,7 +792,7 @@ def _extract_slow_download_url(
# The URL appears as plain text in <span class="bg-gray-200 ...">http://...</span>
for span in soup.find_all("span", class_=lambda c: c and "bg-gray-200" in c):
text = span.get_text(strip=True)
if text.startswith("http://") or text.startswith("https://"):
if text.startswith(("http://", "https://")):
return text
# Try "copy this URL" pattern (legacy)
@@ -875,14 +826,8 @@ def _extract_slow_download_url(
logger.info(f"AA waitlist: {sleep_time}s for {title}")
# Live countdown with status updates
remaining = sleep_time
while remaining > 0:
# Format countdown message with source context
if source_context:
wait_msg = f"{source_context} - Waiting {remaining}s"
else:
wait_msg = f"Waiting {remaining}s"
for remaining in range(sleep_time, 0, -1):
wait_msg = f"{source_context} - Waiting {remaining}s" if source_context else f"Waiting {remaining}s"
if status_callback:
status_callback("resolving", wait_msg)
@@ -891,8 +836,6 @@ def _extract_slow_download_url(
logger.info(f"Cancelled wait for {title}")
return ""
remaining -= 1
# After countdown, update status and re-fetch
if status_callback and source_context:
status_callback("resolving", f"{source_context} - Fetching")
cwa_book_downloader/release_sources/irc/client.py
+15 -97
View File
@@ -72,24 +72,7 @@ class IRCConnectionError(IRCError):
class IRCClient:
"""Minimal IRC client for IRC Highway ebook searches.
Designed for per-request connections - connect, do operation, disconnect.
Not intended for long-lived connections.
Usage:
client = IRCClient(nick="mybot")
client.connect()
client.join_channel("ebooks")
client.send_message("#ebooks", "@search harry potter")
for msg in client.read_messages():
if msg.event == IRCEvent.SEARCH_RESULT:
offer = parse_dcc_send(msg.raw)
break
client.disconnect()
"""
"""Minimal IRC client for per-request ebook searches on IRC Highway."""
def __init__(
self,
@@ -119,16 +102,7 @@ class IRCClient:
return f"cwa_{suffix}"
def connect(self) -> None:
"""Connect to IRC server and authenticate.
Connection sequence:
1. TCP/TLS connect
2. Send USER and NICK
3. Wait for server welcome messages
Raises:
IRCConnectionError: If connection fails
"""
"""Connect to IRC server, send USER/NICK, and wait for welcome."""
logger.info(f"Connecting to {self.server}:{self.port} (TLS={self.use_tls})")
try:
@@ -179,14 +153,7 @@ class IRCClient:
logger.info("Disconnected from IRC")
def join_channel(self, channel: str, wait_for_join: bool = True) -> None:
"""Join an IRC channel.
Args:
channel: Channel name without # prefix
wait_for_join: If True, wait for server confirmation (366 message)
Also captures the channel's user list to track online servers.
"""
"""Join an IRC channel (without # prefix) and capture online servers."""
self._send(f"JOIN #{channel}")
logger.debug(f"Sent JOIN #{channel}")
@@ -228,41 +195,20 @@ class IRCClient:
logger.warning(f"Joined #{channel} (no confirmation received)")
def send_message(self, target: str, message: str) -> None:
"""Send a PRIVMSG to a channel or user.
Args:
target: Channel (with #) or user nick
message: Message content
"""
"""Send a PRIVMSG to a channel or user."""
self._send(f"PRIVMSG {target} :{message}")
logger.debug(f"Sent to {target}: {message[:50]}...")
def send_notice(self, target: str, message: str) -> None:
"""Send a NOTICE to a user.
Args:
target: User nick
message: Notice content
"""
"""Send a NOTICE to a user."""
self._send(f"NOTICE {target} :{message}")
def request_names(self, channel: str) -> None:
"""Request user list for a channel.
Args:
channel: Channel name without # prefix
"""
"""Request user list for a channel (without # prefix)."""
self._send(f"NAMES #{channel}")
def _parse_names_list(self, names_data: str) -> None:
"""Parse NAMES list and extract elevated users (download servers).
IRC NAMES reply format (353):
:server 353 nick = #channel :@user1 +user2 user3 ...
Users with prefixes (~, &, @, %, +) are elevated (ops/voice).
These are the download bots/servers.
"""
"""Parse 353 NAMES reply and extract elevated users (download servers)."""
# Extract the trailing part after the last colon (the actual names)
if ' :' in names_data:
names_part = names_data.split(' :')[-1]
@@ -270,9 +216,6 @@ class IRCClient:
names_part = names_data
for name in names_part.split():
if not name:
continue
# Check if user has an elevated prefix
if name[0] in ELEVATED_PREFIXES:
# Strip the prefix to get the actual nick
@@ -288,11 +231,7 @@ class IRCClient:
self._socket.sendall(data)
def _recv_lines(self) -> Iterator[str]:
"""Receive and yield complete IRC lines.
IRC messages are delimited by \\r\\n. We buffer partial
reads and yield complete lines as they arrive.
"""
"""Receive and yield complete CRLF-delimited IRC lines."""
while True:
# Check if we have a complete line in buffer
while '\r\n' in self._buffer:
@@ -344,11 +283,7 @@ class IRCClient:
return msg
def _classify_event(self, msg: IRCMessage) -> IRCEvent:
"""Classify message into event type.
Uses simple string containment checks for robustness
rather than strict IRC protocol parsing.
"""
"""Classify message into event type using string containment checks."""
raw = msg.raw
trailing = msg.trailing or ""
@@ -370,9 +305,7 @@ class IRCClient:
return IRCEvent.MATCHES_FOUND
# User list (RPL_NAMREPLY and RPL_ENDOFNAMES)
if msg.command == "353":
return IRCEvent.SERVER_LIST
if msg.command == "366":
if msg.command in ("353", "366"):
return IRCEvent.SERVER_LIST
# Server PING
@@ -401,14 +334,7 @@ class IRCClient:
logger.debug(f"Sent VERSION to {sender}")
def read_messages(self, auto_handle: bool = True) -> Iterator[IRCMessage]:
"""Read and yield IRC messages.
Args:
auto_handle: If True, automatically handle PING and VERSION
Yields:
IRCMessage objects for each received message
"""
"""Read and yield IRC messages, optionally auto-handling PING/VERSION."""
for line in self._recv_lines():
msg = self._parse_message(line)
@@ -429,15 +355,7 @@ class IRCClient:
timeout: float = 60.0,
result_type: bool = False,
) -> Optional[DCCOffer]:
"""Wait for a DCC SEND offer.
Args:
timeout: Maximum time to wait in seconds
result_type: If True, wait for SEARCH_RESULT; else BOOK_RESULT
Returns:
DCCOffer if received, None if timeout
"""
"""Wait for a DCC SEND offer. Returns None on timeout or no results."""
target_event = IRCEvent.SEARCH_RESULT if result_type else IRCEvent.BOOK_RESULT
start = time.time()
@@ -459,12 +377,12 @@ class IRCClient:
if msg.event == IRCEvent.NO_RESULTS:
logger.info("Server reports no results")
return None
if msg.event == IRCEvent.BAD_SERVER:
elif msg.event == IRCEvent.BAD_SERVER:
logger.warning("Server unavailable")
return None
if msg.event == IRCEvent.SEARCH_ACCEPTED:
elif msg.event == IRCEvent.SEARCH_ACCEPTED:
logger.info("Search accepted, waiting for results...")
if msg.event == IRCEvent.MATCHES_FOUND:
elif msg.event == IRCEvent.MATCHES_FOUND:
# Extract count from "returned X matches"
if msg.trailing and "returned" in msg.trailing:
try:
cwa_book_downloader/release_sources/irc/dcc.py
+3 -35
View File
@@ -60,29 +60,13 @@ class DCCConnectionError(DCCError):
def int_to_ip(ip_int: int) -> str:
"""Convert 32-bit integer to dotted IP notation.
DCC protocol sends IP addresses as 32-bit unsigned integers
in network byte order (big-endian).
Example: 2760158537 -> "164.132.173.73"
"""
"""Convert 32-bit integer (DCC format) to dotted IP notation."""
packed = struct.pack('>I', ip_int)
return '.'.join(str(b) for b in packed)
def parse_dcc_send(text: str) -> DCCOffer:
"""Parse a DCC SEND message into a DCCOffer.
Args:
text: Full IRC message containing DCC SEND
Returns:
DCCOffer with filename, ip, port, size
Raises:
DCCParseError: If message doesn't match expected format
"""
"""Parse a DCC SEND message into a DCCOffer. Raises DCCParseError on failure."""
match = DCC_REGEX.search(text)
if not match:
raise DCCParseError(f"Invalid DCC SEND format: {text[:100]}")
@@ -107,23 +91,7 @@ def download_dcc(
cancel_flag: Optional[Event] = None,
timeout: float = 30.0,
) -> None:
"""Download file via DCC protocol.
Uses a custom read loop with 4096-byte buffer which is faster than
Python's shutil.copyfileobj since DCC servers don't properly signal EOF.
Args:
offer: Parsed DCC offer with connection details
dest_path: Where to save the file
progress_callback: Called with percentage (0-100) during download
cancel_flag: If set, abort the download
timeout: Socket timeout in seconds
Raises:
DCCConnectionError: Failed to connect to sender
DCCSizeError: Downloaded bytes != expected size
DCCError: Other socket/IO errors
"""
"""Download file via DCC protocol to dest_path. Raises DCCError on failure."""
logger.info(f"DCC connecting to {offer.ip}:{offer.port} for {offer.filename}")
try:
cwa_book_downloader/release_sources/irc/handler.py
+2 -19
View File
@@ -29,20 +29,7 @@ class IRCDownloadHandler(DownloadHandler):
progress_callback: Callable[[float], None],
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[str]:
"""Download a book via IRC DCC.
The task.task_id contains the full IRC request string,
e.g., "!ServerName Author - Title.epub ::INFO:: 2.5MB"
Args:
task: Download task with IRC request info
cancel_flag: Set to cancel download
progress_callback: Report progress 0-100
status_callback: Report status messages
Returns:
Path to downloaded file, or None on failure
"""
"""Download a book via IRC DCC. task.task_id contains the IRC request string."""
download_request = task.task_id
logger.info(f"IRC download: {download_request[:60]}...")
@@ -135,10 +122,6 @@ class IRCDownloadHandler(DownloadHandler):
return None
def cancel(self, task_id: str) -> bool:
"""Cancel an in-progress download.
Note: Actual cancellation is handled via the cancel_flag in download().
This method is for cleanup if the cancel_flag mechanism fails.
"""
"""Cancel an in-progress download (cleanup if cancel_flag fails)."""
logger.debug(f"Cancel requested for IRC task: {task_id}")
return True
cwa_book_downloader/release_sources/irc/parser.py
+8 -31
View File
@@ -7,7 +7,7 @@ import re
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional
from typing import Optional
from cwa_book_downloader.core.config import config
from cwa_book_downloader.core.logger import setup_logger
@@ -29,12 +29,12 @@ ALL_RECOGNIZED_FORMATS = {
}
def _get_supported_formats() -> List[str]:
def _get_supported_formats() -> set[str]:
"""Get user's configured supported formats from settings."""
formats = config.get("SUPPORTED_FORMATS", ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"])
if isinstance(formats, str):
return [fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()]
return [fmt.lower() for fmt in formats]
return {fmt.strip().lower() for fmt in formats.split(",") if fmt.strip()}
return {fmt.lower() for fmt in formats}
# Regex to parse result lines
# Format: !Server Author - Title.format ::INFO:: size
@@ -75,14 +75,7 @@ class SearchResult:
def parse_result_line(line: str) -> Optional[SearchResult]:
"""Parse a single search result line.
Args:
line: Raw line from search results file
Returns:
SearchResult if parseable, None otherwise
"""
"""Parse a single search result line. Returns None if unparseable."""
line = line.strip()
# Must start with !
@@ -148,16 +141,9 @@ def parse_result_line(line: str) -> Optional[SearchResult]:
def parse_results_file(content: str) -> list[SearchResult]:
"""Parse a search results file.
Args:
content: Full file content
Returns:
List of parsed SearchResult objects
"""
"""Parse a search results file into SearchResult objects."""
results = []
supported = set(_get_supported_formats())
supported = _get_supported_formats()
for line in content.splitlines():
result = parse_result_line(line)
@@ -171,16 +157,7 @@ def parse_results_file(content: str) -> list[SearchResult]:
def extract_results_from_zip(zip_path: Path) -> str:
"""Extract and return content from a search results ZIP.
Search results are sent as ZIP files containing a single text file.
Args:
zip_path: Path to downloaded ZIP file
Returns:
Text content of the results file
"""
"""Extract and return text content from a search results ZIP."""
with zipfile.ZipFile(zip_path, 'r') as zf:
# Should contain exactly one text file
names = zf.namelist()
cwa_book_downloader/release_sources/irc/source.py
+6 -29
View File
@@ -70,7 +70,7 @@ class IRCReleaseSource(ReleaseSource):
def __init__(self):
# Track online servers from most recent search
self._online_servers: Optional[list[str]] = None
self._online_servers: Optional[set[str]] = None
@classmethod
def is_available(cls) -> bool:
@@ -78,11 +78,7 @@ class IRCReleaseSource(ReleaseSource):
return config.get("IRC_ENABLED", False)
def get_column_config(self) -> ReleaseColumnConfig:
"""Configure UI columns for IRC results.
Includes online_servers from the most recent search, allowing
the frontend to show status indicators for each server.
"""
"""Configure UI columns for IRC results."""
return ReleaseColumnConfig(
columns=[
ColumnSchema(
@@ -124,17 +120,7 @@ class IRCReleaseSource(ReleaseSource):
languages: Optional[List[str]] = None,
content_type: str = "ebook"
) -> List[Release]:
"""Search IRC Highway for books matching metadata.
Args:
book: Book metadata (title, authors, etc.)
expand_search: Ignored - IRC always uses title+author search
languages: Ignored - IRC doesn't support language filtering
content_type: Ignored - IRC doesn't differentiate content types
Returns:
List of matching releases
"""
"""Search IRC Highway for books matching metadata."""
# Build search query
query = self._build_query(book)
if not query:
@@ -241,15 +227,9 @@ class IRCReleaseSource(ReleaseSource):
}
def _convert_to_releases(self, results: List[SearchResult]) -> List[Release]:
"""Convert parsed results to Release objects.
Results are sorted by:
1. Online status (online servers first)
2. Format priority (epub > mobi > azw3 > ...)
3. Server name (alphabetically)
"""
"""Convert parsed results to Release objects, sorted by online/format/server."""
releases = []
online_servers = self._online_servers or set()
online_servers = self._online_servers if self._online_servers else set()
for result in results:
release = Release(
@@ -287,10 +267,7 @@ class IRCReleaseSource(ReleaseSource):
@staticmethod
def _parse_size(size_str: str) -> Optional[int]:
"""Parse human-readable size to bytes.
Handles formats like: 1.2MB, 1.2M, 500KB, 500K, 1GB, 1G, etc.
"""
"""Parse human-readable size (e.g., '1.2MB', '500K') to bytes."""
if not size_str:
return None
cwa_book_downloader/release_sources/prowlarr/api.py
+23 -82
View File
@@ -1,14 +1,7 @@
"""
Prowlarr API client.
Handles communication with the Prowlarr API for:
- Connection testing
- Indexer listing
- Book search
"""
"""Prowlarr API client for connection testing, indexer listing, and search."""
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import urlencode, urljoin
from urllib.parse import urljoin
import requests
@@ -21,14 +14,6 @@ class ProwlarrClient:
"""Client for interacting with the Prowlarr API."""
def __init__(self, url: str, api_key: str, timeout: int = 30):
"""
Initialize the Prowlarr client.
Args:
url: Base URL of the Prowlarr instance (e.g., http://prowlarr:9696)
api_key: Prowlarr API key
timeout: Request timeout in seconds
"""
self.base_url = url.rstrip("/")
self.api_key = api_key
self.timeout = timeout
@@ -45,22 +30,7 @@ class ProwlarrClient:
params: Optional[Dict[str, Any]] = None,
json_data: Optional[Dict[str, Any]] = None,
) -> Any:
"""
Make an API request to Prowlarr.
Args:
method: HTTP method (GET, POST, etc.)
endpoint: API endpoint (e.g., /api/v1/search)
params: Query parameters
json_data: JSON body data
Returns:
Parsed JSON response
Raises:
requests.RequestException: On network errors
ValueError: On invalid JSON response
"""
"""Make an API request to Prowlarr. Returns parsed JSON response."""
url = urljoin(self.base_url, endpoint)
logger.debug(f"Prowlarr API: {method} {url}")
@@ -94,12 +64,7 @@ class ProwlarrClient:
raise
def test_connection(self) -> Tuple[bool, str]:
"""
Test the connection to Prowlarr.
Returns:
Tuple of (success: bool, message: str)
"""
"""Test connection to Prowlarr. Returns (success, message)."""
logger.info(f"Testing Prowlarr connection to: {self.base_url}")
try:
data = self._request("GET", "/api/v1/system/status")
@@ -117,12 +82,7 @@ class ProwlarrClient:
return False, f"Connection failed: {str(e)}"
def get_indexers(self) -> List[Dict[str, Any]]:
"""
Get all configured indexers.
Returns:
List of indexer configurations with id, name, protocol, enabled status
"""
"""Get all configured indexers."""
try:
indexers = self._request("GET", "/api/v1/indexer")
return indexers
@@ -131,12 +91,7 @@ class ProwlarrClient:
return []
def get_enabled_indexers(self) -> List[Dict[str, Any]]:
"""
Get enabled indexers with book-related info.
Returns:
List of enabled indexers with simplified structure
"""
"""Get enabled indexers with book capability info."""
indexers = self.get_indexers()
result = []
@@ -144,20 +99,9 @@ class ProwlarrClient:
if not idx.get("enable", False):
continue
# Check for book categories
capabilities = idx.get("capabilities", {})
categories = capabilities.get("categories", [])
has_books = False
for cat in categories:
cat_id = cat.get("id", 0)
if 7000 <= cat_id <= 7999:
has_books = True
break
for subcat in cat.get("subCategories", []):
if 7000 <= subcat.get("id", 0) <= 7999:
has_books = True
break
# Check for book categories (7000-7999 range)
categories = idx.get("capabilities", {}).get("categories", [])
has_books = self._has_book_categories(categories)
result.append({
"id": idx.get("id"),
@@ -168,6 +112,17 @@ class ProwlarrClient:
return result
def _has_book_categories(self, categories: List[Dict[str, Any]]) -> bool:
"""Check if any category or subcategory is in the book range (7000-7999)."""
for cat in categories:
cat_id = cat.get("id", 0)
if 7000 <= cat_id <= 7999:
return True
for subcat in cat.get("subCategories", []):
if 7000 <= subcat.get("id", 0) <= 7999:
return True
return False
def search(
self,
query: str,
@@ -175,33 +130,19 @@ class ProwlarrClient:
categories: Optional[List[int]] = None,
limit: int = 100,
) -> List[Dict[str, Any]]:
"""
Search for releases via Prowlarr.
Args:
query: Search query
indexer_ids: Specific indexer IDs to search (required for targeted search)
categories: Category IDs to filter by (optional)
limit: Maximum number of results to return (default: 100)
Returns:
List of search results
"""
"""Search for releases via Prowlarr."""
if not query:
return []
params = {"query": query, "limit": limit}
params: Dict[str, Any] = {"query": query, "limit": limit}
if indexer_ids:
params["indexerIds"] = indexer_ids
if categories:
params["categories"] = categories
endpoint = f"/api/v1/search?{urlencode(params, doseq=True)}"
try:
results = self._request("GET", endpoint)
results = self._request("GET", "/api/v1/search", params=params)
return results if isinstance(results, list) else []
except Exception as e:
logger.error(f"Prowlarr search failed: {e}")
return []
cwa_book_downloader/release_sources/prowlarr/clients/nzbget.py
+1 -2
View File
@@ -4,6 +4,7 @@ NZBGet download client for Prowlarr integration.
Uses NZBGet's JSON-RPC API directly via requests (no external dependency).
"""
import json
from typing import Any, Optional, Tuple
import requests
@@ -60,8 +61,6 @@ class NZBGetClient(DownloadClient):
"""
rpc_url = f"{self.url}/jsonrpc"
# Build JSON-RPC 2.0 request
import json
payload = json.dumps({
"jsonrpc": "2.0",
"id": 1,
cwa_book_downloader/release_sources/prowlarr/clients/qbittorrent.py
+2 -7
View File
@@ -1,6 +1,7 @@
"""qBittorrent download client for Prowlarr integration."""
import time
from types import SimpleNamespace
from typing import List, Optional, Tuple
from cwa_book_downloader.core.config import config
@@ -69,13 +70,7 @@ class QBittorrentClient(DownloadClient):
)
response.raise_for_status()
torrents = response.json()
class TorrentInfo:
def __init__(self, data):
for key, value in data.items():
setattr(self, key, value)
return [TorrentInfo(t) for t in torrents]
return [SimpleNamespace(**t) for t in torrents]
except requests.exceptions.HTTPError as e:
if e.response is not None and e.response.status_code == 403:
logger.warning("qBittorrent auth failed - check credentials")
cwa_book_downloader/release_sources/prowlarr/clients/sabnzbd.py
+45 -47
View File
@@ -19,6 +19,49 @@ from cwa_book_downloader.release_sources.prowlarr.clients import (
logger = setup_logger(__name__)
def _parse_eta(eta_str: str) -> Optional[int]:
"""Parse SABnzbd ETA string (format: 'H:MM:SS') to seconds."""
if not eta_str or eta_str == "0:00:00":
return None
try:
parts = eta_str.split(":")
if len(parts) == 3:
return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
except (ValueError, IndexError):
pass
return None
def _parse_speed(slot: dict) -> Optional[int]:
"""Parse download speed from SABnzbd slot data, returning bytes/sec."""
# Prefer kbpersec field (more reliable numeric value)
kbpersec_str = slot.get("kbpersec", "")
if kbpersec_str:
try:
return int(float(kbpersec_str) * 1024)
except (ValueError, TypeError):
pass
# Fall back to human-readable speed field
speed_str = slot.get("speed", "")
if not speed_str:
return None
try:
speed_parts = speed_str.split()
if len(speed_parts) < 2:
return None
speed_val = float(speed_parts[0])
unit = speed_parts[1].upper()
multipliers = {"K": 1024, "M": 1024**2, "G": 1024**3}
for prefix, mult in multipliers.items():
if prefix in unit:
return int(speed_val * mult)
return int(speed_val)
except (ValueError, IndexError):
return None
@register_client("usenet")
class SABnzbdClient(DownloadClient):
"""SABnzbd download client using REST API."""
@@ -179,59 +222,14 @@ class SABnzbdClient(DownloadClient):
}
state = status_mapping.get(status_text, "downloading")
# Parse ETA (format: "0:01:23" or empty)
eta_str = slot.get("timeleft", "")
eta_seconds = None
if eta_str and eta_str != "0:00:00":
try:
parts = eta_str.split(":")
if len(parts) == 3:
eta_seconds = (
int(parts[0]) * 3600
+ int(parts[1]) * 60
+ int(parts[2])
)
except (ValueError, IndexError):
pass # ETA display is optional
# Parse speed - prefer kbpersec field (more reliable numeric value)
download_speed = None
kbpersec_str = slot.get("kbpersec", "")
if kbpersec_str:
try:
kbpersec = float(kbpersec_str)
download_speed = int(kbpersec * 1024) # Convert KB/s to bytes/s
except (ValueError, TypeError):
pass # Speed display is optional
# Fall back to human-readable speed field if kbpersec not available
if download_speed is None:
speed_str = slot.get("speed", "")
if speed_str:
try:
speed_parts = speed_str.split()
if len(speed_parts) >= 2:
speed_val = float(speed_parts[0])
unit = speed_parts[1].upper()
if "K" in unit:
download_speed = int(speed_val * 1024)
elif "M" in unit:
download_speed = int(speed_val * 1024 * 1024)
elif "G" in unit:
download_speed = int(speed_val * 1024 * 1024 * 1024)
else:
download_speed = int(speed_val)
except (ValueError, IndexError):
pass # Speed display is optional
return DownloadStatus(
progress=percentage,
state=state,
message=status_text.lower().replace("_", " ").title(),
complete=False,
file_path=None,
download_speed=download_speed,
eta=eta_seconds,
download_speed=_parse_speed(slot),
eta=_parse_eta(slot.get("timeleft", "")),
)
# Not in queue, check history
cwa_book_downloader/release_sources/prowlarr/handler.py
+29 -66
View File
@@ -1,10 +1,4 @@
"""
Prowlarr download handler.
Handles downloads from Prowlarr via external download clients.
Supported torrent clients: qBittorrent, Transmission, Deluge.
Supported usenet clients: NZBGet, SABnzbd.
"""
"""Prowlarr download handler - executes downloads via torrent/usenet clients."""
import shutil
from pathlib import Path
@@ -14,6 +8,7 @@ from typing import Callable, Optional
from cwa_book_downloader.core.config import config
from cwa_book_downloader.core.logger import setup_logger
from cwa_book_downloader.core.models import DownloadTask
from cwa_book_downloader.core.utils import is_audiobook
from cwa_book_downloader.release_sources import DownloadHandler, register_handler
from cwa_book_downloader.release_sources.prowlarr.cache import get_release, remove_release
from cwa_book_downloader.release_sources.prowlarr.clients import (
@@ -34,31 +29,38 @@ class ProwlarrHandler(DownloadHandler):
"""Handler for Prowlarr downloads via configured torrent or usenet client."""
def _get_category_for_task(self, client, task: DownloadTask) -> Optional[str]:
"""Get the appropriate category based on content type.
Returns the audiobook-specific category if configured and the task is an audiobook,
otherwise returns None to let the client use its default category.
"""
is_audiobook = task.content_type and "audiobook" in task.content_type.lower()
if not is_audiobook:
"""Get audiobook category if configured and applicable, else None for default."""
if not is_audiobook(task.content_type):
return None
# Client-specific audiobook category config keys
audiobook_key = {
audiobook_keys = {
"qbittorrent": "QBITTORRENT_CATEGORY_AUDIOBOOK",
"transmission": "TRANSMISSION_CATEGORY_AUDIOBOOK",
"deluge": "DELUGE_CATEGORY_AUDIOBOOK",
"nzbget": "NZBGET_CATEGORY_AUDIOBOOK",
"sabnzbd": "SABNZBD_CATEGORY_AUDIOBOOK",
}.get(client.name)
}
audiobook_key = audiobook_keys.get(client.name)
return config.get(audiobook_key, "") or None if audiobook_key else None
if audiobook_key:
audiobook_cat = config.get(audiobook_key, "")
if audiobook_cat:
return audiobook_cat
def _build_progress_message(self, status) -> str:
"""Build a progress message from download status."""
msg = f"{status.progress:.0f}%"
return None # Let client use its default
if status.download_speed and status.download_speed > 0:
speed_mb = status.download_speed / 1024 / 1024
msg += f" ({speed_mb:.1f} MB/s)"
if status.eta and status.eta > 0:
if status.eta < 60:
msg += f" - {status.eta}s left"
elif status.eta < 3600:
msg += f" - {status.eta // 60}m left"
else:
msg += f" - {status.eta // 3600}h {(status.eta % 3600) // 60}m left"
return msg
def download(
self,
@@ -67,18 +69,7 @@ class ProwlarrHandler(DownloadHandler):
progress_callback: Callable[[float], None],
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[str]:
"""
Execute a Prowlarr download.
Args:
task: Download task with task_id (Prowlarr source_id/GUID)
cancel_flag: Event to check for cancellation
progress_callback: Called with progress percentage (0-100)
status_callback: Called with (status, message) for status updates
Returns:
Path to downloaded file if successful, None otherwise
"""
"""Execute download via configured torrent/usenet client. Returns file path or None."""
try:
# Look up the cached release
prowlarr_result = get_release(task.task_id)
@@ -205,24 +196,8 @@ class ProwlarrHandler(DownloadHandler):
client.remove(download_id, delete_files=True)
return None
# Build status message
# If client provided a specific message (e.g., "Stalled", "Fetching metadata"),
# use that. Otherwise, build a progress message.
if status.message:
msg = status.message
else:
msg = f"{status.progress:.0f}%"
if status.download_speed and status.download_speed > 0:
speed_mb = status.download_speed / 1024 / 1024
msg += f" ({speed_mb:.1f} MB/s)"
if status.eta and status.eta > 0:
if status.eta < 60:
msg += f" - {status.eta}s left"
elif status.eta < 3600:
msg += f" - {status.eta // 60}m left"
else:
msg += f" - {status.eta // 3600}h {(status.eta % 3600) // 60}m left"
# Build status message - use client message if provided, else build progress
msg = status.message or self._build_progress_message(status)
status_callback("downloading", msg)
# Wait for next poll (interruptible by cancel)
@@ -271,14 +246,7 @@ class ProwlarrHandler(DownloadHandler):
task: DownloadTask,
status_callback: Callable[[str, Optional[str]], None],
) -> Optional[str]:
"""Handle completed download - stage or return path for orchestrator.
For torrents: returns original path directly (no copy). Orchestrator
will hardlink or copy as needed, avoiding unnecessary staging of
large files like audiobooks.
For usenet: stages to temp directory based on config.
"""
"""Handle completed download. Torrents return original path; usenet stages to temp."""
try:
# For torrents, skip staging - return original path directly
# Orchestrator will hardlink (library mode) or copy (ingest mode) as needed
@@ -321,12 +289,7 @@ class ProwlarrHandler(DownloadHandler):
return None
def cancel(self, task_id: str) -> bool:
"""
Cancel an in-progress download.
Note: Actual cancellation is handled via the cancel_flag in download().
This method is for cleanup if the cancel_flag mechanism fails.
"""
"""Cancel download and clean up cache. Primary cancellation is via cancel_flag."""
logger.debug(f"Cancel requested for Prowlarr task: {task_id}")
# Remove from cache if present
remove_release(task_id)
cwa_book_downloader/release_sources/prowlarr/source.py
+59 -156
View File
@@ -1,9 +1,4 @@
"""
Prowlarr release source implementation.
Implements the ReleaseSource interface to search Prowlarr indexers
for book releases (torrents and usenet).
"""
"""Prowlarr release source - searches indexers for book releases (torrents/usenet)."""
import re
from typing import List, Optional
@@ -60,53 +55,27 @@ ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS
def _extract_format(title: str) -> Optional[str]:
"""
Extract format from release title with smart parsing.
Supports both ebook formats (epub, mobi, etc.) and audiobook formats (m4b, mp3, etc.).
Priority:
1. File extension at end of title or in quotes (e.g., ".azw3", ".m4b")
2. Format keyword in brackets/parentheses (e.g., "[EPUB]", "(MP3)")
3. Format as standalone word (not part of another word)
"""
"""Extract ebook/audiobook format from release title (extension, bracketed, or standalone)."""
title_lower = title.lower()
# 1. Look for file extensions (most reliable) - pattern: .format at word boundary or end
# This catches ".azw3", ".epub", ".m4b", ".mp3", etc.
for fmt in ALL_BOOK_FORMATS:
# Match .format at end of string or followed by non-alphanumeric
pattern = rf'\.{fmt}(?:["\'\s\]\)]|$)'
if re.search(pattern, title_lower):
return fmt
# Pattern priority: file extension > bracketed > standalone word
# Use %s placeholder since {fmt} conflicts with regex syntax
pattern_templates = [
r'\.%s(?:["\'\s\]\)]|$)', # .format at end or followed by delimiter
r'[\[\(\{]%s[\]\)\}]', # [EPUB], (PDF), {mobi}
r'\b%s\b', # standalone word
]
# 2. Look for format in brackets/parentheses (common in release names)
# e.g., "[EPUB]", "(PDF)", "{mobi}", "[M4B]", "(MP3)"
for fmt in ALL_BOOK_FORMATS:
pattern = rf'[\[\(\{{]{fmt}[\]\)\}}]'
if re.search(pattern, title_lower):
return fmt
# 3. Look for format as standalone word (not part of another word)
# e.g., "epub" but not "republic", "m4b" but not "m4b123"
for fmt in ALL_BOOK_FORMATS:
# Match format as whole word
pattern = rf'\b{fmt}\b'
if re.search(pattern, title_lower):
return fmt
for template in pattern_templates:
for fmt in ALL_BOOK_FORMATS:
if re.search(template % fmt, title_lower):
return fmt
return None
def _extract_language(title: str) -> Optional[str]:
"""
Extract language from release title.
Common patterns:
- [German], (French), {Spanish}
- German, French, etc. as standalone words
- Language codes like [DE], [FR], [ES]
"""
"""Extract language code from release title (e.g., [German] -> 'de')."""
title_lower = title.lower()
# Common language names and their codes
@@ -139,87 +108,50 @@ EBOOK_CATEGORY_IDS = {7000, 7020} # 7000 = Books, 7020 = Books/Ebook
def _detect_content_type_from_categories(categories: list, fallback: str = "book") -> str:
"""
Detect content type from Prowlarr category IDs.
Prowlarr returns categories as a list of dicts with 'id' and 'name' keys,
or sometimes just category IDs directly.
Args:
categories: List of category objects from Prowlarr result
fallback: Content type to use if categories don't indicate a specific type
Returns:
"audiobook" if audiobook categories detected, "book" otherwise
"""
"""Detect content type from Prowlarr category IDs. Returns 'audiobook' or 'book'."""
# Normalize fallback - convert "ebook" to "book" for display consistency
if fallback == "ebook":
fallback = "book"
normalized_fallback = "book" if fallback == "ebook" else fallback
if not categories:
return fallback
return normalized_fallback
# Extract category IDs from the nested structure
cat_ids = set()
for cat in categories:
if isinstance(cat, dict):
cat_id = cat.get("id")
if cat_id is not None:
cat_ids.add(cat_id)
elif isinstance(cat, int):
cat_ids.add(cat)
cat_ids = {
cat.get("id") if isinstance(cat, dict) else cat
for cat in categories
if (isinstance(cat, dict) and cat.get("id") is not None) or isinstance(cat, int)
}
# Check for audiobook categories first (more specific)
# Check for audiobook categories first (more specific), then ebook
if cat_ids & AUDIOBOOK_CATEGORY_IDS:
return "audiobook"
# Check for ebook categories
if cat_ids & EBOOK_CATEGORY_IDS:
return "book"
# Fall back to normalized content_type if no recognized categories
return fallback
return normalized_fallback
def _prowlarr_result_to_release(result: dict, search_content_type: str = "ebook") -> Release:
"""
Convert a Prowlarr search result to a Release object.
Uses structured fields from Prowlarr when available:
- protocol: Direct from Prowlarr
- fileName: For format detection (more reliable than title)
- categories: To detect content type (audiobook vs ebook)
- grabs: Download count
Args:
result: Raw Prowlarr API result
search_content_type: Content type from search, used as fallback if
categories don't indicate a specific type
"""
"""Convert a Prowlarr API result to a Release object."""
title = result.get("title", "Unknown")
size_bytes = result.get("size")
download_url = result.get("downloadUrl") or result.get("magnetUrl")
info_url = result.get("infoUrl") or result.get("guid")
indexer = result.get("indexer", "Unknown")
protocol = get_protocol_display(result)
seeders = result.get("seeders")
leechers = result.get("leechers")
categories = result.get("categories", [])
is_torrent = protocol == "torrent"
# Format peers display string: "seeders / leechers"
peers_display = f"{seeders} / {leechers}" if (seeders is not None and leechers is not None) else None
grabs = result.get("grabs")
peers_display = (
f"{seeders} / {leechers}"
if is_torrent and seeders is not None and leechers is not None
else None
)
# For format detection, prefer fileName over title (often cleaner)
file_name = result.get("fileName", "")
format_detected = _extract_format(file_name) if file_name else None
if not format_detected:
format_detected = _extract_format(title)
# Extract language from title (Prowlarr doesn't provide this structured)
language = _extract_language(title)
# Detect content type from categories (per-result), with search type as fallback
categories = result.get("categories", [])
content_type = _detect_content_type_from_categories(categories, search_content_type)
format_detected = _extract_format(file_name) if file_name else _extract_format(title)
# Build the source_id from GUID or generate from indexer + title
source_id = result.get("guid") or f"{indexer}:{hash(title)}"
@@ -232,34 +164,29 @@ def _prowlarr_result_to_release(result: dict, search_content_type: str = "ebook"
source_id=source_id,
title=title,
format=format_detected,
language=language,
language=_extract_language(title),
size=_parse_size(size_bytes),
size_bytes=size_bytes,
download_url=download_url,
info_url=info_url,
download_url=result.get("downloadUrl") or result.get("magnetUrl"),
info_url=result.get("infoUrl") or result.get("guid"),
protocol=protocol,
indexer=indexer,
seeders=seeders if protocol == "torrent" else None,
peers=peers_display if protocol == "torrent" else None,
content_type=content_type, # Detected per-result from categories
seeders=seeders if is_torrent else None,
peers=peers_display,
content_type=_detect_content_type_from_categories(categories, search_content_type),
extra={
"publish_date": result.get("publishDate"),
"categories": categories,
"indexer_id": result.get("indexerId"),
"files": result.get("files"),
"grabs": grabs,
"grabs": result.get("grabs"),
},
)
@register_source("prowlarr")
class ProwlarrSource(ReleaseSource):
"""
Prowlarr release source.
Searches Prowlarr indexers for book releases (torrents and usenet).
Supports both ebooks (category 7000) and audiobooks (category 3030).
"""
"""Prowlarr release source for ebooks and audiobooks."""
name = "prowlarr"
display_name = "Prowlarr"
@@ -371,21 +298,7 @@ class ProwlarrSource(ReleaseSource):
languages: Optional[List[str]] = None,
content_type: str = "ebook"
) -> List[Release]:
"""
Search Prowlarr for releases matching the book.
Makes separate API calls for each selected indexer to ensure
all indexers are properly queried regardless of their capabilities.
Args:
book: Book metadata to search for
expand_search: If True, skip category filtering (broader search)
languages: Ignored - Prowlarr doesn't support language filtering
content_type: "ebook" or "audiobook" - determines search categories
Returns:
List of Release objects
"""
"""Search Prowlarr indexers for releases matching the book."""
client = self._get_client()
if not client:
logger.warning("Prowlarr not configured - skipping search")
@@ -422,43 +335,33 @@ class ProwlarrSource(ReleaseSource):
# Get search categories based on content type
# Audiobooks use 3030 (Audio/Audiobook), ebooks use 7000 (Books)
if content_type == "audiobook":
search_categories = [3030] # Audio/Audiobook category
else:
search_categories = [7000] # Books category
if expand_search:
# Expand search: search all indexers without category filtering
categories = None
self.last_search_type = "expanded"
else:
categories = search_categories
self.last_search_type = "categories"
search_categories = [3030] if content_type == "audiobook" else [7000]
categories = None if expand_search else search_categories
self.last_search_type = "expanded" if expand_search else "categories"
logger.debug(f"Searching Prowlarr: query='{query}', indexers={indexer_ids}, categories={categories}")
all_results = []
try:
def search_indexers(cats: Optional[List[int]]) -> List[dict]:
"""Search all indexers with given categories, collecting results."""
results = []
for indexer_id in indexer_ids:
try:
raw_results = client.search(query=query, indexer_ids=[indexer_id], categories=categories)
if raw_results:
all_results.extend(raw_results)
raw = client.search(query=query, indexer_ids=[indexer_id], categories=cats)
if raw:
results.extend(raw)
except Exception as e:
logger.warning(f"Search failed for indexer {indexer_id}: {e}")
return results
# Auto-expand: if no results with categories and auto-expand enabled, retry without categories
all_results = []
try:
all_results = search_indexers(categories)
# Auto-expand: if no results with categories and auto-expand enabled, retry without
auto_expand_enabled = config.get("PROWLARR_AUTO_EXPAND", False)
logger.debug(f"Auto-expand check: no_results={not all_results}, has_categories={bool(categories)}, auto_expand_enabled={auto_expand_enabled}")
if not all_results and categories and auto_expand_enabled:
logger.info("Prowlarr: no results with category filter, auto-expanding search")
for indexer_id in indexer_ids:
try:
raw_results = client.search(query=query, indexer_ids=[indexer_id], categories=None)
if raw_results:
all_results.extend(raw_results)
except Exception as e:
logger.warning(f"Expanded search failed for indexer {indexer_id}: {e}")
all_results = search_indexers(None)
self.last_search_type = "expanded"
results = [_prowlarr_result_to_release(r, content_type) for r in all_results]
docker-compose.dev.yml
+3 -3
View File
@@ -12,9 +12,9 @@ services:
DEBUG: true
volumes:
- ./.local/config:/config
- ./.local/ingest:/cwa-book-ingest
- ./.local/books:/books
- ./.local/log:/var/log/cwa-book-downloader
- ./.local/tmp:/tmp/cwa-book-downloader
- ./cwa_book_downloader:/app/cwa_book_downloader:ro
# Download client mount (must match your torrent/usenet client's volume)
# - /path/to/downloads:/downloads
# Download client mount - path must match your torrent/usenet client's volume exactly
# - /path/to/downloads:/path/to/downloads
docker-compose.extbp.dev.yml
+3 -3
View File
@@ -15,11 +15,11 @@ services:
EXT_BYPASSER_TIMEOUT: 60000
volumes:
- ./.local/config:/config
- ./.local/ingest:/cwa-book-ingest
- ./.local/books:/books
- ./.local/log:/var/log/cwa-book-downloader
- ./.local/tmp:/tmp/cwa-book-downloader
# Download client mount (must match your torrent/usenet client's volume)
# - /path/to/downloads:/downloads
# Download client mount - path must match your torrent/usenet client's volume exactly
# - /path/to/downloads:/path/to/downloads
flaresolverr:
image: ghcr.io/flaresolverr/flaresolverr:latest
docker-compose.extbp.yml
+3 -3
View File
@@ -11,10 +11,10 @@ services:
- 8084:8084
restart: unless-stopped
volumes:
- /path/to/ingest:/cwa-book-ingest # Book ingest directory
- /path/to/books:/books # Book destination directory
- /path/to/config:/config # App configuration
# Download client mount (must match your torrent/usenet client's volume)
# - /path/to/downloads:/downloads
# Download client mount - path must match your torrent/usenet client's volume exactly
# - /path/to/downloads:/path/to/downloads
flaresolverr:
image: ghcr.io/flaresolverr/flaresolverr:latest
docker-compose.test-clients.yml
+2 -2
View File
@@ -43,8 +43,8 @@ services:
# Config and state
- ./.local/test-clients/cwabd/config:/config
- ./.local/test-clients/cwabd/log:/var/log/cwa-book-downloader
# Ingest directory (where completed books go)
- ./.local/test-clients/ingest:/cwa-book-ingest
# Book destination directory (where completed books go)
- ./.local/test-clients/books:/books
# Staging directory
- ./.local/test-clients/tmp:/tmp/cwa-book-downloader
# CRITICAL: Mount client download directories so cwabd can access completed files
docker-compose.tor.dev.yml
+3 -3
View File
@@ -12,8 +12,8 @@ services:
DEBUG: true
volumes:
- ./.local/config:/config
- ./.local/ingest:/cwa-book-ingest
- ./.local/books:/books
- ./.local/log:/var/log/cwa-book-downloader
- ./.local/tmp:/tmp/cwa-book-downloader
# Download client mount (must match your torrent/usenet client's volume)
# - /path/to/downloads:/downloads
# Download client mount - path must match your torrent/usenet client's volume exactly
# - /path/to/downloads:/path/to/downloads
docker-compose.tor.yml
+3 -3
View File
@@ -15,7 +15,7 @@ services:
- 8084:8084
restart: unless-stopped
volumes:
- /path/to/ingest:/cwa-book-ingest # Book ingest directory
- /path/to/books:/books # Book destination directory
- /path/to/config:/config # App configuration
# Download client mount (must match your torrent/usenet client's volume)
# - /path/to/downloads:/downloads
# Download client mount - path must match your torrent/usenet client's volume exactly
# - /path/to/downloads:/path/to/downloads
docker-compose.yml
+3 -3
View File
@@ -10,7 +10,7 @@ services:
- 8084:8084
restart: unless-stopped
volumes:
- /path/to/ingest:/cwa-book-ingest # Book ingest directory
- /path/to/books:/books # Book destination directory
- /path/to/config:/config # App configuration
# Download client mount (must match your torrent/usenet client's volume)
# - /path/to/downloads:/downloads
# Download client mount - path must match your torrent/usenet client's volume exactly
# - /path/to/downloads:/path/to/downloads
entrypoint.sh
+5
View File
@@ -203,6 +203,11 @@ fi
echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)"
# Set umask for file permissions (default: 0022 = files 644, dirs 755)
UMASK_VALUE=${UMASK:-0022}
echo "Setting umask to $UMASK_VALUE"
umask $UMASK_VALUE
# Stop logging
exec 1>&3 2>&4
exec 3>&- 4>&-
src/frontend/src/App.tsx
+2 -2
View File
@@ -608,8 +608,8 @@ function App() {
/>
{isInitialState && !featureNoticeDismissed && (
<div className="absolute bottom-4 left-0 right-0 text-center text-sm text-gray-500 dark:text-gray-400">
<span>We've added Prowlarr and IRC support for more download options. Check Settings to configure.</span>
<div className="absolute bottom-4 left-0 right-0 px-4 text-center text-sm text-gray-500 dark:text-gray-400">
<span>New: Torrent, Usenet and IRC downloads, Audiobook support and more. Configure in Settings</span>
<button
onClick={handleDismissFeatureNotice}
className="ml-2 text-blue-500 hover:text-blue-600 dark:text-blue-400 dark:hover:text-blue-300 underline"
tests/config/test_environment.py
+55 -44
View File
@@ -97,28 +97,37 @@ class TestSupportedFormats:
def test_default_supported_formats(self):
"""Default formats should include common ebook formats."""
from cwa_book_downloader.config.env import _SUPPORTED_FORMATS
from cwa_book_downloader.core.config import config
# Ensure settings are refreshed to pick up defaults
config.refresh()
formats = config.get("SUPPORTED_FORMATS", [])
# Check some expected defaults
assert "epub" in _SUPPORTED_FORMATS
assert "mobi" in _SUPPORTED_FORMATS
assert "azw3" in _SUPPORTED_FORMATS
assert "epub" in formats
assert "mobi" in formats
assert "azw3" in formats
def test_format_list_is_lowercase(self):
"""Format list should be normalized to lowercase."""
from cwa_book_downloader.config.env import _SUPPORTED_FORMATS
from cwa_book_downloader.core.config import config
# Ensure settings are refreshed to pick up defaults
config.refresh()
formats = config.get("SUPPORTED_FORMATS", [])
# All formats should be lowercase
for fmt in _SUPPORTED_FORMATS.split(","):
for fmt in formats:
assert fmt == fmt.lower()
def test_config_supported_formats_attribute(self):
def test_config_supported_formats_is_list(self):
"""Config should have SUPPORTED_FORMATS as a list."""
from cwa_book_downloader.config.settings import SUPPORTED_FORMATS
from cwa_book_downloader.core.config import config
# Ensure settings are refreshed to pick up defaults
config.refresh()
assert isinstance(SUPPORTED_FORMATS, list)
assert len(SUPPORTED_FORMATS) > 0
assert "epub" in SUPPORTED_FORMATS
formats = config.get("SUPPORTED_FORMATS", [])
assert isinstance(formats, list)
assert len(formats) > 0
assert "epub" in formats
# =============================================================================
@@ -369,25 +378,21 @@ class TestDebugConfiguration:
class TestNetworkConfiguration:
"""Tests for proxy and network settings."""
def test_proxy_settings_stripped(self):
"""Proxy URLs should be stripped of whitespace."""
from cwa_book_downloader.config.env import HTTP_PROXY, HTTPS_PROXY
def test_proxy_settings_default(self):
"""Proxy settings should have sensible defaults."""
from cwa_book_downloader.core.config import config
config.refresh()
# These are already evaluated, but the logic is:
# HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
# So whitespace should be removed
assert HTTP_PROXY == HTTP_PROXY.strip()
assert HTTPS_PROXY == HTTPS_PROXY.strip()
# Default proxy mode should be 'none' (no proxy)
assert config.get("PROXY_MODE", "none") == "none"
def test_tor_mode_disables_other_network_settings(self):
"""Tor mode should disable custom DNS, DOH, and proxies."""
# This is a documentation test - the logic is in env.py:
# if USING_TOR:
# _CUSTOM_DNS = ""
# USE_DOH = False
# HTTP_PROXY = ""
# HTTPS_PROXY = ""
pass
def test_tor_mode_is_detected(self):
"""Tor mode should be detected from container variant."""
from cwa_book_downloader.config.env import TOR_VARIANT_AVAILABLE
# In regular test environment, Tor should not be available
# (unless running in Tor container)
assert isinstance(TOR_VARIANT_AVAILABLE, bool)
# =============================================================================
@@ -400,17 +405,21 @@ class TestConcurrencyConfiguration:
def test_max_concurrent_downloads_default(self):
"""MAX_CONCURRENT_DOWNLOADS should have a sensible default."""
from cwa_book_downloader.config.env import MAX_CONCURRENT_DOWNLOADS
from cwa_book_downloader.core.config import config
config.refresh()
assert MAX_CONCURRENT_DOWNLOADS >= 1
assert MAX_CONCURRENT_DOWNLOADS <= 10 # Reasonable upper bound
max_downloads = config.get("MAX_CONCURRENT_DOWNLOADS", 3)
assert max_downloads >= 1
assert max_downloads <= 10 # Reasonable upper bound
def test_download_progress_interval_default(self):
"""DOWNLOAD_PROGRESS_UPDATE_INTERVAL should have a sensible default."""
from cwa_book_downloader.config.env import DOWNLOAD_PROGRESS_UPDATE_INTERVAL
from cwa_book_downloader.core.config import config
config.refresh()
assert DOWNLOAD_PROGRESS_UPDATE_INTERVAL >= 1
assert DOWNLOAD_PROGRESS_UPDATE_INTERVAL <= 10
interval = config.get("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", 1)
assert interval >= 1
assert interval <= 10
# =============================================================================
@@ -423,22 +432,24 @@ class TestCacheConfiguration:
def test_metadata_cache_ttl_defaults(self):
"""Metadata cache TTLs should have sensible defaults."""
from cwa_book_downloader.config.env import (
METADATA_CACHE_SEARCH_TTL,
METADATA_CACHE_BOOK_TTL,
)
from cwa_book_downloader.core.config import config
config.refresh()
search_ttl = config.get("METADATA_CACHE_SEARCH_TTL", 300)
book_ttl = config.get("METADATA_CACHE_BOOK_TTL", 600)
# Search cache should be shorter than book cache
assert METADATA_CACHE_SEARCH_TTL > 0
assert METADATA_CACHE_BOOK_TTL > 0
assert METADATA_CACHE_SEARCH_TTL <= METADATA_CACHE_BOOK_TTL
assert search_ttl > 0
assert book_ttl > 0
assert search_ttl <= book_ttl
def test_covers_cache_directory(self):
"""Covers cache directory should be under CONFIG_DIR."""
from cwa_book_downloader.config.env import CONFIG_DIR, COVERS_CACHE_DIR
from cwa_book_downloader.config.env import CONFIG_DIR
assert COVERS_CACHE_DIR.parent == CONFIG_DIR
assert COVERS_CACHE_DIR.name == "covers"
covers_dir = CONFIG_DIR / "covers"
assert covers_dir.parent == CONFIG_DIR
assert covers_dir.name == "covers"
# =============================================================================
tests/core/test_download_processing.py
+2 -2
View File
@@ -550,7 +550,7 @@ class TestPostProcessDownload:
assert result is not None
result_path = Path(result)
assert library in result_path.parents or result_path.parent == library
status_cb.assert_called_with("complete", "Complete (library mode)")
status_cb.assert_called_with("complete", "Complete")
def test_direct_mode_skips_library(self, temp_dirs, sample_direct_task):
"""Direct mode skips library mode even when configured."""
@@ -993,7 +993,7 @@ class TestDownloadProcessingIntegration:
assert "The Way of Kings" in result_path.name
# Staging file cleaned up
assert not temp_file.exists()
status_cb.assert_called_with("complete", "Complete (library mode)")
status_cb.assert_called_with("complete", "Complete")
def test_library_fallback_to_ingest(self, temp_dirs, sample_task):
"""Falls back to ingest when library mode fails."""
tests/core/test_hardlink.py
+12 -12
View File
@@ -303,7 +303,7 @@ class TestHardlinkWithLibraryMode:
assert source.exists()
# Temp file should be cleaned up
assert not temp_file.exists()
status_cb.assert_called_with("complete", "Complete (library mode)")
status_cb.assert_called_with("complete", "Complete")
def test_transfer_file_move(self, tmp_path, sample_task):
"""Single file transferred via move."""
@@ -333,7 +333,7 @@ class TestHardlinkWithLibraryMode:
assert result_path.exists()
# Source should NOT exist (moved)
assert not source.exists()
status_cb.assert_called_with("complete", "Complete (library mode)")
status_cb.assert_called_with("complete", "Complete")
def test_transfer_directory_hardlink_multifile(self, tmp_path, sample_task):
"""Directory with multiple files transferred via hardlinks."""
@@ -477,7 +477,7 @@ class TestHardlinkDecisionLogic:
def test_hardlink_enabled_same_filesystem(self, tmp_path, sample_task):
"""Hardlink used when enabled and same filesystem."""
from cwa_book_downloader.download.orchestrator import _process_library_mode
from cwa_book_downloader.download.orchestrator import _process_organize_mode
library = tmp_path / "library"
library.mkdir()
@@ -502,7 +502,7 @@ class TestHardlinkDecisionLogic:
"PROCESSING_MODE": "library",
}.get(key, default))
result = _process_library_mode(staged, sample_task, status_cb)
result = _process_organize_mode(staged, sample_task, status_cb)
assert result is not None
# Source should still exist (hardlinked)
@@ -510,7 +510,7 @@ class TestHardlinkDecisionLogic:
def test_hardlink_disabled_falls_back_to_move(self, tmp_path, sample_task):
"""Move used when hardlink disabled in config."""
from cwa_book_downloader.download.orchestrator import _process_library_mode
from cwa_book_downloader.download.orchestrator import _process_organize_mode
library = tmp_path / "library"
library.mkdir()
@@ -533,7 +533,7 @@ class TestHardlinkDecisionLogic:
"PROCESSING_MODE": "library",
}.get(key, default))
result = _process_library_mode(staged, sample_task, status_cb)
result = _process_organize_mode(staged, sample_task, status_cb)
assert result is not None
# Staged file should be moved (not exist)
@@ -541,7 +541,7 @@ class TestHardlinkDecisionLogic:
def test_no_original_path_uses_staging(self, tmp_path, sample_task):
"""Without original_download_path, moves from staging."""
from cwa_book_downloader.download.orchestrator import _process_library_mode
from cwa_book_downloader.download.orchestrator import _process_organize_mode
library = tmp_path / "library"
library.mkdir()
@@ -562,7 +562,7 @@ class TestHardlinkDecisionLogic:
"PROCESSING_MODE": "library",
}.get(key, default))
result = _process_library_mode(staged, sample_task, status_cb)
result = _process_organize_mode(staged, sample_task, status_cb)
assert result is not None
# Staged file should be moved
@@ -803,7 +803,7 @@ class TestEdgeCases:
def test_nonexistent_source_for_hardlink(self, tmp_path):
"""Missing source file prevents hardlink creation."""
from cwa_book_downloader.download.orchestrator import _process_library_mode
from cwa_book_downloader.download.orchestrator import _process_organize_mode
from cwa_book_downloader.core.models import DownloadTask, SearchMode
task = DownloadTask(
@@ -832,7 +832,7 @@ class TestEdgeCases:
"PROCESSING_MODE": "library",
}.get(key, default))
result = _process_library_mode(staged, task, status_cb)
result = _process_organize_mode(staged, task, status_cb)
# Should fall back to move since original doesn't exist
assert result is not None
@@ -840,7 +840,7 @@ class TestEdgeCases:
def test_permission_denied_library_path(self, tmp_path):
"""Handles permission denied on library path."""
from cwa_book_downloader.download.orchestrator import _process_library_mode
from cwa_book_downloader.download.orchestrator import _process_organize_mode
from cwa_book_downloader.core.models import DownloadTask, SearchMode
task = DownloadTask(
@@ -865,7 +865,7 @@ class TestEdgeCases:
"PROCESSING_MODE": "library",
}.get(key, default))
result = _process_library_mode(staged, task, status_cb)
result = _process_organize_mode(staged, task, status_cb)
# Should return None (fall back to ingest)
assert result is None
tests/core/test_library_processing.py
+4 -8
View File
@@ -18,6 +18,7 @@ from unittest.mock import MagicMock, patch
from cwa_book_downloader.core.models import DownloadTask, SearchMode
from cwa_book_downloader.core.naming import build_library_path, assign_part_numbers
from cwa_book_downloader.core.utils import is_audiobook
class MockConfig:
@@ -118,8 +119,7 @@ class TestContentTypeDetection:
search_mode=SearchMode.UNIVERSAL,
)
content_type = task.content_type.lower() if task.content_type else ""
assert "audiobook" in content_type
assert is_audiobook(task.content_type)
def test_detect_book_content_type(self):
"""Verify ebook detection from content_type field."""
@@ -132,9 +132,7 @@ class TestContentTypeDetection:
search_mode=SearchMode.UNIVERSAL,
)
content_type = task.content_type.lower() if task.content_type else ""
is_audiobook = "audiobook" in content_type
assert not is_audiobook
assert not is_audiobook(task.content_type)
def test_empty_content_type_defaults_to_book(self):
"""Empty content_type should be treated as a book."""
@@ -146,9 +144,7 @@ class TestContentTypeDetection:
search_mode=SearchMode.UNIVERSAL,
)
content_type = task.content_type.lower() if task.content_type else ""
is_audiobook = "audiobook" in content_type
assert not is_audiobook
assert not is_audiobook(task.content_type)
class TestLibraryPathBuilding:
tests/prowlarr/test_failure_scenarios.py
+17 -6
View File
@@ -581,10 +581,19 @@ class TestFileHandlingFailures:
assert recorder.had_error
assert "locate" in recorder.last_message.lower()
def test_permission_denied_on_copy(
self, handler, mock_client, recorder, cancel_flag, sample_task, sample_release
def test_permission_denied_on_move(
self, handler, mock_client, recorder, cancel_flag, sample_task
):
"""Handler should report permission errors during file staging."""
"""Handler should report permission errors during file staging (usenet only - torrents skip staging)."""
# Use usenet protocol - torrents skip staging and return original path directly
# Default usenet action is "move", so we mock shutil.move
usenet_release = {
"guid": "test-task-123",
"title": "Test Book",
"downloadUrl": "https://indexer.example.com/download/123",
"protocol": "usenet",
"indexer": "TestIndexer",
}
mock_client.status_sequence = [
DownloadStatus(
progress=100,
@@ -598,21 +607,23 @@ class TestFileHandlingFailures:
with tempfile.TemporaryDirectory() as tmpdir:
source_file = Path(tmpdir) / "source.epub"
source_file.write_text("test content")
staging_dir = Path(tmpdir) / "staging"
staging_dir.mkdir()
mock_client.get_download_path = lambda x: str(source_file)
with patch(
"cwa_book_downloader.release_sources.prowlarr.handler.get_release",
return_value=sample_release,
return_value=usenet_release,
), patch(
"cwa_book_downloader.release_sources.prowlarr.handler.get_client",
return_value=mock_client,
), patch(
"shutil.copy2",
"cwa_book_downloader.release_sources.prowlarr.handler.shutil.move",
side_effect=PermissionError("Permission denied"),
), patch(
"cwa_book_downloader.download.orchestrator.get_staging_dir",
return_value=Path(tmpdir) / "staging",
return_value=staging_dir,
):
result = handler.download(
task=sample_task,
tests/prowlarr/test_handler.py
+4 -3
View File
@@ -604,7 +604,7 @@ class TestProwlarrHandlerFileStaging:
assert (staged_dir / "cover.jpg").exists()
def test_handles_duplicate_filename(self):
"""Test handling of duplicate filename during staging."""
"""Test handling of duplicate filename during staging (usenet only - torrents skip staging)."""
with tempfile.TemporaryDirectory() as tmp_dir:
source_file = Path(tmp_dir) / "source" / "book.epub"
source_file.parent.mkdir(parents=True)
@@ -628,11 +628,12 @@ class TestProwlarrHandlerFileStaging:
)
mock_client.get_download_path.return_value = str(source_file)
# Use usenet protocol - torrents skip staging and return original path directly
with patch(
"cwa_book_downloader.release_sources.prowlarr.handler.get_release",
return_value={
"protocol": "torrent",
"magnetUrl": "magnet:?xt=urn:btih:abc123",
"protocol": "usenet",
"downloadUrl": "https://indexer.example.com/download/123",
},
), patch(
"cwa_book_downloader.release_sources.prowlarr.handler.get_client",