From a0f8d14c45e193ba6e212041b99341a5ea0e152d Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 28 Dec 2025 22:51:40 +0000 Subject: [PATCH] Hardcover enhancements, refactor and cleanup, PUID/PGID additions (#365) --- Dockerfile | 2 +- README.v2.md | 240 ---------- cwa_book_downloader/core/cache.py | 2 - cwa_book_downloader/core/utils.py | 40 ++ cwa_book_downloader/download/http.py | 4 +- cwa_book_downloader/download/orchestrator.py | 30 +- .../download_clients/__init__.py | 55 --- cwa_book_downloader/main.py | 93 ++-- .../metadata_providers/README.md | 2 +- .../metadata_providers/__init__.py | 59 ++- .../metadata_providers/hardcover.py | 302 +++++++------ .../release_sources/__init__.py | 26 +- .../release_sources/direct_download.py | 158 ++++--- .../release_sources/irc/handler.py | 19 +- .../release_sources/prowlarr/api.py | 14 +- .../prowlarr/clients/__init__.py | 11 + .../prowlarr/clients/deluge.py | 130 ++---- .../prowlarr/clients/nzbget.py | 16 +- .../prowlarr/clients/qbittorrent.py | 90 +--- .../prowlarr/clients/sabnzbd.py | 16 +- .../prowlarr/clients/torrent_utils.py | 60 +++ .../prowlarr/clients/transmission.py | 102 +---- .../release_sources/prowlarr/handler.py | 32 +- .../release_sources/prowlarr/settings.py | 2 +- .../release_sources/prowlarr/source.py | 28 +- .../release_sources/prowlarr/utils.py | 76 ++++ docker-compose.dev.yml | 4 + docker-compose.extbp.dev.yml | 4 + docker-compose.extbp.yml | 8 +- docker-compose.tor.dev.yml | 4 + docker-compose.tor.yml | 6 + docker-compose.yml | 8 +- entrypoint.sh | 72 ++- readme.md | 2 +- src/frontend/src/App.tsx | 9 + .../src/components/LanguageMultiSelect.tsx | 36 +- src/frontend/src/components/ReleaseCell.tsx | 49 +- src/frontend/src/components/ReleaseModal.tsx | 423 +++++++++--------- .../src/components/ResultsSection.tsx | 41 ++ .../src/components/resultsViews/CardView.tsx | 8 +- .../components/resultsViews/CompactView.tsx | 8 +- .../src/components/resultsViews/ListView.tsx | 8 +- src/frontend/src/hooks/useSearch.ts | 133 ++++-- src/frontend/src/services/api.ts | 38 +- src/frontend/src/types/index.ts | 6 + src/frontend/src/utils/colorMaps.ts | 35 ++ src/frontend/src/utils/objectHelpers.ts | 12 + 47 files changed, 1192 insertions(+), 1331 deletions(-) delete mode 100644 README.v2.md create mode 100644 cwa_book_downloader/core/utils.py delete mode 100644 cwa_book_downloader/download_clients/__init__.py create mode 100644 cwa_book_downloader/release_sources/prowlarr/utils.py create mode 100644 src/frontend/src/utils/objectHelpers.ts diff --git a/Dockerfile b/Dockerfile index fb24d8e0..3a6b4c77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ PIP_DEFAULT_TIMEOUT=100 \ NAME=Calibre-Web-Automated-Book-Downloader \ PYTHONPATH=/app \ - # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed + # PUID/PGID will be handled by entrypoint script, but TZ/Locale are still needed LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 diff --git a/README.v2.md b/README.v2.md deleted file mode 100644 index e5ba53a1..00000000 --- a/README.v2.md +++ /dev/null @@ -1,240 +0,0 @@ -# πŸ“š Book Downloader -*calibre-web-automated-book-downloader* - -Book Downloader - -A unified web interface for searching and downloading books from multiple sources β€” all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library. - -**Fully standalone** β€” no external dependencies required. Works great alongside library tools like [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) or [Booklore](https://github.com/booklore-app/booklore) for automatic import. - -## ✨ Features - -- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place -- **Real-Time Progress** - Unified download queue with live status updates across all sources -- **Two Search Modes**: - - **Direct Download** - Search and download from popular web sources - - **Universal Mode** - Search metadata providers (Hardcover, Open Library) for richer book discovery and multi-source downloads *(additional sources in development - coming soon!)* -- **Format Support** - EPUB, MOBI, AZW3, FB2, DJVU, CBZ, CBR and more -- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources -- **PWA Support** - Install as a mobile app for quick access -- **Docker Deployment** - Up and running in minutes - -## πŸ–ΌοΈ Screenshots - -**Home screen** -![Home screen](README_images/homescreen.png 'Home screen') - -**Search results** -![Search results](README_images/search-results.png 'Search results') - -**Multi-source downloads** -![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads') - -**Download queue** -![Download queue](README_images/downloads.png 'Download queue') - -## πŸš€ Quick Start - -### Prerequisites - -- Docker & Docker Compose - -### Installation - -1. Download the docker-compose file: - ```bash - curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.yml - ``` - -2. Start the service: - ```bash - docker compose up -d - ``` - -3. Open `http://localhost:8084` - -That's it! Configure settings through the web interface as needed. - -### Volume Setup - -```yaml -volumes: - - /your/config/path:/config # Config, database, and artwork cache directory - - /your/download/path:/cwa-book-ingest # Downloaded books -``` - -> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import. - -> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors. - -## βš™οΈ Configuration - -### Search Modes - -**Direct Download Mode** (default) -- Works out of the box, no setup required -- Searches a huge library of books directly -- Returns downloadable releases immediately - -**Universal Mode** -- Cleaner search results via metadata providers (Hardcover, Open Library) -- Aggregates releases from multiple configured sources -- Requires manual setup (API keys, additional sources) - -Set the mode via Settings or `SEARCH_MODE` environment variable. - -### Environment Variables - -Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface. - -| Variable | Description | Default | -|----------|-------------|---------| -| `FLASK_PORT` | Web interface port | `8084` | -| `INGEST_DIR` | Book download directory | `/cwa-book-ingest` | -| `TZ` | Container timezone | `UTC` | -| `UID` / `GID` | Runtime user/group ID | `1000` / `100` | -| `SEARCH_MODE` | `direct` or `universal` | `direct` | - -Some of the additional options available in Settings: -- **AA Donator Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads -- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header -- **Content Folders** - Route fiction, non-fiction, comics, etc. to separate directories -- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable -- **Format & Language** - Filter downloads by preferred formats and languages -- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc. - -## 🐳 Docker Variants - -### Standard -```bash -docker compose up -d -``` - -### Tor Variant -Routes all traffic through Tor for enhanced privacy: -```bash -curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.tor.yml -docker compose -f docker-compose.tor.yml up -d -``` - -**Notes:** -- Requires `NET_ADMIN` and `NET_RAW` capabilities -- Timezone is auto-detected from Tor exit node -- Custom DNS/proxy settings are ignored - -### External Cloudflare Resolver -Use FlareSolverr or ByParr instead of the built-in bypasser: -```bash -curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.extbp.yml -docker compose -f docker-compose.extbp.yml up -d -``` - -Configure the resolver URL in Settings under the Cloudflare tab. - -**When to use external vs internal bypasser:** -- **External** is useful if you already run FlareSolverr for other services (saves resources) or if you rarely need bypassing -- **Internal** (default) is faster and more reliable for most users - it's optimized specifically for this application - -## πŸ” Authentication - -Authentication is optional but recommended for shared or exposed instances. Enable in Settings. - -**Alternative**: If you're running Calibre-Web, you can reuse its user database by mounting it: - -```yaml -volumes: - - /path/to/calibre-web/app.db:/auth/app.db:ro -``` - -## Health Monitoring - -The application exposes a health endpoint at `/api/status`. Add a health check to your compose: - -```yaml -healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:8084/api/status"] - interval: 30s - timeout: 30s - retries: 3 -``` - -## Logging - -Logs are available via: -- `docker logs ` -- `/var/log/cwa-book-downloader/` inside the container (when `ENABLE_LOGGING=true`) - -Log level is configurable via Settings or `LOG_LEVEL` environment variable. - -## Development - -```bash -# Frontend development -make install # Install dependencies -make dev # Start Vite dev server (localhost:5173) -make build # Production build -make typecheck # TypeScript checks - -# Backend (Docker) -make up # Start backend via docker-compose.dev.yml -make down # Stop services -make refresh # Rebuild and restart -``` - -The frontend dev server proxies to the backend on port 8084. - -### Architecture - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Web Interface β”‚ -β”‚ (React + TypeScript + Vite) β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Flask Backend β”‚ -β”‚ (REST API + WebSocket) β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Metadata Providersβ”‚ Download Queue β”‚ Cloudflare β”‚ -β”‚ β”‚ & Orchestrator β”‚ Bypass β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ β€’ Hardcover β”‚ β€’ Task scheduling β”‚ β€’ Internal β”‚ -β”‚ β€’ Open Library β”‚ β€’ Progress tracking β”‚ β€’ External β”‚ -β”‚ β”‚ β€’ Retry logic β”‚ (FlareSolverr) β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Release Sources β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ β€’ Direct Download (Anna's Archive β†’ Libgen β†’ Welib) β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Network Layer β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ β€’ Auto DNS rotation β€’ Mirror failover β€’ Resume support β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered. - -## Contributing - -Contributions are welcome! Please file issues or submit pull requests on GitHub. - -> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR! - -## License - -MIT License - see [LICENSE](LICENSE) for details. - -## ⚠️ Disclaimers - -### Copyright Notice - -This tool can access various sources including those that might contain copyrighted material. Users are responsible for: -- Ensuring they have the right to download requested materials -- Respecting copyright laws and intellectual property rights -- Using the tool in compliance with their local regulations - -### Library Integration - -Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads. - -## Support - -For issues or questions, please [file an issue](https://github.com/calibrain/calibre-web-automated-book-downloader/issues) on GitHub. diff --git a/cwa_book_downloader/core/cache.py b/cwa_book_downloader/core/cache.py index a2643fb5..58e91ee0 100644 --- a/cwa_book_downloader/core/cache.py +++ b/cwa_book_downloader/core/cache.py @@ -211,11 +211,9 @@ def cacheable( # Check cache cached = _metadata_cache.get(key) if cached is not None: - logger.debug(f"Cache hit: {key}") return cached # Execute function and cache result - logger.debug(f"Cache miss: {key}") result = func(*args, **kwargs) # Only cache non-None results diff --git a/cwa_book_downloader/core/utils.py b/cwa_book_downloader/core/utils.py new file mode 100644 index 00000000..6bb99ddb --- /dev/null +++ b/cwa_book_downloader/core/utils.py @@ -0,0 +1,40 @@ +""" +Shared utility functions for the CWA Book Downloader. + +Provides common helper functions used across the application. +""" + +import base64 +from typing import Optional + + +def transform_cover_url(cover_url: Optional[str], cache_id: str) -> Optional[str]: + """ + Transform an external cover URL to a local proxy URL when caching is enabled. + + When cover caching is enabled, external cover image URLs are transformed + to local proxy URLs that cache the images on first access. This reduces + external requests and provides a consistent caching layer. + + Args: + cover_url: Original cover URL (external or already local) + cache_id: Unique identifier for the cache entry (e.g., "provider_bookid") + + Returns: + Transformed URL if caching enabled and URL is external, otherwise original URL + """ + if not cover_url: + return cover_url + + # Skip if already a local URL (starts with /) + if cover_url.startswith('/'): + return cover_url + + # Check if cover caching is enabled + from cwa_book_downloader.config.env import is_covers_cache_enabled + if not is_covers_cache_enabled(): + return cover_url + + # Encode the original URL and create a proxy URL + encoded_url = base64.urlsafe_b64encode(cover_url.encode()).decode() + return f"/api/covers/{cache_id}?url={encoded_url}" diff --git a/cwa_book_downloader/download/http.py b/cwa_book_downloader/download/http.py index 52c898b6..625e59f0 100644 --- a/cwa_book_downloader/download/http.py +++ b/cwa_book_downloader/download/http.py @@ -136,7 +136,7 @@ def html_get_page( try: if use_bypasser_now and USE_CF_BYPASS: - logger.info(f"GET (bypasser): {current_url}") + logger.debug(f"GET (bypasser): {current_url}") try: result = get_bypassed_page(current_url, selector, cancel_flag) return result or "" @@ -144,7 +144,7 @@ def html_get_page( logger.warning(f"Bypasser error: {type(e).__name__}: {e}") return "" - logger.info(f"GET: {current_url}") + logger.debug(f"GET: {current_url}") # Try with CF cookies/UA if available (from previous bypass) cookies = {} headers = {} diff --git a/cwa_book_downloader/download/orchestrator.py b/cwa_book_downloader/download/orchestrator.py index 5c32bccb..35a1aa23 100644 --- a/cwa_book_downloader/download/orchestrator.py +++ b/cwa_book_downloader/download/orchestrator.py @@ -481,8 +481,7 @@ def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]: Transforms external preview URLs to local proxy URLs when cover caching is enabled. """ - import base64 - from cwa_book_downloader.config.env import is_covers_cache_enabled + from cwa_book_downloader.core.utils import transform_cover_url result = { key: value for key, value in book.__dict__.items() @@ -490,11 +489,8 @@ def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]: } # Transform external preview URLs to local proxy URLs - # Skip if already a local URL (starts with /) - if result.get('preview') and is_covers_cache_enabled() and not result['preview'].startswith('/'): - original_url = result['preview'] - encoded_url = base64.urlsafe_b64encode(original_url.encode()).decode() - result['preview'] = f"/api/covers/{book.id}?url={encoded_url}" + if result.get('preview'): + result['preview'] = transform_cover_url(result['preview'], book.id) return result @@ -506,16 +502,10 @@ def _task_to_dict(task: DownloadTask) -> Dict[str, Any]: maintaining compatibility with the previous BookInfo-based format. Transforms external preview URLs to local proxy URLs when cover caching is enabled. """ - import base64 - from cwa_book_downloader.config.env import is_covers_cache_enabled - - preview = task.preview + from cwa_book_downloader.core.utils import transform_cover_url # Transform external preview URLs to local proxy URLs - # Skip if already a local URL (starts with /) - if preview and is_covers_cache_enabled() and not preview.startswith('/'): - encoded_url = base64.urlsafe_b64encode(preview.encode()).decode() - preview = f"/api/covers/{task.task_id}?url={encoded_url}" + preview = transform_cover_url(task.preview, task.task_id) return { 'id': task.task_id, @@ -561,9 +551,11 @@ def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]: logger.error(f"Task not found in queue: {task_id}") return None - # Create callbacks that update the orchestrator's tracking - progress_callback = lambda progress: update_download_progress(task_id, progress) - status_callback = lambda status, message=None: update_download_status(task_id, status, message) + def progress_callback(progress: float) -> None: + update_download_progress(task_id, progress) + + def status_callback(status: str, message: Optional[str] = None) -> None: + update_download_status(task_id, status, message) # Get the download handler based on the task's source handler = get_handler(task.source) @@ -852,7 +844,7 @@ def reorder_queue(book_priorities: Dict[str, int]) -> bool: """ return book_queue.reorder_queue(book_priorities) -def get_queue_order() -> List[Dict[str, any]]: +def get_queue_order() -> List[Dict[str, Any]]: """Get current queue order for display.""" return book_queue.get_queue_order() diff --git a/cwa_book_downloader/download_clients/__init__.py b/cwa_book_downloader/download_clients/__init__.py deleted file mode 100644 index 21e23c7d..00000000 --- a/cwa_book_downloader/download_clients/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -"""External download client integrations (qBittorrent, SABnzbd, etc.).""" - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import List, Optional, Tuple -from enum import Enum - - -class DownloadStatus(Enum): - """Status of a download in an external client.""" - QUEUED = "queued" - DOWNLOADING = "downloading" - PAUSED = "paused" - COMPLETED = "completed" - FAILED = "failed" - SEEDING = "seeding" # Torrents only - - -@dataclass -class ClientDownloadProgress: - """Progress info from external download client.""" - status: DownloadStatus - progress: float # 0-100 - download_speed: Optional[int] # bytes/sec - eta: Optional[int] # seconds remaining - save_path: Optional[str] # Where the file will be/is - - -class DownloadClient(ABC): - """Abstract base class for download clients.""" - - @abstractmethod - def add_download(self, url: str, title: str) -> str: - """Add a download (torrent/magnet or NZB URL). Returns download ID for tracking.""" - pass - - @abstractmethod - def get_download(self, download_id: str) -> Optional[ClientDownloadProgress]: - """Get progress of a specific download.""" - pass - - @abstractmethod - def list_downloads(self) -> List[Tuple[str, ClientDownloadProgress]]: - """List all downloads with their progress.""" - pass - - @abstractmethod - def get_completed_path(self, download_id: str) -> Optional[str]: - """Get the path to completed download.""" - pass - - @abstractmethod - def test_connection(self) -> bool: - """Test if the client is reachable and credentials are valid.""" - pass diff --git a/cwa_book_downloader/main.py b/cwa_book_downloader/main.py index 0087a6e6..5948a2ef 100644 --- a/cwa_book_downloader/main.py +++ b/cwa_book_downloader/main.py @@ -100,35 +100,32 @@ def cleanup_old_lockouts() -> None: def is_account_locked(username: str) -> bool: """Check if an account is currently locked due to failed login attempts.""" cleanup_old_lockouts() - + if username not in failed_login_attempts: return False - + lockout_until = failed_login_attempts[username].get('lockout_until') - if lockout_until and datetime.now() < lockout_until: - return True - - return False + return lockout_until is not None and datetime.now() < lockout_until def record_failed_login(username: str, ip_address: str) -> bool: - """ - Record a failed login attempt and lock account if threshold is reached. + """Record a failed login attempt and lock account if threshold is reached. + Returns True if account is now locked, False otherwise. """ if username not in failed_login_attempts: failed_login_attempts[username] = {'count': 0} - + failed_login_attempts[username]['count'] += 1 count = failed_login_attempts[username]['count'] - + logger.warning(f"Failed login attempt {count}/{MAX_LOGIN_ATTEMPTS} for user '{username}' from IP {ip_address}") - + if count >= MAX_LOGIN_ATTEMPTS: lockout_until = datetime.now() + timedelta(minutes=LOCKOUT_DURATION_MINUTES) failed_login_attempts[username]['lockout_until'] = lockout_until logger.warning(f"Account locked for user '{username}' until {lockout_until.strftime('%Y-%m-%d %H:%M:%S')} due to {count} failed login attempts") return True - + return False def clear_failed_logins(username: str) -> None: @@ -139,16 +136,12 @@ def clear_failed_logins(username: str) -> None: def get_auth_mode() -> str: - """ - Determine which authentication mode is active. + """Determine which authentication mode is active. Priority order: 1. Built-in credentials (if configured) -> "builtin" 2. CWA database (if CWA_DB_PATH is set and exists) -> "cwa" 3. No auth required -> "none" - - Returns: - str: "builtin", "cwa", or "none" """ from cwa_book_downloader.core.settings_registry import load_config_file @@ -848,9 +841,7 @@ def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]: return jsonify({"error": "Internal server error"}), 500 def _failed_login_response(username: str, ip_address: str) -> Tuple[Response, int]: - """ - Handle a failed login attempt by recording it and returning the appropriate response. - """ + """Handle a failed login attempt by recording it and returning the appropriate response.""" is_now_locked = record_failed_login(username, ip_address) if is_now_locked: @@ -1093,7 +1084,7 @@ def api_metadata_search() -> Union[Response, Tuple[Response, int]]: Query Parameters: query (str): Search query (required) - limit (int): Maximum number of results (default: 20, max: 50) + limit (int): Maximum number of results (default: 40, max: 100) sort (str): Sort order - relevance, popularity, rating, newest, oldest (default: relevance) [dynamic fields]: Provider-specific search fields passed as query params @@ -1113,9 +1104,14 @@ def api_metadata_search() -> Union[Response, Tuple[Response, int]]: query = request.args.get('query', '').strip() try: - limit = min(int(request.args.get('limit', 20)), 50) + limit = min(int(request.args.get('limit', 40)), 100) except ValueError: - limit = 20 + limit = 40 + + try: + page = max(1, int(request.args.get('page', 1))) + except ValueError: + page = 1 # Parse sort parameter sort_value = request.args.get('sort', 'relevance').lower() @@ -1160,27 +1156,26 @@ def api_metadata_search() -> Union[Response, Tuple[Response, int]]: if not query and not fields: return jsonify({"error": "Either 'query' or search field values are required"}), 400 - options = MetadataSearchOptions(query=query, limit=limit, sort=sort_order, fields=fields) - books = provider.search(options) + options = MetadataSearchOptions(query=query, limit=limit, page=page, sort=sort_order, fields=fields) + search_result = provider.search_paginated(options) # Convert BookMetadata objects to dicts - books_data = [asdict(book) for book in books] + books_data = [asdict(book) for book in search_result.books] # Transform cover_url to local proxy URLs when caching is enabled - from cwa_book_downloader.config.env import is_covers_cache_enabled - if is_covers_cache_enabled(): - import base64 - for book_dict in books_data: - if book_dict.get('cover_url'): - # Encode original URL in the proxy request itself - no need for persistent mapping - cache_id = f"{book_dict['provider']}_{book_dict['provider_id']}" - encoded_url = base64.urlsafe_b64encode(book_dict['cover_url'].encode()).decode() - book_dict['cover_url'] = f"/api/covers/{cache_id}?url={encoded_url}" + from cwa_book_downloader.core.utils import transform_cover_url + for book_dict in books_data: + if book_dict.get('cover_url'): + cache_id = f"{book_dict['provider']}_{book_dict['provider_id']}" + book_dict['cover_url'] = transform_cover_url(book_dict['cover_url'], cache_id) return jsonify({ "books": books_data, "provider": provider.name, - "query": query + "query": query, + "page": search_result.page, + "total_found": search_result.total_found, + "has_more": search_result.has_more }) except Exception as e: logger.error_trace(f"Metadata search error: {e}") @@ -1225,12 +1220,10 @@ def api_metadata_book(provider: str, book_id: str) -> Union[Response, Tuple[Resp book_dict = asdict(book) # Transform cover_url to local proxy URL when caching is enabled - from cwa_book_downloader.config.env import is_covers_cache_enabled - if is_covers_cache_enabled() and book_dict.get('cover_url'): - import base64 + from cwa_book_downloader.core.utils import transform_cover_url + if book_dict.get('cover_url'): cache_id = f"{provider}_{book_id}" - encoded_url = base64.urlsafe_b64encode(book_dict['cover_url'].encode()).decode() - book_dict['cover_url'] = f"/api/covers/{cache_id}?url={encoded_url}" + book_dict['cover_url'] = transform_cover_url(book_dict['cover_url'], cache_id) return jsonify(book_dict) except ValueError as e: @@ -1338,18 +1331,26 @@ def api_releases() -> Union[Response, Tuple[Response, int]]: # Convert book to dict and transform cover_url book_dict = asdict(book) - from cwa_book_downloader.config.env import is_covers_cache_enabled - if is_covers_cache_enabled() and book_dict.get('cover_url'): - import base64 + from cwa_book_downloader.core.utils import transform_cover_url + if book_dict.get('cover_url'): cache_id = f"{provider}_{book_id}" - encoded_url = base64.urlsafe_b64encode(book_dict['cover_url'].encode()).decode() - book_dict['cover_url'] = f"/api/covers/{cache_id}?url={encoded_url}" + book_dict['cover_url'] = transform_cover_url(book_dict['cover_url'], cache_id) + + # Get search info from direct_download source (if it was searched) + search_info = {} + if "direct_download" in source_instances: + dd_source = source_instances["direct_download"] + if hasattr(dd_source, 'last_search_type'): + search_info["direct_download"] = { + "search_type": dd_source.last_search_type + } response = { "releases": releases_data, "book": book_dict, "sources_searched": sources_to_search, "column_config": column_config, + "search_info": search_info, } if errors: diff --git a/cwa_book_downloader/metadata_providers/README.md b/cwa_book_downloader/metadata_providers/README.md index 9020b88a..2c23b78d 100644 --- a/cwa_book_downloader/metadata_providers/README.md +++ b/cwa_book_downloader/metadata_providers/README.md @@ -64,7 +64,7 @@ class MetadataSearchOptions: search_type: SearchType = SearchType.GENERAL # GENERAL, TITLE, AUTHOR, ISBN language: str = None # ISO 639-1 code (e.g., "en") sort: SortOrder = SortOrder.RELEVANCE - limit: int = 20 + limit: int = 40 page: int = 1 ``` diff --git a/cwa_book_downloader/metadata_providers/__init__.py b/cwa_book_downloader/metadata_providers/__init__.py index 97362e21..0040c0d3 100644 --- a/cwa_book_downloader/metadata_providers/__init__.py +++ b/cwa_book_downloader/metadata_providers/__init__.py @@ -97,8 +97,8 @@ def serialize_search_field(search_field: SearchField) -> Dict[str, Any]: "key": search_field.key, "label": search_field.label, "type": _get_field_type_name(search_field), - "placeholder": search_field.placeholder if hasattr(search_field, 'placeholder') else "", - "description": search_field.description if hasattr(search_field, 'description') else "", + "placeholder": getattr(search_field, 'placeholder', ''), + "description": getattr(search_field, 'description', ''), } # Add type-specific properties @@ -125,7 +125,7 @@ class MetadataSearchOptions: search_type: SearchType = SearchType.GENERAL language: Optional[str] = None # ISO 639-1 code (e.g., "en", "fr") sort: SortOrder = SortOrder.RELEVANCE - limit: int = 20 + limit: int = 40 page: int = 1 fields: Dict[str, Any] = field(default_factory=dict) # Custom search field values @@ -172,6 +172,19 @@ class BookMetadata: series_position: Optional[float] = None # This book's position (e.g., 3, 1.5 for novellas) series_count: Optional[int] = None # Total books in the series + # Alternative titles by language (for localized searches) + # Maps language code (e.g., "de", "German") to localized title + titles_by_language: Dict[str, str] = field(default_factory=dict) + + +@dataclass +class SearchResult: + """Result from a metadata search with pagination info.""" + books: List[BookMetadata] + page: int = 1 + total_found: int = 0 # Total matching results (if known) + has_more: bool = False # True if more results available + class MetadataProvider(ABC): """Interface for metadata providers. @@ -224,6 +237,28 @@ class MetadataProvider(ABC): """Check if this provider is configured and available.""" pass + def search_paginated(self, options: MetadataSearchOptions) -> SearchResult: + """Search for books and return results with pagination info. + + Default implementation calls search() and estimates has_more. + Providers should override this to return accurate pagination info. + + Args: + options: Search options including query, type, language, sort, pagination. + + Returns: + SearchResult with books and pagination info. + """ + books = self.search(options) + # Heuristic: if we got exactly limit results, there might be more + has_more = len(books) >= options.limit + return SearchResult( + books=books, + page=options.page, + total_found=0, # Unknown without provider-specific implementation + has_more=has_more + ) + # Provider registry _PROVIDERS: Dict[str, Type[MetadataProvider]] = {} @@ -367,6 +402,13 @@ def get_configured_provider() -> Optional[MetadataProvider]: return get_provider(metadata_provider, **kwargs) +def _get_configured_provider_name() -> str: + """Get the currently configured metadata provider name from config.""" + from cwa_book_downloader.core.config import config as app_config + app_config.refresh() + return app_config.get("METADATA_PROVIDER", "") + + def get_provider_sort_options(provider_name: Optional[str] = None) -> List[Dict[str, str]]: """Get sort options for a metadata provider. @@ -379,9 +421,7 @@ def get_provider_sort_options(provider_name: Optional[str] = None) -> List[Dict[ List of sort option dicts, or default [relevance] if provider not found. """ if provider_name is None: - from cwa_book_downloader.core.config import config as app_config - app_config.refresh() - provider_name = app_config.get("METADATA_PROVIDER", "") + provider_name = _get_configured_provider_name() if provider_name and provider_name in _PROVIDERS: provider_class = _PROVIDERS[provider_name] @@ -407,9 +447,7 @@ def get_provider_search_fields(provider_name: Optional[str] = None) -> List[Dict List of search field dicts, or empty list if provider not found. """ if provider_name is None: - from cwa_book_downloader.core.config import config as app_config - app_config.refresh() - provider_name = app_config.get("METADATA_PROVIDER", "") + provider_name = _get_configured_provider_name() if provider_name and provider_name in _PROVIDERS: provider_class = _PROVIDERS[provider_name] @@ -434,8 +472,7 @@ def get_provider_default_sort(provider_name: Optional[str] = None) -> str: from cwa_book_downloader.core.config import config as app_config if provider_name is None: - app_config.refresh() - provider_name = app_config.get("METADATA_PROVIDER", "") + provider_name = _get_configured_provider_name() if not provider_name: return "relevance" diff --git a/cwa_book_downloader/metadata_providers/hardcover.py b/cwa_book_downloader/metadata_providers/hardcover.py index 5a1b0157..6ec40cd9 100644 --- a/cwa_book_downloader/metadata_providers/hardcover.py +++ b/cwa_book_downloader/metadata_providers/hardcover.py @@ -19,6 +19,7 @@ from cwa_book_downloader.metadata_providers import ( DisplayField, MetadataProvider, MetadataSearchOptions, + SearchResult, SearchType, SortOrder, register_provider, @@ -29,6 +30,7 @@ from cwa_book_downloader.metadata_providers import ( logger = setup_logger(__name__) HARDCOVER_API_URL = "https://api.hardcover.app/v1/graphql" +HARDCOVER_PAGE_SIZE = 25 # Hardcover API returns max 25 results per page # Mapping from abstract sort order to Hardcover sort parameter @@ -51,28 +53,47 @@ SEARCH_TYPE_FIELDS: Dict[SearchType, str] = { def _combine_headline_description(headline: Optional[str], description: Optional[str]) -> Optional[str]: - """Combine headline (tagline) and description into a single description. - - Hardcover stores a short 'headline' (tagline/promotional text) separately - from the main description. This combines them for display. - - Args: - headline: Short promotional text or tagline. - description: Full book synopsis/description. - - Returns: - Combined description with headline as the first line, or just one if only one exists. - """ + """Combine headline (tagline) and description into a single description.""" if headline and description: - # Add headline as first paragraph, followed by description return f"{headline}\n\n{description}" - elif headline: - return headline - elif description: - return description + return headline or description + + +def _extract_cover_url(data: Dict, *keys: str) -> Optional[str]: + """Extract cover URL from data dict, trying multiple keys. + + Handles both string URLs and dict with 'url' key. + """ + for key in keys: + value = data.get(key) + if value: + if isinstance(value, str): + return value + if isinstance(value, dict): + return value.get("url") return None +def _extract_publish_year(data: Dict) -> Optional[int]: + """Extract publish year from release_year or release_date fields.""" + if data.get("release_year"): + try: + return int(data["release_year"]) + except (ValueError, TypeError): + pass + if data.get("release_date"): + try: + return int(str(data["release_date"])[:4]) + except (ValueError, TypeError): + pass + return None + + +def _build_source_url(slug: str) -> Optional[str]: + """Build Hardcover source URL from book slug.""" + return f"https://hardcover.app/books/{slug}" if slug else None + + @register_provider_kwargs("hardcover") def _hardcover_kwargs() -> Dict[str, Any]: """Provide Hardcover-specific constructor kwargs.""" @@ -139,22 +160,35 @@ class HardcoverProvider(MetadataProvider): Returns: List of BookMetadata objects. """ + return self.search_paginated(options).books + + def search_paginated(self, options: MetadataSearchOptions) -> SearchResult: + """Search for books with pagination info. + + Args: + options: Search options (query, type, sort, pagination, fields). + + Returns: + SearchResult with books and pagination info. + """ if not self.api_key: logger.warning("Hardcover API key not configured") - return [] + return SearchResult(books=[], page=options.page, total_found=0, has_more=False) # Handle ISBN search separately if options.search_type == SearchType.ISBN: result = self.search_by_isbn(options.query) - return [result] if result else [] + books = [result] if result else [] + return SearchResult(books=books, page=1, total_found=len(books), has_more=False) - # Build cache key from options (include fields for cache differentiation) + # Build cache key from options (include fields and settings for cache differentiation) fields_key = ":".join(f"{k}={v}" for k, v in sorted(options.fields.items())) - cache_key = f"{options.query}:{options.search_type.value}:{options.sort.value}:{options.limit}:{options.page}:{fields_key}" + exclude_compilations = app_config.get("HARDCOVER_EXCLUDE_COMPILATIONS", False) + cache_key = f"{options.query}:{options.search_type.value}:{options.sort.value}:{options.limit}:{options.page}:{fields_key}:excl_comp={exclude_compilations}" return self._search_cached(cache_key, options) @cacheable(ttl_key="METADATA_CACHE_SEARCH_TTL", ttl_default=300, key_prefix="hardcover:search") - def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> List[BookMetadata]: + def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> SearchResult: """Cached search implementation. Args: @@ -162,69 +196,35 @@ class HardcoverProvider(MetadataProvider): options: Search options. Returns: - List of BookMetadata objects. + SearchResult with books and pagination info. """ # Determine query and fields based on custom search fields - # Field-first search: when a specific field has a value, search that field + # Note: Hardcover API requires 'weights' when using 'fields' parameter author_value = options.fields.get("author", "").strip() title_value = options.fields.get("title", "").strip() series_value = options.fields.get("series", "").strip() - logger.debug(f"Field-first search check: author='{author_value}', title='{title_value}', series='{series_value}'") - - # Determine what to search and which fields to target - # Note: Hardcover API requires 'weights' when using 'fields' parameter if series_value and not author_value and not title_value: - # Series-only search: search series_names field - query = series_value - search_fields = "series_names" - search_weights = "1" - logger.debug(f"Series-only search: query='{query}', fields='{search_fields}'") + query, search_fields, search_weights = series_value, "series_names", "1" elif author_value and not title_value and not series_value: - # Author-only search: search author_names field with author query - query = author_value - search_fields = "author_names" - search_weights = "1" - logger.debug(f"Author-only search: query='{query}', fields='{search_fields}'") + query, search_fields, search_weights = author_value, "author_names", "1" elif title_value and not author_value and not series_value: - # Title-only search: search title fields with title query - query = title_value - search_fields = "title,alternative_titles" - search_weights = "5,1" - logger.debug(f"Title-only search: query='{query}', fields='{search_fields}'") + query, search_fields, search_weights = title_value, "title,alternative_titles", "5,1" elif author_value and title_value and not series_value: - # Author + Title: combine into query, search both fields query = f"{title_value} {author_value}" - search_fields = "title,alternative_titles,author_names" - search_weights = "5,1,3" - logger.debug(f"Combined title+author search: query='{query}', fields='{search_fields}'") + search_fields, search_weights = "title,alternative_titles,author_names", "5,1,3" elif series_value: - # Series with other fields: include series_names in search - parts = [p for p in [series_value, title_value, author_value] if p] - query = " ".join(parts) - search_fields = "series_names,title,alternative_titles,author_names" - search_weights = "5,3,1,2" - logger.debug(f"Combined search with series: query='{query}', fields='{search_fields}'") + query = " ".join(p for p in [series_value, title_value, author_value] if p) + search_fields, search_weights = "series_names,title,alternative_titles,author_names", "5,3,1,2" else: - # No custom fields: use general query with all default fields - query = options.query - search_fields = None - search_weights = None - logger.debug(f"General search: query='{query}', no field restriction") + query, search_fields, search_weights = options.query, None, None - # Build GraphQL query with optional fields/weights parameters + + # Build GraphQL query - include fields/weights parameters only when needed if search_fields: graphql_query = """ query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String, $fields: String, $weights: String) { - search( - query: $query, - query_type: "Book", - per_page: $limit, - page: $page, - sort: $sort, - fields: $fields, - weights: $weights - ) { + search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort, fields: $fields, weights: $weights) { results } } @@ -232,13 +232,7 @@ class HardcoverProvider(MetadataProvider): else: graphql_query = """ query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String) { - search( - query: $query, - query_type: "Book", - per_page: $limit, - page: $page, - sort: $sort - ) { + search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort) { results } } @@ -258,32 +252,34 @@ class HardcoverProvider(MetadataProvider): variables["fields"] = search_fields variables["weights"] = search_weights - logger.debug(f"GraphQL variables: {variables}") try: result = self._execute_query(graphql_query, variables) if not result: logger.debug("Hardcover search: No result from API") - return [] + return SearchResult(books=[], page=options.page, total_found=0, has_more=False) - search_data = result.get("search", {}) - - # Results is a Typesense response object with hits array - results_obj = search_data.get("results", {}) + # Extract hits from Typesense response + results_obj = result.get("search", {}).get("results", {}) if isinstance(results_obj, dict): hits = results_obj.get("hits", []) + found_count = results_obj.get("found", 0) else: hits = results_obj if isinstance(results_obj, list) else [] + found_count = 0 - # Parse the search results - each hit has a 'document' field + # Parse hits, filtering compilations if enabled + exclude_compilations = app_config.get("HARDCOVER_EXCLUDE_COMPILATIONS", False) books = [] for hit in hits: - # Get the document from the hit item = hit.get("document", hit) if isinstance(hit, dict) else hit - if isinstance(item, dict): - book = self._parse_search_result(item) - if book: - books.append(book) + if not isinstance(item, dict): + continue + if exclude_compilations and item.get("compilation"): + continue + book = self._parse_search_result(item) + if book: + books.append(book) # If series order sort is selected and series field is provided, # filter to exact matches and sort by position @@ -291,11 +287,21 @@ class HardcoverProvider(MetadataProvider): books = self._apply_series_ordering(books, series_value) logger.info(f"Hardcover search '{query}' (fields={search_fields}) returned {len(books)} results") - return books + + # Calculate if there are more results + results_so_far = (options.page - 1) * HARDCOVER_PAGE_SIZE + len(hits) + has_more = results_so_far < found_count + + return SearchResult( + books=books, + page=options.page, + total_found=found_count, + has_more=has_more + ) except Exception as e: logger.error(f"Hardcover search error: {e}") - return [] + return SearchResult(books=[], page=options.page, total_found=0, has_more=False) def _apply_series_ordering(self, books: List[BookMetadata], series_name: str) -> List[BookMetadata]: """Filter books to exact series match and sort by series position. @@ -353,6 +359,7 @@ class HardcoverProvider(MetadataProvider): # Use contributions with filter to get only primary authors (not translators/narrators) # Also include cached_contributors as fallback if contributions is empty # Include featured_book_series for series info + # Include editions with titles and languages for localized search support graphql_query = """ query GetBook($id: Int!) { books(where: {id: {_eq: $id}}, limit: 1) { @@ -382,6 +389,14 @@ class HardcoverProvider(MetadataProvider): primary_books_count } } + editions(limit: 20, order_by: {users_count: desc}) { + title + language { + language + code2 + code3 + } + } } } """ @@ -537,37 +552,27 @@ class HardcoverProvider(MetadataProvider): if not book_id or not title: return None - # Extract authors from various possible fields + # Extract authors - use contribution_types to filter author_names if available authors = [] - if "author_names" in item: - authors = item["author_names"] if isinstance(item["author_names"], list) else [item["author_names"]] - elif "cached_contributors" in item: - for contrib in item.get("cached_contributors", []): - if isinstance(contrib, dict) and contrib.get("name"): - authors.append(contrib["name"]) - elif isinstance(contrib, str): - authors.append(contrib) - # Get cover URL - cover_url = None - if "image" in item and item["image"]: - cover_url = item["image"] if isinstance(item["image"], str) else item["image"].get("url") + author_names = item.get("author_names", []) + if isinstance(author_names, str): + author_names = [author_names] - # Extract year - prefer release_year if available, fall back to release_date - publish_year = None - if "release_year" in item and item["release_year"]: - try: - publish_year = int(item["release_year"]) - except (ValueError, TypeError): - pass - elif "release_date" in item and item["release_date"]: - try: - publish_year = int(str(item["release_date"])[:4]) - except (ValueError, TypeError): - pass + contribution_types = item.get("contribution_types", []) - slug = item.get("slug", "") - source_url = f"https://hardcover.app/books/{slug}" if slug else None + # If we have parallel arrays, filter to only "Author" contributions + if contribution_types and len(contribution_types) == len(author_names): + for name, contrib_type in zip(author_names, contribution_types): + if contrib_type == "Author": + authors.append(name) + elif author_names: + # No contribution_types or length mismatch - use all names as fallback + authors = author_names + + cover_url = _extract_cover_url(item, "image") + publish_year = _extract_publish_year(item) + source_url = _build_source_url(item.get("slug", "")) # Build display fields from Hardcover-specific data display_fields = [] @@ -622,8 +627,6 @@ class HardcoverProvider(MetadataProvider): contributions = book.get("contributions") or [] cached_contributors = book.get("cached_contributors") or [] - logger.debug(f"_parse_book [{book.get('id')}]: contributions={contributions}, cached_contributors={cached_contributors}") - # Try contributions first (filtered to "Author" role only - cleaner data) for contrib in contributions: author = contrib.get("author", {}) @@ -643,27 +646,8 @@ class HardcoverProvider(MetadataProvider): elif isinstance(contrib, str): authors.append(contrib) - logger.debug(f"_parse_book [{book.get('id')}]: final authors={authors}") - - # Get cover URL from cached_image (jsonb) or image relationship - cover_url = None - if book.get("cached_image"): - cached = book["cached_image"] - if isinstance(cached, dict): - cover_url = cached.get("url") - elif isinstance(cached, str): - cover_url = cached - elif book.get("image"): - img = book["image"] - cover_url = img if isinstance(img, str) else img.get("url") - - # Extract year from release_date - publish_year = None - if book.get("release_date"): - try: - publish_year = int(str(book["release_date"])[:4]) - except (ValueError, TypeError): - pass + cover_url = _extract_cover_url(book, "cached_image", "image") + publish_year = _extract_publish_year(book) # Extract genres from cached_tags genres = [] @@ -694,8 +678,7 @@ class HardcoverProvider(MetadataProvider): if isbn_10 and isbn_13: break - slug = book.get("slug", "") - source_url = f"https://hardcover.app/books/{slug}" if slug else None + source_url = _build_source_url(book.get("slug", "")) # Combine headline and description if both present headline = book.get("headline") @@ -714,6 +697,30 @@ class HardcoverProvider(MetadataProvider): series_name = series_data.get("name") series_count = series_data.get("primary_books_count") + # Extract titles by language from editions + # This allows searching with localized titles when language filter is active + titles_by_language: Dict[str, str] = {} + editions = book.get("editions", []) + for edition in editions: + edition_title = edition.get("title") + lang_data = edition.get("language") + if edition_title and lang_data: + # Store by various language identifiers for flexible matching + # Language name (e.g., "German", "English") + lang_name = lang_data.get("language") + # 2-letter code (e.g., "de", "en") + code2 = lang_data.get("code2") + # 3-letter code (e.g., "deu", "eng") + code3 = lang_data.get("code3") + + # Store with all available keys (first title wins for each language) + if lang_name and lang_name not in titles_by_language: + titles_by_language[lang_name] = edition_title + if code2 and code2 not in titles_by_language: + titles_by_language[code2] = edition_title + if code3 and code3 not in titles_by_language: + titles_by_language[code3] = edition_title + return BookMetadata( provider="hardcover", provider_id=str(book["id"]), @@ -730,10 +737,11 @@ class HardcoverProvider(MetadataProvider): series_name=series_name, series_position=series_position, series_count=series_count, + titles_by_language=titles_by_language, ) -def _test_hardcover_connection(current_values: Dict[str, Any] = None) -> Dict[str, Any]: +def _test_hardcover_connection(current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Test the Hardcover API connection using current form values.""" from cwa_book_downloader.core.config import config as app_config @@ -742,10 +750,8 @@ def _test_hardcover_connection(current_values: Dict[str, Any] = None) -> Dict[st # Use current form values first, fall back to saved config api_key = current_values.get("HARDCOVER_API_KEY") or app_config.get("HARDCOVER_API_KEY", "") - # Debug: log key info key_len = len(api_key) if api_key else 0 - key_preview = f"{api_key[:10]}...{api_key[-10:]}" if key_len > 20 else "(too short)" - logger.info(f"Hardcover test: key length={key_len}, preview={key_preview}") + logger.debug(f"Hardcover test: key length={key_len}") if not api_key: # Clear any stored username since there's no key @@ -855,4 +861,10 @@ def hardcover_settings(): default="relevance", env_supported=False, # UI-only setting ), + CheckboxField( + key="HARDCOVER_EXCLUDE_COMPILATIONS", + label="Exclude Compilations", + description="Filter out compilations, anthologies, and omnibus editions from search results", + default=False, + ), ] diff --git a/cwa_book_downloader/release_sources/__init__.py b/cwa_book_downloader/release_sources/__init__.py index ea8c8aa4..f434441e 100644 --- a/cwa_book_downloader/release_sources/__init__.py +++ b/cwa_book_downloader/release_sources/__init__.py @@ -343,30 +343,22 @@ def list_available_sources() -> List[dict]: Returns all sources (not just available ones) so the frontend can show appropriate UI for disabled/unconfigured sources instead of hiding them. - - Each source includes: - - name: Source identifier (e.g., 'prowlarr') - - display_name: Human-readable name (e.g., 'Prowlarr') - - enabled: Whether the source is available for use """ - return [ - { + result = [] + for name, src_class in _SOURCES.items(): + instance = src_class() + result.append({ "name": name, - "display_name": src().display_name, - "enabled": src().is_available(), - } - for name, src in _SOURCES.items() - ] + "display_name": instance.display_name, + "enabled": instance.is_available(), + }) + return result def get_source_display_name(name: str) -> str: - """Get display name for a source by its identifier. - - Falls back to title-cased name if source not found. - """ + """Get display name for a source by its identifier.""" if name in _SOURCES: return _SOURCES[name]().display_name - # Fallback: convert snake_case to Title Case return name.replace('_', ' ').title() diff --git a/cwa_book_downloader/release_sources/direct_download.py b/cwa_book_downloader/release_sources/direct_download.py index 580aac04..29e511b7 100644 --- a/cwa_book_downloader/release_sources/direct_download.py +++ b/cwa_book_downloader/release_sources/direct_download.py @@ -7,7 +7,7 @@ import re import time from pathlib import Path from threading import Event -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Tuple from urllib.parse import quote from bs4 import BeautifulSoup, NavigableString, Tag @@ -36,7 +36,7 @@ from cwa_book_downloader.release_sources import ( logger = setup_logger(__name__) _aa_slow_rotation = itertools.count() -_url_source_types: dict[str, str] = {} +_url_source_types: Dict[str, str] = {} if DEBUG_SKIP_SOURCES: logger.warning("DEBUG_SKIP_SOURCES active: skipping sources %s", DEBUG_SKIP_SOURCES) @@ -67,20 +67,23 @@ _MD5_URL_TEMPLATES = { "welib": "https://welib.org/md5/{md5}", } -def _get_source_priority() -> list[dict]: +def _get_source_priority() -> List[Dict]: """Get the current source priority configuration.""" return config.get("SOURCE_PRIORITY") or [] def _is_source_enabled(source_id: str) -> bool: - """Check if a source is enabled in the priority config.""" + """Check if a source is enabled in the priority config. + + Returns False for unknown sources. + """ for item in _get_source_priority(): if item["id"] == source_id: return item.get("enabled", True) - return False # Unknown sources are disabled + return False -def _get_enabled_source_order() -> list[str]: +def _get_enabled_source_order() -> List[str]: """Get ordered list of enabled source IDs.""" return [ item["id"] @@ -92,13 +95,12 @@ def _get_enabled_source_order() -> list[str]: def _get_source_position(source_id: str) -> int: """Get the position of a source in the priority list (lower = higher priority). - Returns a high number if source not found or disabled. + Returns 999 if source not found or disabled. """ - priority = _get_source_priority() - for i, item in enumerate(priority): + for i, item in enumerate(_get_source_priority()): if item["id"] == source_id and item.get("enabled", True): return i - return 999 # Not found or disabled + return 999 class SearchUnavailable(Exception): @@ -267,10 +269,10 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div") divs = list(data.children) - slow_urls_no_waitlist: list[str] = [] - slow_urls_with_waitlist: list[str] = [] + slow_urls_no_waitlist: List[str] = [] + slow_urls_with_waitlist: List[str] = [] - def _append_unique(lst: list[str], href: str) -> None: + def _append_unique(lst: List[str], href: str) -> None: if href and href not in lst: lst.append(href) @@ -472,7 +474,7 @@ def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]: } -def _get_source_info(link: str) -> tuple[str, str]: +def _get_source_info(link: str) -> Tuple[str, str]: """Get source label and friendly name for a download link. Args: @@ -504,7 +506,7 @@ def _friendly_source_name(link: str) -> str: return _get_source_info(link)[1] -def _fetch_aa_page_urls(book_info: BookInfo, urls_by_source: dict[str, list[str]]) -> None: +def _fetch_aa_page_urls(book_info: BookInfo, urls_by_source: Dict[str, List[str]]) -> None: """Fetch and parse AA page, populating urls_by_source dict. Groups existing book_info.download_urls by source type. If book_info @@ -539,9 +541,9 @@ def _get_urls_for_source( selector: network.AAMirrorSelector, cancel_flag: Optional[Event], status_callback: Optional[Callable[[str, Optional[str]], None]], - urls_by_source: dict[str, list[str]], + urls_by_source: Dict[str, List[str]], aa_page_fetched: bool -) -> list[str]: +) -> List[str]: """Get URLs for a specific source, fetching lazily if needed.""" # AA Fast - generate URL dynamically if source_id == "aa-fast": @@ -628,7 +630,7 @@ def _try_download_url( return None -def _get_download_urls_from_welib(book_id: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> list[str]: +def _get_download_urls_from_welib(book_id: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> List[str]: """Get download URLs from welib.org (bypasser required).""" if not _is_source_enabled("welib"): return [] @@ -925,6 +927,16 @@ class DirectDownloadSource(ReleaseSource): name = "direct_download" display_name = "Anna's Archive" + def __init__(self): + # Tracks which search method was used in the last search() call + # "isbn" = ISBN search returned results, "title_author" = title+author was used + self._last_search_type: str = "title_author" + + @property + def last_search_type(self) -> str: + """Returns the search type used in the last search() call.""" + return self._last_search_type + @classmethod def get_column_config(cls) -> ReleaseColumnConfig: """Column configuration for Direct Download source. @@ -975,81 +987,76 @@ class DirectDownloadSource(ReleaseSource): """ Search for releases using the book's metadata. + Priority: ISBN search first (most precise), then title+author fallback. + For non-English languages, uses localized titles from book.titles_by_language. + Args: book: Book metadata from provider - expand_search: If True, skip ISBN and use title+author search directly. - Useful when ISBN search returns few results. - languages: Optional list of language codes to filter by. - If provided, overrides book.language and default settings. - - Default behavior (expand_search=False): - 1. If ISBN available, try ISBN search first (most precise) - 2. If no results or no ISBN, fall back to title+author search - - Expanded search (expand_search=True): - - Skip ISBN, go straight to title+author search (finds more editions) + expand_search: If True, skip ISBN and use title+author directly + languages: Language codes to filter by (overrides book.language/config) """ - # Determine language filter: explicit languages param > book.language > default - lang_filter = languages if languages else ([book.language] if book.language else None) + # Language filter: explicit param > book.language > config default + lang_filter = languages or ([book.language] if book.language else config.BOOK_LANGUAGE) - # Expanded search skips ISBN and goes straight to title+author + # Reset search type tracking + self._last_search_type = "title_author" + + # ISBN search first (unless expand_search requested) if not expand_search: - # Try ISBN search first if available isbn = book.isbn_13 or book.isbn_10 if isbn: - logger.debug(f"Searching direct downloads by ISBN: {isbn}") + logger.debug(f"Searching by ISBN: {isbn}") filters = SearchFilters(isbn=[isbn]) if lang_filter: filters.lang = lang_filter - try: - book_infos = search_books(isbn, filters) - if book_infos: - logger.info(f"Found {len(book_infos)} releases via ISBN search") - return [_book_info_to_release(bi) for bi in book_infos] - logger.debug("No results from ISBN search, falling back to title+author") + results = search_books(isbn, filters) + if results: + logger.info(f"Found {len(results)} releases via ISBN") + self._last_search_type = "isbn" + return [_book_info_to_release(bi) for bi in results] + logger.debug("No ISBN results, falling back to title+author") except SearchUnavailable: - logger.warning("Direct download search unavailable during ISBN search") raise except Exception as e: - logger.warning(f"ISBN search failed, falling back to title+author: {e}") + logger.warning(f"ISBN search failed: {e}") - # Title + author search (fallback or expanded mode) - query_parts = [] - if book.title: - query_parts.append(book.title) - if book.authors: - query_parts.append(book.authors[0]) # Use first author + # Title + author fallback + author = book.authors[0] if book.authors else "" - query = " ".join(query_parts) - if not query.strip(): - logger.warning("No search query available for book") - return [] + # Group languages by localized title to avoid duplicate searches + if lang_filter and book.titles_by_language: + title_to_langs: Dict[str, List[str]] = {} + for lang in lang_filter: + title = book.titles_by_language.get(lang, book.title) + title_to_langs.setdefault(title, []).append(lang) + searches = list(title_to_langs.items()) + else: + searches = [(book.title, lang_filter)] - logger.debug(f"Searching direct downloads by title+author: {query}") - filters = SearchFilters() - if lang_filter: - filters.lang = lang_filter + # Execute searches with deduplication + seen_ids: set = set() + all_results: List[BookInfo] = [] - try: - book_infos = search_books(query, filters) - logger.info(f"Found {len(book_infos)} releases via title+author search") - return [_book_info_to_release(bi) for bi in book_infos] - except SearchUnavailable: - logger.warning("Direct download search unavailable") - raise - except Exception as e: - logger.error(f"Error searching direct download source: {e}") - raise + for title, langs in searches: + query = f"{title} {author}".strip() + if not query: + continue - def search_raw(self, query: str, filters: SearchFilters) -> List[BookInfo]: - """ - Raw search using existing query format - for backward compatibility. + logger.debug(f"Searching: query='{query}', langs={langs}") + filters = SearchFilters(lang=langs) if langs else SearchFilters() + try: + for bi in search_books(query, filters): + if bi.id not in seen_ids: + seen_ids.add(bi.id) + all_results.append(bi) + except SearchUnavailable: + raise + except Exception as e: + logger.error(f"Search error: {e}") - This is used by the existing "Direct Download Only" mode which doesn't - go through the metadata provider layer. - """ - return search_books(query, filters) + logger.info(f"Found {len(all_results)} releases via title+author") + return [_book_info_to_release(bi) for bi in all_results] def is_available(self) -> bool: """Direct download is always available.""" @@ -1092,6 +1099,7 @@ class DirectDownloadHandler(DownloadHandler): # Check for cancellation before starting if cancel_flag.is_set(): logger.info(f"Download cancelled before starting: {task.task_id}") + status_callback("cancelled", "Cancelled") return None # Create BookInfo from task data - NO AA page fetch here @@ -1116,6 +1124,7 @@ class DirectDownloadHandler(DownloadHandler): except Exception as e: if cancel_flag.is_set(): logger.info(f"Download cancelled during error handling: {task.task_id}") + status_callback("cancelled", "Cancelled") else: logger.error(f"Error downloading book: {e}") status_callback("error", str(e)) @@ -1145,6 +1154,7 @@ class DirectDownloadHandler(DownloadHandler): # Check cancellation before download if cancel_flag.is_set(): logger.info(f"Download cancelled before download call: {book_info.id}") + status_callback("cancelled", "Cancelled") return None # Execute download via _download_book (handles cascade and bypass) @@ -1162,6 +1172,7 @@ class DirectDownloadHandler(DownloadHandler): logger.info(f"Download cancelled during download: {book_info.id}") if book_path.exists(): book_path.unlink() + status_callback("cancelled", "Cancelled") return None if not success_url: @@ -1174,6 +1185,7 @@ class DirectDownloadHandler(DownloadHandler): except Exception as e: if cancel_flag.is_set(): logger.info(f"Download cancelled during error handling: {book_info.id}") + status_callback("cancelled", "Cancelled") else: logger.error(f"Error downloading book: {e}") return None diff --git a/cwa_book_downloader/release_sources/irc/handler.py b/cwa_book_downloader/release_sources/irc/handler.py index a677a18d..837332e0 100644 --- a/cwa_book_downloader/release_sources/irc/handler.py +++ b/cwa_book_downloader/release_sources/irc/handler.py @@ -47,14 +47,22 @@ class IRCDownloadHandler(DownloadHandler): logger.info(f"IRC download: {download_request[:60]}...") nick = config.get("IRC_NICK") or None - client = None + def check_cancelled() -> bool: + """Check if cancelled and handle cleanup.""" + if not cancel_flag.is_set(): + return False + if client: + client.disconnect() + status_callback("cancelled", "Cancelled") + return True + try: # Phase 1: Connect to IRC status_callback("resolving", "Connecting to IRC...") - if cancel_flag.is_set(): + if check_cancelled(): return None client = IRCClient(nick=nick) @@ -64,8 +72,7 @@ class IRCDownloadHandler(DownloadHandler): # Phase 2: Send download request status_callback("resolving", "Requesting file from bot...") - if cancel_flag.is_set(): - client.disconnect() + if check_cancelled(): return None # Send the full request line to the channel @@ -81,8 +88,7 @@ class IRCDownloadHandler(DownloadHandler): client.disconnect() return None - if cancel_flag.is_set(): - client.disconnect() + if check_cancelled(): return None # Phase 4: Download via DCC @@ -108,6 +114,7 @@ class IRCDownloadHandler(DownloadHandler): if cancel_flag.is_set(): # Clean up partial download staging_path.unlink(missing_ok=True) + status_callback("cancelled", "Cancelled") return None logger.info(f"Download complete: {staging_path}") diff --git a/cwa_book_downloader/release_sources/prowlarr/api.py b/cwa_book_downloader/release_sources/prowlarr/api.py index 8414ae2c..601f1d03 100644 --- a/cwa_book_downloader/release_sources/prowlarr/api.py +++ b/cwa_book_downloader/release_sources/prowlarr/api.py @@ -190,19 +190,13 @@ class ProwlarrClient: if not query: return [] - # Build query string with repeated params for arrays - query_parts = [f"query={requests.utils.quote(query)}", f"limit={limit}"] - + params = {"query": query, "limit": limit} if indexer_ids: - for idx_id in indexer_ids: - query_parts.append(f"indexerIds={idx_id}") - + params["indexerIds"] = indexer_ids if categories: - for cat_id in categories: - query_parts.append(f"categories={cat_id}") + params["categories"] = categories - query_string = "&".join(query_parts) - endpoint = f"/api/v1/search?{query_string}" + endpoint = f"/api/v1/search?{urlencode(params, doseq=True)}" try: results = self._request("GET", endpoint) diff --git a/cwa_book_downloader/release_sources/prowlarr/clients/__init__.py b/cwa_book_downloader/release_sources/prowlarr/clients/__init__.py index d253a966..5c66f760 100644 --- a/cwa_book_downloader/release_sources/prowlarr/clients/__init__.py +++ b/cwa_book_downloader/release_sources/prowlarr/clients/__init__.py @@ -45,6 +45,17 @@ class DownloadStatus: download_speed: Optional[int] = None # Bytes per second eta: Optional[int] = None # Seconds remaining + @classmethod + def error(cls, message: str) -> "DownloadStatus": + """Create an error status.""" + return cls( + progress=0, + state=DownloadState.ERROR, + message=message, + complete=False, + file_path=None, + ) + def __post_init__(self): """Validate and normalize state.""" # Normalize string states to enum diff --git a/cwa_book_downloader/release_sources/prowlarr/clients/deluge.py b/cwa_book_downloader/release_sources/prowlarr/clients/deluge.py index 3efe1c40..e2369410 100644 --- a/cwa_book_downloader/release_sources/prowlarr/clients/deluge.py +++ b/cwa_book_downloader/release_sources/prowlarr/clients/deluge.py @@ -8,9 +8,7 @@ configurable via DELUGE_PORT), which requires the daemon to have """ import base64 -from typing import Optional, Tuple - -import requests +from typing import Any, Optional, Tuple from cwa_book_downloader.core.config import config from cwa_book_downloader.core.logger import setup_logger @@ -20,13 +18,17 @@ from cwa_book_downloader.release_sources.prowlarr.clients import ( register_client, ) from cwa_book_downloader.release_sources.prowlarr.clients.torrent_utils import ( - extract_hash_from_magnet, - extract_info_hash_from_torrent, + extract_torrent_info, ) logger = setup_logger(__name__) +def _decode(value: Any) -> Any: + """Decode bytes to string if needed (Deluge returns bytes for strings).""" + return value.decode('utf-8') if isinstance(value, bytes) else value + + @register_client("torrent") class DelugeClient(DownloadClient): """Deluge download client using deluge-client RPC library.""" @@ -107,40 +109,15 @@ class DelugeClient(DownloadClient): try: self._ensure_connected() - # Use configured category if not explicitly provided category = category or self._category - # Try to extract hash from magnet URL before adding - expected_hash = extract_hash_from_magnet(url) - if expected_hash: - logger.debug(f"Extracted hash from magnet: {expected_hash}") + torrent_info = extract_torrent_info(url) + if not torrent_info.is_magnet and not torrent_info.torrent_data: + raise Exception("Failed to fetch torrent file") - is_magnet = url.startswith("magnet:") - logger.debug(f"Adding torrent - URL type: {'magnet' if is_magnet else 'torrent file'}") - - torrent_data = None - - # For non-magnet URLs, fetch the .torrent file - if not is_magnet: - logger.debug(f"Fetching torrent file from: {url[:80]}...") - try: - resp = requests.get(url, timeout=30) - resp.raise_for_status() - torrent_data = resp.content - expected_hash = extract_info_hash_from_torrent(torrent_data) - if expected_hash: - logger.debug(f"Extracted hash from torrent file: {expected_hash}") - else: - logger.warning("Could not extract hash from torrent file") - except Exception as e: - logger.warning(f"Failed to fetch torrent file: {e}") - raise - - # Add options options = {} - # Add the torrent - if is_magnet: + if torrent_info.is_magnet: # Add magnet link torrent_id = self._client.call( 'core.add_torrent_magnet', @@ -148,8 +125,7 @@ class DelugeClient(DownloadClient): options, ) else: - # Add from torrent file content (base64 encoded) - filedump = base64.b64encode(torrent_data).decode('ascii') + filedump = base64.b64encode(torrent_info.torrent_data).decode('ascii') torrent_id = self._client.call( 'core.add_torrent_file', f"{name}.torrent", @@ -158,9 +134,7 @@ class DelugeClient(DownloadClient): ) if torrent_id: - # Deluge returns bytes, decode to string - if isinstance(torrent_id, bytes): - torrent_id = torrent_id.decode('utf-8') + torrent_id = _decode(torrent_id) logger.info(f"Added torrent to Deluge: {torrent_id}") return torrent_id.lower() @@ -192,13 +166,7 @@ class DelugeClient(DownloadClient): ) if not status: - return DownloadStatus( - progress=0, - state="error", - message="Torrent not found", - complete=False, - file_path=None, - ) + return DownloadStatus.error("Torrent not found") # Deluge states: Downloading, Seeding, Paused, Checking, Queued, Error, Moving state_map = { @@ -212,10 +180,7 @@ class DelugeClient(DownloadClient): 'Allocating': ('downloading', 'Allocating space'), } - deluge_state = status.get(b'state', b'Unknown') - if isinstance(deluge_state, bytes): - deluge_state = deluge_state.decode('utf-8') - + deluge_state = _decode(status.get(b'state', b'Unknown')) state, message = state_map.get(deluge_state, ('unknown', deluge_state)) progress = status.get(b'progress', 0) complete = progress >= 100 @@ -223,20 +188,14 @@ class DelugeClient(DownloadClient): if complete: message = "Download complete" - # Get ETA if available and reasonable eta = status.get(b'eta') - if eta and eta > 604800: # More than 1 week + if eta and eta > 604800: eta = None - # Build file path for completed downloads file_path = None if complete: - save_path = status.get(b'save_path', b'') - name = status.get(b'name', b'') - if isinstance(save_path, bytes): - save_path = save_path.decode('utf-8') - if isinstance(name, bytes): - name = name.decode('utf-8') + save_path = _decode(status.get(b'save_path', b'')) + name = _decode(status.get(b'name', b'')) if save_path and name: file_path = f"{save_path}/{name}" @@ -254,13 +213,7 @@ class DelugeClient(DownloadClient): self._connected = False error_type = type(e).__name__ logger.error(f"Deluge get_status failed ({error_type}): {e}") - return DownloadStatus( - progress=0, - state="error", - message=f"{error_type}: {e}", - complete=False, - file_path=None, - ) + return DownloadStatus.error(f"{error_type}: {e}") def remove(self, download_id: str, delete_files: bool = False) -> bool: """ @@ -316,12 +269,8 @@ class DelugeClient(DownloadClient): ) if status: - save_path = status.get(b'save_path', b'') - name = status.get(b'name', b'') - if isinstance(save_path, bytes): - save_path = save_path.decode('utf-8') - if isinstance(name, bytes): - name = name.decode('utf-8') + save_path = _decode(status.get(b'save_path', b'')) + name = _decode(status.get(b'name', b'')) if save_path and name: return f"{save_path}/{name}" return None @@ -333,50 +282,25 @@ class DelugeClient(DownloadClient): return None def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]: - """ - Check if a torrent for this URL already exists in Deluge. - - Args: - url: Magnet link or .torrent URL - - Returns: - Tuple of (info_hash, status) if found, None if not found. - """ + """Check if a torrent for this URL already exists in Deluge.""" try: self._ensure_connected() - # Try to extract hash from magnet URL - expected_hash = extract_hash_from_magnet(url) - - # If not a magnet, try to fetch and parse the .torrent file - if not expected_hash and not url.startswith("magnet:"): - logger.debug(f"Fetching torrent file to check for existing: {url[:80]}...") - try: - resp = requests.get(url, timeout=30) - resp.raise_for_status() - expected_hash = extract_info_hash_from_torrent(resp.content) - except Exception as e: - logger.debug(f"Could not fetch torrent file: {e}") - return None - - if not expected_hash: - logger.debug("Could not extract hash from URL") + torrent_info = extract_torrent_info(url) + if not torrent_info.info_hash: return None - # Check if this torrent exists in Deluge status = self._client.call( 'core.get_torrent_status', - expected_hash, + torrent_info.info_hash, ['state'], ) if status: - full_status = self.get_status(expected_hash) - logger.debug(f"Found existing torrent in Deluge: {expected_hash} (state: {full_status.state})") - return (expected_hash, full_status) + full_status = self.get_status(torrent_info.info_hash) + return (torrent_info.info_hash, full_status) return None - except Exception as e: self._connected = False logger.debug(f"Error checking for existing torrent: {e}") diff --git a/cwa_book_downloader/release_sources/prowlarr/clients/nzbget.py b/cwa_book_downloader/release_sources/prowlarr/clients/nzbget.py index 0e463382..1d109c4f 100644 --- a/cwa_book_downloader/release_sources/prowlarr/clients/nzbget.py +++ b/cwa_book_downloader/release_sources/prowlarr/clients/nzbget.py @@ -247,23 +247,11 @@ class NZBGetClient(DownloadClient): ) # Not found in queue or history - return DownloadStatus( - progress=0, - state="error", - message="Download not found", - complete=False, - file_path=None, - ) + return DownloadStatus.error("Download not found") except Exception as e: error_type = type(e).__name__ logger.error(f"NZBGet get_status failed ({error_type}): {e}") - return DownloadStatus( - progress=0, - state="error", - message=f"{error_type}: {e}", - complete=False, - file_path=None, - ) + return DownloadStatus.error(f"{error_type}: {e}") def remove(self, download_id: str, delete_files: bool = False) -> bool: """ diff --git a/cwa_book_downloader/release_sources/prowlarr/clients/qbittorrent.py b/cwa_book_downloader/release_sources/prowlarr/clients/qbittorrent.py index 1a23686d..0c831ea1 100644 --- a/cwa_book_downloader/release_sources/prowlarr/clients/qbittorrent.py +++ b/cwa_book_downloader/release_sources/prowlarr/clients/qbittorrent.py @@ -7,8 +7,6 @@ Uses the qbittorrent-api library to communicate with qBittorrent's Web API. import time from typing import Optional, Tuple -import requests - from cwa_book_downloader.core.config import config from cwa_book_downloader.core.logger import setup_logger from cwa_book_downloader.release_sources.prowlarr.clients import ( @@ -17,8 +15,7 @@ from cwa_book_downloader.release_sources.prowlarr.clients import ( register_client, ) from cwa_book_downloader.release_sources.prowlarr.clients.torrent_utils import ( - extract_hash_from_magnet, - extract_info_hash_from_torrent, + extract_torrent_info, ) logger = setup_logger(__name__) @@ -91,32 +88,9 @@ class QBittorrentClient(DownloadClient): if "Conflict" not in type(e).__name__ and "409" not in str(e): logger.debug(f"Could not create category '{category}': {type(e).__name__}: {e}") - # Try to extract hash from magnet URL before adding - expected_hash = extract_hash_from_magnet(url) - if expected_hash: - logger.debug(f"Extracted hash from magnet: {expected_hash}") - - is_magnet = url.startswith("magnet:") - logger.debug(f"Adding torrent - URL type: {'magnet' if is_magnet else 'torrent file'}") - - torrent_data = None - - # For non-magnet URLs, fetch the .torrent file to extract the hash - if not is_magnet and not expected_hash: - logger.debug(f"Fetching torrent file from: {url[:80]}...") - try: - resp = requests.get(url, timeout=30) - resp.raise_for_status() - torrent_data = resp.content - expected_hash = extract_info_hash_from_torrent(torrent_data) - if expected_hash: - logger.debug(f"Extracted hash from torrent file: {expected_hash}") - else: - logger.warning("Could not extract hash from torrent file") - except Exception as e: - logger.warning(f"Failed to fetch torrent file: {e}") - - logger.debug(f"Expected hash: {expected_hash}") + torrent_info = extract_torrent_info(url) + expected_hash = torrent_info.info_hash + torrent_data = torrent_info.torrent_data # Add the torrent - use file content if we have it, otherwise URL if torrent_data: @@ -177,13 +151,7 @@ class QBittorrentClient(DownloadClient): try: torrents = self._client.torrents_info(torrent_hashes=download_id) if not torrents: - return DownloadStatus( - progress=0, - state="error", - message="Torrent not found", - complete=False, - file_path=None, - ) + return DownloadStatus.error("Torrent not found") torrent = torrents[0] @@ -233,13 +201,7 @@ class QBittorrentClient(DownloadClient): except Exception as e: error_type = type(e).__name__ logger.error(f"qBittorrent get_status failed ({error_type}): {e}") - return DownloadStatus( - progress=0, - state="error", - message=f"{error_type}: {e}", - complete=False, - file_path=None, - ) + return DownloadStatus.error(f"{error_type}: {e}") def remove(self, download_id: str, delete_files: bool = False) -> bool: """ @@ -287,46 +249,18 @@ class QBittorrentClient(DownloadClient): return None def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]: - """ - Check if a torrent for this URL already exists in qBittorrent. - - Extracts the info_hash from the magnet link or .torrent file and - checks if qBittorrent already has this torrent. - - Args: - url: Magnet link or .torrent URL - - Returns: - Tuple of (info_hash, status) if found, None if not found. - """ + """Check if a torrent for this URL already exists in qBittorrent.""" try: - # Try to extract hash from magnet URL - expected_hash = extract_hash_from_magnet(url) - - # If not a magnet, try to fetch and parse the .torrent file - if not expected_hash and not url.startswith("magnet:"): - logger.debug(f"Fetching torrent file to check for existing: {url[:80]}...") - try: - resp = requests.get(url, timeout=30) - resp.raise_for_status() - expected_hash = extract_info_hash_from_torrent(resp.content) - except Exception as e: - logger.debug(f"Could not fetch torrent file: {e}") - return None - - if not expected_hash: - logger.debug("Could not extract hash from URL") + torrent_info = extract_torrent_info(url) + if not torrent_info.info_hash: return None - # Check if this torrent exists in qBittorrent - torrents = self._client.torrents_info(torrent_hashes=expected_hash) + torrents = self._client.torrents_info(torrent_hashes=torrent_info.info_hash) if torrents: - status = self.get_status(expected_hash) - logger.debug(f"Found existing torrent in qBittorrent: {expected_hash} (state: {status.state})") - return (expected_hash, status) + status = self.get_status(torrent_info.info_hash) + return (torrent_info.info_hash, status) return None - except Exception as e: logger.debug(f"Error checking for existing torrent: {e}") return None diff --git a/cwa_book_downloader/release_sources/prowlarr/clients/sabnzbd.py b/cwa_book_downloader/release_sources/prowlarr/clients/sabnzbd.py index ee99f024..72f34775 100644 --- a/cwa_book_downloader/release_sources/prowlarr/clients/sabnzbd.py +++ b/cwa_book_downloader/release_sources/prowlarr/clients/sabnzbd.py @@ -264,23 +264,11 @@ class SABnzbdClient(DownloadClient): ) # Not found - return DownloadStatus( - progress=0, - state="error", - message="Download not found", - complete=False, - file_path=None, - ) + return DownloadStatus.error("Download not found") except Exception as e: error_type = type(e).__name__ logger.error(f"SABnzbd get_status failed ({error_type}): {e}") - return DownloadStatus( - progress=0, - state="error", - message=f"{error_type}: {e}", - complete=False, - file_path=None, - ) + return DownloadStatus.error(f"{error_type}: {e}") def remove(self, download_id: str, delete_files: bool = False) -> bool: """ diff --git a/cwa_book_downloader/release_sources/prowlarr/clients/torrent_utils.py b/cwa_book_downloader/release_sources/prowlarr/clients/torrent_utils.py index fd1902ad..6cf88280 100644 --- a/cwa_book_downloader/release_sources/prowlarr/clients/torrent_utils.py +++ b/cwa_book_downloader/release_sources/prowlarr/clients/torrent_utils.py @@ -13,14 +13,74 @@ See BEP-3: http://bittorrent.org/beps/bep_0003.html import base64 import hashlib import re +from dataclasses import dataclass from typing import Optional, Tuple from urllib.parse import parse_qs, urlparse +import requests + from cwa_book_downloader.core.logger import setup_logger logger = setup_logger(__name__) +@dataclass +class TorrentInfo: + """Parsed information from a torrent URL.""" + + info_hash: Optional[str] + """40-character lowercase hex info_hash, or None if extraction failed.""" + + torrent_data: Optional[bytes] + """Raw .torrent file content, only populated for .torrent URLs.""" + + is_magnet: bool + """True if the URL was a magnet link.""" + + +def extract_torrent_info(url: str, fetch_torrent: bool = True) -> TorrentInfo: + """ + Extract torrent info_hash and optionally fetch torrent file content. + + Handles both magnet links and .torrent file URLs. For magnet links, + extracts the hash directly. For .torrent URLs, optionally fetches + the file and parses it to extract the hash. + + Args: + url: Magnet link or .torrent URL + fetch_torrent: If True, fetch .torrent URLs to extract hash. + Set to False if you only need the hash from magnets. + + Returns: + TorrentInfo with extracted hash and optional torrent data. + """ + is_magnet = url.startswith("magnet:") + + # Try to extract hash from magnet URL + if is_magnet: + info_hash = extract_hash_from_magnet(url) + return TorrentInfo(info_hash=info_hash, torrent_data=None, is_magnet=True) + + # Not a magnet - try to fetch and parse the .torrent file + if not fetch_torrent: + return TorrentInfo(info_hash=None, torrent_data=None, is_magnet=False) + + try: + logger.debug(f"Fetching torrent file from: {url[:80]}...") + resp = requests.get(url, timeout=30) + resp.raise_for_status() + torrent_data = resp.content + info_hash = extract_info_hash_from_torrent(torrent_data) + if info_hash: + logger.debug(f"Extracted hash from torrent file: {info_hash}") + else: + logger.warning("Could not extract hash from torrent file") + return TorrentInfo(info_hash=info_hash, torrent_data=torrent_data, is_magnet=False) + except Exception as e: + logger.debug(f"Could not fetch torrent file: {e}") + return TorrentInfo(info_hash=None, torrent_data=None, is_magnet=False) + + def parse_transmission_url(url: str) -> Tuple[str, int, str]: """ Parse a Transmission URL into host, port, and RPC path. diff --git a/cwa_book_downloader/release_sources/prowlarr/clients/transmission.py b/cwa_book_downloader/release_sources/prowlarr/clients/transmission.py index 1467f5c2..ac574d28 100644 --- a/cwa_book_downloader/release_sources/prowlarr/clients/transmission.py +++ b/cwa_book_downloader/release_sources/prowlarr/clients/transmission.py @@ -6,8 +6,6 @@ Uses the transmission-rpc library to communicate with Transmission's RPC API. from typing import Optional, Tuple -import requests - from cwa_book_downloader.core.config import config from cwa_book_downloader.core.logger import setup_logger from cwa_book_downloader.release_sources.prowlarr.clients import ( @@ -16,8 +14,7 @@ from cwa_book_downloader.release_sources.prowlarr.clients import ( register_client, ) from cwa_book_downloader.release_sources.prowlarr.clients.torrent_utils import ( - extract_hash_from_magnet, - extract_info_hash_from_torrent, + extract_torrent_info, parse_transmission_url, ) @@ -86,60 +83,24 @@ class TransmissionClient(DownloadClient): Exception: If adding fails. """ try: - # Use configured category if not explicitly provided category = category or self._category - # Try to extract hash from magnet URL before adding - expected_hash = extract_hash_from_magnet(url) - if expected_hash: - logger.debug(f"Extracted hash from magnet: {expected_hash}") + torrent_info = extract_torrent_info(url) - is_magnet = url.startswith("magnet:") - logger.debug(f"Adding torrent - URL type: {'magnet' if is_magnet else 'torrent file'}") - - torrent_data = None - - # For non-magnet URLs, fetch the .torrent file to extract the hash - if not is_magnet and not expected_hash: - logger.debug(f"Fetching torrent file from: {url[:80]}...") - try: - resp = requests.get(url, timeout=30) - resp.raise_for_status() - torrent_data = resp.content - expected_hash = extract_info_hash_from_torrent(torrent_data) - if expected_hash: - logger.debug(f"Extracted hash from torrent file: {expected_hash}") - else: - logger.warning("Could not extract hash from torrent file") - except Exception as e: - logger.warning(f"Failed to fetch torrent file: {e}") - - logger.debug(f"Expected hash: {expected_hash}") - - # Add the torrent - if torrent_data: - # Add from torrent file content (pass raw bytes, library handles encoding) + if torrent_info.torrent_data: torrent = self._client.add_torrent( - torrent=torrent_data, + torrent=torrent_info.torrent_data, labels=[category], ) else: - # Add from URL or magnet torrent = self._client.add_torrent( torrent=url, labels=[category], ) - # Get the hash from the returned torrent torrent_hash = torrent.hashString.lower() logger.info(f"Added torrent to Transmission: {torrent_hash}") - # Verify hash matches if we extracted one - if expected_hash and torrent_hash != expected_hash: - logger.warning( - f"Hash mismatch: expected {expected_hash}, got {torrent_hash}" - ) - return torrent_hash except Exception as e: @@ -215,24 +176,11 @@ class TransmissionClient(DownloadClient): ) except KeyError: - # Torrent not found - return DownloadStatus( - progress=0, - state="error", - message="Torrent not found", - complete=False, - file_path=None, - ) + return DownloadStatus.error("Torrent not found") except Exception as e: error_type = type(e).__name__ logger.error(f"Transmission get_status failed ({error_type}): {e}") - return DownloadStatus( - progress=0, - state="error", - message=f"{error_type}: {e}", - complete=False, - file_path=None, - ) + return DownloadStatus.error(f"{error_type}: {e}") def remove(self, download_id: str, delete_files: bool = False) -> bool: """ @@ -281,44 +229,18 @@ class TransmissionClient(DownloadClient): return None def find_existing(self, url: str) -> Optional[Tuple[str, DownloadStatus]]: - """ - Check if a torrent for this URL already exists in Transmission. - - Args: - url: Magnet link or .torrent URL - - Returns: - Tuple of (info_hash, status) if found, None if not found. - """ + """Check if a torrent for this URL already exists in Transmission.""" try: - # Try to extract hash from magnet URL - expected_hash = extract_hash_from_magnet(url) - - # If not a magnet, try to fetch and parse the .torrent file - if not expected_hash and not url.startswith("magnet:"): - logger.debug(f"Fetching torrent file to check for existing: {url[:80]}...") - try: - resp = requests.get(url, timeout=30) - resp.raise_for_status() - expected_hash = extract_info_hash_from_torrent(resp.content) - except Exception as e: - logger.debug(f"Could not fetch torrent file: {e}") - return None - - if not expected_hash: - logger.debug("Could not extract hash from URL") + torrent_info = extract_torrent_info(url) + if not torrent_info.info_hash: return None - # Check if this torrent exists in Transmission try: - torrent = self._client.get_torrent(expected_hash) - status = self.get_status(expected_hash) - logger.debug(f"Found existing torrent in Transmission: {expected_hash} (state: {status.state})") - return (expected_hash, status) + self._client.get_torrent(torrent_info.info_hash) + status = self.get_status(torrent_info.info_hash) + return (torrent_info.info_hash, status) except KeyError: - # Torrent not found return None - except Exception as e: logger.debug(f"Error checking for existing torrent: {e}") return None diff --git a/cwa_book_downloader/release_sources/prowlarr/handler.py b/cwa_book_downloader/release_sources/prowlarr/handler.py index 7db0c22f..a96c7d42 100644 --- a/cwa_book_downloader/release_sources/prowlarr/handler.py +++ b/cwa_book_downloader/release_sources/prowlarr/handler.py @@ -21,6 +21,7 @@ from cwa_book_downloader.release_sources.prowlarr.clients import ( get_client, list_configured_clients, ) +from cwa_book_downloader.release_sources.prowlarr.utils import get_protocol, get_unique_path logger = setup_logger(__name__) @@ -28,17 +29,6 @@ logger = setup_logger(__name__) POLL_INTERVAL = 2 -def _determine_protocol(result: dict) -> str: - """Determine download protocol from Prowlarr result.""" - # Prowlarr provides protocol directly - just use it - protocol = result.get("protocol", "").lower() - if protocol == "torrent": - return "torrent" - if protocol == "usenet": - return "usenet" - return "unknown" - - @register_handler("prowlarr") class ProwlarrHandler(DownloadHandler): """Handler for Prowlarr downloads via configured torrent or usenet client.""" @@ -76,7 +66,7 @@ class ProwlarrHandler(DownloadHandler): return None # Determine protocol - protocol = _determine_protocol(prowlarr_result) + protocol = get_protocol(prowlarr_result) if protocol == "unknown": status_callback("error", "Could not determine download protocol") return None @@ -102,7 +92,7 @@ class ProwlarrHandler(DownloadHandler): # If already complete, skip straight to file handling if existing_status.complete: logger.info(f"Existing download is complete, copying file directly") - status_callback("processing", "Found existing download, copying to library...") + status_callback("resolving", "Found existing download, copying to library...") source_path = client.get_download_path(download_id) if not source_path: @@ -256,7 +246,7 @@ class ProwlarrHandler(DownloadHandler): The orchestrator will find and filter book files. """ try: - status_callback("processing", "Staging file...") + status_callback("resolving", "Staging file...") # Torrents: copy to preserve seeding. Usenet: configurable. if protocol == "torrent": @@ -270,12 +260,7 @@ class ProwlarrHandler(DownloadHandler): if source_path.is_dir(): # Multi-file download: stage entire directory # Orchestrator will extract book files - staged_path = staging_dir / source_path.name - if staged_path.exists(): - counter = 1 - while staged_path.exists(): - staged_path = staging_dir / f"{source_path.name}_{counter}" - counter += 1 + staged_path = get_unique_path(staging_dir, source_path.name) if use_copy: shutil.copytree(str(source_path), str(staged_path)) @@ -284,12 +269,7 @@ class ProwlarrHandler(DownloadHandler): logger.debug(f"Staged directory: {staged_path.name}") else: # Single file download - staged_path = staging_dir / source_path.name - if staged_path.exists(): - counter = 1 - while staged_path.exists(): - staged_path = staging_dir / f"{source_path.stem}_{counter}{source_path.suffix}" - counter += 1 + staged_path = get_unique_path(staging_dir, source_path.stem, source_path.suffix) if use_copy: shutil.copy2(str(source_path), str(staged_path)) diff --git a/cwa_book_downloader/release_sources/prowlarr/settings.py b/cwa_book_downloader/release_sources/prowlarr/settings.py index 227342fd..0f68f676 100644 --- a/cwa_book_downloader/release_sources/prowlarr/settings.py +++ b/cwa_book_downloader/release_sources/prowlarr/settings.py @@ -321,7 +321,7 @@ def prowlarr_config_settings(): description="Select which indexers to search. πŸ“š = has book categories. Leave empty to search all.", options=_get_indexer_options, default=[], - show_when={"field": "PROWLARR_URL", "notEmpty": True}, + show_when={"field": "PROWLARR_ENABLED", "value": True}, ), ] diff --git a/cwa_book_downloader/release_sources/prowlarr/source.py b/cwa_book_downloader/release_sources/prowlarr/source.py index 566d05ad..01766b3c 100644 --- a/cwa_book_downloader/release_sources/prowlarr/source.py +++ b/cwa_book_downloader/release_sources/prowlarr/source.py @@ -25,6 +25,7 @@ from cwa_book_downloader.release_sources import ( ) from cwa_book_downloader.release_sources.prowlarr.api import ProwlarrClient from cwa_book_downloader.release_sources.prowlarr.cache import cache_release +from cwa_book_downloader.release_sources.prowlarr.utils import get_protocol_display logger = setup_logger(__name__) @@ -90,31 +91,6 @@ def _extract_format(title: str) -> Optional[str]: return None -def _get_protocol(result: dict) -> str: - """ - Get protocol from Prowlarr result. - - Uses the protocol field directly if available, otherwise infers from URL. - Returns user-friendly labels: "torrent" or "nzb". - """ - # Prowlarr provides protocol directly - use it - protocol = result.get("protocol", "").lower() - if protocol == "usenet": - return "nzb" - if protocol == "torrent": - return "torrent" - - # Fallback: infer from download URL - download_url = result.get("downloadUrl") or result.get("magnetUrl") or "" - url_lower = download_url.lower() - if url_lower.startswith("magnet:") or ".torrent" in url_lower: - return "torrent" - if ".nzb" in url_lower: - return "nzb" - - return "unknown" - - def _extract_language(title: str) -> Optional[str]: """ Extract language from release title. @@ -164,7 +140,7 @@ def _prowlarr_result_to_release(result: dict) -> Release: download_url = result.get("downloadUrl") or result.get("magnetUrl") info_url = result.get("infoUrl") or result.get("guid") indexer = result.get("indexer", "Unknown") - protocol = _get_protocol(result) + protocol = get_protocol_display(result) seeders = result.get("seeders") leechers = result.get("leechers") # Format peers display string: "seeders / leechers" diff --git a/cwa_book_downloader/release_sources/prowlarr/utils.py b/cwa_book_downloader/release_sources/prowlarr/utils.py new file mode 100644 index 00000000..bbe27cf0 --- /dev/null +++ b/cwa_book_downloader/release_sources/prowlarr/utils.py @@ -0,0 +1,76 @@ +""" +Shared utilities for Prowlarr release source. + +Provides common helper functions used across the Prowlarr plugin. +""" + +from pathlib import Path +from typing import Optional + + +def get_protocol(result: dict) -> str: + """ + Get the download protocol from a Prowlarr result. + + Uses the protocol field directly if available, otherwise infers from URL. + + Args: + result: Prowlarr search result dictionary + + Returns: + Protocol string: "torrent", "usenet", or "unknown" + """ + # Prowlarr provides protocol directly - use it + protocol = result.get("protocol", "").lower() + if protocol in ("torrent", "usenet"): + return protocol + + # Fallback: infer from download URL + download_url = result.get("downloadUrl") or result.get("magnetUrl") or "" + url_lower = download_url.lower() + if url_lower.startswith("magnet:") or ".torrent" in url_lower: + return "torrent" + if ".nzb" in url_lower: + return "usenet" + + return "unknown" + + +def get_protocol_display(result: dict) -> str: + """ + Get a user-friendly display label for the protocol. + + Args: + result: Prowlarr search result dictionary + + Returns: + Display label: "torrent", "nzb", or "unknown" + """ + protocol = get_protocol(result) + if protocol == "usenet": + return "nzb" + return protocol + + +def get_unique_path(staging_dir: Path, name: str, suffix: str = "") -> Path: + """ + Generate a unique path in staging_dir, appending _N if needed. + + Args: + staging_dir: Directory to create the path in + name: Base name for the file/directory + suffix: Optional suffix (e.g., ".epub" for files) + + Returns: + Unique Path that doesn't exist in staging_dir + """ + staged_path = staging_dir / (name + suffix) + if not staged_path.exists(): + return staged_path + + counter = 1 + while True: + staged_path = staging_dir / f"{name}_{counter}{suffix}" + if not staged_path.exists(): + return staged_path + counter += 1 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 27f43f67..bd74a410 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -17,3 +17,7 @@ services: - ./.local/tmp:/tmp/cwa-book-downloader # Mount source code for development (no rebuild needed for Python code changes) - ./cwa_book_downloader:/app/cwa_book_downloader:ro + # Download client volume - required for Prowlarr/torrent/usenet integration + # IMPORTANT: Both sides of this mount must match your download client's volume mount exactly. + # Example: if qBittorrent has "/mnt/storage/downloads:/data/torrents", use the same here: + # - /mnt/storage/downloads:/data/torrents diff --git a/docker-compose.extbp.dev.yml b/docker-compose.extbp.dev.yml index 13203e7a..9c943bd7 100644 --- a/docker-compose.extbp.dev.yml +++ b/docker-compose.extbp.dev.yml @@ -18,6 +18,10 @@ services: - ./.local/ingest:/cwa-book-ingest - ./.local/log:/var/log/cwa-book-downloader - ./.local/tmp:/tmp/cwa-book-downloader + # Download client volume - required for Prowlarr/torrent/usenet integration + # IMPORTANT: Both sides of this mount must match your download client's volume mount exactly. + # Example: if qBittorrent has "/mnt/storage/downloads:/data/torrents", use the same here: + # - /mnt/storage/downloads:/data/torrents flaresolverr: image: ghcr.io/flaresolverr/flaresolverr:latest diff --git a/docker-compose.extbp.yml b/docker-compose.extbp.yml index e0c992fb..f7a66a5a 100644 --- a/docker-compose.extbp.yml +++ b/docker-compose.extbp.yml @@ -5,8 +5,8 @@ services: environment: TZ: America/New_York EXT_BYPASSER_URL: http://flaresolverr:8191 - # UID: 1000 - # GID: 100 + # PUID: 1000 + # PGID: 1000 # CWA_DB_PATH: /auth/app.db ports: - 8084:8084 @@ -15,6 +15,10 @@ services: - /tmp/data/calibre-web/ingest:/cwa-book-ingest - /path/to/config:/config # - /cwa/config/path/app.db:/auth/app.db:ro + # Download client volume - required for Prowlarr/torrent/usenet integration + # IMPORTANT: Both sides of this mount must match your download client's volume mount exactly. + # Example: if qBittorrent has "/mnt/storage/downloads:/data/torrents", use the same here: + # - /mnt/storage/downloads:/data/torrents flaresolverr: image: ghcr.io/flaresolverr/flaresolverr:latest diff --git a/docker-compose.tor.dev.yml b/docker-compose.tor.dev.yml index 6419b369..a495b967 100644 --- a/docker-compose.tor.dev.yml +++ b/docker-compose.tor.dev.yml @@ -15,3 +15,7 @@ services: - ./.local/ingest:/cwa-book-ingest - ./.local/log:/var/log/cwa-book-downloader - ./.local/tmp:/tmp/cwa-book-downloader + # Download client volume - required for Prowlarr/torrent/usenet integration + # IMPORTANT: Both sides of this mount must match your download client's volume mount exactly. + # Example: if qBittorrent has "/mnt/storage/downloads:/data/torrents", use the same here: + # - /mnt/storage/downloads:/data/torrents diff --git a/docker-compose.tor.yml b/docker-compose.tor.yml index fbde64fc..a4d91c1a 100644 --- a/docker-compose.tor.yml +++ b/docker-compose.tor.yml @@ -6,6 +6,8 @@ services: FLASK_PORT: 8084 TZ: America/New_York USING_TOR: true + # PUID: 1000 + # PGID: 1000 # CWA_DB_PATH: /auth/app.db cap_add: - NET_ADMIN @@ -17,3 +19,7 @@ services: - /tmp/data/calibre-web/ingest:/cwa-book-ingest - /path/to/config:/config # - /cwa/config/path/app.db:/auth/app.db:ro + # Download client volume - required for Prowlarr/torrent/usenet integration + # IMPORTANT: Both sides of this mount must match your download client's volume mount exactly. + # Example: if qBittorrent has "/mnt/storage/downloads:/data/torrents", use the same here: + # - /mnt/storage/downloads:/data/torrents diff --git a/docker-compose.yml b/docker-compose.yml index ed006963..17a74323 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,8 @@ services: container_name: calibre-web-automated-book-downloader environment: TZ: America/New_York - # UID: 1000 - # GID: 100 + # PUID: 1000 + # PGID: 1000 # CWA_DB_PATH: /auth/app.db ports: - 8084:8084 @@ -13,3 +13,7 @@ services: volumes: - /tmp/data/calibre-web/ingest:/cwa-book-ingest # This is where the books will be downloaded and ingested by your book management application - /path/to/config:/config # Configuration files and database + # Download client volume - required for Prowlarr/torrent/usenet integration + # IMPORTANT: Both sides of this mount must match your download client's volume mount exactly. + # Example: if qBittorrent has "/mnt/storage/downloads:/data/torrents", use the same here: + # - /mnt/storage/downloads:/data/torrents diff --git a/entrypoint.sh b/entrypoint.sh index 045aed7c..51ad6cad 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -28,30 +28,53 @@ if [ "$TZ" ]; then ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone fi -# Set UID if not set -if [ -z "$UID" ]; then - UID=1000 +# Determine user ID with proper precedence: +# 1. PUID (LinuxServer.io standard - recommended) +# 2. UID (legacy, for backward compatibility with existing installs) +# 3. Default to 1000 +# +# Note: $UID is a bash builtin that's always set. We use `printenv` to detect +# if UID was explicitly set as an environment variable (e.g., via docker-compose). +if [ -n "$PUID" ]; then + RUN_UID="$PUID" + echo "Using PUID=$RUN_UID" +elif printenv UID >/dev/null 2>&1; then + RUN_UID="$(printenv UID)" + echo "Using UID=$RUN_UID (legacy - consider migrating to PUID)" +else + RUN_UID=1000 + echo "Using default UID=$RUN_UID" fi -# Set GID if not set -if [ -z "$GID" ]; then - GID=100 +# Determine group ID with proper precedence: +# 1. PGID (LinuxServer.io standard - recommended) +# 2. GID (legacy, for backward compatibility with existing installs) +# 3. Default to 1000 +if [ -n "$PGID" ]; then + RUN_GID="$PGID" + echo "Using PGID=$RUN_GID" +elif [ -n "$GID" ]; then + RUN_GID="$GID" + echo "Using GID=$RUN_GID (legacy - consider migrating to PGID)" +else + RUN_GID=1000 + echo "Using default GID=$RUN_GID" fi -if ! getent group "$GID" >/dev/null; then - echo "Adding group $GID with name appuser" - groupadd -g "$GID" appuser +if ! getent group "$RUN_GID" >/dev/null; then + echo "Adding group $RUN_GID with name appuser" + groupadd -g "$RUN_GID" appuser fi # Create user if it doesn't exist -if ! id -u "$UID" >/dev/null 2>&1; then - echo "Adding user $UID with name appuser" - useradd -u "$UID" -g "$GID" -d /app -s /sbin/nologin appuser +if ! id -u "$RUN_UID" >/dev/null 2>&1; then + echo "Adding user $RUN_UID with name appuser" + useradd -u "$RUN_UID" -g "$RUN_GID" -d /app -s /sbin/nologin appuser fi # Get username for the UID (whether we just created it or it existed) -USERNAME=$(getent passwd "$UID" | cut -d: -f1) -echo "Username for UID $UID is $USERNAME" +USERNAME=$(getent passwd "$RUN_UID" | cut -d: -f1) +echo "Username for UID $RUN_UID is $USERNAME" test_write() { folder=$1 @@ -93,9 +116,9 @@ make_writable() { change_ownership() { folder=$1 mkdir -p $folder - echo "Changing ownership of $folder to $USERNAME:$GID" - chown -R "${UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..." - chown -R ":${GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..." + echo "Changing ownership of $folder to $USERNAME:$RUN_GID" + chown -R "${RUN_UID}" "${folder}" || echo "Failed to change user ownership for ${folder}, continuing..." + chown -R ":${RUN_GID}" "${folder}" || echo "Failed to change group ownership for ${folder}, continuing..." } change_ownership /app @@ -103,6 +126,7 @@ change_ownership /var/log/cwa-book-downloader change_ownership /tmp/cwa-book-downloader # Test write to all folders +make_writable ${CONFIG_DIR:-/config} make_writable /cwa-book-ingest # Always run Gunicorn (even when DEBUG=true) to ensure Socket.IO WebSocket @@ -165,13 +189,17 @@ if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" fi -# Hacky way to verify /tmp has at least 1MB of space and is writable/readable +# Verify /tmp has at least 1MB of space and is writable/readable echo "Verifying /tmp has enough space" rm -f /tmp/test.cwa-bd -for i in {1..150000}; do printf "%04d\n" $i; done > /tmp/test.cwa-bd -sum=$(python3 -c "print(sum(int(l.strip()) for l in open('/tmp/test.cwa-bd').readlines()))") -[ "$sum" == 11250075000 ] && echo "Success: /tmp is writable" || (echo "Failure: /tmp is not writable" && exit 1) -rm /tmp/test.cwa-bd +if dd if=/dev/zero of=/tmp/test.cwa-bd bs=1M count=1 2>/dev/null && \ + [ "$(wc -c < /tmp/test.cwa-bd)" -eq 1048576 ]; then + rm -f /tmp/test.cwa-bd + echo "Success: /tmp is writable and readable" +else + echo "Failure: /tmp is not writable or has insufficient space" + exit 1 +fi echo "Running command: '$command' as '$USERNAME' (debug=$is_debug)" diff --git a/readme.md b/readme.md index af41c350..a0238935 100644 --- a/readme.md +++ b/readme.md @@ -92,7 +92,7 @@ Environment variables work for initial setup and Docker deployments. They serve | `FLASK_PORT` | Web interface port | `8084` | | `INGEST_DIR` | Book download directory | `/cwa-book-ingest` | | `TZ` | Container timezone | `UTC` | -| `UID` / `GID` | Runtime user/group ID | `1000` / `100` | +| `PUID` / `PGID` | Runtime user/group ID (also supports legacy `UID`/`GID`) | `1000` / `1000` | | `SEARCH_MODE` | `direct` or `universal` | `direct` | Some of the additional options available in Settings: diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index bf7543ed..6075350e 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -85,6 +85,11 @@ function App() { handleSortChange, searchFieldValues, updateSearchFieldValue, + // Pagination (universal mode) + hasMore, + isLoadingMore, + loadMore, + totalFound, } = useSearch({ showToast, setIsAuthenticated, @@ -591,6 +596,10 @@ function App() { sortValue={advancedFilters.sort} onSortChange={(value) => handleSortChange(value, config)} metadataSortOptions={config?.metadata_sort_options} + hasMore={hasMore} + isLoadingMore={isLoadingMore} + onLoadMore={() => loadMore(config)} + totalFound={totalFound} /> {selectedBook && ( diff --git a/src/frontend/src/components/LanguageMultiSelect.tsx b/src/frontend/src/components/LanguageMultiSelect.tsx index 2e044755..358b257e 100644 --- a/src/frontend/src/components/LanguageMultiSelect.tsx +++ b/src/frontend/src/components/LanguageMultiSelect.tsx @@ -1,6 +1,5 @@ import { Language } from '../types'; import { - formatDefaultLanguageLabel, LANGUAGE_OPTION_ALL, LANGUAGE_OPTION_DEFAULT, normalizeLanguageSelection, @@ -24,28 +23,40 @@ export const LanguageMultiSelect = ({ label, placeholder, }: LanguageMultiSelectProps) => { - const defaultLabel = formatDefaultLanguageLabel(defaultLanguageCodes, options); const defaultCodeSet = new Set(defaultLanguageCodes); - const nonDefaultLanguages = options.filter(lang => !defaultCodeSet.has(lang.code)); - const selectableValues = [LANGUAGE_OPTION_DEFAULT, ...nonDefaultLanguages.map(lang => lang.code)]; + // Get default languages with their full info + const defaultLanguages = options.filter(lang => defaultCodeSet.has(lang.code)); + const nonDefaultLanguages = options.filter(lang => !defaultCodeSet.has(lang.code)); + + // All selectable values (individual language codes, not LANGUAGE_OPTION_DEFAULT) + const selectableValues = [...defaultLanguageCodes, ...nonDefaultLanguages.map(lang => lang.code)]; + + // Build option list: All, then defaults (marked), then others const optionList: DropdownListOption[] = [ { value: LANGUAGE_OPTION_ALL, label: 'All languages', }, - { - value: LANGUAGE_OPTION_DEFAULT, - label: defaultLabel, - }, + // Each default language as a separate option + ...defaultLanguages.map(lang => ({ + value: lang.code, + label: `${lang.language} (default)`, + })), + // Non-default languages ...nonDefaultLanguages.map(lang => ({ value: lang.code, label: lang.language, })), ]; + // Expand LANGUAGE_OPTION_DEFAULT to individual default codes for comparison + const expandedValue = value.flatMap(v => + v === LANGUAGE_OPTION_DEFAULT ? defaultLanguageCodes : [v] + ); + const includesAllSelection = value.includes(LANGUAGE_OPTION_ALL); - const effectiveValue = includesAllSelection ? selectableValues : value; + const effectiveValue = includesAllSelection ? selectableValues : expandedValue; const selectedSet = new Set(effectiveValue); const isAllSelected = selectableValues.every(code => selectedSet.has(code)); const displayedValue = isAllSelected ? [LANGUAGE_OPTION_ALL, ...effectiveValue] : effectiveValue; @@ -57,11 +68,8 @@ export const LanguageMultiSelect = ({ const labels: string[] = []; - if (selectedSet.has(LANGUAGE_OPTION_DEFAULT)) { - labels.push(defaultLabel); - } - - nonDefaultLanguages.forEach(lang => { + // Check each language + options.forEach(lang => { if (selectedSet.has(lang.code)) { labels.push(lang.language); } diff --git a/src/frontend/src/components/ReleaseCell.tsx b/src/frontend/src/components/ReleaseCell.tsx index bc9497cd..b9173b4e 100644 --- a/src/frontend/src/components/ReleaseCell.tsx +++ b/src/frontend/src/components/ReleaseCell.tsx @@ -1,5 +1,6 @@ -import { ColumnSchema, ColumnColorHint, Release } from '../types'; -import { getFormatColor, getLanguageColor, getDownloadTypeColor, ColorStyle } from '../utils/colorMaps'; +import { ColumnSchema, Release } from '../types'; +import { getColorStyleFromHint } from '../utils/colorMaps'; +import { getNestedValue } from '../utils/objectHelpers'; interface ReleaseCellProps { column: ColumnSchema; @@ -8,48 +9,6 @@ interface ReleaseCellProps { onlineServers?: string[]; // For IRC: list of online server nicks to show status indicator } -/** - * Get a nested value from an object using dot-notation path. - * e.g., getNestedValue(obj, "extra.language") returns obj.extra.language - */ -const getNestedValue = (obj: Record, path: string): unknown => { - return path.split('.').reduce((current, key) => { - if (current && typeof current === 'object') { - return (current as Record)[key]; - } - return undefined; - }, obj as unknown); -}; - -const DEFAULT_COLOR_STYLE: ColorStyle = { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300' }; - -/** - * Get the color style for a value based on the color hint. - */ -const getColorStyle = (value: string, colorHint?: ColumnColorHint | null): ColorStyle => { - if (!colorHint) return DEFAULT_COLOR_STYLE; - - if (colorHint.type === 'static') { - // For static hints, assume it's a bg class and pair with default text - return { bg: colorHint.value, text: 'text-gray-700 dark:text-gray-300' }; - } - - if (colorHint.type === 'map') { - switch (colorHint.value) { - case 'format': - return getFormatColor(value); - case 'language': - return getLanguageColor(value); - case 'download_type': - return getDownloadTypeColor(value); - default: - return DEFAULT_COLOR_STYLE; - } - } - - return DEFAULT_COLOR_STYLE; -}; - /** * Generic cell renderer for release list columns. * Renders different column types (text, badge, size, number, seeders) based on schema. @@ -77,7 +36,7 @@ export const ReleaseCell = ({ column, release, compact = false, onlineServers }: if (compact) { return {displayValue}; } - const colorStyle = getColorStyle(value, column.color_hint); + const colorStyle = getColorStyleFromHint(value, column.color_hint); return (
{value !== column.fallback ? ( diff --git a/src/frontend/src/components/ReleaseModal.tsx b/src/frontend/src/components/ReleaseModal.tsx index 4e496e09..f7eef2d3 100644 --- a/src/frontend/src/components/ReleaseModal.tsx +++ b/src/frontend/src/components/ReleaseModal.tsx @@ -1,12 +1,13 @@ import { useEffect, useState, useCallback, useMemo, useRef } from 'react'; -import { Book, Release, ReleaseSource, ReleasesResponse, Language, StatusData, ButtonStateInfo, ColumnSchema, ReleaseColumnConfig, LeadingCellConfig, ColumnColorHint, SearchStatusData } from '../types'; +import { Book, Release, ReleaseSource, ReleasesResponse, Language, StatusData, ButtonStateInfo, ColumnSchema, ReleaseColumnConfig, LeadingCellConfig, SearchStatusData } from '../types'; import { getReleases, getReleaseSources } from '../services/api'; import { useSocket } from '../contexts/SocketContext'; import { Dropdown } from './Dropdown'; import { DropdownList } from './DropdownList'; import { BookDownloadButton } from './BookDownloadButton'; import { ReleaseCell } from './ReleaseCell'; -import { getFormatColor, getLanguageColor, getDownloadTypeColor, ColorStyle } from '../utils/colorMaps'; +import { getColorStyleFromHint } from '../utils/colorMaps'; +import { getNestedValue } from '../utils/objectHelpers'; import { LanguageMultiSelect } from './LanguageMultiSelect'; import { LANGUAGE_OPTION_ALL, LANGUAGE_OPTION_DEFAULT, getLanguageFilterValues } from '../utils/languageFilters'; @@ -15,14 +16,15 @@ import { LANGUAGE_OPTION_ALL, LANGUAGE_OPTION_DEFAULT, getLanguageFilterValues } // This persists across modal open/close cycles const releaseCache = new Map(); -const getCacheKey = (provider: string, providerId: string, source: string): string => - `${provider}:${providerId}:${source}`; +function getCacheKey(provider: string, providerId: string, source: string): string { + return `${provider}:${providerId}:${source}`; +} // Default cache TTL (5 minutes) - sources can override via column_config.cache_ttl_seconds const DEFAULT_CACHE_TTL_MS = 5 * 60 * 1000; const cacheTimestamps = new Map(); -const getCachedReleases = (provider: string, providerId: string, source: string): ReleasesResponse | null => { +function getCachedReleases(provider: string, providerId: string, source: string): ReleasesResponse | null { const key = getCacheKey(provider, providerId, source); const timestamp = cacheTimestamps.get(key); const cached = releaseCache.get(key); @@ -45,13 +47,13 @@ const getCachedReleases = (provider: string, providerId: string, source: string) cacheTimestamps.delete(key); return null; -}; +} -const setCachedReleases = (provider: string, providerId: string, source: string, data: ReleasesResponse): void => { +function setCachedReleases(provider: string, providerId: string, source: string, data: ReleasesResponse): void { const key = getCacheKey(provider, providerId, source); releaseCache.set(key, data); cacheTimestamps.set(key, Date.now()); -}; +} // Default column configuration (fallback when backend doesn't provide one) const DEFAULT_COLUMN_CONFIG: ReleaseColumnConfig = { @@ -104,40 +106,9 @@ interface ReleaseModalProps { onSearchSeries?: (seriesName: string) => void; // Callback to search for series } -// Color map lookup for dynamic color hints -const COLOR_MAP_HANDLERS: Record ColorStyle> = { - format: getFormatColor, - language: getLanguageColor, - download_type: getDownloadTypeColor, -}; - -const DEFAULT_COLOR_STYLE: ColorStyle = { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300' }; - -// Helper to get color style based on color hint -function getLeadingCellColor(value: string, colorHint?: ColumnColorHint): ColorStyle { - if (!colorHint) return DEFAULT_COLOR_STYLE; - - if (colorHint.type === 'map') { - const handler = COLOR_MAP_HANDLERS[colorHint.value]; - return handler ? handler(value) : DEFAULT_COLOR_STYLE; - } - - // Static color hint - assume it's a bg class - return { bg: colorHint.value, text: 'text-gray-700 dark:text-gray-300' }; -} - -// Helper to get nested value from object using dot notation path -function getNestedValue(obj: Record, path: string): unknown { - return path.split('.').reduce((current, key) => { - if (current && typeof current === 'object') { - return (current as Record)[key]; - } - return undefined; - }, obj as unknown); -} // 5-star rating display with partial fill support -const StarRating = ({ rating, maxRating = 5 }: { rating: number; maxRating?: number }) => { +function StarRating({ rating, maxRating = 5 }: { rating: number; maxRating?: number }) { // Normalize rating to 0-5 scale if needed const normalizedRating = Math.min(Math.max(rating, 0), maxRating); @@ -170,7 +141,7 @@ const StarRating = ({ rating, maxRating = 5 }: { rating: number; maxRating?: num })}
); -}; +} // Thumbnail component for release rows const ReleaseThumbnail = ({ preview, title }: { preview?: string; title?: string }) => { @@ -231,7 +202,7 @@ const LeadingCell = ({ if (cellType === 'badge' && config?.key) { const value = getNestedValue(release as unknown as Record, config.key); const displayValue = value ? String(value) : ''; - const colorStyle = getLeadingCellColor(displayValue, config.color_hint); + const colorStyle = getColorStyleFromHint(displayValue, config.color_hint); const text = config.uppercase ? displayValue.toUpperCase() : displayValue; return ( @@ -396,149 +367,159 @@ const ReleaseRow = ({ }; // Shimmer block with wave animation - same as DownloadsSidebar -const ShimmerBlock = ({ className }: { className: string }) => ( -
+function ShimmerBlock({ className }: { className: string }) { + return (
-
-); + className={`rounded bg-gray-200 dark:bg-gray-800 relative overflow-hidden ${className}`} + > +
+
+ ); +} // Loading skeleton for releases - matches ReleaseRow layout -const ReleaseSkeleton = () => ( -
- {[1, 2, 3, 4, 5].map((i) => ( -
-
- {/* Thumbnail skeleton */} - +function ReleaseSkeleton() { + return ( +
+ {[1, 2, 3, 4, 5].map((i) => ( +
+
+ {/* Thumbnail skeleton */} + - {/* Title and author skeleton */} -
- - -
- - {/* Language badge skeleton - desktop */} -
- -
- - {/* Format badge skeleton - desktop */} -
- -
- - {/* Size skeleton - desktop */} -
- -
- - {/* Mobile info + action skeleton */} -
- {/* Mobile: format + size inline */} -
- - + {/* Title and author skeleton */} +
+ +
- {/* Action button skeleton */} - + {/* Language badge skeleton - desktop */} +
+ +
+ + {/* Format badge skeleton - desktop */} +
+ +
+ + {/* Size skeleton - desktop */} +
+ +
+ + {/* Mobile info + action skeleton */} +
+ {/* Mobile: format + size inline */} +
+ + +
+ + {/* Action button skeleton */} + +
-
- ))} -
-); + ))} +
+ ); +} // Empty state component -const EmptyState = ({ message }: { message: string }) => ( -
-
- - - +function EmptyState({ message }: { message: string }) { + return ( +
+
+ + + +
+

{message}

-

{message}

-
-); + ); +} // Configure source CTA -const ConfigureSourceCTA = ({ sourceName }: { sourceName: string }) => ( -
-
- - - - +function ConfigureSourceCTA({ sourceName }: { sourceName: string }) { + return ( +
+
+ + + + +
+

+ {sourceName} Not Configured +

+

+ Configure {sourceName} in Settings to enable searching this source. +

-

- {sourceName} Not Configured -

-

- Configure {sourceName} in Settings to enable searching this source. -

-
-); + ); +} // Error state component -const ErrorState = ({ message }: { message: string }) => ( -
-
- - - +function ErrorState({ message }: { message: string }) { + return ( +
+
+ + + +
+

+ Error Loading Releases +

+

{message}

-

- Error Loading Releases -

-

{message}

-
-); + ); +} export const ReleaseModal = ({ @@ -736,11 +717,15 @@ export const ReleaseModal = ({ if (sources.length > 0) { const enabledSources = sources.filter(s => s.enabled); const defaultIsEnabled = defaultReleaseSource && enabledSources.some(s => s.name === defaultReleaseSource); - const defaultSource = defaultIsEnabled - ? defaultReleaseSource - : enabledSources.length > 0 - ? enabledSources[0].name - : sources[0].name; // Fallback to first source if none enabled + + let defaultSource: string; + if (defaultIsEnabled) { + defaultSource = defaultReleaseSource; + } else if (enabledSources.length > 0) { + defaultSource = enabledSources[0].name; + } else { + defaultSource = sources[0].name; // Fallback to first source if none enabled + } setActiveTab(defaultSource); } } catch (err) { @@ -807,9 +792,15 @@ export const ReleaseModal = ({ setExpandedBySource((prev) => ({ ...prev, [activeTab]: true })); try { + // Resolve language codes for the API call (same logic as Apply button) + const langCodes = getLanguageFilterValues(languageFilter, bookLanguages, defaultLanguages); + const languagesParam = (langCodes === null || langCodes?.includes(LANGUAGE_OPTION_ALL)) + ? undefined + : langCodes; + // Fetch with expand_search=true (title+author search) const expandedResponse = await getReleases( - provider, bookId, activeTab, book.title, book.author, true + provider, bookId, activeTab, book.title, book.author, true, languagesParam ); // Merge with existing results, deduplicating by source_id @@ -836,7 +827,7 @@ export const ReleaseModal = ({ } finally { setLoadingBySource((prev) => ({ ...prev, [activeTab]: false })); } - }, [activeTab, book]); + }, [activeTab, book, languageFilter, bookLanguages, defaultLanguages]); // Build list of tabs to show // All sources come from backend with their enabled status @@ -961,6 +952,18 @@ export const ReleaseModal = ({ return DEFAULT_COLUMN_CONFIG; }, [releasesBySource, activeTab]); + // Pre-compute display field lookups to avoid repeated .find() calls in JSX + const displayFields = useMemo(() => { + if (!book?.display_fields) return null; + + const starField = book.display_fields.find(f => f.icon === 'star'); + const ratingsField = book.display_fields.find(f => f.icon === 'ratings'); + const usersField = book.display_fields.find(f => f.icon === 'users'); + const pagesField = book.display_fields.find(f => f.icon === 'book'); + + return { starField, ratingsField, usersField, pagesField }; + }, [book?.display_fields]); + // Get button state for a release based on its source_id const getButtonState = useCallback( (releaseId: string): ButtonStateInfo => { @@ -1019,6 +1022,7 @@ export const ReleaseModal = ({ const currentTabLoading = loadingBySource[activeTab] ?? false; const currentTabError = errorBySource[activeTab] ?? null; const currentTabEnabled = allTabs.find((t) => t.name === activeTab)?.enabled ?? false; + const isInitialLoading = currentTabLoading || (releasesBySource[activeTab] === undefined && !currentTabError); return (
{book.year && {book.year}} - {book.display_fields?.find(f => f.icon === 'star') && (() => { - const starField = book.display_fields!.find(f => f.icon === 'star')!; - const ratingsField = book.display_fields?.find(f => f.icon === 'ratings'); - return ( - - - {starField.value} - {ratingsField && ( - ({ratingsField.value}) - )} - - ); - })()} - {book.display_fields?.find(f => f.icon === 'users') && ( + {displayFields?.starField && ( + + + {displayFields.starField.value} + {displayFields.ratingsField && ( + ({displayFields.ratingsField.value}) + )} + + )} + {displayFields?.usersField && ( - {book.display_fields.find(f => f.icon === 'users')?.value} readers + {displayFields.usersField.value} readers )} - {book.display_fields?.find(f => f.icon === 'book') && ( - {book.display_fields.find(f => f.icon === 'book')?.value} pages + {displayFields?.pagesField && ( + {displayFields.pagesField.value} pages )}
@@ -1366,26 +1366,14 @@ export const ReleaseModal = ({ {/* Release list content */}
- {!currentTabEnabled ? ( + {sourcesLoading ? ( + + ) : !currentTabEnabled ? ( t.name === activeTab)?.displayName || activeTab} /> - ) : currentTabLoading && filteredReleases.length === 0 ? ( - // Initial loading - show full skeleton -
- - {/* Search status - bottom center */} - {searchStatus && searchStatus.source === activeTab && ( -
-
- {searchStatus.phase !== 'complete' && searchStatus.phase !== 'error' && ( -
- )} - {searchStatus.message} -
-
- )} -
+ ) : isInitialLoading && filteredReleases.length === 0 ? ( + ) : currentTabError ? ( ) : filteredReleases.length === 0 && !currentTabLoading ? ( @@ -1413,8 +1401,11 @@ export const ReleaseModal = ({ /> ))}
- {/* Expand search button or loading indicator */} - {activeTab === 'direct_download' && !expandedBySource[activeTab] && !currentTabLoading && ( + {/* Expand search button - only show if ISBN search was used (otherwise we already did title+author) */} + {activeTab === 'direct_download' && + releasesBySource[activeTab]?.search_info?.direct_download?.search_type === 'isbn' && + !expandedBySource[activeTab] && + !currentTabLoading && (
)}
+ + {/* Sticky search status indicator - stays at bottom of visible scroll area */} + {searchStatus && searchStatus.source === activeTab && currentTabLoading && ( +
+
+ {searchStatus.phase !== 'complete' && searchStatus.phase !== 'error' && ( +
+ )} + {searchStatus.message} +
+
+ )}
diff --git a/src/frontend/src/components/ResultsSection.tsx b/src/frontend/src/components/ResultsSection.tsx index 337de96a..9c47a4b0 100644 --- a/src/frontend/src/components/ResultsSection.tsx +++ b/src/frontend/src/components/ResultsSection.tsx @@ -25,6 +25,11 @@ interface ResultsSectionProps { sortValue: string; onSortChange: (value: string) => void; metadataSortOptions?: SortOption[]; + // Pagination (universal mode) + hasMore?: boolean; + isLoadingMore?: boolean; + onLoadMore?: () => void; + totalFound?: number; } export const ResultsSection = ({ @@ -38,6 +43,10 @@ export const ResultsSection = ({ sortValue, onSortChange, metadataSortOptions, + hasMore, + isLoadingMore, + onLoadMore, + totalFound, }: ResultsSectionProps) => { const { searchMode } = useSearchMode(); const [viewMode, setViewMode] = useState<'card' | 'compact' | 'list'>(() => { @@ -210,6 +219,38 @@ export const ResultsSection = ({ {books.length === 0 && (
No results found.
)} + + {/* Load More button (universal mode pagination) */} + {searchMode === 'universal' && hasMore && onLoadMore && ( +
+ + {totalFound !== undefined && totalFound > 0 && ( + + Showing {books.length} of {totalFound} results + + )} +
+ )} ); }; diff --git a/src/frontend/src/components/resultsViews/CardView.tsx b/src/frontend/src/components/resultsViews/CardView.tsx index 164b45b2..4374b7ca 100644 --- a/src/frontend/src/components/resultsViews/CardView.tsx +++ b/src/frontend/src/components/resultsViews/CardView.tsx @@ -60,7 +60,13 @@ export const CardView = ({ book, onDetails, onDownload, onGetReleases, buttonSta
{/* Series position badge */} {showSeriesPosition && book.series_position != null && ( -
+
#{book.series_position}
)} diff --git a/src/frontend/src/components/resultsViews/CompactView.tsx b/src/frontend/src/components/resultsViews/CompactView.tsx index 8015c633..efd830a8 100644 --- a/src/frontend/src/components/resultsViews/CompactView.tsx +++ b/src/frontend/src/components/resultsViews/CompactView.tsx @@ -61,7 +61,13 @@ export const CompactView = ({ book, onDetails, onDownload, onGetReleases, button
{/* Series position badge */} {showSeriesPosition && book.series_position != null && ( -
+
#{book.series_position}
)} diff --git a/src/frontend/src/components/resultsViews/ListView.tsx b/src/frontend/src/components/resultsViews/ListView.tsx index 9386738a..d9f1af16 100644 --- a/src/frontend/src/components/resultsViews/ListView.tsx +++ b/src/frontend/src/components/resultsViews/ListView.tsx @@ -123,7 +123,13 @@ export const ListView = ({ books, onDetails, onDownload, onGetReleases, getButto

{showSeriesPosition && book.series_position != null && ( - + #{book.series_position} )} diff --git a/src/frontend/src/hooks/useSearch.ts b/src/frontend/src/hooks/useSearch.ts index 713563a1..2b0692d3 100644 --- a/src/frontend/src/hooks/useSearch.ts +++ b/src/frontend/src/hooks/useSearch.ts @@ -1,4 +1,4 @@ -import { useState, useCallback } from 'react'; +import { useState, useCallback, useRef } from 'react'; import { useNavigate } from 'react-router-dom'; import { Book, AppConfig, AdvancedFilterState } from '../types'; import { searchBooks, searchMetadata, AuthenticationError } from '../services/api'; @@ -36,6 +36,11 @@ interface UseSearchReturn { // Universal mode search field values searchFieldValues: SearchFieldValues; updateSearchFieldValue: (key: string, value: string | number | boolean) => void; + // Pagination (universal mode only) + hasMore: boolean; + isLoadingMore: boolean; + loadMore: (config: AppConfig | null) => Promise; + totalFound: number; } export function useSearch(options: UseSearchOptions): UseSearchReturn { @@ -60,23 +65,46 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { // Universal mode: provider-specific search field values const [searchFieldValues, setSearchFieldValues] = useState({}); + // Pagination state (universal mode only) + const [currentPage, setCurrentPage] = useState(1); + const [hasMore, setHasMore] = useState(false); + const [isLoadingMore, setIsLoadingMore] = useState(false); + const [totalFound, setTotalFound] = useState(0); + + // Store last search params for loadMore + const lastSearchParamsRef = useRef<{ + query: string; + sort: string; + fieldValues: SearchFieldValues; + } | null>(null); + const updateAdvancedFilters = useCallback((updates: Partial) => { setAdvancedFilters(prev => ({ ...prev, ...updates })); }, []); const updateSearchFieldValue = useCallback((key: string, value: string | number | boolean) => { - console.log('[useSearch] updateSearchFieldValue:', key, '=', value); - setSearchFieldValues(prev => { - const next = { ...prev, [key]: value }; - console.log('[useSearch] searchFieldValues updated:', next); - return next; - }); + setSearchFieldValues(prev => ({ ...prev, [key]: value })); }, []); const resetSortFilter = useCallback(() => { setAdvancedFilters(prev => ({ ...prev, sort: '' })); }, []); + // Helper to handle authentication and other errors consistently + const handleSearchError = useCallback((error: unknown, context: string) => { + if (error instanceof AuthenticationError) { + setIsAuthenticated(false); + if (authRequired) { + navigate('/login', { replace: true }); + } + return; + } + + console.error(`${context}:`, error); + const message = error instanceof Error ? error.message : context; + showToast(message, 'error'); + }, [setIsAuthenticated, authRequired, navigate, showToast]); + const handleSearch = useCallback(async ( query: string, config: AppConfig | null, @@ -97,21 +125,13 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { const hasSeriesSearch = typeof seriesValue === 'string' && seriesValue.trim() !== ''; const sort = hasSeriesSearch ? 'series_order' : (params.get('sort') || 'relevance'); - // Debug logging - console.log('[useSearch] Universal mode search:', { - query, - searchQuery, - sort, - searchFieldValues, - fieldValues, - effectiveFieldValues, - hasFieldValues, - }); - if (!searchQuery && !hasFieldValues) { - console.log('[useSearch] Early return: no query and no field values'); setBooks([]); setLastSearchQuery(''); + setHasMore(false); + setTotalFound(0); + setCurrentPage(1); + lastSearchParamsRef.current = null; return; } @@ -122,26 +142,27 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { setIsSearching(true); setLastSearchQuery(query); + // Reset pagination for new search + setCurrentPage(1); + setHasMore(false); + setTotalFound(0); try { - const results = await searchMetadata(searchQuery, 20, sort, effectiveFieldValues); - if (results.length > 0) { - setBooks(results); + const result = await searchMetadata(searchQuery, 40, sort, effectiveFieldValues, 1); + if (result.books.length > 0) { + setBooks(result.books); + setHasMore(result.hasMore); + setTotalFound(result.totalFound); + // Store params for loadMore + lastSearchParamsRef.current = { query: searchQuery, sort, fieldValues: effectiveFieldValues }; } else { + setBooks([]); + setHasMore(false); + setTotalFound(0); showToast('No results found', 'error'); } } catch (error) { - if (error instanceof AuthenticationError) { - setIsAuthenticated(false); - if (authRequired) { - navigate('/login', { replace: true }); - } - } else { - console.error('Search failed:', error); - // API now returns user-friendly error messages directly - const message = error instanceof Error ? error.message : 'Search failed'; - showToast(message, 'error'); - } + handleSearchError(error, 'Search failed'); } finally { setIsSearching(false); } @@ -167,10 +188,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { } } catch (error) { if (error instanceof AuthenticationError) { - setIsAuthenticated(false); - if (authRequired) { - navigate('/login', { replace: true }); - } + handleSearchError(error, 'Search failed'); } else { console.error('Search failed:', error); const message = error instanceof Error ? error.message : 'Search failed'; @@ -182,7 +200,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { } finally { setIsSearching(false); } - }, [showToast, setIsAuthenticated, authRequired, navigate, searchFieldValues]); + }, [showToast, setIsAuthenticated, authRequired, navigate, searchFieldValues, handleSearchError]); const handleResetSearch = useCallback((config: AppConfig | null) => { setBooks([]); @@ -204,8 +222,42 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { // Reset universal mode search field values setSearchFieldValues({}); + + // Reset pagination + setCurrentPage(1); + setHasMore(false); + setTotalFound(0); + lastSearchParamsRef.current = null; }, [onSearchReset]); + // Load more results (universal mode pagination) + const loadMore = useCallback(async (config: AppConfig | null) => { + const searchMode = config?.search_mode || 'direct'; + if (searchMode !== 'universal') return; + if (!lastSearchParamsRef.current) return; + if (isLoadingMore || !hasMore) return; + + const { query, sort, fieldValues } = lastSearchParamsRef.current; + const nextPage = currentPage + 1; + + setIsLoadingMore(true); + + try { + const result = await searchMetadata(query, 40, sort, fieldValues, nextPage); + if (result.books.length > 0) { + setBooks(prev => [...prev, ...result.books]); + setHasMore(result.hasMore); + setCurrentPage(nextPage); + } else { + setHasMore(false); + } + } catch (error) { + handleSearchError(error, 'Failed to load more results'); + } finally { + setIsLoadingMore(false); + } + }, [currentPage, hasMore, isLoadingMore, handleSearchError]); + const handleSortChange = useCallback((value: string, config: AppConfig | null) => { updateAdvancedFilters({ sort: value }); if (!lastSearchQuery) return; @@ -241,5 +293,10 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { // Universal mode search field values searchFieldValues, updateSearchFieldValue, + // Pagination (universal mode only) + hasMore, + isLoadingMore, + loadMore, + totalFound, }; } diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index d6ac6c1f..0a5266ce 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -78,23 +78,31 @@ interface MetadataSearchResponse { books: MetadataBookData[]; provider: string; query: string; + page?: number; + total_found?: number; + has_more?: boolean; +} + +// Metadata search result with pagination info +export interface MetadataSearchResult { + books: Book[]; + page: number; + totalFound: number; + hasMore: boolean; } // Search metadata providers and normalize to Book format export const searchMetadata = async ( query: string, - limit: number = 20, + limit: number = 40, sort: string = 'relevance', - fields: Record = {} -): Promise => { + fields: Record = {}, + page: number = 1 +): Promise => { const hasFields = Object.values(fields).some(v => v !== '' && v !== false); - // Debug logging - console.log('[api.searchMetadata] Called with:', { query, limit, sort, fields, hasFields }); - if (!query && !hasFields) { - console.log('[api.searchMetadata] Early return: no query and no fields'); - return []; + return { books: [], page: 1, totalFound: 0, hasMore: false }; } const params = new URLSearchParams(); @@ -103,6 +111,7 @@ export const searchMetadata = async ( } params.set('limit', String(limit)); params.set('sort', sort); + params.set('page', String(page)); // Add custom search field values Object.entries(fields).forEach(([key, value]) => { @@ -111,13 +120,14 @@ export const searchMetadata = async ( } }); - const requestUrl = `${API.metadataSearch}?${params.toString()}`; - console.log('[api.searchMetadata] Request URL:', requestUrl); + const response = await fetchJSON(`${API.metadataSearch}?${params.toString()}`); - const response = await fetchJSON(requestUrl); - console.log('[api.searchMetadata] Response:', response); - - return response.books.map(transformMetadataToBook); + return { + books: response.books.map(transformMetadataToBook), + page: response.page || page, + totalFound: response.total_found || 0, + hasMore: response.has_more || false, + }; }; export const getBookInfo = async (id: string): Promise => { diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts index c41a0691..66ead740 100644 --- a/src/frontend/src/types/index.ts +++ b/src/frontend/src/types/index.ts @@ -248,6 +248,11 @@ export interface Release { extra?: Record; // Source-specific metadata } +// Search info returned by release sources +export interface SourceSearchInfo { + search_type: 'isbn' | 'title_author'; +} + // Response from /api/releases endpoint export interface ReleasesResponse { releases: Release[]; @@ -265,6 +270,7 @@ export interface ReleasesResponse { sources_searched: string[]; errors?: string[]; column_config?: ReleaseColumnConfig | null; // Plugin-driven column configuration + search_info?: Record; // Per-source search metadata } // Search status update from WebSocket (for ReleaseModal loading state) diff --git a/src/frontend/src/utils/colorMaps.ts b/src/frontend/src/utils/colorMaps.ts index 08d7cd61..24a1d4fa 100644 --- a/src/frontend/src/utils/colorMaps.ts +++ b/src/frontend/src/utils/colorMaps.ts @@ -65,4 +65,39 @@ export function getDownloadTypeColor(downloadType?: string): ColorStyle { return DOWNLOAD_TYPE_COLORS[downloadType.toLowerCase()] || DEFAULT_DOWNLOAD_TYPE_COLOR; } +/** + * Color hint type for dynamic column coloring. + */ +interface ColumnColorHint { + type: 'static' | 'map'; + value: string; +} + +/** + * Get the color style for a value based on a color hint. + * Supports both static color classes and dynamic map lookups. + */ +export function getColorStyleFromHint(value: string, colorHint?: ColumnColorHint | null): ColorStyle { + if (!colorHint) return FALLBACK_COLOR; + + if (colorHint.type === 'static') { + return { bg: colorHint.value, text: 'text-gray-700 dark:text-gray-300' }; + } + + if (colorHint.type === 'map') { + switch (colorHint.value) { + case 'format': + return getFormatColor(value); + case 'language': + return getLanguageColor(value); + case 'download_type': + return getDownloadTypeColor(value); + default: + return FALLBACK_COLOR; + } + } + + return FALLBACK_COLOR; +} + export type { ColorStyle }; diff --git a/src/frontend/src/utils/objectHelpers.ts b/src/frontend/src/utils/objectHelpers.ts new file mode 100644 index 00000000..5cf0ffb3 --- /dev/null +++ b/src/frontend/src/utils/objectHelpers.ts @@ -0,0 +1,12 @@ +/** + * Get a nested value from an object using dot-notation path. + * e.g., getNestedValue(obj, "extra.language") returns obj.extra.language + */ +export function getNestedValue(obj: Record, path: string): unknown { + return path.split('.').reduce((current, key) => { + if (current && typeof current === 'object') { + return (current as Record)[key]; + } + return undefined; + }, obj as unknown); +}