diff --git a/.gitignore b/.gitignore index e9019636..9d548301 100644 --- a/.gitignore +++ b/.gitignore @@ -227,3 +227,4 @@ pyrightconfig.json # End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python /downloaded_files +/.local/ diff --git a/Dockerfile b/Dockerfile index bd31b7c8..55256bdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -131,7 +131,7 @@ RUN apt-get update && \ python3-tk # install additional dependencies -COPY requirements-cwa-bd.txt . +COPY requirements-cwa-bd.txt ./ RUN pip install --no-cache-dir -r requirements-cwa-bd.txt && \ # Clean root's pip cache rm -rf /root/.cache diff --git a/README_images/details.png b/README_images/details.png deleted file mode 100644 index 61e2ebaa..00000000 Binary files a/README_images/details.png and /dev/null differ diff --git a/README_images/downloading.png b/README_images/downloading.png deleted file mode 100644 index ccf7a099..00000000 Binary files a/README_images/downloading.png and /dev/null differ diff --git a/README_images/downloads.png b/README_images/downloads.png new file mode 100644 index 00000000..3ec8c4b3 Binary files /dev/null and b/README_images/downloads.png differ diff --git a/README_images/homescreen.png b/README_images/homescreen.png new file mode 100644 index 00000000..258d4c11 Binary files /dev/null and b/README_images/homescreen.png differ diff --git a/README_images/main_page.png b/README_images/main_page.png deleted file mode 100644 index 6f452e59..00000000 Binary files a/README_images/main_page.png and /dev/null differ diff --git a/README_images/multi-source.png b/README_images/multi-source.png new file mode 100644 index 00000000..112933bc Binary files /dev/null and b/README_images/multi-source.png differ diff --git a/README_images/search-results.png b/README_images/search-results.png new file mode 100644 index 00000000..3d6353c6 Binary files /dev/null and b/README_images/search-results.png differ diff --git a/README_images/search.png b/README_images/search.png deleted file mode 100644 index 291dfd77..00000000 Binary files a/README_images/search.png and /dev/null differ diff --git a/README_images/status.png b/README_images/status.png deleted file mode 100644 index 940498be..00000000 Binary files a/README_images/status.png and /dev/null differ diff --git a/app.py b/app.py deleted file mode 100644 index 59a41a71..00000000 --- a/app.py +++ /dev/null @@ -1,872 +0,0 @@ -"""Flask web application for book download service with URL rewrite support.""" - -import io -import logging -import os -import sqlite3 -import time -from datetime import datetime, timedelta -from functools import wraps -from typing import Any, Dict, Tuple, Union - -from flask import Flask, jsonify, request, send_file, send_from_directory, session -from flask_cors import CORS -from flask_socketio import SocketIO, emit -from werkzeug.middleware.proxy_fix import ProxyFix -from werkzeug.security import check_password_hash -from werkzeug.wrappers import Response - -import backend -from book_manager import SearchUnavailable -from config import BOOK_LANGUAGE, SUPPORTED_FORMATS, _SUPPORTED_BOOK_LANGUAGE -from env import ( - BUILD_VERSION, CALIBRE_WEB_URL, CWA_DB_PATH, DEBUG, FLASK_HOST, FLASK_PORT, - RELEASE_VERSION, USING_EXTERNAL_BYPASSER, -) -from logger import setup_logger -from models import SearchFilters -from websocket_manager import ws_manager - -logger = setup_logger(__name__) -app = Flask(__name__) -app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore -app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 # Disable caching -app.config['APPLICATION_ROOT'] = '/' - -# Socket.IO async mode. -# We run this app under Gunicorn with a gevent websocket worker (even when DEBUG=true), -# so Socket.IO should always use gevent here. -async_mode = 'gevent' - -# Initialize Flask-SocketIO with reverse proxy support -socketio = SocketIO( - app, - cors_allowed_origins="*", - async_mode=async_mode, - logger=False, - engineio_logger=False, - # Reverse proxy / Traefik compatibility settings - path='/socket.io', - ping_timeout=60, # Time to wait for pong response - ping_interval=25, # Send ping every 25 seconds - # Allow both websocket and polling for better compatibility - transports=['websocket', 'polling'], - # Enable CORS for all origins (you can restrict this in production) - allow_upgrades=True, - # Important for proxies that buffer - http_compression=True -) - -# Initialize WebSocket manager -ws_manager.init_app(app, socketio) -logger.info(f"Flask-SocketIO initialized with async_mode='{async_mode}'") - -# Rate limiting for login attempts -# Structure: {username: {'count': int, 'lockout_until': datetime}} -failed_login_attempts: Dict[str, Dict[str, Any]] = {} -MAX_LOGIN_ATTEMPTS = 10 -LOCKOUT_DURATION_MINUTES = 30 - -def cleanup_old_lockouts() -> None: - """Remove expired lockout entries to prevent memory buildup.""" - current_time = datetime.now() - expired_users = [ - username for username, data in failed_login_attempts.items() - if 'lockout_until' in data and data['lockout_until'] < current_time - ] - for username in expired_users: - logger.info(f"Lockout expired for user: {username}") - del failed_login_attempts[username] - -def is_account_locked(username: str) -> bool: - """Check if an account is currently locked due to failed login attempts.""" - cleanup_old_lockouts() - - if username not in failed_login_attempts: - return False - - lockout_until = failed_login_attempts[username].get('lockout_until') - if lockout_until and datetime.now() < lockout_until: - return True - - return False - -def record_failed_login(username: str, ip_address: str) -> bool: - """ - Record a failed login attempt and lock account if threshold is reached. - Returns True if account is now locked, False otherwise. - """ - if username not in failed_login_attempts: - failed_login_attempts[username] = {'count': 0} - - failed_login_attempts[username]['count'] += 1 - count = failed_login_attempts[username]['count'] - - logger.warning(f"Failed login attempt {count}/{MAX_LOGIN_ATTEMPTS} for user '{username}' from IP {ip_address}") - - if count >= MAX_LOGIN_ATTEMPTS: - lockout_until = datetime.now() + timedelta(minutes=LOCKOUT_DURATION_MINUTES) - failed_login_attempts[username]['lockout_until'] = lockout_until - logger.warning(f"Account locked for user '{username}' until {lockout_until.strftime('%Y-%m-%d %H:%M:%S')} due to {count} failed login attempts") - return True - - return False - -def clear_failed_logins(username: str) -> None: - """Clear failed login attempts for a user after successful login.""" - if username in failed_login_attempts: - del failed_login_attempts[username] - logger.debug(f"Cleared failed login attempts for user: {username}") - -# Enable CORS in development mode for local frontend development -if DEBUG: - CORS(app, resources={ - r"/*": { - "origins": ["http://localhost:5173", "http://127.0.0.1:5173"], - "supports_credentials": True, - "allow_headers": ["Content-Type", "Authorization"], - "methods": ["GET", "POST", "PUT", "DELETE", "OPTIONS"] - } - }) - -# Custom log filter to exclude routine status endpoint polling and WebSocket noise -class StatusEndpointFilter(logging.Filter): - """Filter out routine status endpoint requests and WebSocket upgrade errors to reduce log noise.""" - def filter(self, record): - if hasattr(record, 'getMessage'): - message = record.getMessage() - # Exclude GET /api/status requests (polling noise) - if 'GET /api/status' in message: - return False - # Exclude WebSocket upgrade errors (benign - falls back to polling) - if 'write() before start_response' in message: - return False - # Exclude the Error on request line that precedes WebSocket errors - if 'Error on request:' in message and record.levelno == logging.ERROR: - return False - return True - - -class WebSocketErrorFilter(logging.Filter): - """Filter out WebSocket upgrade errors that occur in Werkzeug dev server. - - These errors are benign - Flask-SocketIO automatically falls back to polling transport. - The error occurs because Werkzeug's built-in server doesn't fully support WebSocket upgrades. - """ - def filter(self, record): - # Filter out the AssertionError traceback for WebSocket upgrades - if record.levelno == logging.ERROR: - message = record.getMessage() if hasattr(record, 'getMessage') else str(record.msg) - # Filter out the full traceback that includes the WebSocket assertion error - if 'write() before start_response' in message: - return False - # Also filter the "Error on request" header that precedes it - if hasattr(record, 'exc_info') and record.exc_info: - exc_type = record.exc_info[0] - if exc_type and exc_type.__name__ == 'AssertionError': - # Check if it's the WebSocket-related assertion - exc_value = record.exc_info[1] - if exc_value and 'write() before start_response' in str(exc_value): - return False - return True - -# Flask logger -app.logger.handlers = logger.handlers -app.logger.setLevel(logger.level) -# Also handle Werkzeug's logger -werkzeug_logger = logging.getLogger('werkzeug') -werkzeug_logger.handlers = logger.handlers -werkzeug_logger.setLevel(logger.level) -# Add filters to suppress routine status endpoint polling logs and WebSocket upgrade errors -werkzeug_logger.addFilter(StatusEndpointFilter()) -werkzeug_logger.addFilter(WebSocketErrorFilter()) - -# Set up authentication defaults -# The secret key will reset every time we restart, which will -# require users to authenticate again - -# Session cookie security - set to 'true' if exclusively using HTTPS -session_cookie_secure_env = os.getenv('SESSION_COOKIE_SECURE', 'false').lower() -SESSION_COOKIE_SECURE = session_cookie_secure_env in ['true', 'yes', '1'] - -app.config.update( - SECRET_KEY = os.urandom(64), - SESSION_COOKIE_HTTPONLY = True, - SESSION_COOKIE_SAMESITE = 'Lax', - SESSION_COOKIE_SECURE = SESSION_COOKIE_SECURE, - PERMANENT_SESSION_LIFETIME = 604800 # 7 days in seconds -) - -logger.info(f"Session cookie secure setting: {SESSION_COOKIE_SECURE} (from env: {session_cookie_secure_env})") - -def login_required(f): - @wraps(f) - def decorated_function(*args, **kwargs): - # If the CWA_DB_PATH variable exists, but isn't a valid - # path, return a server error - if CWA_DB_PATH is not None and not os.path.isfile(CWA_DB_PATH): - logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path") - return jsonify({"error": "Internal Server Error"}), 500 - - # If no database is configured, allow access - if not CWA_DB_PATH: - return f(*args, **kwargs) - - # Check if user has a valid session - if 'user_id' not in session: - return jsonify({"error": "Unauthorized"}), 401 - - return f(*args, **kwargs) - return decorated_function - - -# Serve frontend static files -@app.route('/assets/') -def serve_frontend_assets(filename: str) -> Response: - """ - Serve static assets from the built frontend. - """ - return send_from_directory(os.path.join(app.root_path, 'frontend-dist', 'assets'), filename) - -@app.route('/') -def index() -> Response: - """ - Serve the React frontend application. - Authentication is handled by the React app itself. - """ - return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), 'index.html') - -@app.route('/logo.png') -def logo() -> Response: - """ - Serve logo from built frontend assets. - """ - return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), - 'logo.png', mimetype='image/png') - -@app.route('/favicon.ico') -@app.route('/favico') -def favicon(_: Any = None) -> Response: - """ - Serve favicon from built frontend assets. - """ - return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), - 'favicon.ico', mimetype='image/vnd.microsoft.icon') - -# Register bypasser warmup callback for when first WebSocket client connects -# and shutdown callback for when all clients disconnect -if not USING_EXTERNAL_BYPASSER: - from cloudflare_bypasser import warmup as bypasser_warmup, shutdown_if_idle as bypasser_shutdown - ws_manager.register_on_first_connect(bypasser_warmup) - ws_manager.register_on_all_disconnect(bypasser_shutdown) - logger.info("Registered Cloudflare bypasser warmup/shutdown on WebSocket connect/disconnect") - -if DEBUG: - import subprocess - if USING_EXTERNAL_BYPASSER: - STOP_GUI = lambda: None - else: - from cloudflare_bypasser import _reset_driver as STOP_GUI - @app.route('/api/debug', methods=['GET']) - @login_required - def debug() -> Union[Response, Tuple[Response, int]]: - """ - This will run the /app/genDebug.sh script, which will generate a debug zip with all the logs - The file will be named /tmp/cwa-book-downloader-debug.zip - And then return it to the user - """ - try: - # Run the debug script - logger.info("Debug endpoint called, stopping GUI and generating debug info...") - STOP_GUI() - time.sleep(1) - result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True) - if result.returncode != 0: - raise Exception(f"Debug script failed: {result.stderr}") - logger.info(f"Debug script executed: {result.stdout}") - debug_file_path = result.stdout.strip().split('\n')[-1] - if not os.path.exists(debug_file_path): - logger.error(f"Debug zip file not found at: {debug_file_path}") - return jsonify({"error": "Failed to generate debug information"}), 500 - - logger.info(f"Sending debug file: {debug_file_path}") - # Return the file to the user - return send_file( - debug_file_path, - mimetype='application/zip', - download_name=os.path.basename(debug_file_path), - as_attachment=True - ) - except subprocess.CalledProcessError as e: - logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}") - return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500 - except Exception as e: - logger.error_trace(f"Debug endpoint error: {e}") - return jsonify({"error": str(e)}), 500 - -if DEBUG: - @app.route('/api/restart', methods=['GET']) - @login_required - def restart() -> Union[Response, Tuple[Response, int]]: - """ - Restart the application - """ - os._exit(0) - -@app.route('/api/search', methods=['GET']) -@login_required -def api_search() -> Union[Response, Tuple[Response, int]]: - """ - Search for books matching the provided query. - - Query Parameters: - query (str): Search term (ISBN, title, author, etc.) - isbn (str): Book ISBN - author (str): Book Author - title (str): Book Title - lang (str): Book Language - sort (str): Order to sort results - content (str): Content type of book - format (str): File format filter (pdf, epub, mobi, azw3, fb2, djvu, cbz, cbr) - - Returns: - flask.Response: JSON array of matching books or error response. - """ - query = request.args.get('query', '') - - filters = SearchFilters( - isbn = request.args.getlist('isbn'), - author = request.args.getlist('author'), - title = request.args.getlist('title'), - lang = request.args.getlist('lang'), - sort = request.args.get('sort'), - content = request.args.getlist('content'), - format = request.args.getlist('format'), - ) - - if not query and not any(vars(filters).values()): - return jsonify([]) - - try: - books = backend.search_books(query, filters) - return jsonify(books) - except SearchUnavailable as e: - logger.warning(f"Search unavailable: {e}") - return jsonify({"error": str(e)}), 503 - except Exception as e: - logger.error_trace(f"Search error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/info', methods=['GET']) -@login_required -def api_info() -> Union[Response, Tuple[Response, int]]: - """ - Get detailed book information. - - Query Parameters: - id (str): Book identifier (MD5 hash) - - Returns: - flask.Response: JSON object with book details, or an error message. - """ - book_id = request.args.get('id', '') - if not book_id: - return jsonify({"error": "No book ID provided"}), 400 - - try: - book = backend.get_book_info(book_id) - if book: - return jsonify(book) - return jsonify({"error": "Book not found"}), 404 - except Exception as e: - logger.error_trace(f"Info error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/download', methods=['GET']) -@login_required -def api_download() -> Union[Response, Tuple[Response, int]]: - """ - Queue a book for download. - - Query Parameters: - id (str): Book identifier (MD5 hash) - - Returns: - flask.Response: JSON status object indicating success or failure. - """ - book_id = request.args.get('id', '') - if not book_id: - return jsonify({"error": "No book ID provided"}), 400 - - try: - priority = int(request.args.get('priority', 0)) - success = backend.queue_book(book_id, priority) - if success: - return jsonify({"status": "queued", "priority": priority}) - return jsonify({"error": "Failed to queue book"}), 500 - except Exception as e: - logger.error_trace(f"Download error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/config', methods=['GET']) -@login_required -def api_config() -> Union[Response, Tuple[Response, int]]: - """ - Get application configuration for frontend. - """ - try: - config = { - "calibre_web_url": CALIBRE_WEB_URL, - "debug": DEBUG, - "build_version": BUILD_VERSION, - "release_version": RELEASE_VERSION, - "book_languages": _SUPPORTED_BOOK_LANGUAGE, - "default_language": BOOK_LANGUAGE, - "supported_formats": SUPPORTED_FORMATS - } - return jsonify(config) - except Exception as e: - logger.error_trace(f"Config error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/health', methods=['GET']) -def api_health() -> Union[Response, Tuple[Response, int]]: - """ - Health check endpoint for container orchestration. - No authentication required. - - Returns: - flask.Response: JSON with status "ok". - """ - return jsonify({"status": "ok"}) - -@app.route('/api/status', methods=['GET']) -@login_required -def api_status() -> Union[Response, Tuple[Response, int]]: - """ - Get current download queue status. - - Returns: - flask.Response: JSON object with queue status. - """ - try: - status = backend.queue_status() - return jsonify(status) - except Exception as e: - logger.error_trace(f"Status error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/localdownload', methods=['GET']) -@login_required -def api_local_download() -> Union[Response, Tuple[Response, int]]: - """ - Download an EPUB file from local storage if available. - - Query Parameters: - id (str): Book identifier (MD5 hash) - - Returns: - flask.Response: The EPUB file if found, otherwise an error response. - """ - book_id = request.args.get('id', '') - if not book_id: - return jsonify({"error": "No book ID provided"}), 400 - - try: - file_data, book_info = backend.get_book_data(book_id) - if file_data is None: - # Book data not found or not available - return jsonify({"error": "File not found"}), 404 - file_name = book_info.get_filename() - # Prepare the file for sending to the client - data = io.BytesIO(file_data) - return send_file( - data, - download_name=file_name, - as_attachment=True - ) - - except Exception as e: - logger.error_trace(f"Local download error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/download//cancel', methods=['DELETE']) -@login_required -def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]: - """ - Cancel a download. - - Path Parameters: - book_id (str): Book identifier to cancel - - Returns: - flask.Response: JSON status indicating success or failure. - """ - try: - success = backend.cancel_download(book_id) - if success: - return jsonify({"status": "cancelled", "book_id": book_id}) - return jsonify({"error": "Failed to cancel download or book not found"}), 404 - except Exception as e: - logger.error_trace(f"Cancel download error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/queue//priority', methods=['PUT']) -@login_required -def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]: - """ - Set priority for a queued book. - - Path Parameters: - book_id (str): Book identifier - - Request Body: - priority (int): New priority level (lower number = higher priority) - - Returns: - flask.Response: JSON status indicating success or failure. - """ - try: - data = request.get_json() - if not data or 'priority' not in data: - return jsonify({"error": "Priority not provided"}), 400 - - priority = int(data['priority']) - success = backend.set_book_priority(book_id, priority) - - if success: - return jsonify({"status": "updated", "book_id": book_id, "priority": priority}) - return jsonify({"error": "Failed to update priority or book not found"}), 404 - except ValueError: - return jsonify({"error": "Invalid priority value"}), 400 - except Exception as e: - logger.error_trace(f"Set priority error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/queue/reorder', methods=['POST']) -@login_required -def api_reorder_queue() -> Union[Response, Tuple[Response, int]]: - """ - Bulk reorder queue by setting new priorities. - - Request Body: - book_priorities (dict): Mapping of book_id to new priority - - Returns: - flask.Response: JSON status indicating success or failure. - """ - try: - data = request.get_json() - if not data or 'book_priorities' not in data: - return jsonify({"error": "book_priorities not provided"}), 400 - - book_priorities = data['book_priorities'] - if not isinstance(book_priorities, dict): - return jsonify({"error": "book_priorities must be a dictionary"}), 400 - - # Validate all priorities are integers - for book_id, priority in book_priorities.items(): - if not isinstance(priority, int): - return jsonify({"error": f"Invalid priority for book {book_id}"}), 400 - - success = backend.reorder_queue(book_priorities) - - if success: - return jsonify({"status": "reordered", "updated_count": len(book_priorities)}) - return jsonify({"error": "Failed to reorder queue"}), 500 - except Exception as e: - logger.error_trace(f"Reorder queue error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/queue/order', methods=['GET']) -@login_required -def api_queue_order() -> Union[Response, Tuple[Response, int]]: - """ - Get current queue order for display. - - Returns: - flask.Response: JSON array of queued books with their order and priorities. - """ - try: - queue_order = backend.get_queue_order() - return jsonify({"queue": queue_order}) - except Exception as e: - logger.error_trace(f"Queue order error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/downloads/active', methods=['GET']) -@login_required -def api_active_downloads() -> Union[Response, Tuple[Response, int]]: - """ - Get list of currently active downloads. - - Returns: - flask.Response: JSON array of active download book IDs. - """ - try: - active_downloads = backend.get_active_downloads() - return jsonify({"active_downloads": active_downloads}) - except Exception as e: - logger.error_trace(f"Active downloads error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.route('/api/queue/clear', methods=['DELETE']) -@login_required -def api_clear_completed() -> Union[Response, Tuple[Response, int]]: - """ - Clear all completed, errored, or cancelled books from tracking. - - Returns: - flask.Response: JSON with count of removed books. - """ - try: - removed_count = backend.clear_completed() - - # Broadcast status update after clearing - if ws_manager: - ws_manager.broadcast_status_update(backend.queue_status()) - - return jsonify({"status": "cleared", "removed_count": removed_count}) - except Exception as e: - logger.error_trace(f"Clear completed error: {e}") - return jsonify({"error": str(e)}), 500 - -@app.errorhandler(404) -def not_found_error(error: Exception) -> Union[Response, Tuple[Response, int]]: - """ - Handle 404 (Not Found) errors. - - Args: - error (HTTPException): The 404 error raised by Flask. - - Returns: - flask.Response: JSON error message with 404 status. - """ - logger.warning(f"404 error: {request.url} : {error}") - return jsonify({"error": "Resource not found"}), 404 - -@app.errorhandler(500) -def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]: - """ - Handle 500 (Internal Server) errors. - - Args: - error (HTTPException): The 500 error raised by Flask. - - Returns: - flask.Response: JSON error message with 500 status. - """ - logger.error_trace(f"500 error: {error}") - return jsonify({"error": "Internal server error"}), 500 - -@app.route('/api/auth/login', methods=['POST']) -def api_login() -> Union[Response, Tuple[Response, int]]: - """ - Login endpoint that validates credentials and creates a session. - Includes rate limiting: 10 failed attempts = 30 minute lockout. - - Request Body: - username (str): Username - password (str): Password - remember_me (bool): Whether to extend session duration - - Returns: - flask.Response: JSON with success status or error message. - """ - try: - # Get client IP address (handles reverse proxy forwarding) - ip_address = request.headers.get('X-Forwarded-For', request.remote_addr) - if ip_address and ',' in ip_address: - # X-Forwarded-For can contain multiple IPs, take the first one - ip_address = ip_address.split(',')[0].strip() - - data = request.get_json() - if not data: - return jsonify({"error": "No data provided"}), 400 - - username = data.get('username', '').strip() - password = data.get('password', '') - remember_me = data.get('remember_me', False) - - if not username or not password: - return jsonify({"error": "Username and password are required"}), 400 - - # Check if account is locked due to failed login attempts - if is_account_locked(username): - lockout_until = failed_login_attempts[username].get('lockout_until') - remaining_time = (lockout_until - datetime.now()).total_seconds() / 60 - logger.warning(f"Login attempt blocked for locked account '{username}' from IP {ip_address}") - return jsonify({ - "error": f"Account temporarily locked due to multiple failed login attempts. Try again in {int(remaining_time)} minutes." - }), 429 - - # If the database doesn't exist, authentication always succeeds - if not CWA_DB_PATH: - session['user_id'] = username - session.permanent = remember_me - clear_failed_logins(username) - logger.info(f"Login successful for user '{username}' from IP {ip_address} (no DB configured)") - return jsonify({"success": True}) - - # If the CWA_DB_PATH variable exists, but isn't a valid path, return error - if not os.path.isfile(CWA_DB_PATH): - logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path") - return jsonify({"error": "Database configuration error"}), 500 - - # Validate credentials against database - try: - db_path = os.fspath(CWA_DB_PATH) - db_uri = f"file:{db_path}?mode=ro&immutable=1" - conn = sqlite3.connect(db_uri, uri=True) - cur = conn.cursor() - cur.execute("SELECT password FROM user WHERE name = ?", (username,)) - row = cur.fetchone() - conn.close() - - # Check if user exists and password is correct - if not row or not row[0] or not check_password_hash(row[0], password): - # Record failed login attempt - is_now_locked = record_failed_login(username, ip_address) - - if is_now_locked: - return jsonify({ - "error": f"Account locked due to {MAX_LOGIN_ATTEMPTS} failed login attempts. Try again in {LOCKOUT_DURATION_MINUTES} minutes." - }), 429 - else: - attempts_remaining = MAX_LOGIN_ATTEMPTS - failed_login_attempts[username]['count'] - # Only show attempts remaining when 5 or fewer attempts remain (after 6+ failed attempts) - if attempts_remaining <= 5: - return jsonify({ - "error": f"Invalid username or password. {attempts_remaining} attempts remaining." - }), 401 - else: - return jsonify({ - "error": "Invalid username or password." - }), 401 - - # Successful authentication - create session and clear failed attempts - session['user_id'] = username - session.permanent = remember_me - clear_failed_logins(username) - logger.info(f"Login successful for user '{username}' from IP {ip_address} (remember_me={remember_me})") - return jsonify({"success": True}) - - except Exception as e: - logger.error_trace(f"Database error during login: {e}") - return jsonify({"error": "Authentication system error"}), 500 - - except Exception as e: - logger.error_trace(f"Login error: {e}") - return jsonify({"error": "Login failed"}), 500 - -@app.route('/api/auth/logout', methods=['POST']) -def api_logout() -> Union[Response, Tuple[Response, int]]: - """ - Logout endpoint that clears the session. - - Returns: - flask.Response: JSON with success status. - """ - try: - # Get client IP address (handles reverse proxy forwarding) - ip_address = request.headers.get('X-Forwarded-For', request.remote_addr) - if ip_address and ',' in ip_address: - ip_address = ip_address.split(',')[0].strip() - - username = session.get('user_id', 'unknown') - session.clear() - logger.info(f"Logout successful for user '{username}' from IP {ip_address}") - return jsonify({"success": True}) - except Exception as e: - logger.error_trace(f"Logout error: {e}") - return jsonify({"error": "Logout failed"}), 500 - -@app.route('/api/auth/check', methods=['GET']) -def api_auth_check() -> Union[Response, Tuple[Response, int]]: - """ - Check if user has a valid session. - - Returns: - flask.Response: JSON with authentication status and whether auth is required. - """ - try: - # If no database is configured, authentication is not required - if not CWA_DB_PATH: - return jsonify({ - "authenticated": True, - "auth_required": False - }) - - # Check if user has a valid session - is_authenticated = 'user_id' in session - return jsonify({ - "authenticated": is_authenticated, - "auth_required": True - }) - except Exception as e: - logger.error_trace(f"Auth check error: {e}") - return jsonify({ - "authenticated": False, - "auth_required": True - }) - -# Catch-all route for React Router (must be last) -# This handles client-side routing by serving index.html for any unmatched routes -@app.route('/') -def catch_all(path: str) -> Response: - """ - Serve the React app for any route not matched by API endpoints. - This allows React Router to handle client-side routing. - Authentication is handled by the React app itself. - """ - # If the request is for an API endpoint or static file, let it 404 - if path.startswith('api/') or path.startswith('assets/'): - return jsonify({"error": "Resource not found"}), 404 - # Otherwise serve the React app - return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), 'index.html') - -# WebSocket event handlers -@socketio.on('connect') -def handle_connect(): - """Handle client connection.""" - logger.info("WebSocket client connected") - - # Track the connection (triggers warmup callbacks on first connect) - ws_manager.client_connected() - - # Send initial status to the newly connected client - try: - status = backend.queue_status() - emit('status_update', status) - except Exception as e: - logger.error(f"Error sending initial status: {e}") - -@socketio.on('disconnect') -def handle_disconnect(): - """Handle client disconnection.""" - logger.info("WebSocket client disconnected") - - # Track the disconnection - ws_manager.client_disconnected() - -@socketio.on('request_status') -def handle_status_request(): - """Handle manual status request from client.""" - try: - status = backend.queue_status() - emit('status_update', status) - except Exception as e: - logger.error(f"Error handling status request: {e}") - emit('error', {'message': 'Failed to get status'}) - -logger.log_resource_usage() - -if __name__ == '__main__': - logger.info(f"Starting Flask application with WebSocket support on {FLASK_HOST}:{FLASK_PORT} (debug={DEBUG})") - socketio.run( - app, - host=FLASK_HOST, - port=FLASK_PORT, - debug=DEBUG, - allow_unsafe_werkzeug=True # For development only - ) diff --git a/backend.py b/backend.py deleted file mode 100644 index d542ce4b..00000000 --- a/backend.py +++ /dev/null @@ -1,513 +0,0 @@ -"""Backend logic for the book download application.""" - -import os -import random -import shutil -import subprocess -import threading -import time -from concurrent.futures import Future, ThreadPoolExecutor -from pathlib import Path -from threading import Event, Lock -from typing import Any, Dict, List, Optional, Tuple - -import book_manager -from book_manager import SearchUnavailable -from config import CUSTOM_SCRIPT -from env import ( - DOWNLOAD_PATHS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL, INGEST_DIR, - MAIN_LOOP_SLEEP_TIME, MAX_CONCURRENT_DOWNLOADS, TMP_DIR, USE_BOOK_TITLE, -) -from logger import setup_logger -from models import BookInfo, QueueStatus, SearchFilters, book_queue - -logger = setup_logger(__name__) - -# WebSocket manager (initialized by app.py) -try: - from websocket_manager import ws_manager -except ImportError: - ws_manager = None - -# Progress update throttling - track last broadcast time per book -_progress_last_broadcast: Dict[str, float] = {} -_progress_lock = Lock() - -# Stall detection - track last activity time per download -_last_activity: Dict[str, float] = {} -STALL_TIMEOUT = 300 # 5 minutes without progress/status update = stalled - -def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]: - """Search for books matching the query. - - Args: - query: Search term - filters: Search filters object - - Returns: - List[Dict]: List of book information dictionaries - """ - try: - books = book_manager.search_books(query, filters) - return [_book_info_to_dict(book) for book in books] - except SearchUnavailable as e: - logger.warning(f"Search unavailable: {e}") - raise - except Exception as e: - logger.error_trace(f"Error searching books: {e}") - return [] - -def get_book_info(book_id: str) -> Optional[Dict[str, Any]]: - """Get detailed information for a specific book. - - Args: - book_id: Book identifier - - Returns: - Optional[Dict]: Book information dictionary if found - """ - try: - book = book_manager.get_book_info(book_id) - return _book_info_to_dict(book) - except Exception as e: - logger.error_trace(f"Error getting book info: {e}") - return None - -def queue_book(book_id: str, priority: int = 0) -> bool: - """Add a book to the download queue with specified priority. - - Args: - book_id: Book identifier - priority: Priority level (lower number = higher priority) - - Returns: - bool: True if book was successfully queued - """ - try: - book_info = book_manager.get_book_info(book_id) - book_queue.add(book_id, book_info, priority) - logger.info(f"Book queued with priority {priority}: {book_info.title}") - - # Broadcast status update via WebSocket - if ws_manager: - ws_manager.broadcast_status_update(queue_status()) - - return True - except Exception as e: - logger.error_trace(f"Error queueing book: {e}") - return False - -def queue_status() -> Dict[str, Dict[str, Any]]: - """Get current status of the download queue. - - Returns: - Dict: Queue status organized by status type with serialized book data - """ - status = book_queue.get_status() - for _, books in status.items(): - for _, book_info in books.items(): - if book_info.download_path: - if not os.path.exists(book_info.download_path): - book_info.download_path = None - - # Convert Enum keys to strings and BookInfo objects to dicts for JSON serialization - return { - status_type.value: { - book_id: _book_info_to_dict(book_info) - for book_id, book_info in books.items() - } - for status_type, books in status.items() - } - -def get_book_data(book_id: str) -> Tuple[Optional[bytes], BookInfo]: - """Get book data for a specific book, including its title. - - Args: - book_id: Book identifier - - Returns: - Tuple[Optional[bytes], str]: Book data if available, and the book title - """ - try: - book_info = book_queue._book_data[book_id] - path = book_info.download_path - with open(path, "rb") as f: - return f.read(), book_info - except Exception as e: - logger.error_trace(f"Error getting book data: {e}") - if book_info: - book_info.download_path = None - return None, book_info if book_info else BookInfo(id=book_id, title="Unknown") - -def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]: - """Convert BookInfo object to dictionary representation.""" - return { - key: value for key, value in book.__dict__.items() - if value is not None - } - -def _prepare_download_folder(book_info: BookInfo) -> Path: - """Prepare final content-type subdir""" - content = book_info.content - content_dir = DOWNLOAD_PATHS.get(content) if content and content in DOWNLOAD_PATHS else INGEST_DIR - os.makedirs(content_dir, exist_ok=True) - return content_dir - -def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Optional[str]: - """Download and process a book with cancellation support. - - Args: - book_id: Book identifier - cancel_flag: Threading event to signal cancellation - - Returns: - str: Path to the downloaded book if successful, None otherwise - """ - try: - # Check for cancellation before starting - if cancel_flag.is_set(): - logger.info(f"Download cancelled before starting: {book_id}") - return None - - book_info = book_queue._book_data[book_id] - logger.info(f"Starting download: {book_info.title}") - - if not book_info.download_urls: - raise ValueError(f"No download URLs available for {book_id}") - - # get_filename() resolves format as side effect - full_name = book_info.get_filename() - book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}" - book_path = TMP_DIR / book_name - - # Check cancellation before download - if cancel_flag.is_set(): - logger.info(f"Download cancelled before book manager call: {book_id}") - return None - - progress_callback = lambda progress: update_download_progress(book_id, progress) - status_callback = lambda status, message=None: update_download_status(book_id, status, message) - - # Set status to resolving immediately when processing starts - update_download_status(book_id, "resolving") - - success_download_url = book_manager.download_book(book_info, book_path, progress_callback, cancel_flag, status_callback) - - # Stop progress updates - cancel_flag.wait(0.1) # Brief pause for progress thread cleanup - - if cancel_flag.is_set(): - logger.info(f"Download cancelled during download: {book_id}") - # Clean up partial download - if book_path.exists(): - book_path.unlink() - return None - - if not success_download_url: - raise Exception("Unknown error downloading book") - - # Check cancellation before post-processing - if cancel_flag.is_set(): - logger.info(f"Download cancelled before post-processing: {book_id}") - if book_path.exists(): - book_path.unlink() - return None - - logger.debug(f"Post-processing download: {book_info.title}") - - if CUSTOM_SCRIPT: - logger.info(f"Running custom script: {CUSTOM_SCRIPT}") - subprocess.run([CUSTOM_SCRIPT, book_path]) - - # Regenerate filename with fallback to successful download URL for format - full_name = book_info.get_filename(success_download_url) - book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}" - - final_dir = _prepare_download_folder(book_info) - intermediate_path = final_dir / f"{book_id}.crdownload" - final_path = final_dir / book_name - - # Handle file already exists - add suffix to avoid overwrite - if final_path.exists(): - base = final_path.stem - ext = final_path.suffix - counter = 1 - while final_path.exists(): - final_path = final_dir / f"{base}_{counter}{ext}" - counter += 1 - logger.info(f"File already exists, saving as: {final_path.name}") - - if os.path.exists(book_path): - logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}") - try: - shutil.move(book_path, intermediate_path) - except Exception as e: - try: - logger.debug(f"Error moving book: {e}, will try copying instead") - shutil.move(book_path, intermediate_path) - except Exception as e: - logger.debug(f"Error copying book: {e}, will try copying without permissions instead") - shutil.copyfile(book_path, intermediate_path) - os.remove(book_path) - - # Final cancellation check before completing - if cancel_flag.is_set(): - logger.info(f"Download cancelled before final rename: {book_id}") - if intermediate_path.exists(): - intermediate_path.unlink() - return None - - os.rename(intermediate_path, final_path) - logger.info(f"Download completed successfully: {book_info.title}") - - return str(final_path) - except Exception as e: - if cancel_flag.is_set(): - logger.info(f"Download cancelled during error handling: {book_id}") - else: - logger.error_trace(f"Error downloading book: {e}") - return None - -def update_download_progress(book_id: str, progress: float) -> None: - """Update download progress with throttled WebSocket broadcasts. - - Progress is always stored in the queue, but WebSocket broadcasts are - throttled to avoid flooding clients with updates. Broadcasts occur: - - At most once per DOWNLOAD_PROGRESS_UPDATE_INTERVAL seconds - - Always at 0% (start) and 100% (complete) - - On significant progress jumps (>10%) - """ - book_queue.update_progress(book_id, progress) - - # Track activity for stall detection - with _progress_lock: - _last_activity[book_id] = time.time() - - # Broadcast progress via WebSocket with throttling - if ws_manager: - current_time = time.time() - should_broadcast = False - - with _progress_lock: - last_broadcast = _progress_last_broadcast.get(book_id, 0) - last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0) - time_elapsed = current_time - last_broadcast - - # Always broadcast at start (0%) or completion (>=99%) - if progress <= 1 or progress >= 99: - should_broadcast = True - # Broadcast if enough time has passed (convert interval from seconds) - elif time_elapsed >= DOWNLOAD_PROGRESS_UPDATE_INTERVAL: - should_broadcast = True - # Broadcast on significant progress jumps (>10%) - elif progress - last_progress >= 10: - should_broadcast = True - - if should_broadcast: - _progress_last_broadcast[book_id] = current_time - _progress_last_broadcast[f"{book_id}_progress"] = progress - - if should_broadcast: - ws_manager.broadcast_download_progress(book_id, progress, 'downloading') - -def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None: - """Update download status with optional detailed message. - - Args: - book_id: Book identifier - status: Status string (e.g., 'resolving', 'downloading') - message: Optional detailed status message for UI display - """ - # Map string status to QueueStatus enum - status_map = { - 'queued': QueueStatus.QUEUED, - 'resolving': QueueStatus.RESOLVING, - 'downloading': QueueStatus.DOWNLOADING, - 'complete': QueueStatus.COMPLETE, - 'available': QueueStatus.AVAILABLE, - 'error': QueueStatus.ERROR, - 'done': QueueStatus.DONE, - 'cancelled': QueueStatus.CANCELLED, - } - - queue_status_enum = status_map.get(status.lower()) - if queue_status_enum: - book_queue.update_status(book_id, queue_status_enum) - - # Track activity for stall detection - with _progress_lock: - _last_activity[book_id] = time.time() - - # Update status message if provided (empty string clears the message) - if message is not None: - book_queue.update_status_message(book_id, message) - - # Broadcast status update via WebSocket - if ws_manager: - ws_manager.broadcast_status_update(queue_status()) - -def cancel_download(book_id: str) -> bool: - """Cancel a download. - - Args: - book_id: Book identifier to cancel - - Returns: - bool: True if cancellation was successful - """ - result = book_queue.cancel_download(book_id) - - # Broadcast status update via WebSocket - if result and ws_manager and ws_manager.is_enabled(): - ws_manager.broadcast_status_update(queue_status()) - - return result - -def set_book_priority(book_id: str, priority: int) -> bool: - """Set priority for a queued book. - - Args: - book_id: Book identifier - priority: New priority level (lower = higher priority) - - Returns: - bool: True if priority was successfully changed - """ - return book_queue.set_priority(book_id, priority) - -def reorder_queue(book_priorities: Dict[str, int]) -> bool: - """Bulk reorder queue. - - Args: - book_priorities: Dict mapping book_id to new priority - - Returns: - bool: True if reordering was successful - """ - return book_queue.reorder_queue(book_priorities) - -def get_queue_order() -> List[Dict[str, any]]: - """Get current queue order for display.""" - return book_queue.get_queue_order() - -def get_active_downloads() -> List[str]: - """Get list of currently active downloads.""" - return book_queue.get_active_downloads() - -def clear_completed() -> int: - """Clear all completed downloads from tracking.""" - return book_queue.clear_completed() - -def _cleanup_progress_tracking(book_id: str) -> None: - """Clean up progress tracking data for a completed/cancelled download.""" - with _progress_lock: - _progress_last_broadcast.pop(book_id, None) - _progress_last_broadcast.pop(f"{book_id}_progress", None) - _last_activity.pop(book_id, None) - -def _process_single_download(book_id: str, cancel_flag: Event) -> None: - """Process a single download job.""" - try: - # Status will be updated through callbacks during download process - # (resolving -> downloading -> complete) - download_path = _download_book_with_cancellation(book_id, cancel_flag) - - # Clean up progress tracking - _cleanup_progress_tracking(book_id) - - if cancel_flag.is_set(): - book_queue.update_status(book_id, QueueStatus.CANCELLED) - # Broadcast cancellation - if ws_manager: - ws_manager.broadcast_status_update(queue_status()) - return - - if download_path: - book_queue.update_download_path(book_id, download_path) - new_status = QueueStatus.COMPLETE - else: - new_status = QueueStatus.ERROR - - book_queue.update_status(book_id, new_status) - - # Broadcast final status (completed or error) - if ws_manager: - ws_manager.broadcast_status_update(queue_status()) - - - except Exception as e: - # Clean up progress tracking even on error - _cleanup_progress_tracking(book_id) - - if not cancel_flag.is_set(): - logger.error_trace(f"Error in download processing: {e}") - book_queue.update_status(book_id, QueueStatus.ERROR) - # Set error message if not already set by download_book() - if book_id in book_queue._book_data and not book_queue._book_data[book_id].status_message: - book_queue.update_status_message(book_id, f"Download failed: {type(e).__name__}: {str(e)}") - else: - logger.info(f"Download cancelled: {book_id}") - book_queue.update_status(book_id, QueueStatus.CANCELLED) - - # Broadcast error/cancelled status - if ws_manager: - ws_manager.broadcast_status_update(queue_status()) - -def concurrent_download_loop() -> None: - """Main download coordinator using ThreadPoolExecutor for concurrent downloads.""" - logger.info(f"Starting concurrent download loop with {MAX_CONCURRENT_DOWNLOADS} workers") - - with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_DOWNLOADS, thread_name_prefix="BookDownload") as executor: - active_futures: Dict[Future, str] = {} # Track active download futures - - while True: - # Clean up completed futures - completed_futures = [f for f in active_futures if f.done()] - for future in completed_futures: - book_id = active_futures.pop(future) - try: - future.result() # This will raise any exceptions from the worker - except Exception as e: - logger.error_trace(f"Future exception for {book_id}: {e}") - - # Check for stalled downloads (no activity in STALL_TIMEOUT seconds) - current_time = time.time() - with _progress_lock: - for future, book_id in list(active_futures.items()): - last_active = _last_activity.get(book_id, current_time) - if current_time - last_active > STALL_TIMEOUT: - logger.warning(f"Download stalled for {book_id}, cancelling") - book_queue.cancel_download(book_id) - book_queue.update_status_message(book_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)") - - # Start new downloads if we have capacity - while len(active_futures) < MAX_CONCURRENT_DOWNLOADS: - next_download = book_queue.get_next() - if not next_download: - break - - # Stagger concurrent downloads to avoid rate limiting on shared download servers - # Only delay if other downloads are already active - if active_futures: - stagger_delay = random.uniform(2, 5) - logger.debug(f"Staggering download start by {stagger_delay:.1f}s") - time.sleep(stagger_delay) - - book_id, cancel_flag = next_download - - # Submit download job to thread pool - future = executor.submit(_process_single_download, book_id, cancel_flag) - active_futures[future] = book_id - - # Brief sleep to prevent busy waiting - time.sleep(MAIN_LOOP_SLEEP_TIME) - -# Start concurrent download coordinator -download_coordinator_thread = threading.Thread( - target=concurrent_download_loop, - daemon=True, - name="DownloadCoordinator" -) -download_coordinator_thread.start() - -logger.info(f"Download system initialized with {MAX_CONCURRENT_DOWNLOADS} concurrent workers") diff --git a/config.py b/config.py deleted file mode 100644 index 9b2d99f7..00000000 --- a/config.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Configuration settings for the book downloader application.""" - -import os -from pathlib import Path -import json -import env -from logger import setup_logger - -logger = setup_logger(__name__) - -for key, value in env.__dict__.items(): - if not key.startswith('_'): - if key == "AA_DONATOR_KEY" and value.strip() != "": - value = "REDACTED" - logger.info(f"{key}: {value}") - -with open("data/book-languages.json") as file: - _SUPPORTED_BOOK_LANGUAGE = json.load(file) - -# Directory settings -BASE_DIR = Path(__file__).resolve().parent -logger.info(f"BASE_DIR: {BASE_DIR}") -if env.ENABLE_LOGGING: - env.LOG_DIR.mkdir(exist_ok=True) - -# Create necessary directories -env.TMP_DIR.mkdir(exist_ok=True) -env.INGEST_DIR.mkdir(exist_ok=True) - -CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev -logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}") -logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}") -logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}") - -# Network settings -_custom_dns = env._CUSTOM_DNS.lower().strip() -_doh_server = "" - -if _custom_dns == "auto" or _custom_dns == "": - # Auto mode - DNS provider rotation handled by network.py - # Starts with system DNS, switches to providers from DNS_PROVIDERS on failure - CUSTOM_DNS = [] - _doh_server = "" - logger.info("CUSTOM_DNS: auto (starts with system DNS, rotates on failure)") -elif _custom_dns == "google": - CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860:0000:0000:0000:0000:8888", "2001:4860:4860:0000:0000:0000:0000:8844"] - _doh_server = "https://dns.google/resolve" - logger.info(f"CUSTOM_DNS: google {CUSTOM_DNS}") -elif _custom_dns == "quad9": - CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:00fe:0000:0000:0000:0000:0000:00fe", "2620:00fe:0000:0000:0000:0000:0000:0009"] - _doh_server = "https://dns.quad9.net/dns-query" - logger.info(f"CUSTOM_DNS: quad9 {CUSTOM_DNS}") -elif _custom_dns == "cloudflare": - CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700:0000:0000:0000:0000:1111", "2606:4700:4700:0000:0000:0000:0000:1001"] - _doh_server = "https://cloudflare-dns.com/dns-query" - logger.info(f"CUSTOM_DNS: cloudflare {CUSTOM_DNS}") -elif _custom_dns == "opendns": - CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:0119:0035:0000:0000:0000:0000:0035", "2620:0119:0053:0000:0000:0000:0000:0053"] - _doh_server = "https://doh.opendns.com/dns-query" - logger.info(f"CUSTOM_DNS: opendns {CUSTOM_DNS}") -else: - # Custom DNS IPs provided by user - _custom_dns_ip = _custom_dns.split(",") - CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()] - logger.info(f"CUSTOM_DNS: custom {CUSTOM_DNS}") -DOH_SERVER = _doh_server -if env.USE_DOH: - DOH_SERVER = _doh_server -else: - DOH_SERVER = "" -logger.info(f"DOH_SERVER: {DOH_SERVER}") - -# Warn about external bypasser DNS limitations -if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS: - logger.warning( - "Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, " - "not this application's custom DNS settings. If you experience DNS-related blocks, " - "configure DNS at the Docker/system level for your FlareSolverr container, " - "or consider using the internal bypasser which integrates with the app's DNS system." - ) - -# Proxy settings -PROXIES = {} -if env.HTTP_PROXY: - PROXIES["http"] = env.HTTP_PROXY -if env.HTTPS_PROXY: - PROXIES["https"] = env.HTTPS_PROXY -logger.info(f"PROXIES: {PROXIES}") - -# Anna's Archive settings -AA_BASE_URL = env._AA_BASE_URL -AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"] -AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(",")) -AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()] - -# File format settings -SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",") -logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}") - -# Complex language processing logic kept in config.py -BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',') -BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]] -if len(BOOK_LANGUAGE) == 0: - BOOK_LANGUAGE = ['en'] - -# Custom script settings with validation logic -CUSTOM_SCRIPT = env._CUSTOM_SCRIPT -if CUSTOM_SCRIPT: - if not os.path.exists(CUSTOM_SCRIPT): - logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist") - CUSTOM_SCRIPT = "" - elif not os.access(CUSTOM_SCRIPT, os.X_OK): - logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable") - CUSTOM_SCRIPT = "" - -# Debugging settings -if not env.USING_EXTERNAL_BYPASSER: - # Virtual display settings for debugging internal cloudflare bypasser - VIRTUAL_SCREEN_SIZE = (1024, 768) - RECORDING_DIR = env.LOG_DIR / "recording" - if env.DEBUG: - RECORDING_DIR.mkdir(parents=True, exist_ok=True) diff --git a/cwa_book_downloader/__init__.py b/cwa_book_downloader/__init__.py new file mode 100644 index 00000000..d9948443 --- /dev/null +++ b/cwa_book_downloader/__init__.py @@ -0,0 +1 @@ +"""CWA Book Downloader - book search and download service.""" diff --git a/cwa_book_downloader/__main__.py b/cwa_book_downloader/__main__.py new file mode 100644 index 00000000..578f5f10 --- /dev/null +++ b/cwa_book_downloader/__main__.py @@ -0,0 +1,7 @@ +"""Package entry point for `python -m cwa_book_downloader`.""" + +from cwa_book_downloader.main import app, socketio +from cwa_book_downloader.config.env import FLASK_HOST, FLASK_PORT, DEBUG + +if __name__ == "__main__": + socketio.run(app, host=FLASK_HOST, port=FLASK_PORT, debug=DEBUG) diff --git a/cwa_book_downloader/api/__init__.py b/cwa_book_downloader/api/__init__.py new file mode 100644 index 00000000..a52228f2 --- /dev/null +++ b/cwa_book_downloader/api/__init__.py @@ -0,0 +1 @@ +"""API module - WebSocket handling.""" diff --git a/websocket_manager.py b/cwa_book_downloader/api/websocket.py similarity index 97% rename from websocket_manager.py rename to cwa_book_downloader/api/websocket.py index 134d8df2..89d74876 100644 --- a/websocket_manager.py +++ b/cwa_book_downloader/api/websocket.py @@ -4,13 +4,14 @@ import logging import threading from typing import Optional, Dict, Any, Callable, List -from flask_socketio import SocketIO, emit +from flask_socketio import SocketIO logger = logging.getLogger(__name__) + class WebSocketManager: """Manages WebSocket connections and broadcasts.""" - + def __init__(self): self.socketio: Optional[SocketIO] = None self._enabled = False @@ -19,22 +20,22 @@ class WebSocketManager: self._on_first_connect_callbacks: List[Callable[[], None]] = [] self._on_all_disconnect_callbacks: List[Callable[[], None]] = [] self._needs_rewarm = False # Flag to trigger warmup callbacks on next connect - + def init_app(self, app, socketio: SocketIO): """Initialize the WebSocket manager with Flask-SocketIO instance.""" self.socketio = socketio self._enabled = True logger.info("WebSocket manager initialized") - + def register_on_first_connect(self, callback: Callable[[], None]): """Register a callback to be called when the first client connects. - + This is useful for warming up resources (like the Cloudflare bypasser) when a user starts using the web UI. """ self._on_first_connect_callbacks.append(callback) logger.debug(f"Registered on_first_connect callback: {callback.__name__}") - + def register_on_all_disconnect(self, callback: Callable[[], None]): """Register a callback to be called when all clients disconnect. @@ -53,7 +54,7 @@ class WebSocketManager: with self._connection_lock: self._needs_rewarm = True logger.debug("Warmup requested for next client connect") - + def client_connected(self): """Track a new client connection. Call this from the connect event handler.""" with self._connection_lock: @@ -79,16 +80,16 @@ class WebSocketManager: thread.start() except Exception as e: logger.error(f"Error in on_first_connect callback {callback.__name__}: {e}") - + def client_disconnected(self): """Track a client disconnection. Call this from the disconnect event handler.""" with self._connection_lock: self._connection_count = max(0, self._connection_count - 1) current_count = self._connection_count is_now_zero = current_count == 0 - + logger.debug(f"Client disconnected. Active connections: {current_count}") - + # If all clients have disconnected, trigger cleanup callbacks if is_now_zero: logger.info("All clients disconnected, triggering disconnect callbacks...") @@ -97,37 +98,37 @@ class WebSocketManager: callback() except Exception as e: logger.error(f"Error in on_all_disconnect callback {callback.__name__}: {e}") - + def get_connection_count(self) -> int: """Get the current number of active WebSocket connections.""" with self._connection_lock: return self._connection_count - + def has_active_connections(self) -> bool: """Check if there are any active WebSocket connections.""" return self.get_connection_count() > 0 - + def is_enabled(self) -> bool: """Check if WebSocket is enabled and ready.""" return self._enabled and self.socketio is not None - + def broadcast_status_update(self, status_data: Dict[str, Any]): """Broadcast status update to all connected clients.""" if not self.is_enabled(): return - + try: # When calling socketio.emit() outside event handlers, it broadcasts by default self.socketio.emit('status_update', status_data) logger.debug(f"Broadcasted status update to all clients") except Exception as e: logger.error(f"Error broadcasting status update: {e}") - + def broadcast_download_progress(self, book_id: str, progress: float, status: str): """Broadcast download progress update for a specific book.""" if not self.is_enabled(): return - + try: data = { 'book_id': book_id, @@ -139,12 +140,12 @@ class WebSocketManager: logger.debug(f"Broadcasted progress for book {book_id}: {progress}%") except Exception as e: logger.error(f"Error broadcasting download progress: {e}") - + def broadcast_notification(self, message: str, notification_type: str = 'info'): """Broadcast a notification message to all clients.""" if not self.is_enabled(): return - + try: data = { 'message': message, @@ -156,5 +157,6 @@ class WebSocketManager: except Exception as e: logger.error(f"Error broadcasting notification: {e}") + # Global WebSocket manager instance ws_manager = WebSocketManager() diff --git a/cwa_book_downloader/bypass/__init__.py b/cwa_book_downloader/bypass/__init__.py new file mode 100644 index 00000000..f3a5339f --- /dev/null +++ b/cwa_book_downloader/bypass/__init__.py @@ -0,0 +1 @@ +"""Cloudflare bypass utilities.""" diff --git a/cloudflare_bypasser_external.py b/cwa_book_downloader/bypass/external_bypasser.py similarity index 84% rename from cloudflare_bypasser_external.py rename to cwa_book_downloader/bypass/external_bypasser.py index d53a8122..c8aaa080 100644 --- a/cloudflare_bypasser_external.py +++ b/cwa_book_downloader/bypass/external_bypasser.py @@ -1,23 +1,22 @@ -from logger import setup_logger +"""External Cloudflare bypasser using FlareSolverr.""" + from threading import Event from typing import Optional, TYPE_CHECKING import requests import time import random +from cwa_book_downloader.core.config import config +from cwa_book_downloader.core.logger import setup_logger + if TYPE_CHECKING: - import network + from cwa_book_downloader.download import network class BypassCancelledException(Exception): """Raised when a bypass operation is cancelled.""" pass -try: - from env import EXT_BYPASSER_PATH, EXT_BYPASSER_TIMEOUT, EXT_BYPASSER_URL -except ImportError: - raise RuntimeError("Failed to import environment variables. Are you using an `extbp` image?") - logger = setup_logger(__name__) # Connection timeout (seconds) - how long to wait for external bypasser to accept connection @@ -34,59 +33,63 @@ BACKOFF_CAP = 10.0 def _fetch_via_bypasser(target_url: str) -> Optional[str]: """Make a single request to the external bypasser service. - + Args: target_url: The URL to fetch through the bypasser - + Returns: HTML content if successful, None otherwise """ - if not EXT_BYPASSER_URL or not EXT_BYPASSER_PATH: + bypasser_url = config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191") + bypasser_path = config.get("EXT_BYPASSER_PATH", "/v1") + bypasser_timeout = config.get("EXT_BYPASSER_TIMEOUT", 60000) + + if not bypasser_url or not bypasser_path: logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.") return None - - bypasser_endpoint = f"{EXT_BYPASSER_URL}{EXT_BYPASSER_PATH}" + + bypasser_endpoint = f"{bypasser_url}{bypasser_path}" headers = {"Content-Type": "application/json"} payload = { "cmd": "request.get", "url": target_url, - "maxTimeout": EXT_BYPASSER_TIMEOUT + "maxTimeout": bypasser_timeout } - - # Calculate read timeout: bypasser timeout (ms → s) + buffer, capped at max - read_timeout = min((EXT_BYPASSER_TIMEOUT / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT) - + + # Calculate read timeout: bypasser timeout (ms -> s) + buffer, capped at max + read_timeout = min((bypasser_timeout / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT) + try: response = requests.post( - bypasser_endpoint, - headers=headers, - json=payload, + bypasser_endpoint, + headers=headers, + json=payload, timeout=(CONNECT_TIMEOUT, read_timeout) ) response.raise_for_status() result = response.json() - + status = result.get('status', 'unknown') message = result.get('message', '') logger.debug(f"External bypasser response for '{target_url}': {status} - {message}") - + # Check for error status (bypasser returns status="error" with solution=null on failure) if status != 'ok': logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}") return None - + solution = result.get('solution') if not solution: logger.warning(f"External bypasser returned empty solution for '{target_url}'") return None - + html = solution.get('response', '') if not html: logger.warning(f"External bypasser returned empty response for '{target_url}'") return None - + return html - + except requests.exceptions.Timeout: logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)") return None @@ -114,8 +117,8 @@ def get_bypassed_page(url: str, selector: Optional["network.AAMirrorSelector"] = Raises: BypassCancelledException: If cancel_flag is set during operation """ - import network - sel = selector or network.AAMirrorSelector() + from cwa_book_downloader.download import network as network_module + sel = selector or network_module.AAMirrorSelector() for attempt in range(1, MAX_RETRY + 1): # Check for cancellation before each attempt diff --git a/cloudflare_bypasser.py b/cwa_book_downloader/bypass/internal_bypasser.py similarity index 93% rename from cloudflare_bypasser.py rename to cwa_book_downloader/bypass/internal_bypasser.py index 525504b2..95ed3567 100644 --- a/cloudflare_bypasser.py +++ b/cwa_book_downloader/bypass/internal_bypasser.py @@ -19,11 +19,24 @@ class BypassCancelledException(Exception): import requests from seleniumbase import Driver -import env -import network -from config import PROXIES, RECORDING_DIR, VIRTUAL_SCREEN_SIZE -from env import AA_DONATOR_KEY, BYPASS_WARMUP_ON_CONNECT, DEBUG, DEFAULT_SLEEP, LOG_DIR, MAX_RETRY -from logger import setup_logger +from cwa_book_downloader.config import env +from cwa_book_downloader.download import network +from cwa_book_downloader.config.settings import RECORDING_DIR, VIRTUAL_SCREEN_SIZE +from cwa_book_downloader.config.env import DEBUG, LOG_DIR +from cwa_book_downloader.core.config import config as app_config +from cwa_book_downloader.core.logger import setup_logger + + +def _get_proxies() -> dict: + """Get current proxy configuration from config singleton.""" + proxies = {} + http_proxy = app_config.get("HTTP_PROXY", "") + https_proxy = app_config.get("HTTPS_PROXY", "") + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + return proxies logger = setup_logger(__name__) @@ -167,15 +180,15 @@ def _get_page_info(sb) -> tuple[str, str, str]: """Extract page title, body text, and current URL safely.""" try: title = sb.get_title().lower() - except: + except Exception: title = "" try: body = sb.get_text("body").lower() - except: + except Exception: body = "" try: current_url = sb.get_current_url() - except: + except Exception: current_url = "" return title, body, current_url @@ -277,7 +290,7 @@ def _bypass_method_1(sb) -> bool: except Exception as e2: logger.debug(f"Method 1 failed on second try: {e2}") try: - time.sleep(DEFAULT_SLEEP) + time.sleep(app_config.DEFAULT_SLEEP) sb.uc_gui_click_captcha() time.sleep(5) return _is_bypassed(sb) @@ -304,7 +317,7 @@ def _bypass_method_2(sb) -> bool: try: sb.click_if_visible("body", timeout=5) time.sleep(5) - except: + except Exception: pass return _is_bypassed(sb) @@ -326,7 +339,7 @@ def _bypass_method_3(sb) -> bool: time.sleep(2) sb.scroll_to_top() time.sleep(3) - except: + except Exception: pass # Check if this helped @@ -337,7 +350,7 @@ def _bypass_method_3(sb) -> bool: try: sb.uc_gui_click_captcha() time.sleep(5) - except: + except Exception: pass return _is_bypassed(sb) @@ -376,7 +389,7 @@ def _bypass_ddos_guard_method_1(sb) -> bool: time.sleep(random.uniform(3, 5)) if _is_bypassed(sb): return True - except: + except Exception: continue return False @@ -413,11 +426,12 @@ def _bypass_ddos_guard_method_2(sb) -> bool: logger.debug(f"DDOS-Guard method 2 failed: {e}") return False -def _bypass(sb, max_retries: int = MAX_RETRY, cancel_flag: Optional[Event] = None) -> bool: +def _bypass(sb, max_retries: Optional[int] = None, cancel_flag: Optional[Event] = None) -> bool: """Bypass function with strategies for Cloudflare and DDOS-Guard protection. Returns True if bypass succeeded, False otherwise. """ + max_retries = max_retries if max_retries is not None else app_config.MAX_RETRY cloudflare_methods = [_bypass_method_1, _bypass_method_2, _bypass_method_3] ddos_guard_methods = [_bypass_ddos_guard_method_1, _bypass_ddos_guard_method_2] @@ -444,7 +458,7 @@ def _bypass(sb, max_retries: int = MAX_RETRY, cancel_flag: Optional[Event] = Non logger.info(f"Bypass attempt {try_count + 1}/{max_retries} using {method.__name__}") # Progressive backoff with cancellation checks - wait_time = min(DEFAULT_SLEEP * try_count, 15) + wait_time = min(app_config.DEFAULT_SLEEP * try_count, 15) if wait_time > 0: logger.info(f"Waiting {wait_time}s before trying...") # Check cancellation during wait (check every second) @@ -494,8 +508,9 @@ def _get_chromium_args(): ]) # Add proxy settings if configured - if PROXIES: - proxy_url = PROXIES.get('https') or PROXIES.get('http') + proxies = _get_proxies() + if proxies: + proxy_url = proxies.get('https') or proxies.get('http') if proxy_url: arguments.append(f'--proxy-server={proxy_url}') @@ -537,7 +552,8 @@ def _get_chromium_args(): return arguments -def _get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): +def _get(url, retry: Optional[int] = None, cancel_flag: Optional[Event] = None): + retry = retry if retry is not None else app_config.MAX_RETRY # Check for cancellation before starting if cancel_flag and cancel_flag.is_set(): logger.info("Bypass cancelled before starting") @@ -549,8 +565,8 @@ def _get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): # Enhanced page loading with better error handling logger.debug("Opening URL with SeleniumBase...") - sb.uc_open_with_reconnect(url, DEFAULT_SLEEP) - time.sleep(DEFAULT_SLEEP) + sb.uc_open_with_reconnect(url, app_config.DEFAULT_SLEEP) + time.sleep(app_config.DEFAULT_SLEEP) # Check for cancellation after page load if cancel_flag and cancel_flag.is_set(): @@ -578,7 +594,7 @@ def _get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): try: page_text = sb.get_text("body")[:500] + "..." if len(sb.get_text("body")) > 500 else sb.get_text("body") logger.debug(f"Page content: {page_text}") - except: + except Exception: pass except BypassCancelledException: @@ -593,7 +609,7 @@ def _get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): _reset_driver() raise e - logger.warning(f"Failed to bypass Cloudflare (retry {MAX_RETRY - retry + 1}/{MAX_RETRY}): {error_details}") + logger.warning(f"Failed to bypass Cloudflare (retry {app_config.MAX_RETRY - retry + 1}/{app_config.MAX_RETRY}): {error_details}") logger.debug(f"Stack trace: {stack_trace}") # Reset driver on certain errors @@ -609,8 +625,9 @@ def _get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): return _get(url, retry - 1, cancel_flag) -def get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): +def get(url, retry: Optional[int] = None, cancel_flag: Optional[Event] = None): """Fetch a URL with protection bypass.""" + retry = retry if retry is not None else app_config.MAX_RETRY global LAST_USED with LOCKED: # Check for cookies AFTER acquiring lock - another request may have @@ -619,7 +636,7 @@ def get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): cookies = get_cf_cookies_for_domain(parsed.hostname or "") if cookies: try: - response = requests.get(url, cookies=cookies, proxies=PROXIES, timeout=(5, 10)) + response = requests.get(url, cookies=cookies, proxies=_get_proxies(), timeout=(5, 10)) if response.status_code == 200: logger.debug(f"Cookies available after lock wait - skipped Chrome") LAST_USED = time.time() @@ -641,7 +658,7 @@ def _init_driver(): driver = Driver(uc=True, headless=False, size=f"{VIRTUAL_SCREEN_SIZE[0]},{VIRTUAL_SCREEN_SIZE[1]}", chromium_arg=chromium_args) driver.set_page_load_timeout(60) DRIVER = driver - time.sleep(DEFAULT_SLEEP) + time.sleep(app_config.DEFAULT_SLEEP) return driver def _ensure_display_initialized(): @@ -657,7 +674,7 @@ def _ensure_display_initialized(): display.start() DISPLAY["xvfb"] = display logger.info("Virtual display started") - time.sleep(DEFAULT_SLEEP) + time.sleep(app_config.DEFAULT_SLEEP) _reset_pyautogui_display_state() @@ -825,14 +842,14 @@ def _cleanup_driver(): # Check for active UI connections try: - from websocket_manager import ws_manager + from cwa_book_downloader.api.websocket import ws_manager has_active_clients = ws_manager.has_active_connections() except ImportError: ws_manager = None has_active_clients = False # Use longer timeout when UI is connected (user might be browsing) - timeout_minutes = env.BYPASS_RELEASE_INACTIVE_MIN + timeout_minutes = app_config.BYPASS_RELEASE_INACTIVE_MIN if has_active_clients: timeout_minutes *= 4 # 20 min default when UI open vs 5 min after disconnect @@ -851,7 +868,7 @@ def _cleanup_driver(): def _cleanup_loop(): while True: _cleanup_driver() - time.sleep(max(env.BYPASS_RELEASE_INACTIVE_MIN / 2, 1)) + time.sleep(max(app_config.BYPASS_RELEASE_INACTIVE_MIN / 2, 1)) def _init_cleanup_thread(): cleanup_thread = threading.Thread(target=_cleanup_loop) @@ -877,19 +894,19 @@ def warmup(): """ global DRIVER, LAST_USED - if not BYPASS_WARMUP_ON_CONNECT: + if not app_config.get("BYPASS_WARMUP_ON_CONNECT", True): logger.debug("Bypasser warmup disabled via BYPASS_WARMUP_ON_CONNECT") return - + if not env.DOCKERMODE: logger.debug("Bypasser warmup skipped - not in Docker mode") return - + if not env.USE_CF_BYPASS: logger.debug("Bypasser warmup skipped - CF bypass disabled") return - - if AA_DONATOR_KEY: + + if app_config.get("AA_DONATOR_KEY", ""): logger.debug("Bypasser warmup skipped - AA donator key set (fast downloads available)") return @@ -941,7 +958,7 @@ def shutdown_if_idle(): # Start the inactivity countdown LAST_USED = time.time() - logger.info(f"All clients disconnected - bypasser will shut down after {env.BYPASS_RELEASE_INACTIVE_MIN} min of inactivity") + logger.info(f"All clients disconnected - bypasser will shut down after {app_config.BYPASS_RELEASE_INACTIVE_MIN} min of inactivity") _init_cleanup_thread() @@ -975,7 +992,7 @@ def get_bypassed_page(url: str, selector: Optional[network.AAMirrorSelector] = N if cookies: try: logger.debug(f"Trying request with cached cookies before Chrome: {attempt_url}") - response = requests.get(attempt_url, cookies=cookies, proxies=PROXIES, timeout=(5, 10)) + response = requests.get(attempt_url, cookies=cookies, proxies=_get_proxies(), timeout=(5, 10)) if response.status_code == 200: logger.debug(f"Cached cookies worked, skipped Chrome bypass") return response.text diff --git a/cwa_book_downloader/config/__init__.py b/cwa_book_downloader/config/__init__.py new file mode 100644 index 00000000..1e5b7f9d --- /dev/null +++ b/cwa_book_downloader/config/__init__.py @@ -0,0 +1 @@ +"""Configuration module - environment variables and settings.""" diff --git a/env.py b/cwa_book_downloader/config/env.py similarity index 70% rename from env.py rename to cwa_book_downloader/config/env.py index a07d6178..3190d736 100644 --- a/env.py +++ b/cwa_book_downloader/config/env.py @@ -1,14 +1,20 @@ +"""Environment variable parsing. No local dependencies - import first.""" + import os +import shutil from pathlib import Path + def string_to_bool(s: str) -> bool: return s.lower() in ["true", "yes", "1", "y"] + # Authentication and session settings SESSION_COOKIE_SECURE_ENV = os.getenv("SESSION_COOKIE_SECURE", "false") CWA_DB = os.getenv("CWA_DB_PATH") CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None +CONFIG_DIR = Path(os.getenv("CONFIG_DIR", "/config")) LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/")) LOG_DIR = LOG_ROOT / "cwa-book-downloader" TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader")) @@ -72,7 +78,7 @@ MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3")) DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "1")) DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false")) _CUSTOM_DNS = os.getenv("CUSTOM_DNS", "auto").strip() -USE_DOH = string_to_bool(os.getenv("USE_DOH", "false")) +USE_DOH = string_to_bool(os.getenv("USE_DOH", "true")) BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5")) BYPASS_WARMUP_ON_CONNECT = string_to_bool(os.getenv("BYPASS_WARMUP_ON_CONNECT", "true")) @@ -93,6 +99,51 @@ if USING_TOR: HTTP_PROXY = "" HTTPS_PROXY = "" +# Check if this is the Tor variant (has tor binary installed) +# Only the Tor variant image includes the tor binary +TOR_VARIANT_AVAILABLE = shutil.which("tor") is not None + # Calibre-Web URL for navigation button CALIBRE_WEB_URL = os.getenv("CALIBRE_WEB_URL", "").strip() - \ No newline at end of file + +# Metadata provider settings (Stage 2) +# Set to "hardcover" or "openlibrary" to enable metadata-first search mode +METADATA_PROVIDER = os.getenv("METADATA_PROVIDER", "").strip().lower() +HARDCOVER_API_KEY = os.getenv("HARDCOVER_API_KEY", "").strip() + +# Cache TTL settings (in seconds) +METADATA_CACHE_SEARCH_TTL = int(os.getenv("METADATA_CACHE_SEARCH_TTL", "300")) # 5 minutes +METADATA_CACHE_BOOK_TTL = int(os.getenv("METADATA_CACHE_BOOK_TTL", "600")) # 10 minutes + +# Cover image cache settings +def _is_config_dir_writable() -> bool: + """Check if the config directory exists and is writable.""" + try: + if not CONFIG_DIR.exists() or not CONFIG_DIR.is_dir(): + return False + test_file = CONFIG_DIR / ".write_test" + test_file.touch() + test_file.unlink() + return True + except (OSError, PermissionError): + return False + + +def is_covers_cache_enabled() -> bool: + """Check if cover caching is enabled (dynamic, respects settings changes). + + Cache is only enabled if: + 1. The COVERS_CACHE_ENABLED setting is true + 2. The config directory is writable + """ + from cwa_book_downloader.core.config import config + setting_enabled = config.get("COVERS_CACHE_ENABLED", True) + return setting_enabled and _is_config_dir_writable() + + +# Legacy static value - use is_covers_cache_enabled() for dynamic checks +_COVERS_CACHE_ENABLED_ENV = string_to_bool(os.getenv("COVERS_CACHE_ENABLED", "true")) +COVERS_CACHE_ENABLED = _COVERS_CACHE_ENABLED_ENV and _is_config_dir_writable() +COVERS_CACHE_DIR = CONFIG_DIR / "covers" +COVERS_CACHE_TTL = int(os.getenv("COVERS_CACHE_TTL", "0")) # 0 = forever (covers are static) +COVERS_CACHE_MAX_SIZE_MB = int(os.getenv("COVERS_CACHE_MAX_SIZE_MB", "500")) diff --git a/cwa_book_downloader/config/security.py b/cwa_book_downloader/config/security.py new file mode 100644 index 00000000..3223f101 --- /dev/null +++ b/cwa_book_downloader/config/security.py @@ -0,0 +1,152 @@ +"""Authentication settings registration.""" + +from typing import Any, Dict + +from werkzeug.security import generate_password_hash + +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.core.settings_registry import ( + register_settings, + register_on_save, + load_config_file, + TextField, + PasswordField, + CheckboxField, + ActionButton, +) + +logger = setup_logger(__name__) + + +def _clear_builtin_credentials() -> Dict[str, Any]: + """Clear built-in credentials to allow public access.""" + try: + config = load_config_file("security") + config.pop("BUILTIN_USERNAME", None) + config.pop("BUILTIN_PASSWORD_HASH", None) + + # Save the cleared config + from cwa_book_downloader.core.settings_registry import _get_config_file_path, _ensure_config_dir + import json + + _ensure_config_dir("security") + config_path = _get_config_file_path("security") + with open(config_path, 'w') as f: + json.dump(config, f, indent=2) + + logger.info("Cleared credentials") + return {"success": True, "message": "Credentials cleared. The app is now publicly accessible."} + + except Exception as e: + logger.error(f"Failed to clear credentials: {e}") + return {"success": False, "message": f"Failed to clear credentials: {str(e)}"} + + +def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]: + """ + Custom save handler for security settings. + + Handles password validation and hashing: + - If new password is provided, validate confirmation and hash it + - If password fields are empty, preserve existing hash + - Never store raw passwords + + Returns: + Dict with processed values to save and any validation errors. + """ + password = values.get("BUILTIN_PASSWORD", "") + password_confirm = values.get("BUILTIN_PASSWORD_CONFIRM", "") + + # Remove raw password fields - they should never be persisted + values.pop("BUILTIN_PASSWORD", None) + values.pop("BUILTIN_PASSWORD_CONFIRM", None) + + # If password is provided, validate and hash it + if password: + if password != password_confirm: + return { + "error": True, + "message": "Passwords do not match", + "values": values + } + + if len(password) < 4: + return { + "error": True, + "message": "Password must be at least 4 characters", + "values": values + } + + # Hash the password + values["BUILTIN_PASSWORD_HASH"] = generate_password_hash(password) + logger.info("Password hash updated") + + # If no password provided but username is being set, preserve existing hash + elif "BUILTIN_USERNAME" in values: + existing = load_config_file("security") + if "BUILTIN_PASSWORD_HASH" in existing: + values["BUILTIN_PASSWORD_HASH"] = existing["BUILTIN_PASSWORD_HASH"] + + return {"error": False, "values": values} + + +@register_settings("security", "Security", icon="shield", order=5) +def security_settings(): + """Security and authentication settings.""" + from cwa_book_downloader.config.env import CWA_DB_PATH + import os + + cwa_db_available = CWA_DB_PATH and os.path.exists(CWA_DB_PATH) + + fields = [ + TextField( + key="BUILTIN_USERNAME", + label="Username", + description="Set a username and password to require login. Leave both empty for public access.", + placeholder="Enter username", + env_supported=False, + disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."}, + ), + PasswordField( + key="BUILTIN_PASSWORD", + label="Set Password", + description="Fill in to set or change the password.", + placeholder="Enter new password", + env_supported=False, + disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."}, + ), + PasswordField( + key="BUILTIN_PASSWORD_CONFIRM", + label="Confirm Password", + placeholder="Confirm new password", + env_supported=False, + disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."}, + ), + ActionButton( + key="clear_credentials", + label="Clear Credentials", + description="Remove login requirement and make the app publicly accessible.", + style="danger", + callback=_clear_builtin_credentials, + disabled_when={"field": "USE_CWA_AUTH", "value": True, "reason": "Using Calibre-Web database for authentication."}, + ), + CheckboxField( + key="USE_CWA_AUTH", + label="Use Calibre-Web Database", + description=( + "Authenticate using your existing Calibre-Web users instead of the credentials above." + if cwa_db_available + else "Authenticate using your existing Calibre-Web users. Set the CWA_DB_PATH environment variable to your Calibre-Web app.db file to enable this option." + ), + default=False, + env_supported=False, + disabled=not cwa_db_available, + disabled_reason="Set the CWA_DB_PATH environment variable to your Calibre-Web app.db file path to enable this option.", + ), + ] + + return fields + + +# Register the on_save handler for this tab +register_on_save("security", _on_save_security) diff --git a/cwa_book_downloader/config/settings.py b/cwa_book_downloader/config/settings.py new file mode 100644 index 00000000..7ae50b74 --- /dev/null +++ b/cwa_book_downloader/config/settings.py @@ -0,0 +1,676 @@ +"""Core settings registration and derived configuration values.""" + +import os +from pathlib import Path +import json + +from cwa_book_downloader.config import env +from cwa_book_downloader.core.logger import setup_logger + +logger = setup_logger(__name__) + +# Log configuration values at DEBUG level, filtering out module imports and functions +logger.debug("Environment configuration:") +for key, value in env.__dict__.items(): + # Skip private attributes, modules, types, and callables (functions) + if key.startswith('_'): + continue + if isinstance(value, type) or callable(value): + continue + # Don't log module objects (they have __name__ attribute) + if hasattr(value, '__name__') and hasattr(value, '__file__'): + continue + # Redact sensitive values + if key == "AA_DONATOR_KEY" and isinstance(value, str) and value.strip(): + value = "REDACTED" + if key == "HARDCOVER_API_KEY" and isinstance(value, str) and value.strip(): + value = "REDACTED" + logger.debug(f" {key}: {value}") + +# Load supported book languages from data file +# Path is relative to the package root, not this file +_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data" +with open(_DATA_DIR / "book-languages.json") as file: + _SUPPORTED_BOOK_LANGUAGE = json.load(file) + +# Directory settings +BASE_DIR = Path(__file__).resolve().parent.parent.parent +logger.debug(f"BASE_DIR: {BASE_DIR}") +if env.ENABLE_LOGGING: + env.LOG_DIR.mkdir(exist_ok=True) + +# Create necessary directories +env.TMP_DIR.mkdir(exist_ok=True) +env.INGEST_DIR.mkdir(exist_ok=True) + +CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev +logger.debug(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}") +logger.debug(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}") +logger.debug(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}") + +# Network settings - DNS configuration is managed by network.py +# These are placeholder values that will be set when network.init() is called +# The authoritative DNS state lives in network.py and is configured via set_dns_provider() +# Actual DNS provider is determined from config singleton (settings UI) or ENV var +CUSTOM_DNS: list[str] = [] +DOH_SERVER: str = "" + +# Warn about external bypasser DNS limitations +if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS: + logger.warning( + "Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, " + "not this application's custom DNS settings. If you experience DNS-related blocks, " + "configure DNS at the Docker/system level for your FlareSolverr container, " + "or consider using the internal bypasser which integrates with the app's DNS system." + ) + +# Proxy settings +PROXIES = {} +if env.HTTP_PROXY: + PROXIES["http"] = env.HTTP_PROXY +if env.HTTPS_PROXY: + PROXIES["https"] = env.HTTPS_PROXY +logger.debug(f"PROXIES: {PROXIES}") + +# Anna's Archive settings +AA_BASE_URL = env._AA_BASE_URL +AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"] +AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(",")) +AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()] + +# File format settings +SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",") +logger.debug(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}") + +# Complex language processing logic kept in config.py +BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',') +BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]] +if len(BOOK_LANGUAGE) == 0: + BOOK_LANGUAGE = ['en'] + +# Custom script settings with validation logic +CUSTOM_SCRIPT = env._CUSTOM_SCRIPT +if CUSTOM_SCRIPT: + if not os.path.exists(CUSTOM_SCRIPT): + logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist") + CUSTOM_SCRIPT = "" + elif not os.access(CUSTOM_SCRIPT, os.X_OK): + logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable") + CUSTOM_SCRIPT = "" + +# Debugging settings +if not env.USING_EXTERNAL_BYPASSER: + # Virtual display settings for debugging internal cloudflare bypasser + VIRTUAL_SCREEN_SIZE = (1024, 768) + RECORDING_DIR = env.LOG_DIR / "recording" + if env.DEBUG: + RECORDING_DIR.mkdir(parents=True, exist_ok=True) + + +from cwa_book_downloader.core.settings_registry import ( + register_settings, + register_group, + TextField, + PasswordField, + NumberField, + CheckboxField, + SelectField, + MultiSelectField, + HeadingField, + ActionButton, +) + + +register_group( + "direct_download", + "Anna's Archive", + icon="download", + order=20 +) + +register_group( + "metadata_providers", + "Metadata Providers", + icon="book", + order=12 # Between Network (10) and Advanced (15) +) + + +# Build format options from supported formats +_FORMAT_OPTIONS = [ + {"value": "epub", "label": "EPUB"}, + {"value": "mobi", "label": "MOBI"}, + {"value": "azw3", "label": "AZW3"}, + {"value": "pdf", "label": "PDF"}, + {"value": "fb2", "label": "FB2"}, + {"value": "djvu", "label": "DJVU"}, + {"value": "cbz", "label": "CBZ"}, + {"value": "cbr", "label": "CBR"}, + {"value": "txt", "label": "TXT"}, + {"value": "rtf", "label": "RTF"}, + {"value": "doc", "label": "DOC"}, + {"value": "docx", "label": "DOCX"}, +] + + +def _get_metadata_provider_options(): + """Build metadata provider options dynamically from enabled providers only.""" + from cwa_book_downloader.metadata_providers import list_providers, is_provider_enabled + + options = [] + for provider in list_providers(): + # Only show providers that are enabled + if is_provider_enabled(provider["name"]): + options.append({"value": provider["name"], "label": provider["display_name"]}) + + # If no providers enabled, show a placeholder option + if not options: + options = [ + {"value": "", "label": "No providers enabled"}, + ] + + return options + + +def _get_release_source_options(): + """Build release source options dynamically from registered sources.""" + from cwa_book_downloader.release_sources import list_available_sources + + return [ + {"value": source["name"], "label": source["display_name"]} + for source in list_available_sources() + ] + +# Build language options from supported languages +_LANGUAGE_OPTIONS = [{"value": lang["code"], "label": lang["language"]} for lang in _SUPPORTED_BOOK_LANGUAGE] + + +def _clear_covers_cache(current_values: dict) -> dict: + """Clear the cover image cache.""" + try: + from cwa_book_downloader.core.image_cache import get_image_cache, reset_image_cache + + cache = get_image_cache() + count = cache.clear() + + # Reset the singleton so it reinitializes with fresh state + reset_image_cache() + + return { + "success": True, + "message": f"Cleared {count} cached cover images.", + } + except Exception as e: + logger.error(f"Failed to clear cover cache: {e}") + return { + "success": False, + "message": f"Failed to clear cache: {str(e)}", + } + + +@register_settings("general", "General", icon="settings", order=0) +def general_settings(): + """Core application settings.""" + return [ + SelectField( + key="SEARCH_MODE", + label="Search Mode", + description="How you want to search for and download books.", + options=[ + { + "value": "direct", + "label": "Direct (Anna's Archive)", + "description": "Search Anna's Archive and download directly. Works out of the box.", + }, + { + "value": "universal", + "label": "Universal", + "description": "Metadata-based search with downloads from all sources.", + }, + ], + default="direct", + ), + SelectField( + key="METADATA_PROVIDER", + label="Metadata Provider for Universal Search", + description="Choose which metadata provider to use for book searches.", + options=_get_metadata_provider_options, # Callable - evaluated lazily to avoid circular imports + default="openlibrary", + show_when={"field": "SEARCH_MODE", "value": "universal"}, + ), + SelectField( + key="DEFAULT_RELEASE_SOURCE", + label="Default Release Source", + description="The release source tab to open by default in the release modal.", + options=_get_release_source_options, # Callable - evaluated lazily to avoid circular imports + default="direct_download", + env_supported=False, # UI-only setting, not configurable via ENV + show_when={"field": "SEARCH_MODE", "value": "universal"}, + ), + MultiSelectField( + key="SUPPORTED_FORMATS", + label="Supported Formats", + description="Book formats to include in search results.", + options=_FORMAT_OPTIONS, + default=["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"], + ), + MultiSelectField( + key="BOOK_LANGUAGE", + label="Default Book Languages", + description="Default language filter for searches. Can be overridden in advanced search options.", + options=_LANGUAGE_OPTIONS, + default=["en"], + ), + CheckboxField( + key="USE_BOOK_TITLE", + label="Use Book Title as Filename", + description="Save files using book title instead of ID. May cause issues with special characters.", + default=False, + ), + TextField( + key="CALIBRE_WEB_URL", + label="Book Management App URL", + description="Adds a navigation button to your book manager instance (Calibre-Web Automated, Booklore, etc).", + placeholder="http://calibre-web:8083", + ), + NumberField( + key="MAX_CONCURRENT_DOWNLOADS", + label="Max Concurrent Downloads", + description="Maximum number of simultaneous downloads.", + default=3, + min_value=1, + max_value=10, + requires_restart=True, + ), + NumberField( + key="STATUS_TIMEOUT", + label="Status Timeout (seconds)", + description="How long to keep completed/failed downloads in the queue display.", + default=3600, + min_value=60, + max_value=86400, + ), + ] + + +@register_settings("network", "Network", icon="globe", order=10) +def network_settings(): + """Network and connectivity settings.""" + # Check if Tor variant is available and if Tor is currently enabled + tor_available = env.TOR_VARIANT_AVAILABLE + tor_enabled = env.USING_TOR + + # When Tor is enabled (only possible in Tor variant), DNS/proxy settings are overridden + # The Tor variant uses iptables to force ALL traffic through Tor - it cannot be disabled + tor_overrides_network = tor_available # If Tor variant, network settings are always managed by Tor + + return [ + CheckboxField( + key="USING_TOR", + label="Tor Routing", + description=( + "All traffic is routed through Tor in this container variant. This cannot be changed." + if tor_available + else "Tor routing is not available in this container variant." + ), + default=tor_available, # Reflects actual state: True if Tor variant, False otherwise + disabled=True, # Always disabled - Tor state is determined by container variant + disabled_reason=( + "Tor routing is always active in the Tor container variant." + if tor_available + else "Requires the Tor container variant (calibre-web-automated-book-downloader-tor)." + ), + ), + SelectField( + key="CUSTOM_DNS", + label="DNS Provider", + description=( + "Managed by Tor when Tor routing is enabled." + if tor_overrides_network + else "DNS provider for domain resolution. 'Auto' rotates through providers on failure." + ), + options=[ + {"value": "auto", "label": "Auto (Recommended)"}, + {"value": "system", "label": "System"}, + {"value": "google", "label": "Google"}, + {"value": "cloudflare", "label": "Cloudflare"}, + {"value": "quad9", "label": "Quad9"}, + {"value": "opendns", "label": "OpenDNS"}, + {"value": "manual", "label": "Manual"}, + ], + default="auto", + disabled=tor_overrides_network, + disabled_reason="DNS is managed by Tor when Tor routing is enabled.", + ), + TextField( + key="CUSTOM_DNS_MANUAL", + label="Manual DNS Servers", + description="Comma-separated list of DNS server IP addresses (e.g., 8.8.8.8, 1.1.1.1).", + placeholder="8.8.8.8, 1.1.1.1", + disabled=tor_overrides_network, + disabled_reason="DNS is managed by Tor when Tor routing is enabled.", + show_when={"field": "CUSTOM_DNS", "value": "manual"}, + ), + CheckboxField( + key="USE_DOH", + label="Use DNS over HTTPS", + description=( + "Not applicable when Tor routing is enabled." + if tor_overrides_network + else "Use encrypted DNS queries for improved reliability and privacy." + ), + default=True, + disabled=tor_overrides_network, + disabled_reason="DNS over HTTPS is not used when Tor routing is enabled.", + # Hide for manual and system (no DoH endpoint available for custom IPs or system DNS) + show_when={"field": "CUSTOM_DNS", "value": ["auto", "google", "cloudflare", "quad9", "opendns"]}, + # Disable for auto (always uses DoH) + disabled_when={ + "field": "CUSTOM_DNS", + "value": "auto", + "reason": "Auto mode always uses DNS over HTTPS for reliable provider rotation.", + }, + ), + TextField( + key="HTTP_PROXY", + label="HTTP Proxy", + description=( + "Not applicable when Tor routing is enabled." + if tor_overrides_network + else "HTTP proxy URL (e.g., http://proxy:8080). Leave empty for direct connection." + ), + placeholder="http://proxy:8080", + disabled=tor_overrides_network, + disabled_reason="Proxy settings are not used when Tor routing is enabled.", + ), + TextField( + key="HTTPS_PROXY", + label="HTTPS Proxy", + description=( + "Not applicable when Tor routing is enabled." + if tor_overrides_network + else "HTTPS proxy URL. Leave empty for direct connection." + ), + placeholder="http://proxy:8080", + disabled=tor_overrides_network, + disabled_reason="Proxy settings are not used when Tor routing is enabled.", + ), + ] + + +@register_settings("ingest_directories", "Ingest Directories", icon="folder", order=5) +def ingest_directory_settings(): + """Configure where different content types are saved.""" + return [ + TextField( + key="INGEST_DIR", + label="Default Ingest Directory", + description="Default directory for all downloads. Used when no specific directory is set.", + default="/cwa-book-ingest", + required=True, + ), + HeadingField( + key="content_type_directories_heading", + title="Content-Type Directories", + description="Override the default directory for specific content types. Leave empty to use the default.", + ), + TextField( + key="INGEST_DIR_BOOK_FICTION", + label="Fiction Books", + placeholder="/cwa-book-ingest/fiction", + ), + TextField( + key="INGEST_DIR_BOOK_NON_FICTION", + label="Non-Fiction Books", + placeholder="/cwa-book-ingest/non-fiction", + ), + TextField( + key="INGEST_DIR_BOOK_UNKNOWN", + label="Unknown Books", + placeholder="/cwa-book-ingest/unknown", + ), + TextField( + key="INGEST_DIR_MAGAZINE", + label="Magazines", + placeholder="/cwa-book-ingest/magazines", + ), + TextField( + key="INGEST_DIR_COMIC_BOOK", + label="Comic Books", + placeholder="/cwa-book-ingest/comics", + ), + TextField( + key="INGEST_DIR_AUDIOBOOK", + label="Audiobooks", + placeholder="/cwa-book-ingest/audiobooks", + ), + TextField( + key="INGEST_DIR_STANDARDS_DOCUMENT", + label="Standards Documents", + placeholder="/cwa-book-ingest/standards", + ), + TextField( + key="INGEST_DIR_MUSICAL_SCORE", + label="Musical Scores", + placeholder="/cwa-book-ingest/scores", + ), + TextField( + key="INGEST_DIR_OTHER", + label="Other", + placeholder="/cwa-book-ingest/other", + ), + ] + + +@register_settings("download_sources", "Download Sources", icon="download", order=21, group="direct_download") +def download_source_settings(): + """Settings for download source behavior.""" + return [ + SelectField( + key="AA_BASE_URL", + label="Anna's Archive URL", + description="Primary Anna's Archive mirror to use. 'auto' selects automatically.", + options=[ + {"value": "auto", "label": "Auto (Recommended)"}, + {"value": "https://annas-archive.org", "label": "annas-archive.org"}, + {"value": "https://annas-archive.se", "label": "annas-archive.se"}, + {"value": "https://annas-archive.li", "label": "annas-archive.li"}, + ], + default="auto", + ), + TextField( + key="AA_ADDITIONAL_URLS", + label="Additional AA Mirrors", + description="Comma-separated list of additional Anna's Archive mirror URLs.", + placeholder="https://example.com,https://another.com", + ), + PasswordField( + key="AA_DONATOR_KEY", + label="Anna's Archive Donator Key", + description="Optional donator key for faster downloads from Anna's Archive.", + ), + CheckboxField( + key="ALLOW_USE_WELIB", + label="Allow Welib Downloads", + description="Enable Welib as a fallback download source.", + default=True, + ), + CheckboxField( + key="PRIORITIZE_WELIB", + label="Prioritize Welib", + description="Try Welib before other slow download sources.", + default=False, + show_when={"field": "ALLOW_USE_WELIB", "value": True}, + ), + NumberField( + key="MAX_RETRY", + label="Max Retries", + description="Maximum retry attempts for failed downloads.", + default=10, + min_value=1, + max_value=50, + ), + NumberField( + key="DEFAULT_SLEEP", + label="Retry Delay (seconds)", + description="Wait time between download retry attempts.", + default=5, + min_value=1, + max_value=60, + ), + ] + + +@register_settings("cloudflare_bypass", "Cloudflare Bypass", icon="shield", order=22, group="direct_download") +def cloudflare_bypass_settings(): + """Settings for Cloudflare bypass behavior.""" + return [ + CheckboxField( + key="USE_CF_BYPASS", + label="Enable Cloudflare Bypass", + description="Attempt to bypass Cloudflare protection on download sites.", + default=True, + requires_restart=True, + ), + CheckboxField( + key="BYPASS_WARMUP_ON_CONNECT", + label="Warmup on Connect", + description="Pre-warm the bypasser when user connects to Web App UI", + default=True, + ), + NumberField( + key="BYPASS_RELEASE_INACTIVE_MIN", + label="Release Inactive (minutes)", + description="Release bypasser resources after this many minutes of inactivity.", + default=5, + min_value=1, + max_value=60, + ), + CheckboxField( + key="USING_EXTERNAL_BYPASSER", + label="Use External Bypasser", + description="Use FlareSolverr or similar external service instead of built-in bypasser.", + default=False, + requires_restart=True, + ), + TextField( + key="EXT_BYPASSER_URL", + label="External Bypasser URL", + description="URL of the external bypasser service (e.g., FlareSolverr).", + default="http://flaresolverr:8191", + placeholder="http://flaresolverr:8191", + requires_restart=True, + show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True}, + ), + TextField( + key="EXT_BYPASSER_PATH", + label="External Bypasser Path", + description="API path for the external bypasser.", + default="/v1", + placeholder="/v1", + requires_restart=True, + show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True}, + ), + NumberField( + key="EXT_BYPASSER_TIMEOUT", + label="External Bypasser Timeout (ms)", + description="Timeout for external bypasser requests in milliseconds.", + default=60000, + min_value=10000, + max_value=300000, + requires_restart=True, + show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True}, + ), + ] + + +@register_settings("advanced", "Advanced", icon="cog", order=15) +def advanced_settings(): + """Advanced settings for power users.""" + return [ + TextField( + key="CUSTOM_SCRIPT", + label="Custom Script Path", + description="Path to a script to run after each successful download. Must be executable.", + placeholder="/path/to/script.sh", + ), + CheckboxField( + key="DEBUG", + label="Debug Mode", + description="Enable verbose logging. Not recommended for normal use.", + default=False, + requires_restart=True, + ), + SelectField( + key="LOG_LEVEL", + label="Log Level", + description="Logging verbosity level.", + options=[ + {"value": "DEBUG", "label": "Debug"}, + {"value": "INFO", "label": "Info"}, + {"value": "WARNING", "label": "Warning"}, + {"value": "ERROR", "label": "Error"}, + ], + default="INFO", + requires_restart=True, + ), + CheckboxField( + key="ENABLE_LOGGING", + label="Enable File Logging", + description="Write logs to file in addition to console output.", + default=True, + requires_restart=True, + ), + NumberField( + key="MAIN_LOOP_SLEEP_TIME", + label="Queue Check Interval (seconds)", + description="How often the download queue is checked for new items.", + default=5, + min_value=1, + max_value=60, + requires_restart=True, + ), + NumberField( + key="DOWNLOAD_PROGRESS_UPDATE_INTERVAL", + label="Progress Update Interval (seconds)", + description="How often download progress is broadcast to the UI.", + default=1, + min_value=1, + max_value=10, + requires_restart=True, + ), + HeadingField( + key="covers_cache_heading", + title="Cover Image Cache", + description="Cache book cover images locally for faster loading. Works for both Direct Download and Universal mode.", + ), + CheckboxField( + key="COVERS_CACHE_ENABLED", + label="Enable Cover Cache", + description="Cache book covers on the server for faster loading.", + default=True, + ), + NumberField( + key="COVERS_CACHE_TTL", + label="Cache TTL (days)", + description="How long to keep cached covers. Set to 0 to keep forever (recommended for static artwork).", + default=0, + min_value=0, + max_value=365, + ), + NumberField( + key="COVERS_CACHE_MAX_SIZE_MB", + label="Max Cache Size (MB)", + description="Maximum disk space for cached covers. Oldest images are removed when limit is reached.", + default=500, + min_value=50, + max_value=5000, + ), + ActionButton( + key="clear_covers_cache", + label="Clear Cover Cache", + description="Delete all cached cover images.", + style="danger", + callback=_clear_covers_cache, + ), + ] diff --git a/cwa_book_downloader/core/__init__.py b/cwa_book_downloader/core/__init__.py new file mode 100644 index 00000000..07873a33 --- /dev/null +++ b/cwa_book_downloader/core/__init__.py @@ -0,0 +1,5 @@ +"""Core module - shared models, queue, and utilities.""" + +from cwa_book_downloader.core.models import BookInfo, QueueItem, SearchFilters, QueueStatus +from cwa_book_downloader.core.queue import BookQueue, book_queue +from cwa_book_downloader.core.logger import setup_logger diff --git a/cwa_book_downloader/core/cache.py b/cwa_book_downloader/core/cache.py new file mode 100644 index 00000000..cb1b4392 --- /dev/null +++ b/cwa_book_downloader/core/cache.py @@ -0,0 +1,207 @@ +"""Thread-safe in-memory cache with TTL support.""" + +import threading +import time +from dataclasses import dataclass +from functools import wraps +from typing import Any, Callable, Dict, Optional, TypeVar + +from cwa_book_downloader.core.logger import setup_logger + +logger = setup_logger(__name__) + +T = TypeVar("T") + + +@dataclass +class CacheEntry: + """A cached value with expiration time.""" + value: Any + expires_at: float + + +class CacheService: + """Thread-safe in-memory cache with TTL support.""" + + def __init__(self, max_size: int = 1000): + """Initialize cache service. + + Args: + max_size: Maximum number of entries before oldest are evicted. + """ + self._cache: Dict[str, CacheEntry] = {} + self._lock = threading.Lock() + self._max_size = max_size + + def get(self, key: str) -> Optional[Any]: + """Get cached value if not expired. + + Args: + key: Cache key to retrieve. + + Returns: + Cached value or None if not found/expired. + """ + with self._lock: + entry = self._cache.get(key) + if entry is None: + return None + + if time.time() > entry.expires_at: + del self._cache[key] + return None + + return entry.value + + def set(self, key: str, value: Any, ttl: int) -> None: + """Cache value with TTL. + + Args: + key: Cache key. + value: Value to cache. + ttl: Time to live in seconds. + """ + with self._lock: + # Evict oldest entries if at capacity + if len(self._cache) >= self._max_size: + self._evict_oldest() + + self._cache[key] = CacheEntry( + value=value, + expires_at=time.time() + ttl + ) + + def invalidate(self, key: str) -> bool: + """Remove specific cache entry. + + Args: + key: Cache key to remove. + + Returns: + True if entry was removed, False if not found. + """ + with self._lock: + if key in self._cache: + del self._cache[key] + return True + return False + + def clear(self) -> None: + """Clear all cache entries.""" + with self._lock: + self._cache.clear() + + def cleanup_expired(self) -> int: + """Remove all expired entries. + + Returns: + Number of entries removed. + """ + with self._lock: + now = time.time() + expired_keys = [ + key for key, entry in self._cache.items() + if entry.expires_at < now + ] + for key in expired_keys: + del self._cache[key] + return len(expired_keys) + + def _evict_oldest(self) -> None: + """Evict oldest entries (by expiration time) to make room. + + Called with lock held. + """ + if not self._cache: + return + + # Remove ~10% of entries, oldest first + entries_to_remove = max(1, len(self._cache) // 10) + sorted_entries = sorted( + self._cache.items(), + key=lambda x: x[1].expires_at + ) + + for key, _ in sorted_entries[:entries_to_remove]: + del self._cache[key] + + def stats(self) -> Dict[str, int]: + """Get cache statistics. + + Returns: + Dict with size and max_size. + """ + with self._lock: + return { + "size": len(self._cache), + "max_size": self._max_size + } + + +# Global cache instance for metadata providers +_metadata_cache = CacheService(max_size=1000) + + +def get_metadata_cache() -> CacheService: + """Get the global metadata cache instance.""" + return _metadata_cache + + +def cache_key(*args, **kwargs) -> str: + """Generate cache key from arguments. + + Args: + *args: Positional arguments to include in key. + **kwargs: Keyword arguments to include in key. + + Returns: + String cache key. + """ + parts = [str(arg) for arg in args] + parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items())) + return ":".join(parts) + + +def cacheable(ttl: int, key_prefix: str = ""): + """Decorator for caching function results. + + Args: + ttl: Time to live in seconds. + key_prefix: Optional prefix for cache keys. + + Usage: + @cacheable(ttl=300, key_prefix="hardcover:search") + def search(self, query: str, limit: int = 20): + ... + """ + def decorator(func: Callable[..., T]) -> Callable[..., T]: + @wraps(func) + def wrapper(*args, **kwargs) -> T: + # Generate cache key from function name and arguments + # Skip 'self' argument if present (first arg of method) + cache_args = args[1:] if args and hasattr(args[0], func.__name__) else args + + key = cache_key( + key_prefix or func.__name__, + *cache_args, + **kwargs + ) + + # Check cache + cached = _metadata_cache.get(key) + if cached is not None: + logger.debug(f"Cache hit: {key}") + return cached + + # Execute function and cache result + logger.debug(f"Cache miss: {key}") + result = func(*args, **kwargs) + + # Only cache non-None results + if result is not None: + _metadata_cache.set(key, result, ttl) + + return result + + return wrapper + return decorator diff --git a/cwa_book_downloader/core/config.py b/cwa_book_downloader/core/config.py new file mode 100644 index 00000000..dc1be74c --- /dev/null +++ b/cwa_book_downloader/core/config.py @@ -0,0 +1,182 @@ +"""Configuration singleton with ENV > config file > default resolution.""" + +from threading import Lock +from typing import Any, Dict, Optional + +# Import lazily to avoid circular imports +_registry_module = None +_env_module = None + + +def _get_registry(): + """Lazy import of settings registry to avoid circular imports.""" + global _registry_module + if _registry_module is None: + from cwa_book_downloader.core import settings_registry + _registry_module = settings_registry + return _registry_module + + +def _get_env(): + """Lazy import of env module for fallback values.""" + global _env_module + if _env_module is None: + from cwa_book_downloader.config import env + _env_module = env + return _env_module + + +class Config: + """ + Dynamic configuration singleton that provides live settings access. + + Settings are resolved with priority: ENV var > config file > default. + Values are cached for performance and can be refreshed when settings change. + """ + + _instance: Optional['Config'] = None + _lock = Lock() + + def __new__(cls) -> 'Config': + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + if self._initialized: + return + self._cache: Dict[str, Any] = {} + self._field_map: Dict[str, tuple] = {} # key -> (field, tab_name) + self._cache_lock = Lock() + self._initialized = True + self._loaded = False + + def _ensure_loaded(self) -> None: + """Ensure settings are loaded from the registry.""" + if self._loaded: + return + with self._cache_lock: + if self._loaded: + return + self._load_settings() + + def _load_settings(self) -> None: + """Load all settings from the registry.""" + # Ensure all plugin settings are registered before loading + # This handles cases where config is accessed before plugins are imported + try: + import cwa_book_downloader.release_sources # noqa: F401 + import cwa_book_downloader.metadata_providers # noqa: F401 + except ImportError: + pass + + registry = _get_registry() + + # On first load, sync ENV values to config files + # This ensures ENV values persist even if ENV vars are later removed + if not hasattr(self, '_env_synced'): + registry.sync_env_to_config() + self._env_synced = True + + # Build field map from all registered tabs + self._field_map.clear() + self._cache.clear() + + for tab in registry.get_all_settings_tabs(): + for field in tab.fields: + # Skip action buttons and headings - they don't have values + if isinstance(field, (registry.ActionButton, registry.HeadingField)): + continue + + key = field.key + self._field_map[key] = (field, tab.name) + + # Load current value + value = registry.get_setting_value(field, tab.name) + self._cache[key] = value + + self._loaded = True + + def refresh(self) -> None: + """ + Refresh all cached settings from config files. + + Call this after settings are updated via the UI to ensure + the config singleton reflects the new values. + """ + with self._cache_lock: + self._loaded = False + self._load_settings() + + def get(self, key: str, default: Any = None) -> Any: + """ + Get a setting value by key. + + Args: + key: The setting key (e.g., 'MAX_RETRY') + default: Default value if setting not found + + Returns: + The setting value, or default if not found + """ + self._ensure_loaded() + return self._cache.get(key, default) + + def __getattr__(self, name: str) -> Any: + """ + Allow attribute-style access to settings. + + Example: config.MAX_RETRY instead of config.get('MAX_RETRY') + """ + # Avoid recursion for internal attributes + if name.startswith('_'): + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") + + self._ensure_loaded() + + if name in self._cache: + return self._cache[name] + + # Fallback to env module for settings not in registry + # This ensures backward compatibility during migration + env = _get_env() + if hasattr(env, name): + return getattr(env, name) + + raise AttributeError(f"Setting '{name}' not found in config or env") + + def is_from_env(self, key: str) -> bool: + """ + Check if a setting's value comes from an environment variable. + + Args: + key: The setting key + + Returns: + True if the value is set via ENV var, False otherwise + """ + self._ensure_loaded() + + if key not in self._field_map: + return False + + field, _ = self._field_map[key] + registry = _get_registry() + return registry.is_value_from_env(field) + + def get_all(self) -> Dict[str, Any]: + """ + Get all cached settings as a dictionary. + + Returns: + Dict of all setting keys to their current values + """ + self._ensure_loaded() + return dict(self._cache) + + +# Global singleton instance +config = Config() diff --git a/cwa_book_downloader/core/image_cache.py b/cwa_book_downloader/core/image_cache.py new file mode 100644 index 00000000..bf3c601a --- /dev/null +++ b/cwa_book_downloader/core/image_cache.py @@ -0,0 +1,505 @@ +"""Disk-based image cache with LRU eviction.""" + +import json +import os +import threading +import time +from io import BytesIO +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +import requests + +from cwa_book_downloader.core.logger import setup_logger + +logger = setup_logger(__name__) + +# Image type detection via magic bytes +IMAGE_SIGNATURES = { + b'\xff\xd8\xff': ('image/jpeg', 'jpg'), + b'\x89PNG\r\n\x1a\n': ('image/png', 'png'), + b'GIF87a': ('image/gif', 'gif'), + b'GIF89a': ('image/gif', 'gif'), + b'RIFF': ('image/webp', 'webp'), # WebP starts with RIFF +} + +# HTTP headers for image fetching +FETCH_HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36', + 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', +} + +# Maximum image size to fetch (5 MB) +MAX_IMAGE_SIZE = 5 * 1024 * 1024 + +# Negative cache TTL (for failed fetches) - 1 hour +NEGATIVE_CACHE_TTL = 3600 + + +def _detect_image_type(data: bytes) -> Optional[Tuple[str, str]]: + """Detect image type from magic bytes. + + Args: + data: Image data bytes + + Returns: + Tuple of (content_type, extension) or None if not recognized + """ + for signature, (content_type, ext) in IMAGE_SIGNATURES.items(): + if data.startswith(signature): + return content_type, ext + + # Special case for WebP - check for WEBP after RIFF + if data.startswith(b'RIFF') and len(data) > 12 and data[8:12] == b'WEBP': + return 'image/webp', 'webp' + + return None + + +class ImageCacheService: + """Persistent image cache with LRU eviction and TTL support.""" + + def __init__(self, cache_dir: Path, max_size_mb: int = 500, ttl_seconds: int = 0): + """Initialize the image cache. + + Args: + cache_dir: Directory to store cached images + max_size_mb: Maximum cache size in megabytes + ttl_seconds: Time-to-live in seconds (0 = forever) + """ + self.cache_dir = cache_dir + self.max_size_bytes = max_size_mb * 1024 * 1024 + self.ttl_seconds = ttl_seconds + self.index_path = cache_dir / "cache_index.json" + self._lock = threading.RLock() + self._index: Dict[str, Dict[str, Any]] = {} + + # Stats tracking + self._hits = 0 + self._misses = 0 + + # Ensure cache directory exists + self.cache_dir.mkdir(parents=True, exist_ok=True) + + # Load existing index + self._load_index() + + def _load_index(self) -> None: + """Load cache index from disk.""" + try: + if self.index_path.exists(): + with open(self.index_path, 'r') as f: + self._index = json.load(f) + except (json.JSONDecodeError, IOError) as e: + logger.warning(f"Failed to load cache index, starting fresh: {e}") + self._index = {} + + def _save_index(self) -> None: + """Save cache index to disk.""" + try: + # Write to temp file first, then rename for atomicity + temp_path = self.index_path.with_suffix('.tmp') + with open(temp_path, 'w') as f: + json.dump(self._index, f) + temp_path.rename(self.index_path) + except IOError as e: + logger.error(f"Failed to save cache index: {e}") + + def _get_image_path(self, cache_id: str, ext: str) -> Path: + """Get the file path for a cached image.""" + return self.cache_dir / f"{cache_id}.{ext}" + + def _is_expired(self, entry: Dict[str, Any]) -> bool: + """Check if a cache entry is expired.""" + if self.ttl_seconds == 0: + return False + + cached_at = entry.get('cached_at', 0) + return (time.time() - cached_at) > self.ttl_seconds + + def _is_negative_expired(self, entry: Dict[str, Any]) -> bool: + """Check if a negative cache entry is expired.""" + if not entry.get('negative', False): + return False + + cached_at = entry.get('cached_at', 0) + return (time.time() - cached_at) > NEGATIVE_CACHE_TTL + + def _calculate_total_size(self) -> int: + """Calculate total size of cached images.""" + return sum(entry.get('size', 0) for entry in self._index.values()) + + def _evict_if_needed(self, required_space: int = 0) -> None: + """Evict old entries if cache is over size limit. + + Uses LRU eviction based on accessed_at timestamp. + """ + current_size = self._calculate_total_size() + target_size = self.max_size_bytes - required_space + + if current_size <= target_size: + return + + # Sort entries by accessed_at (oldest first) + sorted_entries = sorted( + self._index.items(), + key=lambda x: x[1].get('accessed_at', 0) + ) + + evicted_count = 0 + for cache_id, entry in sorted_entries: + if current_size <= target_size: + break + + # Delete the image file + ext = entry.get('ext', 'jpg') + image_path = self._get_image_path(cache_id, ext) + try: + if image_path.exists(): + image_path.unlink() + except IOError as e: + logger.warning(f"Failed to delete cached image {cache_id}: {e}") + + # Update tracking + current_size -= entry.get('size', 0) + del self._index[cache_id] + evicted_count += 1 + + if evicted_count > 0: + logger.info(f"Evicted {evicted_count} images from cache (LRU)") + self._save_index() + + def get(self, cache_id: str) -> Optional[Tuple[bytes, str]]: + """Get a cached image. + + Args: + cache_id: Cache key (book ID or composite key) + + Returns: + Tuple of (image_data, content_type) or None if not cached/expired + """ + with self._lock: + entry = self._index.get(cache_id) + + if not entry: + # Try reloading from disk (handles multiprocess case) + self._load_index() + entry = self._index.get(cache_id) + + if not entry: + self._misses += 1 + return None + + # Check for negative cache (failed fetch) + if entry.get('negative', False): + if self._is_negative_expired(entry): + # Negative cache expired, allow retry + del self._index[cache_id] + self._save_index() + self._misses += 1 + return None + # Still in negative cache, return None (don't retry) + return None + + # Check for expired entry + if self._is_expired(entry): + # Remove expired entry + ext = entry.get('ext', 'jpg') + image_path = self._get_image_path(cache_id, ext) + try: + if image_path.exists(): + image_path.unlink() + except IOError: + pass + del self._index[cache_id] + self._save_index() + self._misses += 1 + return None + + # Try to read the cached image + ext = entry.get('ext', 'jpg') + content_type = entry.get('content_type', 'image/jpeg') + image_path = self._get_image_path(cache_id, ext) + + try: + if not image_path.exists(): + # File missing, remove from index + del self._index[cache_id] + self._save_index() + self._misses += 1 + return None + + with open(image_path, 'rb') as f: + data = f.read() + + # Update accessed time + entry['accessed_at'] = time.time() + self._save_index() + + self._hits += 1 + return data, content_type + + except IOError as e: + logger.warning(f"Failed to read cached image {cache_id}: {e}") + self._misses += 1 + return None + + def put(self, cache_id: str, data: bytes, content_type: str) -> bool: + """Store an image in the cache. + + Args: + cache_id: Cache key + data: Image data bytes + content_type: MIME type of the image + + Returns: + True if stored successfully + """ + with self._lock: + # Detect image type for extension + detected = _detect_image_type(data) + if detected: + content_type, ext = detected + else: + # Fall back to content-type header + if 'jpeg' in content_type or 'jpg' in content_type: + ext = 'jpg' + elif 'png' in content_type: + ext = 'png' + elif 'gif' in content_type: + ext = 'gif' + elif 'webp' in content_type: + ext = 'webp' + else: + ext = 'jpg' # Default + + image_size = len(data) + + # Evict if needed to make room + self._evict_if_needed(image_size) + + # Write image to disk + image_path = self._get_image_path(cache_id, ext) + try: + with open(image_path, 'wb') as f: + f.write(data) + except IOError as e: + logger.warning(f"Failed to write cached image {cache_id}: {e}") + return False + + # Update index + now = time.time() + self._index[cache_id] = { + 'ext': ext, + 'content_type': content_type, + 'size': image_size, + 'cached_at': now, + 'accessed_at': now, + 'negative': False, + } + self._save_index() + return True + + def put_negative(self, cache_id: str) -> None: + """Store a negative cache entry (failed fetch). + + Args: + cache_id: Cache key + """ + with self._lock: + self._index[cache_id] = { + 'negative': True, + 'cached_at': time.time(), + } + self._save_index() + + def delete(self, cache_id: str) -> bool: + """Delete a single cache entry. + + Args: + cache_id: Cache key + + Returns: + True if entry existed and was deleted + """ + with self._lock: + entry = self._index.get(cache_id) + if not entry: + return False + + # Delete file if it exists + if not entry.get('negative', False): + ext = entry.get('ext', 'jpg') + image_path = self._get_image_path(cache_id, ext) + try: + if image_path.exists(): + image_path.unlink() + except IOError as e: + logger.warning(f"Failed to delete cached image {cache_id}: {e}") + + del self._index[cache_id] + self._save_index() + return True + + def clear(self) -> int: + """Clear all cached images. + + Returns: + Number of entries cleared + """ + with self._lock: + count = len(self._index) + + # Delete all image files + for cache_id, entry in self._index.items(): + if not entry.get('negative', False): + ext = entry.get('ext', 'jpg') + image_path = self._get_image_path(cache_id, ext) + try: + if image_path.exists(): + image_path.unlink() + except IOError: + pass + + # Clear index + self._index = {} + self._save_index() + + # Reset stats + self._hits = 0 + self._misses = 0 + + logger.info(f"Cleared {count} entries from image cache") + return count + + def stats(self) -> Dict[str, Any]: + """Get cache statistics. + + Returns: + Dict with size, count, hit rate, etc. + """ + with self._lock: + total_size = self._calculate_total_size() + entry_count = len(self._index) + negative_count = sum(1 for e in self._index.values() if e.get('negative', False)) + total_requests = self._hits + self._misses + hit_rate = (self._hits / total_requests * 100) if total_requests > 0 else 0 + + return { + 'entry_count': entry_count, + 'negative_count': negative_count, + 'total_size_bytes': total_size, + 'total_size_mb': round(total_size / (1024 * 1024), 2), + 'max_size_mb': self.max_size_bytes / (1024 * 1024), + 'hits': self._hits, + 'misses': self._misses, + 'hit_rate': round(hit_rate, 1), + } + + def fetch_and_cache(self, cache_id: str, url: str) -> Optional[Tuple[bytes, str]]: + """Fetch an image from URL and cache it. + + Args: + cache_id: Cache key + url: URL to fetch from + + Returns: + Tuple of (image_data, content_type) or None on failure + """ + try: + + response = requests.get( + url, + timeout=(5, 10), + headers=FETCH_HEADERS, + stream=True, + ) + response.raise_for_status() + + # Validate content type + content_type = response.headers.get('content-type', '') + if not content_type.startswith('image/'): + logger.warning(f"Invalid content type for cover: {content_type}") + self.put_negative(cache_id) + return None + + # Read with size limit + data = BytesIO() + for chunk in response.iter_content(chunk_size=8192): + data.write(chunk) + if data.tell() > MAX_IMAGE_SIZE: + logger.warning(f"Cover image too large: {url}") + self.put_negative(cache_id) + return None + + image_data = data.getvalue() + + if not image_data: + logger.warning(f"Empty image response: {url}") + self.put_negative(cache_id) + return None + + # Store in cache + if self.put(cache_id, image_data, content_type): + # Get the actual content type from detection + detected = _detect_image_type(image_data) + if detected: + content_type = detected[0] + return image_data, content_type + + return None + + except requests.exceptions.Timeout: + logger.warning(f"Timeout fetching cover: {url}") + # Don't cache timeout - it's transient + return None + except requests.exceptions.HTTPError as e: + if e.response is not None and e.response.status_code == 404: + self.put_negative(cache_id) + else: + logger.warning(f"HTTP error fetching cover: {e}") + return None + except Exception as e: + logger.warning(f"Error fetching cover: {e}") + return None + + +# Singleton instance (initialized lazily when config is available) +_instance: Optional[ImageCacheService] = None +_instance_lock = threading.Lock() + + +def get_image_cache() -> ImageCacheService: + """Get the singleton image cache instance. + + Lazily initializes using config values. + """ + global _instance + + if _instance is None: + with _instance_lock: + if _instance is None: + from cwa_book_downloader.core.config import config + from cwa_book_downloader.config.env import CONFIG_DIR + + cache_dir = CONFIG_DIR / "covers" + max_size_mb = config.get("COVERS_CACHE_MAX_SIZE_MB", 500) + ttl_days = config.get("COVERS_CACHE_TTL", 0) + ttl_seconds = ttl_days * 86400 if ttl_days > 0 else 0 + + _instance = ImageCacheService( + cache_dir=cache_dir, + max_size_mb=max_size_mb, + ttl_seconds=ttl_seconds, + ) + logger.info(f"Initialized image cache: {cache_dir} (max {max_size_mb}MB, TTL {ttl_days} days)") + + return _instance + + +def reset_image_cache() -> None: + """Reset the singleton instance (for testing or config changes).""" + global _instance + with _instance_lock: + _instance = None diff --git a/logger.py b/cwa_book_downloader/core/logger.py similarity index 95% rename from logger.py rename to cwa_book_downloader/core/logger.py index 440f30fe..8300637a 100644 --- a/logger.py +++ b/cwa_book_downloader/core/logger.py @@ -1,15 +1,17 @@ -"""Centralized logging configuration for the book downloader application.""" +"""Logging configuration and custom logger with error tracing.""" import logging import sys from pathlib import Path from logging.handlers import RotatingFileHandler -from env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL from typing import Any +from cwa_book_downloader.config.env import LOG_FILE, ENABLE_LOGGING, LOG_LEVEL + + class CustomLogger(logging.Logger): """Custom logger class with additional error_trace method.""" - + def error_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None: """Log an error message with full stack trace.""" self.log_resource_usage() @@ -21,7 +23,7 @@ class CustomLogger(logging.Logger): self.log_resource_usage() kwargs.pop('exc_info', None) self.warning(msg, *args, exc_info=True, **kwargs) - + def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None: """Log an info message (stack trace only if exception active).""" kwargs.pop('exc_info', None) @@ -35,7 +37,7 @@ class CustomLogger(logging.Logger): # Only include exc_info if there's actually an exception has_exception = sys.exc_info()[0] is not None self.debug(msg, *args, exc_info=has_exception, **kwargs) - + def log_resource_usage(self): import psutil memory = psutil.virtual_memory() @@ -47,17 +49,17 @@ class CustomLogger(logging.Logger): def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger: """Set up and configure a logger instance. - + Args: name: The name of the logger instance log_file: Optional path to log file. If None, logs only to stdout/stderr - + Returns: CustomLogger: Configured logger instance with error_trace method """ # Register our custom logger class logging.setLoggerClass(CustomLogger) - + # Create logger as CustomLogger instance logger = CustomLogger(name) log_level = logging.INFO @@ -72,7 +74,7 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger: elif LOG_LEVEL == "CRITICAL": log_level = logging.CRITICAL logger.setLevel(log_level) - + formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' ) @@ -83,13 +85,13 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger: console_handler.setLevel(log_level) console_handler.addFilter(lambda record: record.levelno < logging.ERROR) # Only allow logs below ERROR to stdout logger.addHandler(console_handler) - + # Error handler for stderr error_handler = logging.StreamHandler(sys.stderr) error_handler.setLevel(logging.ERROR) # Error and above go to stderr error_handler.setFormatter(formatter) logger.addHandler(error_handler) - + # File handler if log file is specified try: if ENABLE_LOGGING: @@ -107,4 +109,3 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> CustomLogger: logger.error_trace(f"Failed to create log file: {e}", exc_info=True) return logger - diff --git a/cwa_book_downloader/core/models.py b/cwa_book_downloader/core/models.py new file mode 100644 index 00000000..6fbfa70d --- /dev/null +++ b/cwa_book_downloader/core/models.py @@ -0,0 +1,139 @@ +"""Data structures and models used across the application.""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional +from enum import Enum +import re +import time + + +class QueueStatus(str, Enum): + """Enum for possible book queue statuses.""" + QUEUED = "queued" + RESOLVING = "resolving" + DOWNLOADING = "downloading" + COMPLETE = "complete" + AVAILABLE = "available" + ERROR = "error" + DONE = "done" + CANCELLED = "cancelled" + + +@dataclass +class QueueItem: + """Queue item with priority and metadata.""" + book_id: str + priority: int + added_time: float + + def __lt__(self, other): + """Compare items for priority queue (lower priority number = higher precedence).""" + if self.priority != other.priority: + return self.priority < other.priority + return self.added_time < other.added_time + + +@dataclass +class DownloadTask: + """Source-agnostic download task for the queue. + + This replaces BookInfo in the queue, providing a unified interface + for both Direct Download and Universal modes. The handler uses task_id + to fetch whatever source-specific data it needs internally. + """ + task_id: str # Unique ID (e.g., AA MD5 hash, Prowlarr GUID) + source: str # Handler name ("direct_download", "prowlarr") + title: str # Display title for queue sidebar + + # Display info for queue sidebar + author: Optional[str] = None + format: Optional[str] = None + size: Optional[str] = None + preview: Optional[str] = None + + # Runtime state + priority: int = 0 + added_time: float = field(default_factory=time.time) + progress: float = 0.0 + status: QueueStatus = QueueStatus.QUEUED + status_message: Optional[str] = None + download_path: Optional[str] = None + + def __lt__(self, other): + """Compare tasks for priority queue (lower priority number = higher precedence).""" + if self.priority != other.priority: + return self.priority < other.priority + return self.added_time < other.added_time + + +@dataclass +class BookInfo: + """Data class representing book information.""" + id: str + title: str + preview: Optional[str] = None + author: Optional[str] = None + publisher: Optional[str] = None + year: Optional[str] = None + language: Optional[str] = None + content: Optional[str] = None + format: Optional[str] = None + size: Optional[str] = None + info: Optional[Dict[str, List[str]]] = None + description: Optional[str] = None + download_urls: List[str] = field(default_factory=list) + download_path: Optional[str] = None + priority: int = 0 + progress: Optional[float] = None + status_message: Optional[str] = None # Detailed status message for UI display + added_time: Optional[float] = None # Timestamp when added to queue + source: str = "direct_download" # Release source handler to use for downloads + + def get_filename(self, fallback_url: Optional[str] = None) -> str: + """Build sanitized filename: 'Author - Title (Year).format' + + Resolves format from self.format, download_urls, or fallback_url. + + Args: + fallback_url: URL to extract format from if not already known + + Returns: + Sanitized filename safe for filesystem use + """ + # Resolve format if needed + if not self.format: + for url in (self.download_urls[0] if self.download_urls else None, fallback_url): + if url: + ext = url.split(".")[-1].lower() + if ext and len(ext) <= 5 and ext.isalnum(): + self.format = ext + break + + # Build filename + parts = [] + if self.author: + parts.append(self.author) + parts.append(" - ") + parts.append(self.title) + if self.year: + parts.append(f" ({self.year})") + + filename = "".join(parts) + filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245] + + if self.format: + filename = f"{filename}.{self.format}" + + return filename + + +@dataclass +class SearchFilters: + """Filters for book search queries.""" + isbn: Optional[List[str]] = None + author: Optional[List[str]] = None + title: Optional[List[str]] = None + lang: Optional[List[str]] = None + sort: Optional[str] = None + content: Optional[List[str]] = None + format: Optional[List[str]] = None diff --git a/cwa_book_downloader/core/queue.py b/cwa_book_downloader/core/queue.py new file mode 100644 index 00000000..faa93645 --- /dev/null +++ b/cwa_book_downloader/core/queue.py @@ -0,0 +1,365 @@ +"""Thread-safe download queue manager with priority support and cancellation.""" + +import queue +import time +from datetime import datetime, timedelta +from pathlib import Path +from threading import Lock, Event +from typing import Dict, List, Optional, Tuple, Any + +from cwa_book_downloader.core.config import config as app_config +from cwa_book_downloader.core.models import QueueStatus, QueueItem, DownloadTask + + +class BookQueue: + """Thread-safe download queue manager with priority support and cancellation. + + Stores DownloadTask objects which are source-agnostic download descriptors. + Works with both Direct Download and Universal modes. + """ + + def __init__(self) -> None: + self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue() + self._lock = Lock() + self._status: dict[str, QueueStatus] = {} + self._task_data: dict[str, DownloadTask] = {} + self._status_timestamps: dict[str, datetime] = {} # Track when each status was last updated + self._cancel_flags: dict[str, Event] = {} # Cancellation flags for active downloads + self._active_downloads: dict[str, bool] = {} # Track currently downloading tasks + + @property + def _status_timeout(self) -> timedelta: + """Get status timeout from config (allows live updates).""" + return timedelta(seconds=app_config.get("STATUS_TIMEOUT", 3600)) + + def add(self, task: DownloadTask) -> bool: + """Add a download task to the queue. + + Args: + task: The download task to queue (includes task_id, priority, etc.) + + Returns: + True if added successfully, False if already exists + """ + with self._lock: + task_id = task.task_id + + # Don't add if already exists and not in error/done state + if task_id in self._status and self._status[task_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]: + return False + + # Ensure added_time is set + if task.added_time == 0: + task.added_time = time.time() + + queue_item = QueueItem(task_id, task.priority, task.added_time) + self._queue.put(queue_item) + self._task_data[task_id] = task + self._update_status(task_id, QueueStatus.QUEUED) + return True + + def get_next(self) -> Optional[Tuple[str, Event]]: + """Get next task ID from queue with cancellation flag. + + Returns: + Tuple of (task_id, cancel_flag) or None if queue is empty + """ + # Use iterative approach to avoid stack overflow if many items are cancelled + while True: + try: + queue_item = self._queue.get_nowait() + task_id = queue_item.book_id # QueueItem uses book_id as the ID field + + with self._lock: + # Check if task was cancelled while in queue + if task_id in self._status and self._status[task_id] == QueueStatus.CANCELLED: + continue # Skip cancelled items, try next + + # Create cancellation flag for this download + cancel_flag = Event() + self._cancel_flags[task_id] = cancel_flag + self._active_downloads[task_id] = True + + return task_id, cancel_flag + except queue.Empty: + return None + + def get_task(self, task_id: str) -> Optional[DownloadTask]: + """Get a task by its ID. + + Args: + task_id: The task identifier + + Returns: + The DownloadTask if found, None otherwise + """ + with self._lock: + return self._task_data.get(task_id) + + def _update_status(self, book_id: str, status: QueueStatus) -> None: + """Internal method to update status and timestamp.""" + self._status[book_id] = status + self._status_timestamps[book_id] = datetime.now() + + def update_status(self, book_id: str, status: QueueStatus) -> None: + """Update status of a book in the queue.""" + with self._lock: + self._update_status(book_id, status) + + # Clean up active download tracking when finished + if status in [QueueStatus.COMPLETE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]: + self._active_downloads.pop(book_id, None) + self._cancel_flags.pop(book_id, None) + + def update_download_path(self, task_id: str, download_path: str) -> None: + """Update the download path of a task in the queue.""" + with self._lock: + if task_id in self._task_data: + self._task_data[task_id].download_path = download_path + + def update_progress(self, task_id: str, progress: float) -> None: + """Update download progress for a task.""" + with self._lock: + if task_id in self._task_data: + self._task_data[task_id].progress = progress + + def update_status_message(self, task_id: str, message: str) -> None: + """Update detailed status message for a task.""" + with self._lock: + if task_id in self._task_data: + self._task_data[task_id].status_message = message + + def get_status(self) -> Dict[QueueStatus, Dict[str, DownloadTask]]: + """Get current queue status grouped by status.""" + self.refresh() + with self._lock: + result: Dict[QueueStatus, Dict[str, DownloadTask]] = {status: {} for status in QueueStatus} + for task_id, status in self._status.items(): + if task_id in self._task_data: + result[status][task_id] = self._task_data[task_id] + return result + + def get_queue_order(self) -> List[Dict[str, Any]]: + """Get current queue order for display.""" + with self._lock: + queue_items = [] + + # Get items from priority queue without removing them + temp_items = [] + while not self._queue.empty(): + try: + item = self._queue.get_nowait() + temp_items.append(item) + task_id = item.book_id # QueueItem uses book_id as the ID field + if task_id in self._task_data: + task = self._task_data[task_id] + queue_items.append({ + 'id': task_id, + 'title': task.title, + 'author': task.author, + 'priority': item.priority, + 'added_time': item.added_time, + 'status': self._status.get(task_id, QueueStatus.QUEUED) + }) + except queue.Empty: + break + + # Put items back in queue + for item in temp_items: + self._queue.put(item) + + return sorted(queue_items, key=lambda x: (x['priority'], x['added_time'])) + + def cancel_download(self, task_id: str) -> bool: + """Cancel a download or clear a completed/errored item. + + Args: + task_id: Task identifier to cancel or clear + + Returns: + bool: True if cancellation/clearing was successful + """ + with self._lock: + current_status = self._status.get(task_id) + + # Allow cancellation during any active state + if current_status in [QueueStatus.RESOLVING, QueueStatus.DOWNLOADING]: + # Signal active download to stop + if task_id in self._cancel_flags: + self._cancel_flags[task_id].set() + self._update_status(task_id, QueueStatus.CANCELLED) + return True + elif current_status == QueueStatus.QUEUED: + # Remove from queue and mark as cancelled + self._update_status(task_id, QueueStatus.CANCELLED) + return True + elif current_status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]: + # Clear completed/errored/cancelled items from tracking + self._status.pop(task_id, None) + self._status_timestamps.pop(task_id, None) + self._task_data.pop(task_id, None) + self._cancel_flags.pop(task_id, None) + self._active_downloads.pop(task_id, None) + return True + + return False + + def set_priority(self, task_id: str, new_priority: int) -> bool: + """Change the priority of a queued task. + + Args: + task_id: Task identifier + new_priority: New priority level (lower = higher priority) + + Returns: + bool: True if priority was successfully changed + """ + with self._lock: + if task_id not in self._status or self._status[task_id] != QueueStatus.QUEUED: + return False + + # Remove task from queue and re-add with new priority + temp_items = [] + found = False + + while not self._queue.empty(): + try: + item = self._queue.get_nowait() + if item.book_id == task_id: # QueueItem uses book_id as the ID field + # Create new item with updated priority + new_item = QueueItem(task_id, new_priority, item.added_time) + temp_items.append(new_item) + found = True + # Update task data priority + if task_id in self._task_data: + self._task_data[task_id].priority = new_priority + else: + temp_items.append(item) + except queue.Empty: + break + + # Put all items back + for item in temp_items: + self._queue.put(item) + + return found + + def reorder_queue(self, task_priorities: Dict[str, int]) -> bool: + """Bulk reorder queue by setting new priorities. + + Args: + task_priorities: Dict mapping task_id to new priority + + Returns: + bool: True if reordering was successful + """ + with self._lock: + # Extract all items from queue + all_items = [] + while not self._queue.empty(): + try: + item = self._queue.get_nowait() + task_id = item.book_id # QueueItem uses book_id as the ID field + # Update priority if specified + if task_id in task_priorities: + new_priority = task_priorities[task_id] + item = QueueItem(task_id, new_priority, item.added_time) + # Update task data priority + if task_id in self._task_data: + self._task_data[task_id].priority = new_priority + all_items.append(item) + except queue.Empty: + break + + # Put all items back with updated priorities + for item in all_items: + self._queue.put(item) + + return True + + def get_active_downloads(self) -> List[str]: + """Get list of currently active download task IDs.""" + with self._lock: + return list(self._active_downloads.keys()) + + def has_pending_work(self) -> bool: + """Check if there are any active downloads or queued items. + + This is useful for determining if the bypasser should stay active + even when the UI is closed. + + Returns: + bool: True if there are active downloads or queued items + """ + with self._lock: + # Check for active downloads + if self._active_downloads: + return True + + # Check for queued items (excluding cancelled ones) + for task_id, status in self._status.items(): + if status == QueueStatus.QUEUED: + return True + + return False + + def clear_completed(self) -> int: + """Remove all completed, errored, or cancelled tasks from tracking. + + Returns: + int: Number of tasks removed + """ + with self._lock: + to_remove = [] + for task_id, status in self._status.items(): + if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]: + to_remove.append(task_id) + + removed_count = len(to_remove) + for task_id in to_remove: + self._status.pop(task_id, None) + self._status_timestamps.pop(task_id, None) + self._task_data.pop(task_id, None) + self._cancel_flags.pop(task_id, None) + self._active_downloads.pop(task_id, None) + + return removed_count + + def refresh(self) -> None: + """Remove any tasks that are done downloading or have stale status.""" + with self._lock: + current_time = datetime.now() + + # Create a list of items to remove to avoid modifying dict during iteration + to_remove = [] + + for task_id, status in self._status.items(): + task = self._task_data.get(task_id) + if not task: + continue + + path = task.download_path + if path and not Path(path).exists(): + task.download_path = None + path = None + + # Check for completed downloads + if status == QueueStatus.AVAILABLE: + if not path: + self._update_status(task_id, QueueStatus.DONE) + + # Check for stale status entries + last_update = self._status_timestamps.get(task_id) + if last_update and (current_time - last_update) > self._status_timeout: + if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]: + to_remove.append(task_id) + + # Remove stale entries + for task_id in to_remove: + del self._status[task_id] + del self._status_timestamps[task_id] + if task_id in self._task_data: + del self._task_data[task_id] + +# Global instance of BookQueue +book_queue = BookQueue() diff --git a/cwa_book_downloader/core/settings_registry.py b/cwa_book_downloader/core/settings_registry.py new file mode 100644 index 00000000..36e0da35 --- /dev/null +++ b/cwa_book_downloader/core/settings_registry.py @@ -0,0 +1,748 @@ +"""Plugin settings registry with config file persistence.""" + +import json +import os +from dataclasses import dataclass, field, asdict +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Type, Union +from threading import Lock + +from cwa_book_downloader.core.logger import setup_logger + +logger = setup_logger(__name__) + + +@dataclass +class FieldBase: + """Base class for all settings fields.""" + key: str # Environment variable / config key + label: str # Display label in UI + description: str = "" # Help text + default: Any = None # Default value if not set + required: bool = False # Whether field must have a value + env_var: Optional[str] = None # Override env var name (defaults to key) + env_supported: bool = True # Whether this setting can be set via ENV var (False = UI-only) + disabled: bool = False # Whether field is disabled/greyed out + disabled_reason: str = "" # Explanation shown when disabled + show_when: Optional[Dict[str, Any]] = None # Conditional visibility: {"field": "key", "value": "expected"} + disabled_when: Optional[Dict[str, Any]] = None # Conditional disable: {"field": "key", "value": "expected", "reason": "..."} + requires_restart: bool = False # Whether changing this setting requires a container restart + + def get_env_var_name(self) -> str: + """Get the environment variable name for this field.""" + return self.env_var or self.key + + def get_field_type(self) -> str: + """Get the field type name for serialization.""" + return self.__class__.__name__ + + +@dataclass +class TextField(FieldBase): + """Single-line text input.""" + placeholder: str = "" + max_length: Optional[int] = None + + +@dataclass +class PasswordField(FieldBase): + """Password input (masked in UI, not returned in API responses).""" + placeholder: str = "" + + +@dataclass +class NumberField(FieldBase): + """Numeric input.""" + min_value: Optional[float] = None + max_value: Optional[float] = None + step: float = 1 + default: float = 0 + + +@dataclass +class CheckboxField(FieldBase): + """Boolean checkbox.""" + default: bool = False + + +@dataclass +class SelectField(FieldBase): + """Single-choice dropdown.""" + # Options can be a list or a callable that returns a list (for lazy evaluation) + options: Any = field(default_factory=list) # [{value: "", label: ""}] or callable + + +@dataclass +class MultiSelectField(FieldBase): + """Multiple-choice selection.""" + # Options can be a list or a callable that returns a list (for lazy evaluation) + options: Any = field(default_factory=list) # [{value: "", label: ""}] or callable + default: List[str] = field(default_factory=list) + + +@dataclass +class ActionButton: + """ + Button that triggers a callback function. + + Used for actions like "Test Connection" that execute code + and return success/error status. + """ + key: str # Action identifier + label: str # Button text + description: str = "" # Help text + style: str = "default" # "default", "primary", "danger" + callback: Optional[Callable[[], Dict[str, Any]]] = None # Returns {"success": bool, "message": str} + disabled: bool = False # Whether button is disabled/greyed out + disabled_reason: str = "" # Explanation shown when disabled + show_when: Optional[Dict[str, Any]] = None # Conditional visibility: {"field": "key", "value": "expected"} + disabled_when: Optional[Dict[str, Any]] = None # Conditional disable: {"field": "key", "value": "expected", "reason": "..."} + + def get_field_type(self) -> str: + return "ActionButton" + + +@dataclass +class HeadingField: + """ + Display-only heading with title and description. + + Used to add section titles and descriptive text to settings pages. + Not an input field - purely for display. + """ + key: str # Unique identifier + title: str # Heading title + description: str = "" # Description text (supports markdown-style links) + link_url: str = "" # Optional URL for a link + link_text: str = "" # Text for the link (defaults to URL if not provided) + + def get_field_type(self) -> str: + return "HeadingField" + + +# Type alias for all field types +SettingsField = Union[TextField, PasswordField, NumberField, CheckboxField, SelectField, MultiSelectField, ActionButton, HeadingField] + + +@dataclass +class SettingsTab: + """A tab/section in the settings UI.""" + name: str # Internal name (used in URLs) + display_name: str # Display name in UI + fields: List[SettingsField] = field(default_factory=list) + icon: Optional[str] = None # Icon name for UI + order: int = 100 # Sort order (lower = earlier) + group: Optional[str] = None # Group name this tab belongs to + + +@dataclass +class SettingsGroup: + """A collapsible group of settings tabs in the UI.""" + name: str # Internal name + display_name: str # Display name in UI + icon: Optional[str] = None # Icon name for UI + order: int = 100 # Sort order (lower = earlier) + + +_SETTINGS_REGISTRY: Dict[str, SettingsTab] = {} +_GROUPS_REGISTRY: Dict[str, SettingsGroup] = {} +_ON_SAVE_HANDLERS: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {} +_REGISTRY_LOCK = Lock() + + +def register_group( + name: str, + display_name: str, + icon: Optional[str] = None, + order: int = 100 +) -> None: + """ + Register a settings group. + + Groups are collapsible containers for related settings tabs. + + Args: + name: Internal name for the group (e.g., "direct_download") + display_name: Display name in UI (e.g., "Direct Download") + icon: Optional icon name for the UI + order: Sort order (lower numbers appear first) + + Example: + register_group("direct_download", "Direct Download", icon="download", order=20) + """ + with _REGISTRY_LOCK: + group = SettingsGroup( + name=name, + display_name=display_name, + icon=icon, + order=order, + ) + _GROUPS_REGISTRY[name] = group + logger.debug(f"Registered settings group: {name}") + + +def register_settings( + name: str, + display_name: str, + icon: Optional[str] = None, + order: int = 100, + group: Optional[str] = None +): + """ + Decorator to register settings for a plugin/module. + + The decorated function should return a list of SettingsField objects. + + Args: + name: Internal name for the settings tab (e.g., "hardcover") + display_name: Display name in UI (e.g., "Hardcover") + icon: Optional icon name for the UI + order: Sort order (lower numbers appear first) + group: Optional group name this tab belongs to + + Example: + @register_settings("hardcover", "Hardcover", icon="book", order=20, group="metadata_providers") + def hardcover_settings(): + return [ + PasswordField(key="HARDCOVER_API_KEY", label="API Key", required=True), + ] + """ + def decorator(func: Callable[[], List[SettingsField]]): + with _REGISTRY_LOCK: + fields = func() + tab = SettingsTab( + name=name, + display_name=display_name, + fields=fields, + icon=icon, + order=order, + group=group, + ) + _SETTINGS_REGISTRY[name] = tab + logger.debug(f"Registered settings tab: {name} ({len(fields)} fields)" + + (f" in group {group}" if group else "")) + return func + return decorator + + +def register_on_save( + tab_name: str, + handler: Callable[[Dict[str, Any]], Dict[str, Any]] +) -> None: + """ + Register a custom on_save handler for a settings tab. + + The handler is called before saving settings and can: + - Validate values (return {"error": True, "message": "..."}) + - Transform values (e.g., hash passwords) + - Add computed values + + Args: + tab_name: The settings tab name to register the handler for. + handler: Callable that takes values dict and returns: + {"error": bool, "message": str (if error), "values": dict} + + Example: + def _on_save_security(values: Dict[str, Any]) -> Dict[str, Any]: + password = values.pop("password", "") + if password: + values["password_hash"] = hash_password(password) + return {"error": False, "values": values} + + register_on_save("security", _on_save_security) + """ + with _REGISTRY_LOCK: + _ON_SAVE_HANDLERS[tab_name] = handler + logger.debug(f"Registered on_save handler for tab: {tab_name}") + + +def get_on_save_handler(tab_name: str) -> Optional[Callable[[Dict[str, Any]], Dict[str, Any]]]: + """Get the on_save handler for a settings tab, if any.""" + return _ON_SAVE_HANDLERS.get(tab_name) + + +def get_settings_tab(name: str) -> Optional[SettingsTab]: + """Get a specific settings tab by name.""" + return _SETTINGS_REGISTRY.get(name) + + +def get_all_settings_tabs() -> List[SettingsTab]: + """Get all registered settings tabs, sorted by order.""" + return sorted(_SETTINGS_REGISTRY.values(), key=lambda t: (t.order, t.name)) + + +def list_registered_settings() -> List[str]: + """List all registered settings tab names.""" + return list(_SETTINGS_REGISTRY.keys()) + + +def _get_config_dir() -> Path: + """Get the config directory path.""" + from cwa_book_downloader.config.env import CONFIG_DIR + return Path(CONFIG_DIR) + + +def _get_config_file_path(tab_name: str) -> Path: + """Get the config file path for a settings tab.""" + config_dir = _get_config_dir() + if tab_name == "general": + return config_dir / "settings.json" + else: + plugins_dir = config_dir / "plugins" + return plugins_dir / f"{tab_name}.json" + + +def _ensure_config_dir(tab_name: str) -> None: + """Ensure the config directory exists.""" + config_path = _get_config_file_path(tab_name) + config_path.parent.mkdir(parents=True, exist_ok=True) + + +def load_config_file(tab_name: str) -> Dict[str, Any]: + """ + Load settings from a config file. + + Args: + tab_name: The settings tab name. + + Returns: + Dict of setting key -> value from config file. + """ + config_path = _get_config_file_path(tab_name) + + if not config_path.exists(): + return {} + + try: + with open(config_path, 'r') as f: + return json.load(f) + except json.JSONDecodeError as e: + logger.error(f"Invalid JSON in config file {config_path}: {e}") + return {} + + +def save_config_file(tab_name: str, values: Dict[str, Any]) -> bool: + """ + Save settings to a config file. + + Args: + tab_name: The settings tab name. + values: Dict of setting key -> value to save. + + Returns: + True if save succeeded, False otherwise. + """ + try: + _ensure_config_dir(tab_name) + config_path = _get_config_file_path(tab_name) + + # Load existing config and merge + existing = load_config_file(tab_name) + existing.update(values) + + with open(config_path, 'w') as f: + json.dump(existing, f, indent=2) + + logger.info(f"Saved settings to {config_path}") + return True + except Exception as e: + logger.error(f"Error saving config file for {tab_name}: {e}") + return False + + +def sync_env_to_config() -> None: + """ + Sync environment variable values to config files. + + This ensures that when ENV vars are set, their values are persisted to config. + When ENV vars are later removed, the config file retains the last known values. + + Called once during application startup. + """ + for tab in get_all_settings_tabs(): + values_to_sync = {} + + for field in tab.fields: + # Skip non-value fields + if isinstance(field, (ActionButton, HeadingField)): + continue + + # Skip fields that don't support ENV vars + if not getattr(field, 'env_supported', True): + continue + + # Check if ENV var is set + env_var_name = field.get_env_var_name() + env_value = os.environ.get(env_var_name) + + if env_value is not None: + # Parse the ENV value to the appropriate type + parsed_value = _parse_env_value(env_value, field) + values_to_sync[field.key] = parsed_value + + # Save synced values to config file (merge with existing) + if values_to_sync: + save_config_file(tab.name, values_to_sync) + logger.debug(f"Synced {len(values_to_sync)} ENV values to {tab.name} config: {list(values_to_sync.keys())}") + + +def get_setting_value(field: SettingsField, tab_name: str) -> Any: + """ + Get the current value for a settings field. + + Priority: env var > config file > default + + Args: + field: The settings field. + tab_name: The settings tab name (for config file lookup). + + Returns: + The resolved value. + """ + if isinstance(field, (ActionButton, HeadingField)): + return None # Actions and headings don't have values + + # 1. Check environment variable (if supported for this field) + if field.env_supported: + env_var_name = field.get_env_var_name() + env_value = os.environ.get(env_var_name) + if env_value is not None: + return _parse_env_value(env_value, field) + + # 2. Check config file + config = load_config_file(tab_name) + if field.key in config: + return config[field.key] + + # 3. Return default + return field.default + + +def _parse_env_value(value: str, field: SettingsField) -> Any: + """Parse an environment variable value to the appropriate type.""" + if isinstance(field, CheckboxField): + return value.lower() in ('true', '1', 'yes', 'on') + elif isinstance(field, NumberField): + try: + if '.' in value: + return float(value) + return int(value) + except ValueError: + return field.default + elif isinstance(field, MultiSelectField): + return [v.strip() for v in value.split(',') if v.strip()] + else: + return value + + +def is_value_from_env(field: SettingsField) -> bool: + """Check if a field's value comes from an environment variable.""" + if isinstance(field, (ActionButton, HeadingField)): + return False + # UI-only settings never come from ENV (env_supported=False) + # Default to True for backwards compatibility + env_supported = getattr(field, 'env_supported', True) + if env_supported is False: + return False + env_var_name = field.get_env_var_name() + return env_var_name in os.environ + + +def serialize_field(field: SettingsField, tab_name: str, include_value: bool = True) -> Dict[str, Any]: + """ + Serialize a field for API response. + + Args: + field: The settings field. + tab_name: The settings tab name. + include_value: Whether to include the current value. + + Returns: + Dict representation of the field. + """ + # HeadingField has a different structure - handle separately + if isinstance(field, HeadingField): + result = { + "key": field.key, + "type": field.get_field_type(), + "title": field.title, + "description": field.description, + } + if field.link_url: + result["linkUrl"] = field.link_url + result["linkText"] = field.link_text or field.link_url + return result + + result = { + "key": field.key, + "label": field.label, + "type": field.get_field_type(), + "description": getattr(field, 'description', ''), + "required": getattr(field, 'required', False), + "disabled": getattr(field, 'disabled', False), + "disabledReason": getattr(field, 'disabled_reason', ''), + "requiresRestart": getattr(field, 'requires_restart', False), + } + + # Add conditional visibility if specified + show_when = getattr(field, 'show_when', None) + if show_when: + result["showWhen"] = show_when + + # Add conditional disable if specified + disabled_when = getattr(field, 'disabled_when', None) + if disabled_when: + result["disabledWhen"] = disabled_when + + # Add type-specific properties + if isinstance(field, TextField): + result["placeholder"] = field.placeholder + if field.max_length: + result["maxLength"] = field.max_length + elif isinstance(field, PasswordField): + result["placeholder"] = field.placeholder + elif isinstance(field, NumberField): + result["min"] = field.min_value + result["max"] = field.max_value + result["step"] = field.step + elif isinstance(field, (SelectField, MultiSelectField)): + # Support callable options for lazy evaluation (avoids circular imports) + options = field.options() if callable(field.options) else field.options + result["options"] = options + elif isinstance(field, ActionButton): + result["style"] = field.style + result["description"] = field.description + + if include_value and not isinstance(field, (ActionButton, HeadingField)): + value = get_setting_value(field, tab_name) + result["value"] = value if value is not None else "" + result["fromEnv"] = is_value_from_env(field) + + return result + + +def serialize_tab(tab: SettingsTab, include_values: bool = True) -> Dict[str, Any]: + """Serialize a settings tab for API response.""" + return { + "name": tab.name, + "displayName": tab.display_name, + "icon": tab.icon, + "order": tab.order, + "group": tab.group, + "fields": [serialize_field(f, tab.name, include_values) for f in tab.fields], + } + + +def serialize_group(group: SettingsGroup) -> Dict[str, Any]: + """Serialize a settings group for API response.""" + return { + "name": group.name, + "displayName": group.display_name, + "icon": group.icon, + "order": group.order, + } + + +def get_all_groups() -> List[SettingsGroup]: + """Get all registered settings groups, sorted by order.""" + return sorted(_GROUPS_REGISTRY.values(), key=lambda g: (g.order, g.name)) + + +def serialize_all_settings(include_values: bool = True) -> Dict[str, Any]: + """Serialize all settings for API response.""" + tabs = get_all_settings_tabs() + groups = get_all_groups() + return { + "tabs": [serialize_tab(t, include_values) for t in tabs], + "groups": [serialize_group(g) for g in groups], + } + + +def execute_action(tab_name: str, action_key: str, current_values: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """ + Execute an action button's callback. + + Args: + tab_name: The settings tab name. + action_key: The action key to execute. + current_values: Optional dict of current form values (unsaved). + Passed to callbacks that accept it. + + Returns: + Dict with "success" (bool) and "message" (str). + """ + import inspect + + tab = get_settings_tab(tab_name) + if not tab: + return {"success": False, "message": f"Unknown settings tab: {tab_name}"} + + for field in tab.fields: + if isinstance(field, ActionButton) and field.key == action_key: + if field.callback: + try: + # Check if callback accepts current_values parameter + sig = inspect.signature(field.callback) + if 'current_values' in sig.parameters: + return field.callback(current_values=current_values or {}) + else: + return field.callback() + except Exception as e: + logger.error(f"Action {action_key} failed: {e}") + return {"success": False, "message": str(e)} + else: + return {"success": False, "message": "Action has no callback defined"} + + return {"success": False, "message": f"Unknown action: {action_key}"} + + +def _sync_metadata_provider_selection() -> None: + """ + Sync the METADATA_PROVIDER setting based on enabled providers. + + Called after saving metadata provider settings to auto-select + the first enabled provider if the current selection is invalid. + """ + try: + from cwa_book_downloader.metadata_providers import sync_metadata_provider_selection + sync_metadata_provider_selection() + except ImportError: + pass # Metadata providers module not available + + +def _apply_dns_settings(config) -> None: + """ + Apply DNS settings changes to the network module. + + This ensures DNS changes take effect immediately without requiring + a container restart. + """ + try: + from cwa_book_downloader.download import network + + provider = config.get("CUSTOM_DNS", "auto") + use_doh = config.get("USE_DOH", False) + manual_servers = None + + if provider == "manual": + manual_dns = config.get("CUSTOM_DNS_MANUAL", "") + if manual_dns: + # Parse comma-separated server list + manual_servers = [s.strip() for s in manual_dns.split(",") if s.strip()] + + network.set_dns_provider(provider, manual_servers, use_doh=use_doh) + except ImportError: + pass # Network module not available + except Exception as e: + logger.warning(f"Failed to apply DNS settings: {e}") + + +def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]: + """ + Update settings for a tab. + + Only updates values that are not set via environment variables. + + Args: + tab_name: The settings tab name. + values: Dict of key -> value to update. + + Returns: + Dict with "success" (bool), "message" (str), "updated" (list of keys), + and "requiresRestart" (bool) indicating if any changed setting requires restart. + """ + tab = get_settings_tab(tab_name) + if not tab: + return {"success": False, "message": f"Unknown settings tab: {tab_name}", "updated": [], "requiresRestart": False} + + # Build a map of field keys to fields (exclude non-value fields) + field_map = {f.key: f for f in tab.fields if not isinstance(f, (ActionButton, HeadingField))} + + # Filter out values that are set via env vars or unknown + values_to_save = {} + skipped_env = [] + skipped_unknown = [] + restart_required_keys = [] + + for key, value in values.items(): + if key not in field_map: + skipped_unknown.append(key) + continue + + field = field_map[key] + if is_value_from_env(field): + skipped_env.append(key) + continue + + # Handle password fields - only update if a new value is provided + if isinstance(field, PasswordField) and not value: + continue + + values_to_save[key] = value + + # Track if this field requires restart + if getattr(field, 'requires_restart', False): + restart_required_keys.append(key) + + if not values_to_save: + message = "No settings to update" + if skipped_env: + message += f". Skipped (set via env): {', '.join(skipped_env)}" + return {"success": True, "message": message, "updated": [], "requiresRestart": False} + + # Call on_save handler if registered (for custom validation/transformation) + on_save_handler = get_on_save_handler(tab_name) + if on_save_handler: + try: + result = on_save_handler(values_to_save.copy()) + if result.get("error"): + return { + "success": False, + "message": result.get("message", "Validation failed"), + "updated": [], + "requiresRestart": False + } + # Use the transformed values + values_to_save = result.get("values", values_to_save) + except Exception as e: + logger.error(f"on_save handler for {tab_name} failed: {e}") + return { + "success": False, + "message": f"Save handler error: {str(e)}", + "updated": [], + "requiresRestart": False + } + + # Save to config file + if save_config_file(tab_name, values_to_save): + # Refresh the config singleton so live settings take effect immediately + try: + from cwa_book_downloader.core.config import config + config.refresh() + except ImportError: + pass # Config module not yet available during initial setup + + # Apply DNS settings changes live (network tab) + dns_keys = {"CUSTOM_DNS", "CUSTOM_DNS_MANUAL", "USE_DOH"} + if tab_name == "network" and dns_keys.intersection(values_to_save.keys()): + _apply_dns_settings(config) + + # Sync metadata provider selection when a provider's enabled state changes + tab = get_settings_tab(tab_name) + if tab and tab.group == "metadata_providers": + _sync_metadata_provider_selection() + + message = f"Updated {len(values_to_save)} setting(s)" + if skipped_env: + message += f". Skipped (set via env): {', '.join(skipped_env)}" + + requires_restart = len(restart_required_keys) > 0 + return { + "success": True, + "message": message, + "updated": list(values_to_save.keys()), + "requiresRestart": requires_restart, + "restartRequiredFor": restart_required_keys, + } + else: + return {"success": False, "message": "Failed to save settings", "updated": [], "requiresRestart": False} diff --git a/cwa_book_downloader/download/__init__.py b/cwa_book_downloader/download/__init__.py new file mode 100644 index 00000000..d9f13db1 --- /dev/null +++ b/cwa_book_downloader/download/__init__.py @@ -0,0 +1 @@ +"""Download module - HTTP downloads, network, and orchestration.""" diff --git a/downloader.py b/cwa_book_downloader/download/http.py similarity index 91% rename from downloader.py rename to cwa_book_downloader/download/http.py index 3e336e96..fc1a837b 100644 --- a/downloader.py +++ b/cwa_book_downloader/download/http.py @@ -1,4 +1,4 @@ -"""Network operations manager for the book downloader application.""" +"""HTTP download with retry, resume, and Cloudflare bypass support.""" import random import time @@ -10,19 +10,19 @@ from urllib.parse import urlparse import requests from tqdm import tqdm -import network -from config import PROXIES -from env import DEFAULT_SLEEP, MAX_RETRY, USE_CF_BYPASS, USING_EXTERNAL_BYPASSER -from logger import setup_logger +from cwa_book_downloader.download import network +from cwa_book_downloader.config.env import USE_CF_BYPASS, USING_EXTERNAL_BYPASSER +from cwa_book_downloader.core.config import config as app_config +from cwa_book_downloader.core.logger import setup_logger # Import bypasser if enabled if USE_CF_BYPASS: if USING_EXTERNAL_BYPASSER: - from cloudflare_bypasser_external import get_bypassed_page + from cwa_book_downloader.bypass.external_bypasser import get_bypassed_page # External bypasser doesn't share cookies get_cf_cookies_for_domain = lambda domain: {} else: - from cloudflare_bypasser import get_bypassed_page, get_cf_cookies_for_domain + from cwa_book_downloader.bypass.internal_bypasser import get_bypassed_page, get_cf_cookies_for_domain logger = setup_logger(__name__) @@ -30,6 +30,20 @@ logger = setup_logger(__name__) REQUEST_TIMEOUT = (5, 10) # (connect, read) MAX_DOWNLOAD_RETRIES = 2 MAX_RESUME_ATTEMPTS = 3 + + +def _get_proxies() -> dict: + """Get current proxy configuration from config singleton.""" + proxies = {} + http_proxy = app_config.get("HTTP_PROXY", "") + https_proxy = app_config.get("HTTPS_PROXY", "") + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + return proxies + + RETRYABLE_CODES = (429, 500, 502, 503, 504) CONNECTION_ERRORS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.SSLError, requests.exceptions.ChunkedEncodingError) @@ -89,12 +103,13 @@ def _try_rotation(original_url: str, current_url: str, selector: network.AAMirro def html_get_page( url: str, - retry: int = MAX_RETRY, + retry: Optional[int] = None, use_bypasser: bool = False, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None, ) -> str: """Fetch HTML content from a URL with retry mechanism.""" + retry = retry if retry is not None else app_config.MAX_RETRY selector = selector or network.AAMirrorSelector() original_url = url current_url = selector.rewrite(original_url) @@ -122,7 +137,7 @@ def html_get_page( if USE_CF_BYPASS: parsed = urlparse(current_url) cookies = get_cf_cookies_for_domain(parsed.hostname or "") - response = requests.get(current_url, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies) + response = requests.get(current_url, proxies=_get_proxies(), timeout=REQUEST_TIMEOUT, cookies=cookies) response.raise_for_status() time.sleep(1) return response.text @@ -209,7 +224,7 @@ def download_url( cookies = get_cf_cookies_for_domain(parsed.hostname or "") if cookies: logger.debug(f"Using {len(cookies)} cookies for {parsed.hostname}: {list(cookies.keys())}") - response = requests.get(current_url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers) + response = requests.get(current_url, stream=True, proxies=_get_proxies(), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers) response.raise_for_status() if status_callback: @@ -308,7 +323,7 @@ def _try_resume( cookies = get_cf_cookies_for_domain(parsed.hostname or "") resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'} response = requests.get( - url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT, + url, stream=True, proxies=_get_proxies(), timeout=REQUEST_TIMEOUT, headers=resume_headers, cookies=cookies ) diff --git a/network.py b/cwa_book_downloader/download/network.py similarity index 76% rename from network.py rename to cwa_book_downloader/download/network.py index 44f01c63..37956c97 100644 --- a/network.py +++ b/cwa_book_downloader/download/network.py @@ -1,5 +1,6 @@ -"""Network operations manager for the book downloader application.""" +"""DNS rotation, mirror selection, and network utilities.""" +import os import requests import urllib.request from typing import Sequence, Tuple, Any, Union, cast, List, Optional, Callable @@ -10,12 +11,29 @@ import urllib.parse import ssl import ipaddress -from logger import setup_logger -from config import PROXIES, AA_BASE_URL, CUSTOM_DNS, AA_AVAILABLE_URLS, DOH_SERVER -import config -import env +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.config.settings import AA_BASE_URL, AA_AVAILABLE_URLS +from cwa_book_downloader.config import settings as config +from cwa_book_downloader.core.config import config as app_config from datetime import datetime, timedelta + +def _get_proxies() -> dict: + """Get current proxy configuration from config singleton.""" + proxies = {} + http_proxy = app_config.get("HTTP_PROXY", "") + https_proxy = app_config.get("HTTPS_PROXY", "") + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + return proxies + +# DNS state - authoritative values managed by this module +# Other modules should use get_dns_config() to read these +CUSTOM_DNS: List[str] = [] +DOH_SERVER: str = "" + # Try to use gevent locks if available (for gevent worker compatibility) # Fall back to threading locks for non-gevent environments try: @@ -143,7 +161,9 @@ _dns_exhausted_logged = False def _is_auto_dns_mode() -> bool: """Check if DNS is in auto-rotation mode.""" - return env._CUSTOM_DNS.lower().strip() == "auto" and not env.USING_TOR + custom_dns = app_config.get("CUSTOM_DNS", "auto") + using_tor = app_config.get("USING_TOR", False) + return str(custom_dns).lower().strip() == "auto" and not using_tor def _current_dns_label() -> str: @@ -151,9 +171,44 @@ def _current_dns_label() -> str: if _current_dns_index >= 0: return DNS_PROVIDERS[_current_dns_index][0] if CUSTOM_DNS: - return f"custom {CUSTOM_DNS}" + return f"manual ({len(CUSTOM_DNS)} servers)" return "system" + +def get_dns_config() -> dict: + """ + Get the current DNS configuration. + + Returns: + Dict with keys: + - provider: str - Current provider name ('auto', 'system', 'google', 'cloudflare', etc.) + - servers: List[str] - DNS server IPs in use + - doh_url: str - DoH server URL (empty if disabled) + - doh_enabled: bool - Whether DoH is active + - is_auto_mode: bool - Whether auto-rotation is enabled + """ + _ensure_initialized() + + custom_dns = str(app_config.get("CUSTOM_DNS", "auto")).lower().strip() + if _current_dns_index >= 0: + provider = DNS_PROVIDERS[_current_dns_index][0] + elif custom_dns == "auto": + provider = "auto" + elif custom_dns == "system": + provider = "system" + elif custom_dns == "manual": + provider = "manual" + else: + provider = custom_dns + + return { + "provider": provider, + "servers": list(CUSTOM_DNS), + "doh_url": DOH_SERVER, + "doh_enabled": bool(DOH_SERVER), + "is_auto_mode": _is_auto_dns_mode(), + } + # Common helper functions for DNS resolution def _decode_host(host: Union[str, bytes, None]) -> str: """Convert host to string, handling bytes and None cases.""" @@ -300,7 +355,7 @@ class DoHResolver: response = self.session.get( self.base_url, params=params, - proxies=PROXIES, + proxies=_get_proxies(), timeout=10 # Increased from 5s to handle slow network conditions ) response.raise_for_status() @@ -621,6 +676,90 @@ def rotate_dns_and_reset_aa() -> bool: _save_state(aa_url=AA_BASE_URL) return True +def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use_doh: bool | None = None) -> bool: + """ + Set DNS to a specific provider or manual servers. + + Args: + provider: One of 'auto', 'system', 'google', 'cloudflare', 'quad9', 'opendns', 'manual' + manual_servers: List of DNS server IPs when provider is 'manual' + use_doh: Whether to use DNS over HTTPS. If None, uses current USE_DOH config setting. + Note: Auto mode always uses DoH for reliability during rotation. + + Returns: + True if DNS was changed successfully. + """ + global CUSTOM_DNS, DOH_SERVER, _current_dns_index, _dns_exhausted_logged + + provider = provider.lower().strip() + + # Determine DoH preference - use provided value or fall back to config setting + doh_enabled = use_doh if use_doh is not None else app_config.get("USE_DOH", True) + + with _dns_switch_lock: + if provider == "system": + # Use system DNS only - no custom resolver, no failover rotation + _current_dns_index = -1 + _dns_exhausted_logged = False + CUSTOM_DNS = [] + DOH_SERVER = "" + config.CUSTOM_DNS = [] + config.DOH_SERVER = "" + # Restore original system getaddrinfo + socket.getaddrinfo = original_getaddrinfo + logger.info("DNS set to system mode (using OS default resolver)") + _notify_dns_rotation("system", [], "") + return True + + if provider == "auto": + # Reset to auto mode - start with system DNS + # Note: Auto mode always uses DoH when rotating for reliability + _current_dns_index = -1 + _dns_exhausted_logged = False + CUSTOM_DNS = [] + DOH_SERVER = "" + config.CUSTOM_DNS = [] + config.DOH_SERVER = "" + logger.info("DNS set to auto mode (system DNS, will rotate on failure with DoH)") + init_dns_resolvers() + _notify_dns_rotation("auto", [], "") + return True + + if provider == "manual": + if not manual_servers: + logger.warning("Manual DNS requested but no servers provided") + return False + _current_dns_index = -1 # Not using preset providers + CUSTOM_DNS = manual_servers + DOH_SERVER = "" # No DoH for manual servers + config.CUSTOM_DNS = manual_servers + config.DOH_SERVER = "" + logger.info(f"DNS set to manual servers: {manual_servers}") + init_dns_resolvers() + _notify_dns_rotation("manual", manual_servers, "") + return True + + # Find the provider in DNS_PROVIDERS + for i, (name, servers, doh) in enumerate(DNS_PROVIDERS): + if name == provider: + _current_dns_index = i + _dns_exhausted_logged = False + CUSTOM_DNS = servers + # Only set DoH server if DoH is enabled + DOH_SERVER = doh if doh_enabled else "" + config.CUSTOM_DNS = servers + config.DOH_SERVER = DOH_SERVER + doh_status = "DoH enabled" if doh_enabled else "standard DNS" + logger.info(f"DNS set to: {name} ({doh_status})") + _save_state(dns_provider=name) + init_dns_resolvers() + _notify_dns_rotation(name, servers, DOH_SERVER) + return True + + logger.warning(f"Unknown DNS provider: {provider}") + return False + + def init_dns_resolvers(): """Initialize DNS resolvers based on configuration.""" global CUSTOM_DNS, DOH_SERVER @@ -648,19 +787,43 @@ def init_dns_resolvers(): init_doh_resolver(DOH_SERVER) -def _initialize_dns_state() -> None: - """Restore persisted DNS choice or start fresh.""" - global _current_dns_index - - if _is_auto_dns_mode(): - persisted = state.get('dns_provider') if state else None - if persisted: - for i, (name, _, _) in enumerate(DNS_PROVIDERS): - if name == persisted: - _current_dns_index = i - logger.info(f"Restored DNS provider from state: {name}") - return - _current_dns_index = -1 +def _get_initial_dns_config() -> tuple[str, List[str] | None, bool]: + """ + Determine initial DNS configuration from config singleton. + + The config singleton already handles ENV > config file > default priority, + so we just read from config. + + Returns: + Tuple of (provider, manual_servers, use_doh) + """ + provider = str(app_config.get("CUSTOM_DNS", "auto")).lower().strip() + use_doh = app_config.get("USE_DOH", True) + manual_servers = None + + # Check for manual DNS servers in config + if provider == "manual": + manual_dns = str(app_config.get("CUSTOM_DNS_MANUAL", "")).strip() + if manual_dns: + manual_servers = [s.strip() for s in manual_dns.split(",") if s.strip()] + + # Handle legacy format: IPs directly in CUSTOM_DNS setting + if provider and provider not in ("auto", "system", "google", "cloudflare", "quad9", "opendns", "manual", ""): + # Check if it looks like IP addresses + parts = provider.split(",") + potential_ips = [p.strip() for p in parts if p.strip()] + if potential_ips and all(_looks_like_ip(p) for p in potential_ips): + manual_servers = potential_ips + provider = "manual" + logger.info(f"Detected legacy DNS format, treating as manual: {manual_servers}") + + return provider or "auto", manual_servers, use_doh + + +def _looks_like_ip(s: str) -> bool: + """Check if a string looks like an IP address.""" + # Simple heuristic: contains only digits, dots, and colons + return s.replace(".", "").replace(":", "").isdigit() def _initialize_aa_state() -> None: """Restore or probe AA URL state.""" @@ -673,7 +836,7 @@ def _initialize_aa_state() -> None: logger.info(f"AA_BASE_URL: auto, checking available urls {_aa_urls}") for i, url in enumerate(_aa_urls): try: - response = requests.get(url, proxies=PROXIES, timeout=3) + response = requests.get(url, proxies=_get_proxies(), timeout=3) if response.status_code == 200: _current_aa_url_index = i AA_BASE_URL = url @@ -693,23 +856,41 @@ def _initialize_aa_state() -> None: logger.info(f"AA_BASE_URL: {AA_BASE_URL}") def init_dns(force: bool = False) -> None: - """Initialize DNS state and resolvers.""" - global state, _dns_initialized + """Initialize DNS state and resolvers using set_dns_provider() for consistency.""" + global state, _dns_initialized, _current_dns_index if _dns_initialized and not force: return with _init_lock: # Double-check after acquiring lock if _dns_initialized and not force: return - # Set flag BEFORE doing work to prevent recursive calls during init - _dns_initialized = True + # Do work first, set flag after to prevent race conditions try: logger.debug(f"Initializing DNS (using {'gevent' if _using_gevent_locks else 'threading'} locks)") state = _load_state() - _initialize_dns_state() - init_dns_resolvers() + + # Get initial DNS configuration from environment + provider, manual_servers, use_doh = _get_initial_dns_config() + + if provider == "auto": + # Auto mode: check for persisted provider from previous rotation + persisted = state.get('dns_provider') if state else None + if persisted: + for i, (name, _, _) in enumerate(DNS_PROVIDERS): + if name == persisted: + _current_dns_index = i + logger.info(f"Restored DNS provider from state: {name}") + break + # Use init_dns_resolvers() for auto mode to preserve rotation capability + init_dns_resolvers() + else: + # Non-auto mode: use set_dns_provider() for consistent initialization + set_dns_provider(provider, manual_servers, use_doh=use_doh) + + # Only set flag AFTER work completes successfully + _dns_initialized = True except Exception: - _dns_initialized = False + # Flag stays False so retry is possible raise def init_aa(force: bool = False) -> None: @@ -721,13 +902,14 @@ def init_aa(force: bool = False) -> None: # Double-check after acquiring lock if _aa_initialized and not force: return - # Set flag BEFORE doing work to prevent recursive calls during init - _aa_initialized = True + # Do work first, set flag after to prevent race conditions try: state = _load_state() _initialize_aa_state() + # Only set flag AFTER work completes successfully + _aa_initialized = True except Exception: - _aa_initialized = False + # Flag stays False so retry is possible raise def init(force: bool = False) -> None: @@ -744,15 +926,15 @@ def init(force: bool = False) -> None: # Double-check after acquiring lock if _initialized and not force: return - # Set flag BEFORE doing work to prevent recursive calls during init - # (e.g., DNS failover handlers calling back into init) - _initialized = True + # Do the work first, then set flag to prevent race conditions + # where another thread sees _initialized=True but AA_BASE_URL is still "auto" try: init_dns(force=force) init_aa(force=force) + # Only set flag AFTER work completes successfully + _initialized = True except Exception: - # Reset flag on failure so retry is possible - _initialized = False + # Flag stays False so retry is possible raise def get_aa_base_url(): diff --git a/cwa_book_downloader/download/orchestrator.py b/cwa_book_downloader/download/orchestrator.py new file mode 100644 index 00000000..9ac6125b --- /dev/null +++ b/cwa_book_downloader/download/orchestrator.py @@ -0,0 +1,591 @@ +"""Download queue orchestration and worker management.""" + +import os +import random +import threading +import time +from concurrent.futures import Future, ThreadPoolExecutor +from threading import Event, Lock +from typing import Any, Dict, List, Optional, Tuple + +from cwa_book_downloader.release_sources import direct_download +from cwa_book_downloader.release_sources.direct_download import SearchUnavailable +from cwa_book_downloader.core.config import config +from cwa_book_downloader.release_sources import get_handler, get_source_display_name +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.core.models import BookInfo, DownloadTask, QueueStatus, SearchFilters +from cwa_book_downloader.core.queue import book_queue + +logger = setup_logger(__name__) + +# WebSocket manager (initialized by app.py) +try: + from cwa_book_downloader.api.websocket import ws_manager +except ImportError: + ws_manager = None + +# Progress update throttling - track last broadcast time per book +_progress_last_broadcast: Dict[str, float] = {} +_progress_lock = Lock() + +# Stall detection - track last activity time per download +_last_activity: Dict[str, float] = {} +STALL_TIMEOUT = 300 # 5 minutes without progress/status update = stalled + +def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]: + """Search for books matching the query. + + Args: + query: Search term + filters: Search filters object + + Returns: + List[Dict]: List of book information dictionaries + """ + try: + books = direct_download.search_books(query, filters) + return [_book_info_to_dict(book) for book in books] + except SearchUnavailable: + raise + except Exception as e: + logger.error_trace(f"Error searching books: {e}") + raise + +def get_book_info(book_id: str) -> Optional[Dict[str, Any]]: + """Get detailed information for a specific book. + + Args: + book_id: Book identifier + + Returns: + Optional[Dict]: Book information dictionary if found, None if not found + + Raises: + Exception: If there's an error fetching the book info + """ + try: + book = direct_download.get_book_info(book_id) + return _book_info_to_dict(book) + except Exception as e: + logger.error_trace(f"Error getting book info: {e}") + raise + +def queue_book(book_id: str, priority: int = 0, source: str = "direct_download") -> bool: + """Add a book to the download queue with specified priority. + + Fetches display info and creates a DownloadTask. The handler will fetch + the full book details (including download URLs) when processing. + + Args: + book_id: Book identifier (e.g., AA MD5 hash) + priority: Priority level (lower number = higher priority) + source: Release source handler to use (default: direct_download) + + Returns: + bool: True if book was successfully queued + """ + try: + # Fetch book info for display purposes + book_info = direct_download.get_book_info(book_id) + if not book_info: + logger.warning(f"Could not fetch book info for {book_id}") + return False + + # Create a source-agnostic download task + task = DownloadTask( + task_id=book_id, + source=source, + title=book_info.title, + author=book_info.author, + format=book_info.format, + size=book_info.size, + preview=book_info.preview, + priority=priority, + ) + + if not book_queue.add(task): + logger.info(f"Book already in queue: {book_info.title}") + return False + + logger.info(f"Book queued with priority {priority}: {book_info.title}") + + # Broadcast status update via WebSocket + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) + + return True + except Exception as e: + logger.error_trace(f"Error queueing book: {e}") + return False + + +def queue_release(release_data: dict, priority: int = 0) -> bool: + """Add a release to the download queue. + + This is used when downloading from the ReleaseModal where we already have + all the release data from the search - no need to re-fetch. + + Creates a DownloadTask directly from the release data. The handler will + fetch full details when processing. + + Args: + release_data: Release dictionary with source, source_id, title, format, etc. + priority: Priority level (lower number = higher priority) + + Returns: + bool: True if release was successfully queued + """ + try: + source = release_data.get('source', 'direct_download') + extra = release_data.get('extra', {}) + + # Get author and preview from top-level (preferred) or extra (fallback) + author = release_data.get('author') or extra.get('author') + preview = release_data.get('preview') or extra.get('preview') + + # Create a source-agnostic download task from release data + task = DownloadTask( + task_id=release_data['source_id'], + source=source, + title=release_data.get('title', 'Unknown'), + author=author, + format=release_data.get('format'), + size=release_data.get('size'), + preview=preview, + priority=priority, + ) + + if not book_queue.add(task): + logger.info(f"Release already in queue: {task.title}") + return False + + logger.info(f"Release queued with priority {priority}: {task.title}") + + # Broadcast status update via WebSocket + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) + + return True + + except ValueError as e: + # Handler not found for this source + logger.warning(f"Unknown release source: {e}") + return False + except Exception as e: + logger.error_trace(f"Error queueing release: {e}") + return False + +def queue_status() -> Dict[str, Dict[str, Any]]: + """Get current status of the download queue. + + Returns: + Dict: Queue status organized by status type with serialized task data + """ + status = book_queue.get_status() + for _, tasks in status.items(): + for _, task in tasks.items(): + if task.download_path: + if not os.path.exists(task.download_path): + task.download_path = None + + # Convert Enum keys to strings and DownloadTask objects to dicts for JSON serialization + return { + status_type.value: { + task_id: _task_to_dict(task) + for task_id, task in tasks.items() + } + for status_type, tasks in status.items() + } + +def get_book_data(task_id: str) -> Tuple[Optional[bytes], Optional[DownloadTask]]: + """Get downloaded file data for a specific task. + + Args: + task_id: Task identifier + + Returns: + Tuple[Optional[bytes], Optional[DownloadTask]]: File data if available, and the task + """ + task = None + try: + task = book_queue.get_task(task_id) + if not task: + return None, None + + path = task.download_path + if not path: + return None, task + + with open(path, "rb") as f: + return f.read(), task + except Exception as e: + logger.error_trace(f"Error getting book data: {e}") + if task: + task.download_path = None + return None, task + +def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]: + """Convert BookInfo object to dictionary representation. + + Transforms external preview URLs to local proxy URLs when cover caching is enabled. + """ + import base64 + from cwa_book_downloader.config.env import is_covers_cache_enabled + + result = { + key: value for key, value in book.__dict__.items() + if value is not None + } + + # Transform external preview URLs to local proxy URLs + # Skip if already a local URL (starts with /) + if result.get('preview') and is_covers_cache_enabled() and not result['preview'].startswith('/'): + original_url = result['preview'] + encoded_url = base64.urlsafe_b64encode(original_url.encode()).decode() + result['preview'] = f"/api/covers/{book.id}?url={encoded_url}" + + return result + + +def _task_to_dict(task: DownloadTask) -> Dict[str, Any]: + """Convert DownloadTask object to dictionary representation. + + Maps DownloadTask fields to the format expected by the frontend, + maintaining compatibility with the previous BookInfo-based format. + Transforms external preview URLs to local proxy URLs when cover caching is enabled. + """ + import base64 + from cwa_book_downloader.config.env import is_covers_cache_enabled + + preview = task.preview + + # Transform external preview URLs to local proxy URLs + # Skip if already a local URL (starts with /) + if preview and is_covers_cache_enabled() and not preview.startswith('/'): + encoded_url = base64.urlsafe_b64encode(preview.encode()).decode() + preview = f"/api/covers/{task.task_id}?url={encoded_url}" + + return { + 'id': task.task_id, + 'title': task.title, + 'author': task.author, + 'format': task.format, + 'size': task.size, + 'preview': preview, + 'source': task.source, + 'source_display_name': get_source_display_name(task.source), + 'priority': task.priority, + 'added_time': task.added_time, + 'progress': task.progress, + 'status': task.status, + 'status_message': task.status_message, + 'download_path': task.download_path, + } + + +def _download_task(task_id: str, cancel_flag: Event) -> Optional[str]: + """Download a task with cancellation support. + + Delegates to the appropriate handler based on the task's source. + Each handler encapsulates all download and post-processing logic. + + Args: + task_id: Task identifier + cancel_flag: Threading event to signal cancellation + + Returns: + str: Path to the downloaded file if successful, None otherwise + """ + try: + # Check for cancellation before starting + if cancel_flag.is_set(): + logger.info(f"Download cancelled before starting: {task_id}") + return None + + task = book_queue.get_task(task_id) + if not task: + logger.error(f"Task not found in queue: {task_id}") + return None + + # Create callbacks that update the orchestrator's tracking + progress_callback = lambda progress: update_download_progress(task_id, progress) + status_callback = lambda status, message=None: update_download_status(task_id, status, message) + + # Get the download handler based on the task's source + handler = get_handler(task.source) + return handler.download( + task, + cancel_flag, + progress_callback, + status_callback + ) + + except Exception as e: + if cancel_flag.is_set(): + logger.info(f"Download cancelled during error handling: {task_id}") + else: + logger.error_trace(f"Error downloading: {e}") + return None + +def update_download_progress(book_id: str, progress: float) -> None: + """Update download progress with throttled WebSocket broadcasts. + + Progress is always stored in the queue, but WebSocket broadcasts are + throttled to avoid flooding clients with updates. Broadcasts occur: + - At most once per DOWNLOAD_PROGRESS_UPDATE_INTERVAL seconds + - Always at 0% (start) and 100% (complete) + - On significant progress jumps (>10%) + """ + book_queue.update_progress(book_id, progress) + + # Track activity for stall detection + with _progress_lock: + _last_activity[book_id] = time.time() + + # Broadcast progress via WebSocket with throttling + if ws_manager: + current_time = time.time() + should_broadcast = False + + with _progress_lock: + last_broadcast = _progress_last_broadcast.get(book_id, 0) + last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0) + time_elapsed = current_time - last_broadcast + + # Always broadcast at start (0%) or completion (>=99%) + if progress <= 1 or progress >= 99: + should_broadcast = True + # Broadcast if enough time has passed (convert interval from seconds) + elif time_elapsed >= config.DOWNLOAD_PROGRESS_UPDATE_INTERVAL: + should_broadcast = True + # Broadcast on significant progress jumps (>10%) + elif progress - last_progress >= 10: + should_broadcast = True + + if should_broadcast: + _progress_last_broadcast[book_id] = current_time + _progress_last_broadcast[f"{book_id}_progress"] = progress + + if should_broadcast: + ws_manager.broadcast_download_progress(book_id, progress, 'downloading') + +def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None: + """Update download status with optional detailed message. + + Args: + book_id: Book identifier + status: Status string (e.g., 'resolving', 'downloading') + message: Optional detailed status message for UI display + """ + # Map string status to QueueStatus enum + status_map = { + 'queued': QueueStatus.QUEUED, + 'resolving': QueueStatus.RESOLVING, + 'downloading': QueueStatus.DOWNLOADING, + 'complete': QueueStatus.COMPLETE, + 'available': QueueStatus.AVAILABLE, + 'error': QueueStatus.ERROR, + 'done': QueueStatus.DONE, + 'cancelled': QueueStatus.CANCELLED, + } + + queue_status_enum = status_map.get(status.lower()) + if queue_status_enum: + book_queue.update_status(book_id, queue_status_enum) + + # Track activity for stall detection + with _progress_lock: + _last_activity[book_id] = time.time() + + # Update status message if provided (empty string clears the message) + if message is not None: + book_queue.update_status_message(book_id, message) + + # Broadcast status update via WebSocket + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) + +def cancel_download(book_id: str) -> bool: + """Cancel a download. + + Args: + book_id: Book identifier to cancel + + Returns: + bool: True if cancellation was successful + """ + result = book_queue.cancel_download(book_id) + + # Broadcast status update via WebSocket + if result and ws_manager and ws_manager.is_enabled(): + ws_manager.broadcast_status_update(queue_status()) + + return result + +def set_book_priority(book_id: str, priority: int) -> bool: + """Set priority for a queued book. + + Args: + book_id: Book identifier + priority: New priority level (lower = higher priority) + + Returns: + bool: True if priority was successfully changed + """ + return book_queue.set_priority(book_id, priority) + +def reorder_queue(book_priorities: Dict[str, int]) -> bool: + """Bulk reorder queue. + + Args: + book_priorities: Dict mapping book_id to new priority + + Returns: + bool: True if reordering was successful + """ + return book_queue.reorder_queue(book_priorities) + +def get_queue_order() -> List[Dict[str, any]]: + """Get current queue order for display.""" + return book_queue.get_queue_order() + +def get_active_downloads() -> List[str]: + """Get list of currently active downloads.""" + return book_queue.get_active_downloads() + +def clear_completed() -> int: + """Clear all completed downloads from tracking.""" + return book_queue.clear_completed() + +def _cleanup_progress_tracking(task_id: str) -> None: + """Clean up progress tracking data for a completed/cancelled download.""" + with _progress_lock: + _progress_last_broadcast.pop(task_id, None) + _progress_last_broadcast.pop(f"{task_id}_progress", None) + _last_activity.pop(task_id, None) + + +def _process_single_download(task_id: str, cancel_flag: Event) -> None: + """Process a single download job.""" + try: + # Status will be updated through callbacks during download process + # (resolving -> downloading -> complete) + download_path = _download_task(task_id, cancel_flag) + + # Clean up progress tracking + _cleanup_progress_tracking(task_id) + + if cancel_flag.is_set(): + book_queue.update_status(task_id, QueueStatus.CANCELLED) + # Broadcast cancellation + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) + return + + if download_path: + book_queue.update_download_path(task_id, download_path) + new_status = QueueStatus.COMPLETE + else: + new_status = QueueStatus.ERROR + + book_queue.update_status(task_id, new_status) + + # Broadcast final status (completed or error) + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) + + except Exception as e: + # Clean up progress tracking even on error + _cleanup_progress_tracking(task_id) + + if not cancel_flag.is_set(): + logger.error_trace(f"Error in download processing: {e}") + book_queue.update_status(task_id, QueueStatus.ERROR) + # Set error message if not already set by handler + task = book_queue.get_task(task_id) + if task and not task.status_message: + book_queue.update_status_message(task_id, f"Download failed: {type(e).__name__}: {str(e)}") + else: + logger.info(f"Download cancelled: {task_id}") + book_queue.update_status(task_id, QueueStatus.CANCELLED) + + # Broadcast error/cancelled status + if ws_manager: + ws_manager.broadcast_status_update(queue_status()) + +def concurrent_download_loop() -> None: + """Main download coordinator using ThreadPoolExecutor for concurrent downloads.""" + max_workers = config.MAX_CONCURRENT_DOWNLOADS + logger.info(f"Starting concurrent download loop with {max_workers} workers") + + with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="Download") as executor: + active_futures: Dict[Future, str] = {} # Track active download futures + + while True: + # Clean up completed futures + completed_futures = [f for f in active_futures if f.done()] + for future in completed_futures: + task_id = active_futures.pop(future) + try: + future.result() # This will raise any exceptions from the worker + except Exception as e: + logger.error_trace(f"Future exception for {task_id}: {e}") + + # Check for stalled downloads (no activity in STALL_TIMEOUT seconds) + current_time = time.time() + with _progress_lock: + for future, task_id in list(active_futures.items()): + last_active = _last_activity.get(task_id, current_time) + if current_time - last_active > STALL_TIMEOUT: + logger.warning(f"Download stalled for {task_id}, cancelling") + book_queue.cancel_download(task_id) + book_queue.update_status_message(task_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)") + + # Start new downloads if we have capacity + while len(active_futures) < max_workers: + next_download = book_queue.get_next() + if not next_download: + break + + # Stagger concurrent downloads to avoid rate limiting on shared download servers + # Only delay if other downloads are already active + if active_futures: + stagger_delay = random.uniform(2, 5) + logger.debug(f"Staggering download start by {stagger_delay:.1f}s") + time.sleep(stagger_delay) + + task_id, cancel_flag = next_download + + # Submit download job to thread pool + future = executor.submit(_process_single_download, task_id, cancel_flag) + active_futures[future] = task_id + + # Brief sleep to prevent busy waiting + time.sleep(config.MAIN_LOOP_SLEEP_TIME) + +# Download coordinator thread (started explicitly via start()) +_coordinator_thread: Optional[threading.Thread] = None +_started = False + + +def start() -> None: + """Start the download coordinator thread. + + This should be called once during application startup. + Calling multiple times is safe - subsequent calls are no-ops. + """ + global _coordinator_thread, _started + + if _started: + logger.debug("Download coordinator already started") + return + + _coordinator_thread = threading.Thread( + target=concurrent_download_loop, + daemon=True, + name="DownloadCoordinator" + ) + _coordinator_thread.start() + _started = True + + logger.info(f"Download coordinator started with {config.MAX_CONCURRENT_DOWNLOADS} concurrent workers") diff --git a/cwa_book_downloader/download_clients/__init__.py b/cwa_book_downloader/download_clients/__init__.py new file mode 100644 index 00000000..21e23c7d --- /dev/null +++ b/cwa_book_downloader/download_clients/__init__.py @@ -0,0 +1,55 @@ +"""External download client integrations (qBittorrent, SABnzbd, etc.).""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List, Optional, Tuple +from enum import Enum + + +class DownloadStatus(Enum): + """Status of a download in an external client.""" + QUEUED = "queued" + DOWNLOADING = "downloading" + PAUSED = "paused" + COMPLETED = "completed" + FAILED = "failed" + SEEDING = "seeding" # Torrents only + + +@dataclass +class ClientDownloadProgress: + """Progress info from external download client.""" + status: DownloadStatus + progress: float # 0-100 + download_speed: Optional[int] # bytes/sec + eta: Optional[int] # seconds remaining + save_path: Optional[str] # Where the file will be/is + + +class DownloadClient(ABC): + """Abstract base class for download clients.""" + + @abstractmethod + def add_download(self, url: str, title: str) -> str: + """Add a download (torrent/magnet or NZB URL). Returns download ID for tracking.""" + pass + + @abstractmethod + def get_download(self, download_id: str) -> Optional[ClientDownloadProgress]: + """Get progress of a specific download.""" + pass + + @abstractmethod + def list_downloads(self) -> List[Tuple[str, ClientDownloadProgress]]: + """List all downloads with their progress.""" + pass + + @abstractmethod + def get_completed_path(self, download_id: str) -> Optional[str]: + """Get the path to completed download.""" + pass + + @abstractmethod + def test_connection(self) -> bool: + """Test if the client is reachable and credentials are valid.""" + pass diff --git a/cwa_book_downloader/main.py b/cwa_book_downloader/main.py new file mode 100644 index 00000000..80ad4e31 --- /dev/null +++ b/cwa_book_downloader/main.py @@ -0,0 +1,1569 @@ +"""Flask app - routes, WebSocket handlers, and middleware.""" + +import io +import logging +import os +import sqlite3 +import time +from datetime import datetime, timedelta +from functools import wraps +from typing import Any, Dict, Tuple, Union + +from flask import Flask, jsonify, request, send_file, send_from_directory, session +from flask_cors import CORS +from flask_socketio import SocketIO, emit +from werkzeug.middleware.proxy_fix import ProxyFix +from werkzeug.security import check_password_hash +from werkzeug.wrappers import Response + +from cwa_book_downloader.download import orchestrator as backend +from cwa_book_downloader.release_sources.direct_download import SearchUnavailable +from cwa_book_downloader.config.settings import _SUPPORTED_BOOK_LANGUAGE +from cwa_book_downloader.config.env import ( + BUILD_VERSION, CWA_DB_PATH, DEBUG, FLASK_HOST, FLASK_PORT, + RELEASE_VERSION, USING_EXTERNAL_BYPASSER, +) +from cwa_book_downloader.core.config import config as app_config +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.core.models import SearchFilters +from cwa_book_downloader.api.websocket import ws_manager + +logger = setup_logger(__name__) + +# Project root is one level up from this package +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +FRONTEND_DIST = os.path.join(PROJECT_ROOT, 'frontend-dist') + +app = Flask(__name__) +app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore +app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 # Disable caching +app.config['APPLICATION_ROOT'] = '/' + +# Socket.IO async mode. +# We run this app under Gunicorn with a gevent websocket worker (even when DEBUG=true), +# so Socket.IO should always use gevent here. +async_mode = 'gevent' + +# Initialize Flask-SocketIO with reverse proxy support +socketio = SocketIO( + app, + cors_allowed_origins="*", + async_mode=async_mode, + logger=False, + engineio_logger=False, + # Reverse proxy / Traefik compatibility settings + path='/socket.io', + ping_timeout=60, # Time to wait for pong response + ping_interval=25, # Send ping every 25 seconds + # Allow both websocket and polling for better compatibility + transports=['websocket', 'polling'], + # Enable CORS for all origins (you can restrict this in production) + allow_upgrades=True, + # Important for proxies that buffer + http_compression=True +) + +# Initialize WebSocket manager +ws_manager.init_app(app, socketio) +logger.info(f"Flask-SocketIO initialized with async_mode='{async_mode}'") + +# Ensure all plugins are loaded before starting the download coordinator. +# This prevents a race condition where the download loop could try to process +# a queued task before its handler (e.g., prowlarr) is registered. +try: + import cwa_book_downloader.metadata_providers # noqa: F401 + import cwa_book_downloader.release_sources # noqa: F401 + logger.debug("Plugin modules loaded successfully") +except ImportError as e: + logger.warning(f"Failed to import plugin modules: {e}") + +# Start download coordinator +backend.start() + +# Rate limiting for login attempts +# Structure: {username: {'count': int, 'lockout_until': datetime}} +failed_login_attempts: Dict[str, Dict[str, Any]] = {} +MAX_LOGIN_ATTEMPTS = 10 +LOCKOUT_DURATION_MINUTES = 30 + +def cleanup_old_lockouts() -> None: + """Remove expired lockout entries to prevent memory buildup.""" + current_time = datetime.now() + expired_users = [ + username for username, data in failed_login_attempts.items() + if 'lockout_until' in data and data['lockout_until'] < current_time + ] + for username in expired_users: + logger.info(f"Lockout expired for user: {username}") + del failed_login_attempts[username] + +def is_account_locked(username: str) -> bool: + """Check if an account is currently locked due to failed login attempts.""" + cleanup_old_lockouts() + + if username not in failed_login_attempts: + return False + + lockout_until = failed_login_attempts[username].get('lockout_until') + if lockout_until and datetime.now() < lockout_until: + return True + + return False + +def record_failed_login(username: str, ip_address: str) -> bool: + """ + Record a failed login attempt and lock account if threshold is reached. + Returns True if account is now locked, False otherwise. + """ + if username not in failed_login_attempts: + failed_login_attempts[username] = {'count': 0} + + failed_login_attempts[username]['count'] += 1 + count = failed_login_attempts[username]['count'] + + logger.warning(f"Failed login attempt {count}/{MAX_LOGIN_ATTEMPTS} for user '{username}' from IP {ip_address}") + + if count >= MAX_LOGIN_ATTEMPTS: + lockout_until = datetime.now() + timedelta(minutes=LOCKOUT_DURATION_MINUTES) + failed_login_attempts[username]['lockout_until'] = lockout_until + logger.warning(f"Account locked for user '{username}' until {lockout_until.strftime('%Y-%m-%d %H:%M:%S')} due to {count} failed login attempts") + return True + + return False + +def clear_failed_logins(username: str) -> None: + """Clear failed login attempts for a user after successful login.""" + if username in failed_login_attempts: + del failed_login_attempts[username] + logger.debug(f"Cleared failed login attempts for user: {username}") + + +def get_auth_mode() -> str: + """ + Determine which authentication mode is active. + + Priority order: + 1. Built-in credentials (if configured) -> "builtin" + 2. CWA database (if CWA_DB_PATH is set and exists) -> "cwa" + 3. No auth required -> "none" + + Returns: + str: "builtin", "cwa", or "none" + """ + from cwa_book_downloader.core.settings_registry import load_config_file + + # Check for built-in credentials first (they take priority) + try: + security_config = load_config_file("security") + username = security_config.get("BUILTIN_USERNAME") + password_hash = security_config.get("BUILTIN_PASSWORD_HASH") + if username and password_hash: + return "builtin" + except Exception: + pass # If config can't be loaded, fall through to other methods + + # Check for CWA database + if CWA_DB_PATH and os.path.isfile(CWA_DB_PATH): + return "cwa" + + # No auth configured + return "none" + + +# Enable CORS in development mode for local frontend development +if DEBUG: + CORS(app, resources={ + r"/*": { + "origins": ["http://localhost:5173", "http://127.0.0.1:5173"], + "supports_credentials": True, + "allow_headers": ["Content-Type", "Authorization"], + "methods": ["GET", "POST", "PUT", "DELETE", "OPTIONS"] + } + }) + +# Custom log filter to exclude routine status endpoint polling and WebSocket noise +class StatusEndpointFilter(logging.Filter): + """Filter out routine status endpoint requests and WebSocket upgrade errors to reduce log noise.""" + def filter(self, record): + if hasattr(record, 'getMessage'): + message = record.getMessage() + # Exclude GET /api/status requests (polling noise) + if 'GET /api/status' in message: + return False + # Exclude WebSocket upgrade errors (benign - falls back to polling) + if 'write() before start_response' in message: + return False + # Exclude the Error on request line that precedes WebSocket errors + if 'Error on request:' in message and record.levelno == logging.ERROR: + return False + return True + + +class WebSocketErrorFilter(logging.Filter): + """Filter out WebSocket upgrade errors that occur in Werkzeug dev server. + + These errors are benign - Flask-SocketIO automatically falls back to polling transport. + The error occurs because Werkzeug's built-in server doesn't fully support WebSocket upgrades. + """ + def filter(self, record): + # Filter out the AssertionError traceback for WebSocket upgrades + if record.levelno == logging.ERROR: + message = record.getMessage() if hasattr(record, 'getMessage') else str(record.msg) + # Filter out the full traceback that includes the WebSocket assertion error + if 'write() before start_response' in message: + return False + # Also filter the "Error on request" header that precedes it + if hasattr(record, 'exc_info') and record.exc_info: + exc_type = record.exc_info[0] + if exc_type and exc_type.__name__ == 'AssertionError': + # Check if it's the WebSocket-related assertion + exc_value = record.exc_info[1] + if exc_value and 'write() before start_response' in str(exc_value): + return False + return True + +# Flask logger +app.logger.handlers = logger.handlers +app.logger.setLevel(logger.level) +# Also handle Werkzeug's logger +werkzeug_logger = logging.getLogger('werkzeug') +werkzeug_logger.handlers = logger.handlers +werkzeug_logger.setLevel(logger.level) +# Add filters to suppress routine status endpoint polling logs and WebSocket upgrade errors +werkzeug_logger.addFilter(StatusEndpointFilter()) +werkzeug_logger.addFilter(WebSocketErrorFilter()) + +# Set up authentication defaults +# The secret key will reset every time we restart, which will +# require users to authenticate again + +# Session cookie security - set to 'true' if exclusively using HTTPS +session_cookie_secure_env = os.getenv('SESSION_COOKIE_SECURE', 'false').lower() +SESSION_COOKIE_SECURE = session_cookie_secure_env in ['true', 'yes', '1'] + +app.config.update( + SECRET_KEY = os.urandom(64), + SESSION_COOKIE_HTTPONLY = True, + SESSION_COOKIE_SAMESITE = 'Lax', + SESSION_COOKIE_SECURE = SESSION_COOKIE_SECURE, + PERMANENT_SESSION_LIFETIME = 604800 # 7 days in seconds +) + +logger.info(f"Session cookie secure setting: {SESSION_COOKIE_SECURE} (from env: {session_cookie_secure_env})") + +def login_required(f): + @wraps(f) + def decorated_function(*args, **kwargs): + auth_mode = get_auth_mode() + + # If no authentication is configured, allow access + if auth_mode == "none": + return f(*args, **kwargs) + + # If CWA mode and database path is invalid, return error + if auth_mode == "cwa" and CWA_DB_PATH and not os.path.isfile(CWA_DB_PATH): + logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path") + return jsonify({"error": "Internal Server Error"}), 500 + + # Check if user has a valid session + if 'user_id' not in session: + return jsonify({"error": "Unauthorized"}), 401 + + return f(*args, **kwargs) + return decorated_function + + +# Serve frontend static files +@app.route('/assets/') +def serve_frontend_assets(filename: str) -> Response: + """ + Serve static assets from the built frontend. + """ + return send_from_directory(os.path.join(FRONTEND_DIST, 'assets'), filename) + +@app.route('/') +def index() -> Response: + """ + Serve the React frontend application. + Authentication is handled by the React app itself. + """ + return send_from_directory(FRONTEND_DIST, 'index.html') + +@app.route('/logo.png') +def logo() -> Response: + """ + Serve logo from built frontend assets. + """ + return send_from_directory(FRONTEND_DIST, 'logo.png', mimetype='image/png') + +@app.route('/favicon.ico') +@app.route('/favico') +def favicon(_: Any = None) -> Response: + """ + Serve favicon from built frontend assets. + """ + return send_from_directory(FRONTEND_DIST, 'favicon.ico', mimetype='image/vnd.microsoft.icon') + +# Register bypasser warmup callback for when first WebSocket client connects +# and shutdown callback for when all clients disconnect +if not USING_EXTERNAL_BYPASSER: + from cwa_book_downloader.bypass.internal_bypasser import warmup as bypasser_warmup, shutdown_if_idle as bypasser_shutdown + ws_manager.register_on_first_connect(bypasser_warmup) + ws_manager.register_on_all_disconnect(bypasser_shutdown) + logger.info("Registered Cloudflare bypasser warmup/shutdown on WebSocket connect/disconnect") + +if DEBUG: + import subprocess + if USING_EXTERNAL_BYPASSER: + STOP_GUI = lambda: None + else: + from cwa_book_downloader.bypass.internal_bypasser import _reset_driver as STOP_GUI + @app.route('/api/debug', methods=['GET']) + @login_required + def debug() -> Union[Response, Tuple[Response, int]]: + """ + This will run the /app/genDebug.sh script, which will generate a debug zip with all the logs + The file will be named /tmp/cwa-book-downloader-debug.zip + And then return it to the user + """ + try: + # Run the debug script + logger.info("Debug endpoint called, stopping GUI and generating debug info...") + STOP_GUI() + time.sleep(1) + result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True) + if result.returncode != 0: + raise Exception(f"Debug script failed: {result.stderr}") + logger.info(f"Debug script executed: {result.stdout}") + debug_file_path = result.stdout.strip().split('\n')[-1] + if not os.path.exists(debug_file_path): + logger.error(f"Debug zip file not found at: {debug_file_path}") + return jsonify({"error": "Failed to generate debug information"}), 500 + + logger.info(f"Sending debug file: {debug_file_path}") + # Return the file to the user + return send_file( + debug_file_path, + mimetype='application/zip', + download_name=os.path.basename(debug_file_path), + as_attachment=True + ) + except subprocess.CalledProcessError as e: + logger.error_trace(f"Debug script error: {e}, stdout: {e.stdout}, stderr: {e.stderr}") + return jsonify({"error": f"Debug script failed: {e.stderr}"}), 500 + except Exception as e: + logger.error_trace(f"Debug endpoint error: {e}") + return jsonify({"error": str(e)}), 500 + +if DEBUG: + @app.route('/api/restart', methods=['GET']) + @login_required + def restart() -> Union[Response, Tuple[Response, int]]: + """ + Restart the application + """ + os._exit(0) + +@app.route('/api/search', methods=['GET']) +@login_required +def api_search() -> Union[Response, Tuple[Response, int]]: + """ + Search for books matching the provided query. + + Query Parameters: + query (str): Search term (ISBN, title, author, etc.) + isbn (str): Book ISBN + author (str): Book Author + title (str): Book Title + lang (str): Book Language + sort (str): Order to sort results + content (str): Content type of book + format (str): File format filter (pdf, epub, mobi, azw3, fb2, djvu, cbz, cbr) + + Returns: + flask.Response: JSON array of matching books or error response. + """ + query = request.args.get('query', '') + + filters = SearchFilters( + isbn = request.args.getlist('isbn'), + author = request.args.getlist('author'), + title = request.args.getlist('title'), + lang = request.args.getlist('lang'), + sort = request.args.get('sort'), + content = request.args.getlist('content'), + format = request.args.getlist('format'), + ) + + if not query and not any(vars(filters).values()): + return jsonify([]) + + try: + books = backend.search_books(query, filters) + return jsonify(books) + except SearchUnavailable as e: + logger.warning(f"Search unavailable: {e}") + return jsonify({"error": str(e)}), 503 + except Exception as e: + logger.error_trace(f"Search error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/info', methods=['GET']) +@login_required +def api_info() -> Union[Response, Tuple[Response, int]]: + """ + Get detailed book information. + + Query Parameters: + id (str): Book identifier (MD5 hash) + + Returns: + flask.Response: JSON object with book details, or an error message. + """ + book_id = request.args.get('id', '') + if not book_id: + return jsonify({"error": "No book ID provided"}), 400 + + try: + book = backend.get_book_info(book_id) + if book: + return jsonify(book) + return jsonify({"error": "Book not found"}), 404 + except Exception as e: + logger.error_trace(f"Info error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/download', methods=['GET']) +@login_required +def api_download() -> Union[Response, Tuple[Response, int]]: + """ + Queue a book for download. + + Query Parameters: + id (str): Book identifier (MD5 hash) + + Returns: + flask.Response: JSON status object indicating success or failure. + """ + book_id = request.args.get('id', '') + if not book_id: + return jsonify({"error": "No book ID provided"}), 400 + + try: + priority = int(request.args.get('priority', 0)) + success = backend.queue_book(book_id, priority) + if success: + return jsonify({"status": "queued", "priority": priority}) + return jsonify({"error": "Failed to queue book"}), 500 + except Exception as e: + logger.error_trace(f"Download error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/releases/download', methods=['POST']) +@login_required +def api_download_release() -> Union[Response, Tuple[Response, int]]: + """ + Queue a release for download. + + This endpoint is used when downloading from the ReleaseModal where the + frontend already has all the release data from the search results. + + Request Body (JSON): + source (str): Release source (e.g., "direct_download") + source_id (str): ID within the source (e.g., AA MD5 hash) + title (str): Book title + format (str, optional): File format + size (str, optional): Human-readable size + extra (dict, optional): Additional metadata + + Returns: + flask.Response: JSON status object indicating success or failure. + """ + try: + data = request.get_json() + if not data: + return jsonify({"error": "No data provided"}), 400 + + if 'source_id' not in data: + return jsonify({"error": "source_id is required"}), 400 + + priority = data.get('priority', 0) + success = backend.queue_release(data, priority) + + if success: + return jsonify({"status": "queued", "priority": priority}) + return jsonify({"error": "Failed to queue release"}), 500 + except Exception as e: + logger.error_trace(f"Release download error: {e}") + return jsonify({"error": str(e)}), 500 + + +def _is_settings_enabled() -> bool: + """Check if the config directory is mounted and writable.""" + from cwa_book_downloader.config.env import _is_config_dir_writable + return _is_config_dir_writable() + + +@app.route('/api/config', methods=['GET']) +@login_required +def api_config() -> Union[Response, Tuple[Response, int]]: + """ + Get application configuration for frontend. + + Uses the dynamic config singleton to ensure settings changes + are reflected without requiring a container restart. + """ + try: + from cwa_book_downloader.metadata_providers import ( + get_provider_sort_options, + get_provider_search_fields, + ) + + config = { + "calibre_web_url": app_config.get("CALIBRE_WEB_URL", ""), + "debug": DEBUG, + "build_version": BUILD_VERSION, + "release_version": RELEASE_VERSION, + "book_languages": _SUPPORTED_BOOK_LANGUAGE, + "default_language": app_config.BOOK_LANGUAGE, + "supported_formats": app_config.SUPPORTED_FORMATS, + "search_mode": app_config.get("SEARCH_MODE", "direct"), + "metadata_sort_options": get_provider_sort_options(), + "metadata_search_fields": get_provider_search_fields(), + "default_release_source": app_config.get("DEFAULT_RELEASE_SOURCE", "direct_download"), + "settings_enabled": _is_settings_enabled(), + } + return jsonify(config) + except Exception as e: + logger.error_trace(f"Config error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/health', methods=['GET']) +def api_health() -> Union[Response, Tuple[Response, int]]: + """ + Health check endpoint for container orchestration. + No authentication required. + + Returns: + flask.Response: JSON with status "ok". + """ + return jsonify({"status": "ok"}) + +@app.route('/api/status', methods=['GET']) +@login_required +def api_status() -> Union[Response, Tuple[Response, int]]: + """ + Get current download queue status. + + Returns: + flask.Response: JSON object with queue status. + """ + try: + status = backend.queue_status() + return jsonify(status) + except Exception as e: + logger.error_trace(f"Status error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/localdownload', methods=['GET']) +@login_required +def api_local_download() -> Union[Response, Tuple[Response, int]]: + """ + Download an EPUB file from local storage if available. + + Query Parameters: + id (str): Book identifier (MD5 hash) + + Returns: + flask.Response: The EPUB file if found, otherwise an error response. + """ + book_id = request.args.get('id', '') + if not book_id: + return jsonify({"error": "No book ID provided"}), 400 + + try: + file_data, book_info = backend.get_book_data(book_id) + if file_data is None: + # Book data not found or not available + return jsonify({"error": "File not found"}), 404 + file_name = book_info.get_filename() + # Prepare the file for sending to the client + data = io.BytesIO(file_data) + return send_file( + data, + download_name=file_name, + as_attachment=True + ) + + except Exception as e: + logger.error_trace(f"Local download error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/covers/', methods=['GET']) +def api_cover(cover_id: str) -> Union[Response, Tuple[Response, int]]: + """ + Serve a cached book cover image. + + This endpoint proxies and caches cover images from external sources. + Images are cached to disk for faster subsequent requests. + + Path Parameters: + cover_id (str): Cover identifier (book ID or composite key for universal mode) + + Query Parameters: + url (str): Base64-encoded original image URL (required on first request) + + Returns: + flask.Response: Binary image data with appropriate Content-Type, or 404. + """ + try: + import base64 + from cwa_book_downloader.core.image_cache import get_image_cache + from cwa_book_downloader.config.env import is_covers_cache_enabled + + # Check if caching is enabled + if not is_covers_cache_enabled(): + return jsonify({"error": "Cover caching is disabled"}), 404 + + cache = get_image_cache() + + # Try to get from cache first + cached = cache.get(cover_id) + if cached: + image_data, content_type = cached + response = app.response_class( + response=image_data, + status=200, + mimetype=content_type + ) + response.headers['Cache-Control'] = 'public, max-age=86400' + response.headers['X-Cache'] = 'HIT' + return response + + # Cache miss - get URL from query parameter + encoded_url = request.args.get('url') + if not encoded_url: + return jsonify({"error": "Cover URL not provided"}), 404 + + try: + original_url = base64.urlsafe_b64decode(encoded_url).decode() + except Exception as e: + logger.warning(f"Failed to decode cover URL: {e}") + return jsonify({"error": "Invalid cover URL encoding"}), 400 + + # Fetch and cache the image + result = cache.fetch_and_cache(cover_id, original_url) + if not result: + return jsonify({"error": "Failed to fetch cover image"}), 404 + + image_data, content_type = result + response = app.response_class( + response=image_data, + status=200, + mimetype=content_type + ) + response.headers['Cache-Control'] = 'public, max-age=86400' + response.headers['X-Cache'] = 'MISS' + return response + + except Exception as e: + logger.error_trace(f"Cover fetch error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/download//cancel', methods=['DELETE']) +@login_required +def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]: + """ + Cancel a download. + + Path Parameters: + book_id (str): Book identifier to cancel + + Returns: + flask.Response: JSON status indicating success or failure. + """ + try: + success = backend.cancel_download(book_id) + if success: + return jsonify({"status": "cancelled", "book_id": book_id}) + return jsonify({"error": "Failed to cancel download or book not found"}), 404 + except Exception as e: + logger.error_trace(f"Cancel download error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/queue//priority', methods=['PUT']) +@login_required +def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]: + """ + Set priority for a queued book. + + Path Parameters: + book_id (str): Book identifier + + Request Body: + priority (int): New priority level (lower number = higher priority) + + Returns: + flask.Response: JSON status indicating success or failure. + """ + try: + data = request.get_json() + if not data or 'priority' not in data: + return jsonify({"error": "Priority not provided"}), 400 + + priority = int(data['priority']) + success = backend.set_book_priority(book_id, priority) + + if success: + return jsonify({"status": "updated", "book_id": book_id, "priority": priority}) + return jsonify({"error": "Failed to update priority or book not found"}), 404 + except ValueError: + return jsonify({"error": "Invalid priority value"}), 400 + except Exception as e: + logger.error_trace(f"Set priority error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/queue/reorder', methods=['POST']) +@login_required +def api_reorder_queue() -> Union[Response, Tuple[Response, int]]: + """ + Bulk reorder queue by setting new priorities. + + Request Body: + book_priorities (dict): Mapping of book_id to new priority + + Returns: + flask.Response: JSON status indicating success or failure. + """ + try: + data = request.get_json() + if not data or 'book_priorities' not in data: + return jsonify({"error": "book_priorities not provided"}), 400 + + book_priorities = data['book_priorities'] + if not isinstance(book_priorities, dict): + return jsonify({"error": "book_priorities must be a dictionary"}), 400 + + # Validate all priorities are integers + for book_id, priority in book_priorities.items(): + if not isinstance(priority, int): + return jsonify({"error": f"Invalid priority for book {book_id}"}), 400 + + success = backend.reorder_queue(book_priorities) + + if success: + return jsonify({"status": "reordered", "updated_count": len(book_priorities)}) + return jsonify({"error": "Failed to reorder queue"}), 500 + except Exception as e: + logger.error_trace(f"Reorder queue error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/queue/order', methods=['GET']) +@login_required +def api_queue_order() -> Union[Response, Tuple[Response, int]]: + """ + Get current queue order for display. + + Returns: + flask.Response: JSON array of queued books with their order and priorities. + """ + try: + queue_order = backend.get_queue_order() + return jsonify({"queue": queue_order}) + except Exception as e: + logger.error_trace(f"Queue order error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/downloads/active', methods=['GET']) +@login_required +def api_active_downloads() -> Union[Response, Tuple[Response, int]]: + """ + Get list of currently active downloads. + + Returns: + flask.Response: JSON array of active download book IDs. + """ + try: + active_downloads = backend.get_active_downloads() + return jsonify({"active_downloads": active_downloads}) + except Exception as e: + logger.error_trace(f"Active downloads error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.route('/api/queue/clear', methods=['DELETE']) +@login_required +def api_clear_completed() -> Union[Response, Tuple[Response, int]]: + """ + Clear all completed, errored, or cancelled books from tracking. + + Returns: + flask.Response: JSON with count of removed books. + """ + try: + removed_count = backend.clear_completed() + + # Broadcast status update after clearing + if ws_manager: + ws_manager.broadcast_status_update(backend.queue_status()) + + return jsonify({"status": "cleared", "removed_count": removed_count}) + except Exception as e: + logger.error_trace(f"Clear completed error: {e}") + return jsonify({"error": str(e)}), 500 + +@app.errorhandler(404) +def not_found_error(error: Exception) -> Union[Response, Tuple[Response, int]]: + """ + Handle 404 (Not Found) errors. + + Args: + error (HTTPException): The 404 error raised by Flask. + + Returns: + flask.Response: JSON error message with 404 status. + """ + logger.warning(f"404 error: {request.url} : {error}") + return jsonify({"error": "Resource not found"}), 404 + +@app.errorhandler(500) +def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]: + """ + Handle 500 (Internal Server) errors. + + Args: + error (HTTPException): The 500 error raised by Flask. + + Returns: + flask.Response: JSON error message with 500 status. + """ + logger.error_trace(f"500 error: {error}") + return jsonify({"error": "Internal server error"}), 500 + +def _failed_login_response(username: str, ip_address: str) -> Tuple[Response, int]: + """ + Handle a failed login attempt by recording it and returning the appropriate response. + """ + is_now_locked = record_failed_login(username, ip_address) + + if is_now_locked: + return jsonify({ + "error": f"Account locked due to {MAX_LOGIN_ATTEMPTS} failed login attempts. Try again in {LOCKOUT_DURATION_MINUTES} minutes." + }), 429 + + attempts_remaining = MAX_LOGIN_ATTEMPTS - failed_login_attempts[username]['count'] + if attempts_remaining <= 5: + return jsonify({ + "error": f"Invalid username or password. {attempts_remaining} attempts remaining." + }), 401 + + return jsonify({"error": "Invalid username or password."}), 401 + + +@app.route('/api/auth/login', methods=['POST']) +def api_login() -> Union[Response, Tuple[Response, int]]: + """ + Login endpoint that validates credentials and creates a session. + Supports both built-in credentials and CWA database authentication. + Includes rate limiting: 10 failed attempts = 30 minute lockout. + + Request Body: + username (str): Username + password (str): Password + remember_me (bool): Whether to extend session duration + + Returns: + flask.Response: JSON with success status or error message. + """ + from cwa_book_downloader.core.settings_registry import load_config_file + + try: + # Get client IP address (handles reverse proxy forwarding) + ip_address = request.headers.get('X-Forwarded-For', request.remote_addr) + if ip_address and ',' in ip_address: + # X-Forwarded-For can contain multiple IPs, take the first one + ip_address = ip_address.split(',')[0].strip() + + data = request.get_json() + if not data: + return jsonify({"error": "No data provided"}), 400 + + username = data.get('username', '').strip() + password = data.get('password', '') + remember_me = data.get('remember_me', False) + + if not username or not password: + return jsonify({"error": "Username and password are required"}), 400 + + # Check if account is locked due to failed login attempts + if is_account_locked(username): + lockout_until = failed_login_attempts[username].get('lockout_until') + remaining_time = (lockout_until - datetime.now()).total_seconds() / 60 + logger.warning(f"Login attempt blocked for locked account '{username}' from IP {ip_address}") + return jsonify({ + "error": f"Account temporarily locked due to multiple failed login attempts. Try again in {int(remaining_time)} minutes." + }), 429 + + auth_mode = get_auth_mode() + + # If no authentication is configured, authentication always succeeds + if auth_mode == "none": + session['user_id'] = username + session.permanent = remember_me + clear_failed_logins(username) + logger.info(f"Login successful for user '{username}' from IP {ip_address} (no auth configured)") + return jsonify({"success": True}) + + # Built-in authentication mode + if auth_mode == "builtin": + try: + security_config = load_config_file("security") + stored_username = security_config.get("BUILTIN_USERNAME", "") + stored_hash = security_config.get("BUILTIN_PASSWORD_HASH", "") + + # Check credentials + if username == stored_username and check_password_hash(stored_hash, password): + session['user_id'] = username + session.permanent = remember_me + clear_failed_logins(username) + logger.info(f"Login successful for user '{username}' from IP {ip_address} (builtin auth, remember_me={remember_me})") + return jsonify({"success": True}) + else: + return _failed_login_response(username, ip_address) + + except Exception as e: + logger.error_trace(f"Built-in auth error: {e}") + return jsonify({"error": "Authentication system error"}), 500 + + # CWA database authentication mode + if auth_mode == "cwa": + # Validate CWA database path + if not os.path.isfile(CWA_DB_PATH): + logger.error(f"CWA_DB_PATH is set to {CWA_DB_PATH} but this is not a valid path") + return jsonify({"error": "Database configuration error"}), 500 + + try: + db_path = os.fspath(CWA_DB_PATH) + db_uri = f"file:{db_path}?mode=ro&immutable=1" + conn = sqlite3.connect(db_uri, uri=True) + cur = conn.cursor() + cur.execute("SELECT password FROM user WHERE name = ?", (username,)) + row = cur.fetchone() + conn.close() + + # Check if user exists and password is correct + if not row or not row[0] or not check_password_hash(row[0], password): + return _failed_login_response(username, ip_address) + + # Successful authentication - create session and clear failed attempts + session['user_id'] = username + session.permanent = remember_me + clear_failed_logins(username) + logger.info(f"Login successful for user '{username}' from IP {ip_address} (CWA auth, remember_me={remember_me})") + return jsonify({"success": True}) + + except Exception as e: + logger.error_trace(f"CWA database error during login: {e}") + return jsonify({"error": "Authentication system error"}), 500 + + # Should not reach here, but handle gracefully + return jsonify({"error": "Unknown authentication mode"}), 500 + + except Exception as e: + logger.error_trace(f"Login error: {e}") + return jsonify({"error": "Login failed"}), 500 + +@app.route('/api/auth/logout', methods=['POST']) +def api_logout() -> Union[Response, Tuple[Response, int]]: + """ + Logout endpoint that clears the session. + + Returns: + flask.Response: JSON with success status. + """ + try: + # Get client IP address (handles reverse proxy forwarding) + ip_address = request.headers.get('X-Forwarded-For', request.remote_addr) + if ip_address and ',' in ip_address: + ip_address = ip_address.split(',')[0].strip() + + username = session.get('user_id', 'unknown') + session.clear() + logger.info(f"Logout successful for user '{username}' from IP {ip_address}") + return jsonify({"success": True}) + except Exception as e: + logger.error_trace(f"Logout error: {e}") + return jsonify({"error": "Logout failed"}), 500 + +@app.route('/api/auth/check', methods=['GET']) +def api_auth_check() -> Union[Response, Tuple[Response, int]]: + """ + Check if user has a valid session. + + Returns: + flask.Response: JSON with authentication status, whether auth is required, + and which auth mode is active. + """ + try: + auth_mode = get_auth_mode() + + # If no authentication is configured, access is allowed + if auth_mode == "none": + return jsonify({ + "authenticated": True, + "auth_required": False, + "auth_mode": "none" + }) + + # Check if user has a valid session + is_authenticated = 'user_id' in session + return jsonify({ + "authenticated": is_authenticated, + "auth_required": True, + "auth_mode": auth_mode + }) + except Exception as e: + logger.error_trace(f"Auth check error: {e}") + return jsonify({ + "authenticated": False, + "auth_required": True, + "auth_mode": "unknown" + }) + + +@app.route('/api/metadata/providers', methods=['GET']) +@login_required +def api_metadata_providers() -> Union[Response, Tuple[Response, int]]: + """ + Get list of available metadata providers. + + Returns: + flask.Response: JSON with list of providers and their status. + """ + try: + from cwa_book_downloader.metadata_providers import ( + list_providers, + get_provider, + get_provider_kwargs, + ) + + configured_metadata_provider = app_config.get("METADATA_PROVIDER", "") + providers = [] + for info in list_providers(): + provider_info = { + "name": info["name"], + "display_name": info["display_name"], + "requires_auth": info["requires_auth"], + "configured": False, + "available": False, + } + + # Check if provider is configured and available + try: + kwargs = get_provider_kwargs(info["name"]) + provider = get_provider(info["name"], **kwargs) + provider_info["available"] = provider.is_available() + provider_info["configured"] = (info["name"] == configured_metadata_provider) + except Exception: + pass + + providers.append(provider_info) + + return jsonify({ + "providers": providers, + "configured_provider": configured_metadata_provider or None + }) + except Exception as e: + logger.error_trace(f"Metadata providers error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/metadata/search', methods=['GET']) +@login_required +def api_metadata_search() -> Union[Response, Tuple[Response, int]]: + """ + Search for books using the configured metadata provider. + + Query Parameters: + query (str): Search query (required) + limit (int): Maximum number of results (default: 20, max: 50) + sort (str): Sort order - relevance, popularity, rating, newest, oldest (default: relevance) + [dynamic fields]: Provider-specific search fields passed as query params + + Returns: + flask.Response: JSON with list of books from metadata provider. + """ + try: + from cwa_book_downloader.metadata_providers import ( + get_configured_provider, + MetadataSearchOptions, + SortOrder, + CheckboxSearchField, + NumberSearchField, + ) + from dataclasses import asdict + + query = request.args.get('query', '').strip() + + try: + limit = min(int(request.args.get('limit', 20)), 50) + except ValueError: + limit = 20 + + # Parse sort parameter + sort_value = request.args.get('sort', 'relevance').lower() + try: + sort_order = SortOrder(sort_value) + except ValueError: + sort_order = SortOrder.RELEVANCE + + provider = get_configured_provider() + if not provider: + return jsonify({ + "error": "No metadata provider configured", + "message": "No metadata provider configured. Enable one in Settings." + }), 503 + + if not provider.is_available(): + return jsonify({ + "error": f"Metadata provider '{provider.name}' is not available", + "message": f"{provider.display_name} is not available. Check configuration in Settings." + }), 503 + + # Extract custom search field values from query params + fields: Dict[str, Any] = {} + for search_field in provider.search_fields: + value = request.args.get(search_field.key) + if value is not None: + # Strip string values to handle whitespace-only input + value = value.strip() + if value != "": + # Parse value based on field type + if isinstance(search_field, CheckboxSearchField): + fields[search_field.key] = value.lower() in ('true', '1', 'yes', 'on') + elif isinstance(search_field, NumberSearchField): + try: + fields[search_field.key] = int(value) + except ValueError: + pass # Skip invalid numbers + else: + fields[search_field.key] = value + + # Require either a query or at least one field value + if not query and not fields: + return jsonify({"error": "Either 'query' or search field values are required"}), 400 + + options = MetadataSearchOptions(query=query, limit=limit, sort=sort_order, fields=fields) + books = provider.search(options) + + # Convert BookMetadata objects to dicts + books_data = [asdict(book) for book in books] + + # Transform cover_url to local proxy URLs when caching is enabled + from cwa_book_downloader.config.env import is_covers_cache_enabled + if is_covers_cache_enabled(): + import base64 + for book_dict in books_data: + if book_dict.get('cover_url'): + # Encode original URL in the proxy request itself - no need for persistent mapping + cache_id = f"{book_dict['provider']}_{book_dict['provider_id']}" + encoded_url = base64.urlsafe_b64encode(book_dict['cover_url'].encode()).decode() + book_dict['cover_url'] = f"/api/covers/{cache_id}?url={encoded_url}" + + return jsonify({ + "books": books_data, + "provider": provider.name, + "query": query + }) + except Exception as e: + logger.error_trace(f"Metadata search error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/metadata/book//', methods=['GET']) +@login_required +def api_metadata_book(provider: str, book_id: str) -> Union[Response, Tuple[Response, int]]: + """ + Get detailed book information from a metadata provider. + + Path Parameters: + provider (str): Provider name (e.g., "hardcover", "openlibrary") + book_id (str): Book ID in the provider's system + + Returns: + flask.Response: JSON with book details. + """ + try: + from cwa_book_downloader.metadata_providers import ( + get_provider, + is_provider_registered, + get_provider_kwargs, + ) + from dataclasses import asdict + + if not is_provider_registered(provider): + return jsonify({"error": f"Unknown metadata provider: {provider}"}), 400 + + # Get provider instance with appropriate configuration + kwargs = get_provider_kwargs(provider) + prov = get_provider(provider, **kwargs) + + if not prov.is_available(): + return jsonify({"error": f"Provider '{provider}' is not available"}), 503 + + book = prov.get_book(book_id) + if not book: + return jsonify({"error": "Book not found"}), 404 + + book_dict = asdict(book) + + # Transform cover_url to local proxy URL when caching is enabled + from cwa_book_downloader.config.env import is_covers_cache_enabled + if is_covers_cache_enabled() and book_dict.get('cover_url'): + import base64 + cache_id = f"{provider}_{book_id}" + encoded_url = base64.urlsafe_b64encode(book_dict['cover_url'].encode()).decode() + book_dict['cover_url'] = f"/api/covers/{cache_id}?url={encoded_url}" + + return jsonify(book_dict) + except ValueError as e: + return jsonify({"error": str(e)}), 400 + except Exception as e: + logger.error_trace(f"Metadata book error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/releases', methods=['GET']) +@login_required +def api_releases() -> Union[Response, Tuple[Response, int]]: + """ + Search for downloadable releases of a book. + + This endpoint takes book metadata and searches available release sources + (e.g., Anna's Archive, Libgen) for downloadable files. + + Query Parameters: + provider (str): Metadata provider name (required) + book_id (str): Book ID from metadata provider (required) + source (str): Release source to search (optional, default: all) + + Returns: + flask.Response: JSON with list of available releases. + """ + try: + from cwa_book_downloader.metadata_providers import ( + get_provider, + is_provider_registered, + get_provider_kwargs, + ) + from cwa_book_downloader.release_sources import get_source, list_available_sources, serialize_column_config + from dataclasses import asdict + + provider = request.args.get('provider', '').strip() + book_id = request.args.get('book_id', '').strip() + source_filter = request.args.get('source', '').strip() + # Accept title/author from frontend to avoid re-fetching metadata + title_param = request.args.get('title', '').strip() + author_param = request.args.get('author', '').strip() + + if not provider or not book_id: + return jsonify({"error": "Parameters 'provider' and 'book_id' are required"}), 400 + + if not is_provider_registered(provider): + return jsonify({"error": f"Unknown metadata provider: {provider}"}), 400 + + # Get book metadata from provider + kwargs = get_provider_kwargs(provider) + prov = get_provider(provider, **kwargs) + book = prov.get_book(book_id) + + if not book: + return jsonify({"error": "Book not found in metadata provider"}), 404 + + # Override with frontend-provided title/author if available (these come from search results + # which may have more complete data than get_book returns) + if title_param: + book.title = title_param + if author_param: + book.authors = [author_param] if author_param else [] + + # Determine which release sources to search + if source_filter: + sources_to_search = [source_filter] + else: + # Search all available sources + sources_to_search = [src["name"] for src in list_available_sources()] + + # Search each source for releases + all_releases = [] + errors = [] + + for source_name in sources_to_search: + try: + source = get_source(source_name) + releases = source.search(book) + all_releases.extend(releases) + except ValueError: + errors.append(f"Unknown source: {source_name}") + except Exception as e: + logger.warning(f"Release search failed for source {source_name}: {e}") + errors.append(f"{source_name}: {str(e)}") + + # Convert Release objects to dicts + releases_data = [asdict(release) for release in all_releases] + + # Get column config from the first source searched + # (In the UI, releases are shown per-source tab anyway) + column_config = None + if sources_to_search: + try: + first_source = get_source(sources_to_search[0]) + column_config = serialize_column_config(first_source.get_column_config()) + except Exception as e: + logger.warning(f"Failed to get column config: {e}") + + # Convert book to dict and transform cover_url + book_dict = asdict(book) + from cwa_book_downloader.config.env import is_covers_cache_enabled + if is_covers_cache_enabled() and book_dict.get('cover_url'): + import base64 + cache_id = f"{provider}_{book_id}" + encoded_url = base64.urlsafe_b64encode(book_dict['cover_url'].encode()).decode() + book_dict['cover_url'] = f"/api/covers/{cache_id}?url={encoded_url}" + + response = { + "releases": releases_data, + "book": book_dict, + "sources_searched": sources_to_search, + "column_config": column_config, + } + + if errors: + response["errors"] = errors + + return jsonify(response) + except Exception as e: + logger.error_trace(f"Releases search error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/release-sources', methods=['GET']) +@login_required +def api_release_sources() -> Union[Response, Tuple[Response, int]]: + """ + Get available release sources from the plugin registry. + + Returns: + flask.Response: JSON list of available release sources. + """ + try: + from cwa_book_downloader.release_sources import list_available_sources + sources = list_available_sources() + return jsonify(sources) + except Exception as e: + logger.error_trace(f"Release sources error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/settings', methods=['GET']) +@login_required +def api_settings_get_all() -> Union[Response, Tuple[Response, int]]: + """ + Get all settings tabs with their fields and current values. + + Returns: + flask.Response: JSON with all settings tabs. + """ + try: + from cwa_book_downloader.core.settings_registry import serialize_all_settings + + # Ensure settings are registered by importing settings modules + # This triggers the @register_settings decorators + import cwa_book_downloader.config.settings # noqa: F401 + import cwa_book_downloader.config.security # noqa: F401 + + data = serialize_all_settings(include_values=True) + return jsonify(data) + except Exception as e: + logger.error_trace(f"Settings get error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/settings/', methods=['GET']) +@login_required +def api_settings_get_tab(tab_name: str) -> Union[Response, Tuple[Response, int]]: + """ + Get settings for a specific tab. + + Path Parameters: + tab_name (str): Settings tab name (e.g., "general", "hardcover") + + Returns: + flask.Response: JSON with tab settings and values. + """ + try: + from cwa_book_downloader.core.settings_registry import ( + get_settings_tab, + serialize_tab, + ) + + # Ensure settings are registered + import cwa_book_downloader.config.settings # noqa: F401 + import cwa_book_downloader.config.security # noqa: F401 + + tab = get_settings_tab(tab_name) + if not tab: + return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404 + + return jsonify(serialize_tab(tab, include_values=True)) + except Exception as e: + logger.error_trace(f"Settings get tab error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/settings/', methods=['PUT']) +@login_required +def api_settings_update_tab(tab_name: str) -> Union[Response, Tuple[Response, int]]: + """ + Update settings for a specific tab. + + Path Parameters: + tab_name (str): Settings tab name + + Request Body: + JSON object with setting keys and values to update. + + Returns: + flask.Response: JSON with update result. + """ + try: + from cwa_book_downloader.core.settings_registry import ( + get_settings_tab, + update_settings, + ) + + # Ensure settings are registered + import cwa_book_downloader.config.settings # noqa: F401 + import cwa_book_downloader.config.security # noqa: F401 + + tab = get_settings_tab(tab_name) + if not tab: + return jsonify({"error": f"Unknown settings tab: {tab_name}"}), 404 + + values = request.get_json() + if values is None or not isinstance(values, dict): + return jsonify({"error": "Request body must be a JSON object"}), 400 + + # If no values to update, return success with empty updated list + if not values: + return jsonify({"success": True, "message": "No changes to save", "updated": []}) + + result = update_settings(tab_name, values) + + if result["success"]: + return jsonify(result) + else: + return jsonify(result), 400 + except Exception as e: + logger.error_trace(f"Settings update error: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/settings//action/', methods=['POST']) +@login_required +def api_settings_execute_action(tab_name: str, action_key: str) -> Union[Response, Tuple[Response, int]]: + """ + Execute a settings action (e.g., test connection). + + Path Parameters: + tab_name (str): Settings tab name + action_key (str): Action key to execute + + Request Body (optional): + JSON object with current form values (unsaved) + + Returns: + flask.Response: JSON with action result. + """ + try: + from cwa_book_downloader.core.settings_registry import execute_action + + # Ensure settings are registered + import cwa_book_downloader.config.settings # noqa: F401 + import cwa_book_downloader.config.security # noqa: F401 + + # Get current form values if provided (for testing with unsaved values) + current_values = request.get_json(silent=True) or {} + + result = execute_action(tab_name, action_key, current_values) + + if result["success"]: + return jsonify(result) + else: + return jsonify(result), 400 + except Exception as e: + logger.error_trace(f"Settings action error: {e}") + return jsonify({"error": str(e)}), 500 + + +# Catch-all route for React Router (must be last) +# This handles client-side routing by serving index.html for any unmatched routes +@app.route('/') +def catch_all(path: str) -> Response: + """ + Serve the React app for any route not matched by API endpoints. + This allows React Router to handle client-side routing. + Authentication is handled by the React app itself. + """ + # If the request is for an API endpoint or static file, let it 404 + if path.startswith('api/') or path.startswith('assets/'): + return jsonify({"error": "Resource not found"}), 404 + # Otherwise serve the React app + return send_from_directory(FRONTEND_DIST, 'index.html') + +# WebSocket event handlers +@socketio.on('connect') +def handle_connect(): + """Handle client connection.""" + logger.info("WebSocket client connected") + + # Track the connection (triggers warmup callbacks on first connect) + ws_manager.client_connected() + + # Send initial status to the newly connected client + try: + status = backend.queue_status() + emit('status_update', status) + except Exception as e: + logger.error(f"Error sending initial status: {e}") + +@socketio.on('disconnect') +def handle_disconnect(): + """Handle client disconnection.""" + logger.info("WebSocket client disconnected") + + # Track the disconnection + ws_manager.client_disconnected() + +@socketio.on('request_status') +def handle_status_request(): + """Handle manual status request from client.""" + try: + status = backend.queue_status() + emit('status_update', status) + except Exception as e: + logger.error(f"Error handling status request: {e}") + emit('error', {'message': 'Failed to get status'}) + +logger.log_resource_usage() + +if __name__ == '__main__': + logger.info(f"Starting Flask application with WebSocket support on {FLASK_HOST}:{FLASK_PORT} (debug={DEBUG})") + socketio.run( + app, + host=FLASK_HOST, + port=FLASK_PORT, + debug=DEBUG, + allow_unsafe_werkzeug=True # For development only + ) diff --git a/cwa_book_downloader/metadata_providers/README.md b/cwa_book_downloader/metadata_providers/README.md new file mode 100644 index 00000000..9020b88a --- /dev/null +++ b/cwa_book_downloader/metadata_providers/README.md @@ -0,0 +1,318 @@ +# Metadata Providers + +This module provides a plugin architecture for fetching book metadata from various sources with a unified interface. + +## Overview + +Metadata providers allow searching for books and retrieving detailed metadata (title, authors, cover images, descriptions, etc.) from external services. The system uses a decorator-based registration pattern, making it easy to add new providers. + +## Available Providers + +| Provider | Auth Required | Description | +|----------|---------------|-------------| +| **Hardcover** | Yes (API key) | Modern book tracking platform with GraphQL API. Get your key at [hardcover.app/account/api](https://hardcover.app/account/api) | +| **Open Library** | No | Free, open-source library catalog from the Internet Archive. Rate limited to ~100 requests/minute | + +## Core Components + +### BookMetadata + +Dataclass representing a book from a metadata provider: + +```python +@dataclass +class BookMetadata: + provider: str # Internal provider name (e.g., "hardcover") + provider_id: str # ID in that provider's system + title: str + + # Optional fields + provider_display_name: str # Human-readable name (e.g., "Hardcover") + authors: List[str] + isbn_10: str + isbn_13: str + cover_url: str + description: str + publisher: str + publish_year: int + language: str + genres: List[str] + source_url: str # Link to book on provider's site + display_fields: List[DisplayField] # Provider-specific display data +``` + +### DisplayField + +Provider-specific metadata for UI cards (ratings, page counts, reader counts, etc.): + +```python +@dataclass +class DisplayField: + label: str # e.g., "Rating", "Pages", "Readers" + value: str # e.g., "4.5", "496", "8,041" + icon: str # Icon name: "star", "book", "users", "editions" +``` + +### MetadataSearchOptions + +Unified search options that work across all providers: + +```python +@dataclass +class MetadataSearchOptions: + query: str + search_type: SearchType = SearchType.GENERAL # GENERAL, TITLE, AUTHOR, ISBN + language: str = None # ISO 639-1 code (e.g., "en") + sort: SortOrder = SortOrder.RELEVANCE + limit: int = 20 + page: int = 1 +``` + +### SortOrder + +Available sort options (provider support varies): + +| Sort Order | Description | Hardcover | Open Library | +|------------|-------------|-----------|--------------| +| `RELEVANCE` | Best match first (default) | ✓ | ✓ | +| `POPULARITY` | Most popular first | ✓ | ✗ | +| `RATING` | Highest rated first | ✓ | ✗ | +| `NEWEST` | Most recently published | ✓ | ✓ | +| `OLDEST` | Oldest published first | ✓ | ✓ | + +### MetadataProvider (Abstract Base Class) + +All providers must implement this interface: + +```python +class MetadataProvider(ABC): + name: str # Internal identifier + display_name: str # Human-readable name + requires_auth: bool # True if API key required + supported_sorts: List[SortOrder] # Supported sort options + + @abstractmethod + def search(self, options: MetadataSearchOptions) -> List[BookMetadata]: + """Search for books using the provided options.""" + pass + + @abstractmethod + def get_book(self, book_id: str) -> Optional[BookMetadata]: + """Get a specific book by provider ID.""" + pass + + @abstractmethod + def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]: + """Search for a book by ISBN.""" + pass + + @abstractmethod + def is_available(self) -> bool: + """Check if this provider is configured and available.""" + pass +``` + +## Registry Functions + +### Provider Registration + +```python +from cwa_book_downloader.metadata_providers import register_provider + +@register_provider("my_provider") +class MyProvider(MetadataProvider): + ... +``` + +### Getting Providers + +```python +from cwa_book_downloader.metadata_providers import ( + get_provider, + get_configured_provider, + get_provider_kwargs, + list_providers, + is_provider_registered, +) + +# Get specific provider with kwargs +provider = get_provider("hardcover", api_key="...") + +# Get currently configured provider (from settings) +provider = get_configured_provider() + +# Get provider-specific kwargs from config +kwargs = get_provider_kwargs("hardcover") # {"api_key": "..."} + +# List all registered providers +providers = list_providers() +# [{"name": "hardcover", "display_name": "Hardcover", "requires_auth": True}, ...] + +# Check if provider exists +exists = is_provider_registered("hardcover") # True +``` + +### Sort Options + +```python +from cwa_book_downloader.metadata_providers import get_provider_sort_options + +# Get sort options for a specific provider +options = get_provider_sort_options("hardcover") +# [{"value": "relevance", "label": "Most relevant"}, ...] + +# Get sort options for configured provider +options = get_provider_sort_options() # Uses METADATA_PROVIDER from config +``` + +## Creating a New Provider + +1. Create a new file in `cwa_book_downloader/metadata_providers/` (e.g., `my_provider.py`) + +2. Implement the provider: + +```python +from cwa_book_downloader.metadata_providers import ( + BookMetadata, + DisplayField, + MetadataProvider, + MetadataSearchOptions, + SearchType, + SortOrder, + register_provider, +) +from cwa_book_downloader.core.settings_registry import ( + register_settings, + HeadingField, + PasswordField, + ActionButton, +) +from cwa_book_downloader.core.config import config + + +@register_provider("my_provider") +class MyProvider(MetadataProvider): + name = "my_provider" + display_name = "My Provider" + requires_auth = True + supported_sorts = [SortOrder.RELEVANCE, SortOrder.NEWEST] + + def __init__(self, api_key: str = None): + self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "") + + def is_available(self) -> bool: + return bool(self.api_key) + + def search(self, options: MetadataSearchOptions) -> List[BookMetadata]: + # Handle ISBN search separately + if options.search_type == SearchType.ISBN: + result = self.search_by_isbn(options.query) + return [result] if result else [] + + # Implement search logic... + return [] + + def get_book(self, book_id: str) -> Optional[BookMetadata]: + # Implement get book logic... + return None + + def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]: + # Implement ISBN search logic... + return None + + +# Settings for the UI +@register_settings("my_provider", "My Provider", icon="book", order=53, group="metadata_providers") +def my_provider_settings(): + return [ + HeadingField( + key="my_provider_heading", + title="My Provider", + description="Description of your provider", + link_url="https://myprovider.com", + link_text="myprovider.com", + ), + PasswordField( + key="MY_PROVIDER_API_KEY", + label="API Key", + description="Your API key", + required=True, + ), + ActionButton( + key="test_connection", + label="Test Connection", + style="primary", + callback=_test_connection, + ), + ] +``` + +3. Import your provider in `__init__.py`: + +```python +try: + from cwa_book_downloader.metadata_providers import my_provider # noqa: F401 +except ImportError: + pass # Provider is optional +``` + +4. Add provider kwargs to `get_provider_kwargs()` in `__init__.py`: + +```python +def get_provider_kwargs(provider_name: str) -> Dict: + kwargs: Dict = {} + if provider_name == "hardcover": + kwargs["api_key"] = app_config.get("HARDCOVER_API_KEY", "") + elif provider_name == "my_provider": + kwargs["api_key"] = app_config.get("MY_PROVIDER_API_KEY", "") + return kwargs +``` + +## Caching + +Providers should use the `@cacheable` decorator for API calls: + +```python +from cwa_book_downloader.core.cache import cacheable +from cwa_book_downloader.config.env import ( + METADATA_CACHE_SEARCH_TTL, + METADATA_CACHE_BOOK_TTL, +) + +@cacheable(ttl=METADATA_CACHE_SEARCH_TTL, key_prefix="myprovider:search") +def _search_cached(self, cache_key: str, options: MetadataSearchOptions): + # Cached search implementation + pass + +@cacheable(ttl=METADATA_CACHE_BOOK_TTL, key_prefix="myprovider:book") +def get_book(self, book_id: str): + # Cached book lookup + pass +``` + +## Rate Limiting + +For providers with rate limits (like Open Library), implement a rate limiter: + +```python +from cwa_book_downloader.metadata_providers.openlibrary import RateLimiter + +# 90 requests per 60 seconds +rate_limiter = RateLimiter(max_requests=90, window_seconds=60) + +def make_request(self): + rate_limiter.wait_if_needed() # Blocks if rate limited + # ... make request +``` + +## Configuration + +Provider settings are stored in `CONFIG_DIR/plugins/.json` and managed via the Settings UI. See [Plugin Settings Guide](../../docs/plugin-settings.md) for detailed documentation on adding settings to your provider. + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `METADATA_PROVIDER` | `""` | Active metadata provider name | +| `METADATA_CACHE_SEARCH_TTL` | `3600` | Search cache TTL in seconds | +| `METADATA_CACHE_BOOK_TTL` | `86400` | Book lookup cache TTL in seconds | diff --git a/cwa_book_downloader/metadata_providers/__init__.py b/cwa_book_downloader/metadata_providers/__init__.py new file mode 100644 index 00000000..b72b7ec3 --- /dev/null +++ b/cwa_book_downloader/metadata_providers/__init__.py @@ -0,0 +1,456 @@ +"""Metadata provider plugin system - base classes and registry.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field, asdict +from enum import Enum +from typing import List, Optional, Dict, Type, Literal, Any, Union + + +class SearchType(str, Enum): + """Type of search to perform.""" + GENERAL = "general" # Search all fields (title, author, ISBN, etc.) + TITLE = "title" # Search by title only + AUTHOR = "author" # Search by author only + ISBN = "isbn" # Search by ISBN + + +class SortOrder(str, Enum): + """Sort order for search results.""" + RELEVANCE = "relevance" # Best match first (default) + POPULARITY = "popularity" # Most popular first + RATING = "rating" # Highest rated first + NEWEST = "newest" # Most recently published first + OLDEST = "oldest" # Oldest published first + + +# Display labels for sort options +SORT_LABELS: Dict[SortOrder, str] = { + SortOrder.RELEVANCE: "Most relevant", + SortOrder.POPULARITY: "Most popular", + SortOrder.RATING: "Highest rated", + SortOrder.NEWEST: "Newest", + SortOrder.OLDEST: "Oldest", +} + + +@dataclass +class TextSearchField: + """Text input search field.""" + key: str # Field identifier (e.g., "author", "publisher") + label: str # Display label in UI + placeholder: str = "" # Placeholder text + description: str = "" # Help text + + +@dataclass +class NumberSearchField: + """Numeric input search field.""" + key: str + label: str + placeholder: str = "" + description: str = "" + min_value: Optional[int] = None + max_value: Optional[int] = None + step: int = 1 + + +@dataclass +class SelectSearchField: + """Single-choice dropdown search field.""" + key: str + label: str + options: List[Dict[str, str]] = field(default_factory=list) # [{value: "", label: ""}] + placeholder: str = "" + description: str = "" + + +@dataclass +class CheckboxSearchField: + """Boolean checkbox search field.""" + key: str + label: str + description: str = "" + default: bool = False + + +# Type alias for all search field types +SearchField = Union[TextSearchField, NumberSearchField, SelectSearchField, CheckboxSearchField] + + +def _get_field_type_name(search_field: SearchField) -> str: + """Get the type name for a search field.""" + return search_field.__class__.__name__ + + +def serialize_search_field(search_field: SearchField) -> Dict[str, Any]: + """Serialize a search field for API response. + + Args: + search_field: The search field definition. + + Returns: + Dict representation for frontend. + """ + result: Dict[str, Any] = { + "key": search_field.key, + "label": search_field.label, + "type": _get_field_type_name(search_field), + "placeholder": search_field.placeholder if hasattr(search_field, 'placeholder') else "", + "description": search_field.description if hasattr(search_field, 'description') else "", + } + + # Add type-specific properties + if isinstance(search_field, NumberSearchField): + result["min"] = search_field.min_value + result["max"] = search_field.max_value + result["step"] = search_field.step + elif isinstance(search_field, SelectSearchField): + result["options"] = search_field.options + elif isinstance(search_field, CheckboxSearchField): + result["default"] = search_field.default + + return result + + +@dataclass +class MetadataSearchOptions: + """Options for metadata search queries. + + Provides an abstracted interface that works across all metadata providers. + Providers map these options to their specific API parameters. + """ + query: str + search_type: SearchType = SearchType.GENERAL + language: Optional[str] = None # ISO 639-1 code (e.g., "en", "fr") + sort: SortOrder = SortOrder.RELEVANCE + limit: int = 20 + page: int = 1 + fields: Dict[str, Any] = field(default_factory=dict) # Custom search field values + + +@dataclass +class DisplayField: + """A display field for metadata cards. + + Providers can populate these to show provider-specific metadata + like ratings, page counts, reader counts, etc. + """ + label: str # e.g., "Rating", "Pages", "Readers" + value: str # e.g., "4.5", "496", "8,041" + icon: Optional[str] = None # Icon name: "star", "book", "users", "editions" + + +@dataclass +class BookMetadata: + """Book from metadata provider (not a specific release).""" + provider: str # Which provider this came from (internal name) + provider_id: str # ID in that provider's system + title: str + + # Provider display name for UI (e.g., "Open Library" instead of "openlibrary") + provider_display_name: Optional[str] = None + + # Optional - not all providers have all fields + authors: List[str] = field(default_factory=list) + isbn_10: Optional[str] = None + isbn_13: Optional[str] = None + cover_url: Optional[str] = None + description: Optional[str] = None + publisher: Optional[str] = None + publish_year: Optional[int] = None + language: Optional[str] = None + genres: List[str] = field(default_factory=list) + source_url: Optional[str] = None # Link to book on provider's site + + # Provider-specific display fields for cards/lists + display_fields: List[DisplayField] = field(default_factory=list) + + +class MetadataProvider(ABC): + """Interface for metadata providers. + + All metadata providers must implement this interface. The search method + accepts MetadataSearchOptions for unified search across providers. + + Attributes: + name: Internal identifier (e.g., "hardcover") + display_name: Human-readable name (e.g., "Hardcover") + requires_auth: True if API key/authentication is required + supported_sorts: List of SortOrder values this provider supports + search_fields: List of provider-specific search fields + """ + name: str + display_name: str + requires_auth: bool + supported_sorts: List[SortOrder] = [SortOrder.RELEVANCE] + search_fields: List[SearchField] = [] + + @abstractmethod + def search(self, options: MetadataSearchOptions) -> List[BookMetadata]: + """Search for books using the provided options. + + Args: + options: Search options including query, type, language, sort, pagination. + + Returns: + List of BookMetadata matching the search criteria. + + Note: + - If search_type is ISBN, this delegates to search_by_isbn() + - Unsupported sort orders fall back to RELEVANCE + - Language filtering is best-effort (not all providers support it) + """ + pass + + @abstractmethod + def get_book(self, book_id: str) -> Optional[BookMetadata]: + """Get a specific book by provider ID.""" + pass + + @abstractmethod + def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]: + """Search for a book by ISBN.""" + pass + + @abstractmethod + def is_available(self) -> bool: + """Check if this provider is configured and available.""" + pass + + +# Provider registry +_PROVIDERS: Dict[str, Type[MetadataProvider]] = {} +_PROVIDER_KWARGS_FACTORIES: Dict[str, Any] = {} # Callable[[], Dict] + + +def register_provider(name: str): + """Decorator to register a metadata provider.""" + def decorator(cls): + _PROVIDERS[name] = cls + return cls + return decorator + + +def register_provider_kwargs(name: str): + """Decorator to register a provider's kwargs factory. + + The decorated function should return a Dict of kwargs to pass to the + provider constructor. This allows each provider to define its own + configuration requirements without polluting the core module. + + Example: + @register_provider_kwargs("hardcover") + def _hardcover_kwargs() -> Dict: + from cwa_book_downloader.core.config import config + return {"api_key": config.get("HARDCOVER_API_KEY", "")} + """ + def decorator(fn): + _PROVIDER_KWARGS_FACTORIES[name] = fn + return fn + return decorator + + +def get_provider(name: str, **kwargs) -> MetadataProvider: + """Factory - instantiate any registered provider.""" + if name not in _PROVIDERS: + raise ValueError(f"Unknown metadata provider: {name}") + return _PROVIDERS[name](**kwargs) + + +def list_providers() -> List[dict]: + """For settings UI - list available providers with their requirements.""" + return [ + {"name": n, "display_name": c.display_name, "requires_auth": c.requires_auth} + for n, c in _PROVIDERS.items() + ] + + +def get_provider_kwargs(provider_name: str) -> Dict: + """Get provider-specific initialization kwargs based on configuration. + + Looks up the provider's registered kwargs factory and calls it to get + the configuration. Each provider registers its own factory via + @register_provider_kwargs decorator. + + Args: + provider_name: Name of the provider. + + Returns: + Dict of kwargs to pass to provider constructor. + """ + factory = _PROVIDER_KWARGS_FACTORIES.get(provider_name) + if factory: + return factory() + return {} + + +def is_provider_registered(provider_name: str) -> bool: + """Check if a provider is registered. + + Args: + provider_name: Name of the provider. + + Returns: + True if provider is registered, False otherwise. + """ + return provider_name in _PROVIDERS + + +def is_provider_enabled(provider_name: str) -> bool: + """Check if a provider is enabled in settings. + + Each provider has an enabled flag (e.g., HARDCOVER_ENABLED, OPENLIBRARY_ENABLED) + that must be explicitly set to True for the provider to be used. + + Args: + provider_name: Name of the provider. + + Returns: + True if provider is enabled, False otherwise. + """ + from cwa_book_downloader.core.config import config as app_config + + # Refresh config to get latest settings + app_config.refresh() + + # Check the provider-specific enabled flag + enabled_key = f"{provider_name.upper()}_ENABLED" + return app_config.get(enabled_key, False) is True + + +def get_enabled_providers() -> List[str]: + """Get list of all enabled provider names. + + Returns: + List of enabled provider names. + """ + enabled = [] + for name in _PROVIDERS: + if is_provider_enabled(name): + enabled.append(name) + return enabled + + +def get_configured_provider() -> Optional[MetadataProvider]: + """Get the currently configured metadata provider, if any. + + Uses the METADATA_PROVIDER config setting to determine which provider + to instantiate. Returns None if no provider is configured or not enabled. + + Returns: + MetadataProvider instance or None. + """ + from cwa_book_downloader.core.config import config as app_config + + # Refresh config to ensure we have the latest saved settings + app_config.refresh() + + metadata_provider = app_config.get("METADATA_PROVIDER", "") + if not metadata_provider: + return None + + if metadata_provider not in _PROVIDERS: + return None + + # Check if the provider is enabled + if not is_provider_enabled(metadata_provider): + return None + + kwargs = get_provider_kwargs(metadata_provider) + return get_provider(metadata_provider, **kwargs) + + +def get_provider_sort_options(provider_name: Optional[str] = None) -> List[Dict[str, str]]: + """Get sort options for a metadata provider. + + Returns a list of {value, label} dicts suitable for frontend dropdowns. + + Args: + provider_name: Provider name. If None, uses configured provider. + + Returns: + List of sort option dicts, or default [relevance] if provider not found. + """ + if provider_name is None: + from cwa_book_downloader.core.config import config as app_config + app_config.refresh() + provider_name = app_config.get("METADATA_PROVIDER", "") + + if provider_name and provider_name in _PROVIDERS: + provider_class = _PROVIDERS[provider_name] + supported = getattr(provider_class, 'supported_sorts', [SortOrder.RELEVANCE]) + else: + supported = [SortOrder.RELEVANCE] + + return [ + {"value": sort.value, "label": SORT_LABELS.get(sort, sort.value.title())} + for sort in supported + ] + + +def get_provider_search_fields(provider_name: Optional[str] = None) -> List[Dict[str, Any]]: + """Get search fields for a metadata provider. + + Returns a list of serialized search field dicts suitable for frontend rendering. + + Args: + provider_name: Provider name. If None, uses configured provider. + + Returns: + List of search field dicts, or empty list if provider not found. + """ + if provider_name is None: + from cwa_book_downloader.core.config import config as app_config + app_config.refresh() + provider_name = app_config.get("METADATA_PROVIDER", "") + + if provider_name and provider_name in _PROVIDERS: + provider_class = _PROVIDERS[provider_name] + fields = getattr(provider_class, 'search_fields', []) + else: + fields = [] + + return [serialize_search_field(f) for f in fields] + + +def sync_metadata_provider_selection() -> None: + """Sync the METADATA_PROVIDER setting based on enabled providers. + + If the currently selected provider is not enabled (or nothing is selected), + auto-select the first enabled provider. This should be called after + enabling/disabling a provider. + """ + from cwa_book_downloader.core.config import config as app_config + from cwa_book_downloader.core.settings_registry import save_config_file, load_config_file + + app_config.refresh() + + current_provider = app_config.get("METADATA_PROVIDER", "") + enabled = get_enabled_providers() + + # If current provider is valid and enabled, nothing to do + if current_provider and current_provider in enabled: + return + + # Auto-select first enabled provider (or clear if none) + new_provider = enabled[0] if enabled else "" + + if new_provider != current_provider: + # Update the general settings config + general_config = load_config_file("general") + general_config["METADATA_PROVIDER"] = new_provider + save_config_file("general", general_config) + app_config.refresh() + + +# Import provider implementations to trigger registration +# These must be imported AFTER the base classes and registry are defined +try: + from cwa_book_downloader.metadata_providers import hardcover # noqa: F401, E402 +except ImportError: + pass # Hardcover provider is optional + +try: + from cwa_book_downloader.metadata_providers import openlibrary # noqa: F401, E402 +except ImportError: + pass # Open Library provider is optional diff --git a/cwa_book_downloader/metadata_providers/hardcover.py b/cwa_book_downloader/metadata_providers/hardcover.py new file mode 100644 index 00000000..b1e51959 --- /dev/null +++ b/cwa_book_downloader/metadata_providers/hardcover.py @@ -0,0 +1,737 @@ +"""Hardcover.app metadata provider. Requires API key.""" + +import requests +from typing import Any, Dict, List, Optional + +from cwa_book_downloader.core.cache import cacheable +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.core.settings_registry import ( + register_settings, + CheckboxField, + PasswordField, + ActionButton, + HeadingField, +) +from cwa_book_downloader.config.env import ( + METADATA_CACHE_SEARCH_TTL, + METADATA_CACHE_BOOK_TTL, +) +from cwa_book_downloader.core.config import config as app_config +from cwa_book_downloader.metadata_providers import ( + BookMetadata, + DisplayField, + MetadataProvider, + MetadataSearchOptions, + SearchType, + SortOrder, + register_provider, + register_provider_kwargs, + TextSearchField, +) + +logger = setup_logger(__name__) + +HARDCOVER_API_URL = "https://api.hardcover.app/v1/graphql" + + +# Mapping from abstract sort order to Hardcover sort parameter +# Note: release_year is more consistently populated than release_date_i +SORT_MAPPING: Dict[SortOrder, str] = { + SortOrder.RELEVANCE: "_text_match:desc,users_count:desc", + SortOrder.POPULARITY: "users_count:desc", + SortOrder.RATING: "rating:desc", + SortOrder.NEWEST: "release_year:desc", + SortOrder.OLDEST: "release_year:asc", +} + +# Mapping from abstract search type to Hardcover fields parameter +SEARCH_TYPE_FIELDS: Dict[SearchType, str] = { + SearchType.GENERAL: "title,isbns,series_names,author_names,alternative_titles", + SearchType.TITLE: "title,alternative_titles", + SearchType.AUTHOR: "author_names", + # ISBN is handled separately via search_by_isbn() +} + + +def _combine_headline_description(headline: Optional[str], description: Optional[str]) -> Optional[str]: + """Combine headline (tagline) and description into a single description. + + Hardcover stores a short 'headline' (tagline/promotional text) separately + from the main description. This combines them for display. + + Args: + headline: Short promotional text or tagline. + description: Full book synopsis/description. + + Returns: + Combined description with headline as the first line, or just one if only one exists. + """ + if headline and description: + # Add headline as first paragraph, followed by description + return f"{headline}\n\n{description}" + elif headline: + return headline + elif description: + return description + return None + + +@register_provider_kwargs("hardcover") +def _hardcover_kwargs() -> Dict[str, Any]: + """Provide Hardcover-specific constructor kwargs.""" + return {"api_key": app_config.get("HARDCOVER_API_KEY", "")} + + +@register_provider("hardcover") +class HardcoverProvider(MetadataProvider): + """Hardcover.app metadata provider using GraphQL API.""" + + name = "hardcover" + display_name = "Hardcover" + requires_auth = True + supported_sorts = [ + SortOrder.RELEVANCE, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.OLDEST, + ] + search_fields = [ + TextSearchField( + key="author", + label="Author", + description="Search by author name", + ), + TextSearchField( + key="title", + label="Title", + description="Search by book title", + ), + ] + + def __init__(self, api_key: Optional[str] = None): + """Initialize provider with API key. + + Args: + api_key: Hardcover API key. If not provided, uses config singleton. + """ + self.api_key = api_key or app_config.get("HARDCOVER_API_KEY", "") + self.session = requests.Session() + if self.api_key: + self.session.headers.update({ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }) + + def is_available(self) -> bool: + """Check if provider is configured with an API key.""" + return bool(self.api_key) + + def search(self, options: MetadataSearchOptions) -> List[BookMetadata]: + """Search for books using Hardcover's search API. + + Args: + options: Search options (query, type, sort, pagination, fields). + + Returns: + List of BookMetadata objects. + """ + if not self.api_key: + logger.warning("Hardcover API key not configured") + return [] + + # Handle ISBN search separately + if options.search_type == SearchType.ISBN: + result = self.search_by_isbn(options.query) + return [result] if result else [] + + # Build cache key from options (include fields for cache differentiation) + fields_key = ":".join(f"{k}={v}" for k, v in sorted(options.fields.items())) + cache_key = f"{options.query}:{options.search_type.value}:{options.sort.value}:{options.limit}:{options.page}:{fields_key}" + return self._search_cached(cache_key, options) + + @cacheable(ttl=METADATA_CACHE_SEARCH_TTL, key_prefix="hardcover:search") + def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> List[BookMetadata]: + """Cached search implementation. + + Args: + cache_key: Cache key (used by decorator). + options: Search options. + + Returns: + List of BookMetadata objects. + """ + # Determine query and fields based on custom search fields + # Field-first search: when a specific field has a value, search that field + author_value = options.fields.get("author", "").strip() + title_value = options.fields.get("title", "").strip() + + logger.debug(f"Field-first search check: author_value='{author_value}', title_value='{title_value}'") + + # Determine what to search and which fields to target + # Note: Hardcover API requires 'weights' when using 'fields' parameter + if author_value and not title_value: + # Author-only search: search author_names field with author query + query = author_value + search_fields = "author_names" + search_weights = "1" + logger.debug(f"Author-only search: query='{query}', fields='{search_fields}'") + elif title_value and not author_value: + # Title-only search: search title fields with title query + query = title_value + search_fields = "title,alternative_titles" + search_weights = "5,1" + logger.debug(f"Title-only search: query='{query}', fields='{search_fields}'") + elif author_value and title_value: + # Both provided: combine into query, search both fields + query = f"{title_value} {author_value}" + search_fields = "title,alternative_titles,author_names" + search_weights = "5,1,3" + logger.debug(f"Combined search: query='{query}', fields='{search_fields}'") + else: + # No custom fields: use general query with all default fields + query = options.query + search_fields = None + search_weights = None + logger.debug(f"General search: query='{query}', no field restriction") + + # Build GraphQL query with optional fields/weights parameters + if search_fields: + graphql_query = """ + query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String, $fields: String, $weights: String) { + search( + query: $query, + query_type: "Book", + per_page: $limit, + page: $page, + sort: $sort, + fields: $fields, + weights: $weights + ) { + results + } + } + """ + else: + graphql_query = """ + query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String) { + search( + query: $query, + query_type: "Book", + per_page: $limit, + page: $page, + sort: $sort + ) { + results + } + } + """ + + # Map abstract sort order to Hardcover's sort parameter + sort_param = SORT_MAPPING.get(options.sort, SORT_MAPPING[SortOrder.RELEVANCE]) + + variables = { + "query": query, + "limit": options.limit, + "page": options.page, + "sort": sort_param, + } + + if search_fields: + variables["fields"] = search_fields + variables["weights"] = search_weights + + logger.debug(f"GraphQL variables: {variables}") + + try: + result = self._execute_query(graphql_query, variables) + if not result: + logger.debug("Hardcover search: No result from API") + return [] + + search_data = result.get("search", {}) + + # Results is a Typesense response object with hits array + results_obj = search_data.get("results", {}) + if isinstance(results_obj, dict): + hits = results_obj.get("hits", []) + else: + hits = results_obj if isinstance(results_obj, list) else [] + + # Parse the search results - each hit has a 'document' field + books = [] + for hit in hits: + # Get the document from the hit + item = hit.get("document", hit) if isinstance(hit, dict) else hit + if isinstance(item, dict): + book = self._parse_search_result(item) + if book: + books.append(book) + + logger.info(f"Hardcover search '{query}' (fields={search_fields}) returned {len(books)} results") + return books + + except Exception as e: + logger.error(f"Hardcover search error: {e}") + return [] + + @cacheable(ttl=METADATA_CACHE_BOOK_TTL, key_prefix="hardcover:book") + def get_book(self, book_id: str) -> Optional[BookMetadata]: + """Get book details by Hardcover ID. + + Args: + book_id: Hardcover book ID. + + Returns: + BookMetadata or None if not found. + """ + if not self.api_key: + logger.warning("Hardcover API key not configured") + return None + + # Query for specific book by ID + # Note: API has max depth of 3, so use cached_* fields instead of nested relationships + graphql_query = """ + query GetBook($id: Int!) { + books(where: {id: {_eq: $id}}, limit: 1) { + id + title + slug + release_date + headline + description + pages + cached_image + cached_contributors + cached_tags + default_physical_edition { + isbn_10 + isbn_13 + } + } + } + """ + + try: + book_id_int = int(book_id) + result = self._execute_query(graphql_query, {"id": book_id_int}) + if not result: + return None + + books = result.get("books", []) + if not books: + return None + + return self._parse_book(books[0]) + + except ValueError: + logger.error(f"Invalid book ID: {book_id}") + return None + except Exception as e: + logger.error(f"Hardcover get_book error: {e}") + return None + + @cacheable(ttl=METADATA_CACHE_BOOK_TTL, key_prefix="hardcover:isbn") + def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]: + """Search for a book by ISBN. + + Args: + isbn: ISBN-10 or ISBN-13. + + Returns: + BookMetadata or None if not found. + """ + if not self.api_key: + logger.warning("Hardcover API key not configured") + return None + + # Clean ISBN (remove hyphens) + clean_isbn = isbn.replace("-", "").strip() + + # Search for editions with matching ISBN + # Note: API has max depth of 3, so use cached_* fields instead of nested relationships + graphql_query = """ + query SearchByISBN($isbn: String!) { + editions( + where: { + _or: [ + {isbn_10: {_eq: $isbn}}, + {isbn_13: {_eq: $isbn}} + ] + }, + limit: 1 + ) { + isbn_10 + isbn_13 + book { + id + title + slug + release_date + headline + description + pages + cached_image + cached_contributors + cached_tags + } + } + } + """ + + try: + result = self._execute_query(graphql_query, {"isbn": clean_isbn}) + if not result: + return None + + editions = result.get("editions", []) + if not editions: + logger.debug(f"No Hardcover book found for ISBN: {isbn}") + return None + + edition = editions[0] + book_data = edition.get("book", {}) + if not book_data: + return None + + # Add ISBN data from edition to book data + book_data["isbn_10"] = edition.get("isbn_10") + book_data["isbn_13"] = edition.get("isbn_13") + + return self._parse_book(book_data) + + except Exception as e: + logger.error(f"Hardcover ISBN search error: {e}") + return None + + def _execute_query(self, query: str, variables: Dict[str, Any]) -> Optional[Dict]: + """Execute a GraphQL query. + + Args: + query: GraphQL query string. + variables: Query variables. + + Returns: + Response data dict or None on error. + """ + try: + response = self.session.post( + HARDCOVER_API_URL, + json={"query": query, "variables": variables}, + timeout=15 + ) + response.raise_for_status() + + data = response.json() + + if "errors" in data: + logger.error(f"GraphQL errors: {data['errors']}") + return None + + return data.get("data") + + except requests.Timeout: + logger.warning("Hardcover API request timed out") + return None + except requests.HTTPError as e: + if e.response.status_code == 401: + logger.error("Hardcover API key is invalid") + else: + logger.error(f"Hardcover API HTTP error: {e}") + return None + except Exception as e: + logger.error(f"Hardcover API request failed: {e}") + return None + + def _parse_search_result(self, item: Dict) -> Optional[BookMetadata]: + """Parse a search result item into BookMetadata. + + Args: + item: Search result item dict. + + Returns: + BookMetadata or None if parsing fails. + """ + try: + book_id = item.get("id") or item.get("document", {}).get("id") + title = item.get("title") or item.get("document", {}).get("title") + + if not book_id or not title: + return None + + # Extract authors from various possible fields + authors = [] + if "author_names" in item: + authors = item["author_names"] if isinstance(item["author_names"], list) else [item["author_names"]] + elif "cached_contributors" in item: + for contrib in item.get("cached_contributors", []): + if isinstance(contrib, dict) and contrib.get("name"): + authors.append(contrib["name"]) + elif isinstance(contrib, str): + authors.append(contrib) + + # Get cover URL + cover_url = None + if "image" in item and item["image"]: + cover_url = item["image"] if isinstance(item["image"], str) else item["image"].get("url") + + # Extract year - prefer release_year if available, fall back to release_date + publish_year = None + if "release_year" in item and item["release_year"]: + try: + publish_year = int(item["release_year"]) + except (ValueError, TypeError): + pass + elif "release_date" in item and item["release_date"]: + try: + publish_year = int(str(item["release_date"])[:4]) + except (ValueError, TypeError): + pass + + slug = item.get("slug", "") + source_url = f"https://hardcover.app/books/{slug}" if slug else None + + # Build display fields from Hardcover-specific data + display_fields = [] + + # Rating (e.g., "4.5 (3,764)") + rating = item.get("rating") + ratings_count = item.get("ratings_count") + if rating is not None: + rating_str = f"{rating:.1f}" + if ratings_count: + rating_str += f" ({ratings_count:,})" + display_fields.append(DisplayField(label="Rating", value=rating_str, icon="star")) + + # Readers (users who have this book) + users_count = item.get("users_count") + if users_count: + display_fields.append(DisplayField(label="Readers", value=f"{users_count:,}", icon="users")) + + # Combine headline and description if both present + headline = item.get("headline") + description = item.get("description") + full_description = _combine_headline_description(headline, description) + + return BookMetadata( + provider="hardcover", + provider_id=str(book_id), + title=title, + provider_display_name="Hardcover", + authors=authors, + cover_url=cover_url, + description=full_description, + publish_year=publish_year, + source_url=source_url, + display_fields=display_fields, + ) + + except Exception as e: + logger.debug(f"Failed to parse Hardcover search result: {e}") + return None + + def _parse_book(self, book: Dict) -> BookMetadata: + """Parse a book object into BookMetadata. + + Args: + book: Book data dict from GraphQL response. + + Returns: + BookMetadata object. + """ + # Extract authors from cached_contributors (json array) or contributions relationship + authors = [] + if book.get("cached_contributors"): + for contrib in book["cached_contributors"]: + if isinstance(contrib, dict) and contrib.get("name"): + authors.append(contrib["name"]) + elif isinstance(contrib, str): + authors.append(contrib) + elif book.get("contributions"): + # Fallback for contributions relationship (if used) + for contrib in book["contributions"]: + author = contrib.get("author", {}) + if author and author.get("name"): + authors.append(author["name"]) + + # Get cover URL from cached_image (jsonb) or image relationship + cover_url = None + if book.get("cached_image"): + cached = book["cached_image"] + if isinstance(cached, dict): + cover_url = cached.get("url") + elif isinstance(cached, str): + cover_url = cached + elif book.get("image"): + img = book["image"] + cover_url = img if isinstance(img, str) else img.get("url") + + # Extract year from release_date + publish_year = None + if book.get("release_date"): + try: + publish_year = int(str(book["release_date"])[:4]) + except (ValueError, TypeError): + pass + + # Extract genres from cached_tags + genres = [] + for tag in book.get("cached_tags", []): + if isinstance(tag, dict) and tag.get("tag"): + genres.append(tag["tag"]) + elif isinstance(tag, str): + genres.append(tag) + + # Get ISBN from direct fields, default_physical_edition, or editions + isbn_10 = book.get("isbn_10") + isbn_13 = book.get("isbn_13") + + if not isbn_10 and not isbn_13: + # Try default_physical_edition first + edition = book.get("default_physical_edition") + if edition: + isbn_10 = edition.get("isbn_10") + isbn_13 = edition.get("isbn_13") + + # Fallback to editions array + if not isbn_10 and not isbn_13 and book.get("editions"): + for ed in book["editions"]: + if not isbn_10 and ed.get("isbn_10"): + isbn_10 = ed["isbn_10"] + if not isbn_13 and ed.get("isbn_13"): + isbn_13 = ed["isbn_13"] + if isbn_10 and isbn_13: + break + + slug = book.get("slug", "") + source_url = f"https://hardcover.app/books/{slug}" if slug else None + + # Combine headline and description if both present + headline = book.get("headline") + description = book.get("description") + full_description = _combine_headline_description(headline, description) + + return BookMetadata( + provider="hardcover", + provider_id=str(book["id"]), + title=book["title"], + provider_display_name="Hardcover", + authors=authors, + isbn_10=isbn_10, + isbn_13=isbn_13, + cover_url=cover_url, + description=full_description, + publish_year=publish_year, + genres=genres, + source_url=source_url, + ) + + +def _test_hardcover_connection() -> Dict[str, Any]: + """Test the Hardcover API connection.""" + from cwa_book_downloader.core.config import config as app_config + from cwa_book_downloader.core.settings_registry import save_config_file, load_config_file + from cwa_book_downloader.metadata_providers import get_provider_kwargs + + # Refresh config to pick up any recently saved settings + app_config.refresh() + + kwargs = get_provider_kwargs("hardcover") + api_key = kwargs.get("api_key") + + # Debug: log key info + key_len = len(api_key) if api_key else 0 + key_preview = f"{api_key[:10]}...{api_key[-10:]}" if key_len > 20 else "(too short)" + logger.info(f"Hardcover test: key length={key_len}, preview={key_preview}") + + if not api_key: + # Clear any stored username since there's no key + _save_connected_username(None) + return {"success": False, "message": "No API key configured. Save your key and try again."} + + if key_len < 100: + return {"success": False, "message": f"API key seems too short ({key_len} chars). Expected 500+ chars."} + + try: + provider = HardcoverProvider(api_key=api_key) + # Use the 'me' query to test connection (recommended by API docs) + result = provider._execute_query( + "query { me { id, username } }", + {} + ) + if result is not None: + # Handle both single object and array response formats + me_data = result.get("me", {}) + if isinstance(me_data, list) and me_data: + me_data = me_data[0] + username = me_data.get("username", "Unknown") if isinstance(me_data, dict) else "Unknown" + + # Save the username for persistent display + _save_connected_username(username) + + return {"success": True, "message": f"Connected as: {username}"} + else: + _save_connected_username(None) + return {"success": False, "message": "API request failed - check your API key"} + except Exception as e: + logger.exception("Hardcover connection test failed") + _save_connected_username(None) + return {"success": False, "message": f"Connection failed: {str(e)}"} + + +def _save_connected_username(username: Optional[str]) -> None: + """Save or clear the connected username in config.""" + from cwa_book_downloader.core.settings_registry import save_config_file, load_config_file + + config = load_config_file("hardcover") + if username: + config["_connected_username"] = username + else: + config.pop("_connected_username", None) + save_config_file("hardcover", config) + + +def _get_connected_username() -> Optional[str]: + """Get the stored connected username.""" + from cwa_book_downloader.core.settings_registry import load_config_file + + config = load_config_file("hardcover") + return config.get("_connected_username") + + +@register_settings("hardcover", "Hardcover", icon="book", order=51, group="metadata_providers") +def hardcover_settings(): + """Hardcover metadata provider settings.""" + # Check for connected username to show status + connected_user = _get_connected_username() + test_button_description = f"Connected as: {connected_user}" if connected_user else "Verify your API key works" + + return [ + HeadingField( + key="hardcover_heading", + title="Hardcover", + description="A modern book tracking and discovery platform with a comprehensive API.", + link_url="https://hardcover.app", + link_text="hardcover.app", + ), + CheckboxField( + key="HARDCOVER_ENABLED", + label="Enable Hardcover", + description="Enable Hardcover as a metadata provider for book searches", + default=False, + ), + PasswordField( + key="HARDCOVER_API_KEY", + label="API Key", + description="Get your API key from hardcover.app/account/api", + required=True, + env_supported=False, # UI-only setting, no ENV var support + ), + ActionButton( + key="test_connection", + label="Test Connection", + description=test_button_description, + style="primary", + callback=_test_hardcover_connection, + ), + ] diff --git a/cwa_book_downloader/metadata_providers/openlibrary.py b/cwa_book_downloader/metadata_providers/openlibrary.py new file mode 100644 index 00000000..0c4e5c57 --- /dev/null +++ b/cwa_book_downloader/metadata_providers/openlibrary.py @@ -0,0 +1,625 @@ +"""Open Library metadata provider. No API key required, rate limited.""" + +import time +import threading +from collections import deque +from typing import Any, Deque, Dict, List, Optional, Union + +import requests + +from cwa_book_downloader.core.cache import cacheable +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.core.settings_registry import ( + register_settings, + CheckboxField, + ActionButton, + HeadingField, +) +from cwa_book_downloader.config.env import ( + METADATA_CACHE_SEARCH_TTL, + METADATA_CACHE_BOOK_TTL, +) +from cwa_book_downloader.metadata_providers import ( + BookMetadata, + DisplayField, + MetadataProvider, + MetadataSearchOptions, + SearchType, + SortOrder, + register_provider, + TextSearchField, +) + +logger = setup_logger(__name__) + +OPENLIBRARY_BASE_URL = "https://openlibrary.org" +COVERS_BASE_URL = "https://covers.openlibrary.org" + +# Rate limiting: Open Library allows ~100 requests per minute +# We use a sliding window with 90 requests per 60 seconds for safety margin +RATE_LIMIT_REQUESTS = 90 +RATE_LIMIT_WINDOW_SECONDS = 60 + + +class RateLimiter: + """Simple sliding window rate limiter.""" + + def __init__(self, max_requests: int, window_seconds: int): + """Initialize rate limiter. + + Args: + max_requests: Maximum requests allowed in the window. + window_seconds: Time window in seconds. + """ + self.max_requests = max_requests + self.window_seconds = window_seconds + self.timestamps: Deque[float] = deque() + self.lock = threading.Lock() + + def wait_if_needed(self) -> None: + """Block until a request is allowed. + + Thread-safe implementation that calculates wait time with lock held, + then sleeps without holding the lock to avoid blocking other threads. + """ + wait_time = 0 + + # Calculate wait time with lock held + with self.lock: + now = time.time() + cutoff = now - self.window_seconds + + # Remove timestamps outside the window + while self.timestamps and self.timestamps[0] < cutoff: + self.timestamps.popleft() + + if len(self.timestamps) >= self.max_requests: + # Calculate wait time until oldest request falls outside window + wait_time = self.timestamps[0] + self.window_seconds - now + + # Sleep outside the lock to avoid blocking other threads + if wait_time > 0: + logger.debug(f"Rate limited, waiting {wait_time:.2f}s") + time.sleep(wait_time) + + # Re-acquire lock and record request + with self.lock: + # Re-clean timestamps after sleeping + now = time.time() + cutoff = now - self.window_seconds + while self.timestamps and self.timestamps[0] < cutoff: + self.timestamps.popleft() + + # Record this request + self.timestamps.append(time.time()) + + +# Global rate limiter for Open Library +_rate_limiter = RateLimiter(RATE_LIMIT_REQUESTS, RATE_LIMIT_WINDOW_SECONDS) + + +# Mapping from abstract sort order to Open Library sort parameter +# Note: Open Library only supports relevance (default), new, old, random +SORT_MAPPING: Dict[str, Optional[str]] = { + SortOrder.RELEVANCE: None, # Default (no sort param) + SortOrder.NEWEST: "new", + SortOrder.OLDEST: "old", + # POPULARITY and RATING not supported - will fall back to relevance +} + + +@register_provider("openlibrary") +class OpenLibraryProvider(MetadataProvider): + """Open Library metadata provider using REST API.""" + + name = "openlibrary" + display_name = "Open Library" + requires_auth = False + supported_sorts = [ + SortOrder.RELEVANCE, + SortOrder.NEWEST, + SortOrder.OLDEST, + ] + search_fields = [ + TextSearchField( + key="author", + label="Author", + description="Search by author name", + ), + TextSearchField( + key="title", + label="Title", + description="Search by book title", + ), + ] + + def __init__(self): + """Initialize provider.""" + self.session = requests.Session() + + def is_available(self) -> bool: + """Open Library is always available (no auth required).""" + return True + + def search(self, options: MetadataSearchOptions) -> List[BookMetadata]: + """Search for books using Open Library's search API. + + Args: + options: Search options (query, type, sort, language, pagination, fields). + + Returns: + List of BookMetadata objects. + """ + # Handle ISBN search separately + if options.search_type == SearchType.ISBN: + result = self.search_by_isbn(options.query) + return [result] if result else [] + + # Build cache key from options (include fields for cache differentiation) + fields_key = ":".join(f"{k}={v}" for k, v in sorted(options.fields.items())) + cache_key = f"{options.query}:{options.search_type.value}:{options.sort.value}:{options.language}:{options.limit}:{options.page}:{fields_key}" + return self._search_cached(cache_key, options) + + @cacheable(ttl=METADATA_CACHE_SEARCH_TTL, key_prefix="openlibrary:search") + def _search_cached(self, cache_key: str, options: MetadataSearchOptions) -> List[BookMetadata]: + """Cached search implementation. + + Args: + cache_key: Cache key (used by decorator). + options: Search options. + + Returns: + List of BookMetadata objects. + """ + _rate_limiter.wait_if_needed() + + # Build query params + params: Dict[str, Any] = { + "limit": options.limit, + "page": options.page, + "fields": "key,title,author_name,first_publish_year,cover_i,isbn,publisher,language,subject,ratings_average,ratings_count", + } + + # Field-first search: use custom field values when provided + author_value = options.fields.get("author", "").strip() + title_value = options.fields.get("title", "").strip() + + if author_value or title_value: + # Use field-specific search params (Open Library supports both simultaneously) + if author_value: + params["author"] = author_value + if title_value: + params["title"] = title_value + # Also add general query if provided (for additional filtering) + if options.query.strip(): + params["q"] = options.query + elif options.search_type == SearchType.TITLE: + params["title"] = options.query + elif options.search_type == SearchType.AUTHOR: + params["author"] = options.query + else: + # General search + params["q"] = options.query + + # Add sort if supported (fallback to relevance/default if not) + sort = SORT_MAPPING.get(options.sort) + if sort: + params["sort"] = sort + + # Add language preference if specified + if options.language: + params["lang"] = options.language + + try: + response = self.session.get( + f"{OPENLIBRARY_BASE_URL}/search.json", + params=params, + timeout=15 + ) + response.raise_for_status() + data = response.json() + + books = [] + for doc in data.get("docs", []): + book = self._parse_search_doc(doc) + if book: + books.append(book) + + logger.info(f"Open Library search '{options.query}' returned {len(books)} results") + return books + + except requests.Timeout: + logger.warning("Open Library search timed out") + return [] + except requests.HTTPError as e: + if e.response.status_code == 503: + logger.warning("Open Library service unavailable (503)") + else: + logger.error(f"Open Library HTTP error: {e}") + return [] + except Exception as e: + logger.error(f"Open Library search error: {e}") + return [] + + @cacheable(ttl=METADATA_CACHE_BOOK_TTL, key_prefix="openlibrary:book") + def get_book(self, book_id: str) -> Optional[BookMetadata]: + """Get book details by Open Library work ID. + + Args: + book_id: Open Library work ID (e.g., "OL12345W"). + + Returns: + BookMetadata or None if not found. + """ + _rate_limiter.wait_if_needed() + + # Normalize the book_id format + if not book_id.startswith("OL"): + book_id = f"OL{book_id}" + if not book_id.endswith("W"): + book_id = f"{book_id}W" + + try: + response = self.session.get( + f"{OPENLIBRARY_BASE_URL}/works/{book_id}.json", + timeout=15 + ) + response.raise_for_status() + work = response.json() + + return self._parse_work(work, book_id) + + except requests.Timeout: + logger.warning("Open Library get_book timed out") + return None + except requests.HTTPError as e: + if e.response.status_code == 404: + logger.debug(f"Open Library work not found: {book_id}") + else: + logger.error(f"Open Library HTTP error: {e}") + return None + except Exception as e: + logger.error(f"Open Library get_book error: {e}") + return None + + @cacheable(ttl=METADATA_CACHE_BOOK_TTL, key_prefix="openlibrary:isbn") + def search_by_isbn(self, isbn: str) -> Optional[BookMetadata]: + """Search for a book by ISBN. + + Args: + isbn: ISBN-10 or ISBN-13. + + Returns: + BookMetadata or None if not found. + """ + # Clean ISBN + clean_isbn = isbn.replace("-", "").strip() + + _rate_limiter.wait_if_needed() + + try: + # First try the ISBN API which returns edition data + response = self.session.get( + f"{OPENLIBRARY_BASE_URL}/isbn/{clean_isbn}.json", + timeout=15 + ) + response.raise_for_status() + edition = response.json() + + # Get the work key for full book info + works = edition.get("works", []) + if works: + work_key = works[0].get("key", "") + work_id = work_key.split("/")[-1] if work_key else None + + if work_id: + # Fetch full work data + book = self.get_book(work_id) + if book: + # Update with ISBN from edition if not present + # Use dataclasses.replace() to avoid mutating cached object + from dataclasses import replace + updates = {} + if not book.isbn_10: + isbn_10_list = edition.get("isbn_10", []) + if isbn_10_list: + updates["isbn_10"] = isbn_10_list[0] + if not book.isbn_13: + isbn_13_list = edition.get("isbn_13", []) + if isbn_13_list: + updates["isbn_13"] = isbn_13_list[0] + if updates: + return replace(book, **updates) + return book + + # Fallback: parse edition data directly + return self._parse_edition(edition, clean_isbn) + + except requests.HTTPError as e: + if e.response.status_code == 404: + logger.debug(f"Open Library ISBN not found: {isbn}") + else: + logger.error(f"Open Library ISBN search HTTP error: {e}") + return None + except Exception as e: + logger.error(f"Open Library ISBN search error: {e}") + return None + + def _parse_search_doc(self, doc: dict) -> Optional[BookMetadata]: + """Parse a search document into BookMetadata. + + Args: + doc: Search result document from Open Library. + + Returns: + BookMetadata or None if parsing fails. + """ + try: + # Extract work ID from key + key = doc.get("key", "") + work_id = key.split("/")[-1] if key else None + + if not work_id or not doc.get("title"): + return None + + # Get authors + authors = doc.get("author_name", []) + if not isinstance(authors, list): + authors = [authors] if authors else [] + + # Get ISBNs + isbns = doc.get("isbn", []) + isbn_10 = None + isbn_13 = None + for isbn in isbns: + if len(isbn) == 10 and not isbn_10: + isbn_10 = isbn + elif len(isbn) == 13 and not isbn_13: + isbn_13 = isbn + if isbn_10 and isbn_13: + break + + # Get cover URL + cover_id = doc.get("cover_i") + cover_url = f"{COVERS_BASE_URL}/b/id/{cover_id}-L.jpg" if cover_id else None + + # Get publishers (take first one) + publishers = doc.get("publisher", []) + publisher = publishers[0] if publishers else None + + # Get languages (take first one) + languages = doc.get("language", []) + language = languages[0] if languages else None + + # Get subjects as genres (take first 5) + subjects = doc.get("subject", []) + genres = subjects[:5] if subjects else [] + + # Build display fields from Open Library-specific data + display_fields = [] + + # Rating (if available - not always present) + ratings_avg = doc.get("ratings_average") + ratings_count = doc.get("ratings_count") + if ratings_avg is not None and ratings_avg > 0: + rating_str = f"{ratings_avg:.1f}" + if ratings_count: + rating_str += f" ({ratings_count:,})" + display_fields.append(DisplayField(label="Rating", value=rating_str, icon="star")) + + return BookMetadata( + provider="openlibrary", + provider_id=work_id, + title=doc["title"], + provider_display_name="Open Library", + authors=authors, + isbn_10=isbn_10, + isbn_13=isbn_13, + cover_url=cover_url, + publisher=publisher, + publish_year=doc.get("first_publish_year"), + language=language, + genres=genres, + source_url=f"{OPENLIBRARY_BASE_URL}/works/{work_id}", + display_fields=display_fields, + ) + + except Exception as e: + logger.debug(f"Failed to parse Open Library search doc: {e}") + return None + + def _parse_work(self, work: dict, work_id: str) -> Optional[BookMetadata]: + """Parse a work object into BookMetadata. + + Args: + work: Work data from Open Library API. + work_id: The work ID. + + Returns: + BookMetadata or None if parsing fails. + """ + try: + title = work.get("title") + if not title: + return None + + # Get description + description = work.get("description") + if isinstance(description, dict): + description = description.get("value") + + # Get authors (requires additional API calls) + authors = [] + for author_ref in work.get("authors", []): + author_key = None + if isinstance(author_ref, dict): + author_key = author_ref.get("author", {}).get("key") + if author_key: + author_name = self._get_author_name(author_key) + if author_name: + authors.append(author_name) + + # Get cover URL from covers array + cover_url = None + covers = work.get("covers", []) + if covers: + cover_id = covers[0] + cover_url = f"{COVERS_BASE_URL}/b/id/{cover_id}-L.jpg" + + # Get subjects as genres + subjects = work.get("subjects", []) + genres = subjects[:5] if subjects else [] + + return BookMetadata( + provider="openlibrary", + provider_id=work_id, + title=title, + provider_display_name="Open Library", + authors=authors, + cover_url=cover_url, + description=description, + genres=genres, + source_url=f"{OPENLIBRARY_BASE_URL}/works/{work_id}", + ) + + except Exception as e: + logger.debug(f"Failed to parse Open Library work: {e}") + return None + + def _parse_edition(self, edition: dict, isbn: str) -> Optional[BookMetadata]: + """Parse an edition object into BookMetadata (fallback for ISBN lookup). + + Args: + edition: Edition data from Open Library API. + isbn: The ISBN used for lookup. + + Returns: + BookMetadata or None if parsing fails. + """ + try: + title = edition.get("title") + if not title: + return None + + # Get the edition key as ID + key = edition.get("key", "") + edition_id = key.split("/")[-1] if key else isbn + + # Get ISBNs + isbn_10_list = edition.get("isbn_10", []) + isbn_13_list = edition.get("isbn_13", []) + isbn_10 = isbn_10_list[0] if isbn_10_list else None + isbn_13 = isbn_13_list[0] if isbn_13_list else None + + # Get publishers + publishers = edition.get("publishers", []) + publisher = publishers[0] if publishers else None + + # Get cover URL + cover_url = None + covers = edition.get("covers", []) + if covers: + cover_id = covers[0] + cover_url = f"{COVERS_BASE_URL}/b/id/{cover_id}-L.jpg" + + # Get publish date and try to extract year + publish_year = None + publish_date = edition.get("publish_date", "") + if publish_date: + # Try to extract year from various formats + import re + year_match = re.search(r'\b(19|20)\d{2}\b', publish_date) + if year_match: + publish_year = int(year_match.group()) + + return BookMetadata( + provider="openlibrary", + provider_id=edition_id, + title=title, + provider_display_name="Open Library", + isbn_10=isbn_10, + isbn_13=isbn_13, + cover_url=cover_url, + publisher=publisher, + publish_year=publish_year, + source_url=f"{OPENLIBRARY_BASE_URL}{key}" if key else None, + ) + + except Exception as e: + logger.debug(f"Failed to parse Open Library edition: {e}") + return None + + def _get_author_name(self, author_key: str) -> Optional[str]: + """Get author name from author key. + + Args: + author_key: Open Library author key (e.g., "/authors/OL123A"). + + Returns: + Author name or None. + """ + _rate_limiter.wait_if_needed() + + try: + response = self.session.get( + f"{OPENLIBRARY_BASE_URL}{author_key}.json", + timeout=10 + ) + response.raise_for_status() + author = response.json() + return author.get("name") + + except Exception: + # Don't log errors for author lookups - they're supplementary + return None + + +def _test_openlibrary_connection() -> Dict[str, Any]: + """Test the Open Library API connection.""" + try: + provider = OpenLibraryProvider() + # Simple API call to test connectivity + response = provider.session.get( + f"{OPENLIBRARY_BASE_URL}/search.json", + params={"q": "test", "limit": 1}, + timeout=10 + ) + response.raise_for_status() + data = response.json() + if "docs" in data: + return {"success": True, "message": "Successfully connected to Open Library API"} + else: + return {"success": False, "message": "Unexpected response from API"} + except requests.Timeout: + return {"success": False, "message": "Connection timed out"} + except requests.RequestException as e: + return {"success": False, "message": f"Connection failed: {str(e)}"} + except Exception as e: + return {"success": False, "message": f"Error: {str(e)}"} + + +@register_settings("openlibrary", "Open Library", icon="library", order=52, group="metadata_providers") +def openlibrary_settings(): + """Open Library metadata provider settings.""" + return [ + HeadingField( + key="openlibrary_heading", + title="Open Library", + description="An initiative of the Internet Archive. A free, open-source library catalog with millions of books. No API key required.", + link_url="https://openlibrary.org", + link_text="openlibrary.org", + ), + CheckboxField( + key="OPENLIBRARY_ENABLED", + label="Enable Open Library", + description="Enable Open Library as a metadata provider for book searches", + default=False, + ), + ActionButton( + key="test_connection", + label="Test Connection", + description="Verify Open Library API is accessible", + style="primary", + callback=_test_openlibrary_connection, + ), + ] diff --git a/cwa_book_downloader/release_sources/__init__.py b/cwa_book_downloader/release_sources/__init__.py new file mode 100644 index 00000000..c82753c0 --- /dev/null +++ b/cwa_book_downloader/release_sources/__init__.py @@ -0,0 +1,317 @@ +"""Release source plugin system - base classes and registry.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field, asdict +from enum import Enum +from threading import Event +from typing import List, Optional, Dict, Type, Callable, Literal, Any + +from cwa_book_downloader.core.models import DownloadTask +from cwa_book_downloader.metadata_providers import BookMetadata + + +class ReleaseProtocol(str, Enum): + """Protocol for downloading a release.""" + HTTP = "http" # Direct HTTP download + TORRENT = "torrent" # BitTorrent + NZB = "nzb" # Usenet NZB + DCC = "dcc" # IRC DCC + + +@dataclass +class Release: + """A downloadable release - all sources return this same structure.""" + source: str # "direct", "prowlarr", "irc", etc. + source_id: str # ID within that source + title: str + format: Optional[str] = None + language: Optional[str] = None # ISO 639-1 code (e.g., "en", "de", "fr") + size: Optional[str] = None + size_bytes: Optional[int] = None + download_url: Optional[str] = None + info_url: Optional[str] = None # Link to release info page (e.g., tracker) - makes title clickable + protocol: Optional[ReleaseProtocol] = None + indexer: Optional[str] = None # Source name for display + seeders: Optional[int] = None # For torrents + peers: Optional[str] = None # For torrents: "seeders/leechers" display string + extra: Dict = field(default_factory=dict) # Source-specific metadata + + +@dataclass +class DownloadProgress: + """Progress update structure. + + DEPRECATED: This class is deprecated and will be removed. + The new DownloadHandler.download() uses simpler callbacks: + - progress_callback(float) for progress percentage + - status_callback(str, Optional[str]) for status and message + """ + status: str # "queued", "resolving", "downloading", "complete", "failed" + progress: float # 0-100 + status_message: Optional[str] = None + download_speed: Optional[int] = None + eta: Optional[int] = None + save_path: Optional[str] = None + + +# --- Column Schema for Plugin-Driven UI --- + +class ColumnRenderType(str, Enum): + """How the frontend should render the column value.""" + TEXT = "text" # Plain text + BADGE = "badge" # Colored badge (format, language) + SIZE = "size" # File size formatting + NUMBER = "number" # Numeric value + PEERS = "peers" # Peers display: "S/L" with color based on seeder count + + +class ColumnAlign(str, Enum): + """Column alignment options.""" + LEFT = "left" + CENTER = "center" + RIGHT = "right" + + +@dataclass +class ColumnColorHint: + """Color hint for badge-type columns.""" + type: Literal["map", "static"] # "map" uses frontend colorMaps, "static" is fixed class + value: str # Map name ("format", "language") or Tailwind class + + +@dataclass +class ColumnSchema: + """Definition for a single column in the release list.""" + key: str # Data path (e.g., "format", "extra.language") + label: str # Accessibility label + render_type: ColumnRenderType = ColumnRenderType.TEXT + align: ColumnAlign = ColumnAlign.LEFT + width: str = "auto" # CSS width (e.g., "80px", "minmax(0,2fr)") + hide_mobile: bool = False # Hide on small screens + color_hint: Optional[ColumnColorHint] = None # For BADGE render type + fallback: str = "-" # Value to show when data is missing + uppercase: bool = False # Force uppercase display + + +class LeadingCellType(str, Enum): + """Type of leading cell to display in release rows.""" + THUMBNAIL = "thumbnail" # Show book cover image + BADGE = "badge" # Show colored badge (e.g., "Torrent", "Usenet") + NONE = "none" # No leading cell + + +@dataclass +class LeadingCellConfig: + """Configuration for the leading cell in release rows.""" + type: LeadingCellType = LeadingCellType.THUMBNAIL + key: Optional[str] = None # Field path for data (e.g., "extra.preview" or "extra.download_type") + color_hint: Optional[ColumnColorHint] = None # For badge type - maps values to colors + uppercase: bool = False # Force uppercase for badge text + + +@dataclass +class ReleaseColumnConfig: + """Complete column configuration for a release source.""" + columns: List[ColumnSchema] + grid_template: str = "minmax(0,2fr) 60px 80px 80px" # CSS grid-template-columns + leading_cell: Optional[LeadingCellConfig] = None # Defaults to thumbnail mode if None + + +def serialize_column_config(config: ReleaseColumnConfig) -> Dict[str, Any]: + """Serialize column configuration for API response.""" + result: Dict[str, Any] = { + "columns": [ + { + "key": col.key, + "label": col.label, + "render_type": col.render_type.value, + "align": col.align.value, + "width": col.width, + "hide_mobile": col.hide_mobile, + "color_hint": { + "type": col.color_hint.type, + "value": col.color_hint.value + } if col.color_hint else None, + "fallback": col.fallback, + "uppercase": col.uppercase, + } + for col in config.columns + ], + "grid_template": config.grid_template, + } + + # Include leading_cell config if specified + if config.leading_cell: + result["leading_cell"] = { + "type": config.leading_cell.type.value, + "key": config.leading_cell.key, + "color_hint": { + "type": config.leading_cell.color_hint.type, + "value": config.leading_cell.color_hint.value + } if config.leading_cell.color_hint else None, + "uppercase": config.leading_cell.uppercase, + } + + return result + + +def _default_column_config() -> ReleaseColumnConfig: + """Default column configuration used when source doesn't define its own.""" + return ReleaseColumnConfig( + columns=[ + ColumnSchema( + key="extra.language", + label="Language", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="60px", + hide_mobile=False, # Language shown on mobile + color_hint=ColumnColorHint(type="map", value="language"), + uppercase=True, + ), + ColumnSchema( + key="format", + label="Format", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="80px", + hide_mobile=False, # Format shown on mobile + color_hint=ColumnColorHint(type="map", value="format"), + uppercase=True, + ), + ColumnSchema( + key="size", + label="Size", + render_type=ColumnRenderType.SIZE, + align=ColumnAlign.CENTER, + width="80px", + hide_mobile=False, # Size shown on mobile + ), + ], + grid_template="minmax(0,2fr) 60px 80px 80px" + ) + + +class ReleaseSource(ABC): + """Interface for searching a release source.""" + name: str # "direct", "prowlarr" + display_name: str # "Direct Download", "Prowlarr" + + @abstractmethod + def search(self, book: BookMetadata) -> List[Release]: + """Search for releases of a book.""" + pass + + @abstractmethod + def is_available(self) -> bool: + """Check if this source is configured and reachable.""" + pass + + @classmethod + def get_column_config(cls) -> ReleaseColumnConfig: + """Get the column configuration for this source's release list UI. + + Override this method in subclasses to provide custom columns. + Default implementation returns standard columns (language, format, size). + """ + return _default_column_config() + + +class DownloadHandler(ABC): + """Interface for executing downloads from a source. + + Handlers receive a DownloadTask and do everything internally: + - Fetch source-specific data using task.task_id + - Execute the download (HTTP, torrent, usenet, etc.) + - Report progress via callbacks + - Return the final file path + """ + + @abstractmethod + def download( + self, + task: DownloadTask, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, Optional[str]], None] + ) -> Optional[str]: + """ + Execute download. Handler does everything internally. + + Args: + task: The download task with task_id and display info + cancel_flag: Event to check for cancellation + progress_callback: Called with progress percentage (0-100) + status_callback: Called with (status, message) for status updates + + Returns: + Path to downloaded file if successful, None otherwise + """ + pass + + @abstractmethod + def cancel(self, task_id: str) -> bool: + """Cancel an in-progress download.""" + pass + + +# --- Registry --- + +_SOURCES: Dict[str, Type[ReleaseSource]] = {} +_HANDLERS: Dict[str, Type[DownloadHandler]] = {} + + +def register_source(name: str): + """Decorator to register a release source.""" + def decorator(cls): + _SOURCES[name] = cls + return cls + return decorator + + +def register_handler(name: str): + """Decorator to register a download handler.""" + def decorator(cls): + _HANDLERS[name] = cls + return cls + return decorator + + +def get_source(name: str) -> ReleaseSource: + """Get a release source instance by name.""" + if name not in _SOURCES: + raise ValueError(f"Unknown release source: {name}") + return _SOURCES[name]() + + +def get_handler(name: str) -> DownloadHandler: + """Get a download handler instance by name.""" + if name not in _HANDLERS: + raise ValueError(f"Unknown download handler: {name}") + return _HANDLERS[name]() + + +def list_available_sources() -> List[dict]: + """For frontend - list sources that are configured.""" + return [ + {"name": name, "display_name": src().display_name} + for name, src in _SOURCES.items() + if src().is_available() + ] + + +def get_source_display_name(name: str) -> str: + """Get display name for a source by its identifier. + + Falls back to title-cased name if source not found. + """ + if name in _SOURCES: + return _SOURCES[name]().display_name + # Fallback: convert snake_case to Title Case + return name.replace('_', ' ').title() + + +# Import source implementations to trigger registration +# These must be imported AFTER the base classes and registry are defined +from cwa_book_downloader.release_sources import direct_download # noqa: F401, E402 +# from cwa_book_downloader.release_sources import prowlarr # noqa: F401, E402 # TODO: Re-enable when prowlarr plugin is ready diff --git a/book_manager.py b/cwa_book_downloader/release_sources/direct_download.py similarity index 60% rename from book_manager.py rename to cwa_book_downloader/release_sources/direct_download.py index deb412c5..e5a615a9 100644 --- a/book_manager.py +++ b/cwa_book_downloader/release_sources/direct_download.py @@ -1,8 +1,11 @@ -"""Book download manager handling search and retrieval operations.""" +"""Direct download source - Anna's Archive/Libgen with fallback cascade.""" import itertools import json +import os import re +import shutil +import subprocess import time from pathlib import Path from threading import Event @@ -11,45 +14,71 @@ from urllib.parse import quote from bs4 import BeautifulSoup, NavigableString, Tag -import downloader -import network -from config import BOOK_LANGUAGE, SUPPORTED_FORMATS -from env import AA_DONATOR_KEY, ALLOW_USE_WELIB, DEBUG_SKIP_SOURCES, DOWNLOAD_PATHS, PRIORITIZE_WELIB, USE_CF_BYPASS -from logger import setup_logger -from models import BookInfo, SearchFilters +from cwa_book_downloader.download import http as downloader +from cwa_book_downloader.download import network +from cwa_book_downloader.config.env import DEBUG_SKIP_SOURCES, DOWNLOAD_PATHS, TMP_DIR +from cwa_book_downloader.core.config import config +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.core.models import BookInfo, SearchFilters, DownloadTask +from cwa_book_downloader.metadata_providers import BookMetadata +from cwa_book_downloader.release_sources import ( + Release, + ReleaseProtocol, + ReleaseSource, + DownloadHandler, + register_source, + register_handler, + ReleaseColumnConfig, + ColumnSchema, + ColumnRenderType, + ColumnAlign, + ColumnColorHint, +) logger = setup_logger(__name__) -# Round-robin counter for AA slow download source rotation -# Distributes concurrent downloads across different partner mirrors _aa_slow_rotation = itertools.count() +_url_source_types: dict[str, str] = {} if DEBUG_SKIP_SOURCES: logger.warning("DEBUG_SKIP_SOURCES active: skipping sources %s", DEBUG_SKIP_SOURCES) +_DOWNLOAD_SOURCES = [ + ("welib", "Welib", ["welib.org"]), + ("aa-fast", "Anna's Archive (Fast)", ["/dyn/api/fast_download"]), + ("aa-slow-wait", "Anna's Archive (Waitlist)", []), # Matched via _url_source_types + ("aa-slow-nowait", "Anna's Archive", []), # Matched via _url_source_types + ("aa-slow", "Anna's Archive", ["/slow_download/", "annas-"]), # Fallback for untagged AA URLs + ("libgen", "Libgen", ["libgen"]), + ("zlib", "Z-Library", ["z-lib", "zlibrary"]), +] + +_SOURCE_FAILURE_THRESHOLD = 4 +_MIN_VALID_FILE_SIZE = 10 * 1024 + class SearchUnavailable(Exception): """Raised when Anna's Archive cannot be reached via any mirror/DNS.""" pass - def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: """Search for books matching the query. Args: query: Search term (ISBN, title, author, etc.) + filters: Search filters (language, format, content type, etc.) Returns: List[BookInfo]: List of matching books Raises: - Exception: If no books found or parsing fails + SearchUnavailable: If Anna's Archive cannot be reached + Exception: If parsing fails """ query_html = quote(query) if filters.isbn: - # ISBNs are included in query string isbns = " || ".join( [f"('isbn13:{isbn}' || 'isbn10:{isbn}')" for isbn in filters.isbn] ) @@ -57,7 +86,7 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: filters_query = "" - for value in filters.lang or BOOK_LANGUAGE: + for value in filters.lang or config.BOOK_LANGUAGE: if value != "all": filters_query += f"&lang={quote(value)}" @@ -68,12 +97,11 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: for value in filters.content: filters_query += f"&content={quote(value)}" - # Handle format filter - formats_to_use = filters.format if filters.format else SUPPORTED_FORMATS + formats_to_use = filters.format if filters.format else config.SUPPORTED_FORMATS index = 1 for filter_type, filter_values in vars(filters).items(): - if filter_type == "author" or filter_type == "title" and filter_values: + if (filter_type == "author" or filter_type == "title") and filter_values: for value in filter_values: filters_query += ( f"&termtype_{index}={filter_type}&termval_{index}={quote(value)}" @@ -119,19 +147,39 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: books.sort( key=lambda x: ( - SUPPORTED_FORMATS.index(x.format) - if x.format in SUPPORTED_FORMATS - else len(SUPPORTED_FORMATS) + config.SUPPORTED_FORMATS.index(x.format) + if x.format in config.SUPPORTED_FORMATS + else len(config.SUPPORTED_FORMATS) ) ) return books +def get_book_info(book_id: str) -> BookInfo: + """Get detailed information for a specific book. + + Args: + book_id: Book identifier (MD5 hash) + + Returns: + BookInfo: Detailed book information including download URLs + """ + url = f"{network.get_aa_base_url()}/md5/{book_id}" + selector = network.AAMirrorSelector() + html = downloader.html_get_page(url, selector=selector) + + if not html: + raise Exception(f"Failed to fetch book info for ID: {book_id}") + + soup = BeautifulSoup(html, "html.parser") + + return _parse_book_info_page(soup, book_id) + + def _parse_search_result_row(row: Tag) -> Optional[BookInfo]: """Parse a single search result row into a BookInfo object.""" try: - # Skip ad rows if row.text.strip().lower().startswith("your ad here"): return None cells = row.find_all("td") @@ -155,27 +203,6 @@ def _parse_search_result_row(row: Tag) -> Optional[BookInfo]: return None -def get_book_info(book_id: str) -> BookInfo: - """Get detailed information for a specific book. - - Args: - book_id: Book identifier (MD5 hash) - - Returns: - BookInfo: Detailed book information - """ - url = f"{network.get_aa_base_url()}/md5/{book_id}" - selector = network.AAMirrorSelector() - html = downloader.html_get_page(url, selector=selector) - - if not html: - raise Exception(f"Failed to fetch book info for ID: {book_id}") - - soup = BeautifulSoup(html, "html.parser") - - return _parse_book_info_page(soup, book_id) - - def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: """Parse the book info page HTML into a BookInfo object.""" data = soup.select_one("body > main > div:nth-of-type(1)") @@ -196,7 +223,6 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div") divs = list(data.children) - # Collect download URLs by source type (lists preserve page order, dedup inline) slow_urls_no_waitlist: list[str] = [] slow_urls_with_waitlist: list[str] = [] external_urls_libgen: list[str] = [] @@ -220,12 +246,11 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: else: _append_unique(slow_urls_with_waitlist, href) elif 'libgen.li' in href: - # Normalize libgen domains libgen_url = re.sub(r'libgen\.(li|lc|is|bz|st)', 'libgen.gl', href) _append_unique(external_urls_libgen, libgen_url) elif text.startswith("z-lib") and ".onion/" not in href: _append_unique(external_urls_z_lib, href) - except: + except Exception: pass logger.debug( @@ -239,22 +264,16 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: urls = [] - # Priority: reliable sources first, then external fallbacks - # 1. AA slow (no waitlist) - instant but can be slow - # 2. Libgen - instant, external - # 3. AA slow (waitlist) - has countdown timer but faster once started - # Note: Z-Library disabled - download tokens are session-bound - urls += slow_urls_no_waitlist if USE_CF_BYPASS else [] + # Z-Library disabled - download tokens are session-bound + urls += slow_urls_no_waitlist if config.USE_CF_BYPASS else [] urls += external_urls_libgen - urls += slow_urls_with_waitlist if USE_CF_BYPASS else [] + urls += slow_urls_with_waitlist if config.USE_CF_BYPASS else [] for i in range(len(urls)): urls[i] = downloader.get_absolute_url(network.get_aa_base_url(), urls[i]) - # Remove empty urls urls = [url for url in urls if url != ""] - # Tag AA slow URLs with detailed source type for skip/retry tracking base_url = network.get_aa_base_url() for rel_url in slow_urls_no_waitlist: abs_url = downloader.get_absolute_url(base_url, rel_url) @@ -265,7 +284,6 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: if abs_url: _url_source_types[abs_url] = "aa-slow-wait" - # Filter out divs that are not text original_divs = divs divs = [div for div in divs if div.text.strip() != ""] @@ -273,11 +291,11 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: format = "" size = "" content = "" - + for _details in all_details: _details = _details.split(" · ") for f in _details: - if format == "" and f.strip().lower() in SUPPORTED_FORMATS: + if format == "" and f.strip().lower() in config.SUPPORTED_FORMATS: format = f.strip().lower() if size == "" and any(u in f.strip().lower() for u in ["mb", "kb", "gb"]): # Preserve original case but uppercase the unit (e.g., "5.2 mb" -> "5.2 MB") @@ -295,7 +313,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: if size == "" and "." in stripped: # Uppercase any size units size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE) - + book_title = _find_in_divs(divs, "🔍")[0].strip("🔍").strip() # Extract basic information @@ -324,12 +342,9 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: if info.get("Year"): book_info.year = info["Year"][0] - # TODO : - # Backfill missing metadata from original book - # To do this, we need to cache the results of search_books() in some kind of LRU - return book_info + def _find_in_divs(divs: List, text: str, is_class: bool = False) -> List[str]: """Find divs containing text or having a specific class.""" results = [] @@ -341,74 +356,6 @@ def _find_in_divs(divs: List, text: str, is_class: bool = False) -> List[str]: results.append(div.text.strip()) return results -# Download source definitions: (log_label, friendly_name, url_patterns) -_DOWNLOAD_SOURCES = [ - ("welib", "Welib", ["welib.org"]), - ("aa-fast", "Anna's Archive (Fast)", ["/dyn/api/fast_download"]), - ("aa-slow-wait", "Anna's Archive (Waitlist)", []), # Matched via _url_source_types - ("aa-slow-nowait", "Anna's Archive", []), # Matched via _url_source_types - ("aa-slow", "Anna's Archive", ["/slow_download/", "annas-"]), # Fallback for untagged AA URLs - ("libgen", "Libgen", ["libgen"]), - ("zlib", "Z-Library", ["z-lib", "zlibrary"]), -] - -# Track detailed source types for AA slow URLs (populated during get_book_info) -_url_source_types: dict[str, str] = {} - - -def _get_source_info(link: str) -> tuple[str, str]: - """Get source label and friendly name for a download link. - - Args: - link: Download URL - - Returns: - Tuple of (log_label, friendly_name) - """ - # Check detailed source type mapping first (for AA slow distinction) - if link in _url_source_types: - detailed_label = _url_source_types[link] - for log_label, friendly_name, _ in _DOWNLOAD_SOURCES: - if log_label == detailed_label: - return log_label, friendly_name - - for log_label, friendly_name, patterns in _DOWNLOAD_SOURCES: - if patterns and any(pattern in link for pattern in patterns): - return log_label, friendly_name - return "unknown", "Mirror" - - -def _label_source(link: str) -> str: - """Get lightweight source tag for logging/metrics.""" - return _get_source_info(link)[0] - - -def _friendly_source_name(link: str) -> str: - """Get user-friendly name for a download source.""" - return _get_source_info(link)[1] - -def _get_download_urls_from_welib(book_id: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> list[str]: - """Get download URLs from welib.org (bypasser required).""" - if not ALLOW_USE_WELIB: - return [] - url = f"https://welib.org/md5/{book_id}" - logger.info(f"Fetching welib.org download URLs for {book_id}") - try: - html = downloader.html_get_page(url, use_bypasser=True, selector=selector or network.AAMirrorSelector(), cancel_flag=cancel_flag) - except Exception as exc: - logger.error_trace(f"Welib fetch failed for {book_id}: {exc}") - return [] - if not html: - logger.warning(f"Welib page empty for {book_id}") - return [] - - soup = BeautifulSoup(html, "html.parser") - links = [ - downloader.get_absolute_url(url, a["href"]) - for a in soup.find_all("a", href=True) - if "/slow_download/" in a["href"] - ] - return list(dict.fromkeys(links)) # Dedupe while preserving order def _get_next_value_div(label_div: Tag) -> Optional[Tag]: """Find the next sibling div that holds the value for a metadata label.""" @@ -419,6 +366,7 @@ def _get_next_value_div(label_div: Tag) -> Optional[Tag]: sibling = sibling.next_sibling return None + def _extract_book_description(soup: BeautifulSoup) -> Optional[str]: """Extract the primary or alternative description from the book page.""" container = soup.select_one(".js-md5-top-box-description") @@ -456,6 +404,7 @@ def _extract_book_description(soup: BeautifulSoup) -> Optional[str]: return None + def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]: """Extract metadata from book info divs.""" info: Dict[str, List[str]] = {} @@ -472,7 +421,7 @@ def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]: if key not in info: info[key] = set() info[key].add(value) - + # make set into list for key, value in info.items(): info[key] = list(value) @@ -494,19 +443,74 @@ def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]: } -# After N consecutive failures of the same source type, skip remaining sources of that type -SOURCE_FAILURE_THRESHOLD = 4 +def _get_source_info(link: str) -> tuple[str, str]: + """Get source label and friendly name for a download link. -# Minimum valid file size in bytes (10KB) - anything smaller is likely an error page -MIN_VALID_FILE_SIZE = 10 * 1024 + Args: + link: Download URL + + Returns: + Tuple of (log_label, friendly_name) + """ + # Check detailed source type mapping first (for AA slow distinction) + if link in _url_source_types: + detailed_label = _url_source_types[link] + for log_label, friendly_name, _ in _DOWNLOAD_SOURCES: + if log_label == detailed_label: + return log_label, friendly_name + + for log_label, friendly_name, patterns in _DOWNLOAD_SOURCES: + if patterns and any(pattern in link for pattern in patterns): + return log_label, friendly_name + return "unknown", "Mirror" -def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None) -> Optional[str]: +def _label_source(link: str) -> str: + """Get lightweight source tag for logging/metrics.""" + return _get_source_info(link)[0] + + +def _friendly_source_name(link: str) -> str: + """Get user-friendly name for a download source.""" + return _get_source_info(link)[1] + + +def _get_download_urls_from_welib(book_id: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> list[str]: + """Get download URLs from welib.org (bypasser required).""" + if not config.ALLOW_USE_WELIB: + return [] + url = f"https://welib.org/md5/{book_id}" + logger.info(f"Fetching welib.org download URLs for {book_id}") + try: + html = downloader.html_get_page(url, use_bypasser=True, selector=selector or network.AAMirrorSelector(), cancel_flag=cancel_flag) + except Exception as exc: + logger.error_trace(f"Welib fetch failed for {book_id}: {exc}") + return [] + if not html: + logger.warning(f"Welib page empty for {book_id}") + return [] + + soup = BeautifulSoup(html, "html.parser") + links = [ + downloader.get_absolute_url(url, a["href"]) + for a in soup.find_all("a", href=True) + if "/slow_download/" in a["href"] + ] + return list(dict.fromkeys(links)) # Dedupe while preserving order + + +def _download_book( + book_info: BookInfo, + book_path: Path, + progress_callback: Optional[Callable[[float], None]] = None, + cancel_flag: Optional[Event] = None, + status_callback: Optional[Callable[[str, Optional[str]], None]] = None +) -> Optional[str]: """Download a book from available sources. Args: - book_id: Book identifier (MD5 hash) - title: Book title for logging + book_info: Book information with download URLs + book_path: Path to save the downloaded file progress_callback: Optional callback for download progress updates cancel_flag: Optional cancellation flag status_callback: Optional callback for status updates (status, message) @@ -514,18 +518,18 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio Returns: str: Download URL if successful, None otherwise """ - selector = network.AAMirrorSelector() if len(book_info.download_urls) == 0: book_info = get_book_info(book_info.id) download_links = list(book_info.download_urls) - # If AA_DONATOR_KEY is set, use the fast download URL. Else try other sources. - if AA_DONATOR_KEY != "": + # If config.AA_DONATOR_KEY is set, use the fast download URL. Else try other sources. + # Use truthiness check to handle both None and empty string + if config.AA_DONATOR_KEY: download_links.insert( 0, - f"{network.get_aa_base_url()}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}", + f"{network.get_aa_base_url()}/dyn/api/fast_download.json?md5={book_info.id}&key={config.AA_DONATOR_KEY}", ) # Preserve order but drop duplicates to avoid retrying the same host @@ -558,27 +562,27 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio logger.info(f"AA source rotation: nowait={nowait_rotation}, wait={wait_rotation}") links_queue = download_links - + # Fetch welib URLs upfront when prioritized welib_fallback_loaded = "welib" in DEBUG_SKIP_SOURCES # Skip welib entirely if in debug skip list - if USE_CF_BYPASS and PRIORITIZE_WELIB and ALLOW_USE_WELIB and not welib_fallback_loaded: - logger.info("Fetching welib.org download URLs (PRIORITIZE_WELIB enabled)") + if config.USE_CF_BYPASS and config.PRIORITIZE_WELIB and config.ALLOW_USE_WELIB and not welib_fallback_loaded: + logger.info("Fetching welib.org download URLs (config.PRIORITIZE_WELIB enabled)") if status_callback: status_callback("resolving", "Fetching welib sources...") welib_links = _get_download_urls_from_welib(book_info.id, selector=selector, cancel_flag=cancel_flag) if welib_links: links_queue = welib_links + [l for l in links_queue if l not in welib_links] welib_fallback_loaded = True - + total_sources = len(links_queue) - + # Handle case where no download sources are available if total_sources == 0: logger.warning(f"No download sources available for: {book_info.title}") if status_callback: status_callback("error", "No download sources found") return None - + # Track consecutive failures per source type to skip after threshold source_failures: dict[str, int] = {} # Iterate with index so we can append welib links later @@ -595,21 +599,21 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio continue # Skip source types that have failed too many times - if source_failures.get(source_label, 0) >= SOURCE_FAILURE_THRESHOLD: - logger.info("Skipping %s - source type '%s' failed %d times", link, source_label, SOURCE_FAILURE_THRESHOLD) + if source_failures.get(source_label, 0) >= _SOURCE_FAILURE_THRESHOLD: + logger.info("Skipping %s - source type '%s' failed %d times", link, source_label, _SOURCE_FAILURE_THRESHOLD) idx += 1 continue - + try: current_pos = idx + 1 # Update total if we added more sources total_sources = len(links_queue) - + logger.info("Trying download source [%s]: %s (%d/%d)", source_label, link, current_pos, total_sources) - + # Build source context for status messages (e.g., "Welib (1/12)") source_context = f"{friendly_name} (Server #{current_pos})" - + # Update status with simple message showing which source we're trying if status_callback: status_callback("resolving", f"Trying {source_context}") @@ -624,10 +628,10 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio data = downloader.download_url(download_url, book_info.size or "", progress_callback, cancel_flag, selector, status_callback, referer=link) if not data: raise Exception("No data received from download") - + # Validate file size - reject suspiciously small files file_size = data.tell() - if file_size < MIN_VALID_FILE_SIZE: + if file_size < _MIN_VALID_FILE_SIZE: logger.warning(f"Downloaded file too small ({file_size} bytes), likely an error page") raise Exception(f"File too small ({file_size} bytes)") @@ -645,8 +649,8 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio if ( idx >= len(links_queue) and not welib_fallback_loaded - and USE_CF_BYPASS - and ALLOW_USE_WELIB + and config.USE_CF_BYPASS + and config.ALLOW_USE_WELIB ): welib_selector = selector # reuse AA mirror selector for consistency welib_links = _get_download_urls_from_welib(book_info.id, selector=welib_selector, cancel_flag=cancel_flag) @@ -662,13 +666,20 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio # All sources exhausted - report final error to UI if status_callback: status_callback("error", f"All {len(links_queue)} sources failed") - + return None -def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None, selector: Optional[network.AAMirrorSelector] = None, source_context: Optional[str] = None) -> str: +def _get_download_url( + link: str, + title: str, + cancel_flag: Optional[Event] = None, + status_callback: Optional[Callable[[str, Optional[str]], None]] = None, + selector: Optional[network.AAMirrorSelector] = None, + source_context: Optional[str] = None +) -> str: """Extract actual download URL from various source pages. - + Args: link: URL to extract download link from title: Book title for logging @@ -708,7 +719,15 @@ def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None return downloader.get_absolute_url(link, url) -def _extract_slow_download_url(soup: BeautifulSoup, link: str, title: str, cancel_flag: Optional[Event], status_callback, selector, source_context: Optional[str] = None) -> str: +def _extract_slow_download_url( + soup: BeautifulSoup, + link: str, + title: str, + cancel_flag: Optional[Event], + status_callback, + selector, + source_context: Optional[str] = None +) -> str: """Extract download URL from AA slow download pages.""" # Try "Download now" button variations dl_link = soup.find("a", href=True, string="📚 Download now") @@ -753,7 +772,7 @@ def _extract_slow_download_url(soup: BeautifulSoup, link: str, title: str, cance if raw_countdown > MAX_COUNTDOWN_SECONDS: logger.warning(f"Countdown {raw_countdown}s exceeds max, capping at {MAX_COUNTDOWN_SECONDS}s") logger.info(f"Waiting {sleep_time}s for {title}") - + # Live countdown with status updates remaining = sleep_time while remaining > 0: @@ -762,24 +781,371 @@ def _extract_slow_download_url(soup: BeautifulSoup, link: str, title: str, cance wait_msg = f"{source_context} - Waiting {remaining}s" else: wait_msg = f"Waiting {remaining}s" - + if status_callback: status_callback("resolving", wait_msg) - + # Wait 1 second (or until cancelled) if cancel_flag and cancel_flag.wait(timeout=1): logger.info(f"Cancelled wait for {title}") return "" - + remaining -= 1 - + # After countdown, update status and re-fetch if status_callback and source_context: status_callback("resolving", f"{source_context} - Fetching...") - + return _get_download_url(link, title, cancel_flag, status_callback, selector, source_context) # Debug fallback link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]] logger.warning(f"No download URL found. First 10 links: {link_texts}") return "" + + +def _book_info_to_release(book_info: BookInfo) -> Release: + """Convert a BookInfo object to a Release object. + + This bridges the existing BookInfo model (which combines metadata + release info) + to the new Release model (release info only). + """ + return Release( + source="direct_download", + source_id=book_info.id, + title=book_info.title, + format=book_info.format, + size=book_info.size, + download_url=book_info.download_urls[0] if book_info.download_urls else None, + protocol=ReleaseProtocol.HTTP, + indexer="Anna's Archive", + extra={ + "author": book_info.author, + "publisher": book_info.publisher, + "year": book_info.year, + "language": book_info.language, + "content": book_info.content, + "preview": book_info.preview, + "description": book_info.description, + "download_urls": book_info.download_urls, + "info": book_info.info, + } + ) + + +@register_source("direct_download") +class DirectDownloadSource(ReleaseSource): + """ + Direct download source - searches Anna's Archive, Libgen, etc. + + This wraps the search_books() functionality to provide releases + via the plugin interface. + """ + name = "direct_download" + display_name = "Anna's Archive" + + @classmethod + def get_column_config(cls) -> ReleaseColumnConfig: + """Column configuration for Direct Download source. + + Shows language, format, and size badges for each release. + Language is hidden on mobile; format and size are shown. + """ + return ReleaseColumnConfig( + columns=[ + ColumnSchema( + key="extra.language", + label="Language", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="60px", + hide_mobile=False, # Language shown on mobile + color_hint=ColumnColorHint(type="map", value="language"), + uppercase=True, + ), + ColumnSchema( + key="format", + label="Format", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="80px", + hide_mobile=False, # Format shown on mobile + color_hint=ColumnColorHint(type="map", value="format"), + uppercase=True, + ), + ColumnSchema( + key="size", + label="Size", + render_type=ColumnRenderType.SIZE, + align=ColumnAlign.CENTER, + width="80px", + hide_mobile=False, # Size shown on mobile + ), + ], + grid_template="minmax(0,2fr) 60px 80px 80px" + ) + + def search(self, book: BookMetadata) -> List[Release]: + """ + Search for releases using the book's metadata. + + Uses an ISBN-first strategy: + 1. If ISBN available, try ISBN search first (most precise) + 2. If no results or no ISBN, fall back to title+author search + + This approach maximizes accuracy while ensuring we find results. + """ + # Try ISBN search first if available + isbn = book.isbn_13 or book.isbn_10 + if isbn: + logger.debug(f"Searching direct downloads by ISBN: {isbn}") + filters = SearchFilters(isbn=[isbn]) + if book.language: + filters.lang = [book.language] + + try: + book_infos = search_books(isbn, filters) + if book_infos: + logger.info(f"Found {len(book_infos)} releases via ISBN search") + return [_book_info_to_release(bi) for bi in book_infos] + logger.debug(f"No results from ISBN search, falling back to title+author") + except SearchUnavailable: + logger.warning("Direct download search unavailable during ISBN search") + # Fall through to title search + except Exception as e: + logger.warning(f"ISBN search failed, falling back to title+author: {e}") + # Fall through to title search + + # Fallback to title + author search + query_parts = [] + if book.title: + query_parts.append(book.title) + if book.authors: + query_parts.append(book.authors[0]) # Use first author + + query = " ".join(query_parts) + if not query.strip(): + logger.warning("No search query available for book") + return [] + + logger.debug(f"Searching direct downloads by title+author: {query}") + filters = SearchFilters() + if book.language: + filters.lang = [book.language] + + try: + book_infos = search_books(query, filters) + logger.info(f"Found {len(book_infos)} releases via title+author search") + return [_book_info_to_release(bi) for bi in book_infos] + except SearchUnavailable: + logger.warning("Direct download search unavailable") + return [] + except Exception as e: + logger.error(f"Error searching direct download source: {e}") + return [] + + def search_raw(self, query: str, filters: SearchFilters) -> List[BookInfo]: + """ + Raw search using existing query format - for backward compatibility. + + This is used by the existing "Direct Download Only" mode which doesn't + go through the metadata provider layer. + """ + return search_books(query, filters) + + def is_available(self) -> bool: + """Direct download is always available.""" + return True + + +@register_handler("direct_download") +class DirectDownloadHandler(DownloadHandler): + """ + Handler for direct HTTP downloads from Anna's Archive, Libgen, etc. + + Receives a DownloadTask with task_id (AA MD5 hash) and fetches the + book page internally to get download URLs, then cascades through + fallback sources (AA Fast → AA Slow → Libgen → Welib → Z-Lib). + """ + + def download( + self, + task: DownloadTask, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, Optional[str]], None] + ) -> Optional[str]: + """ + Execute a direct HTTP download. + + Uses task.task_id to fetch the book page from Anna's Archive, + extract download URLs, and cascade through fallback sources. + + Args: + task: Download task with task_id (AA MD5 hash) + cancel_flag: Event to check for cancellation + progress_callback: Called with progress percentage (0-100) + status_callback: Called with (status, message) for status updates + + Returns: + Path to downloaded file if successful, None otherwise + """ + try: + # Check for cancellation before starting + if cancel_flag.is_set(): + logger.info(f"Download cancelled before starting: {task.task_id}") + return None + + # Fetch book info from Anna's Archive using task_id + status_callback("resolving", "Fetching book details...") + book_info = get_book_info(task.task_id) + + if not book_info: + status_callback("error", "Could not fetch book details") + return None + + if not book_info.download_urls: + status_callback("error", "No download sources found") + return None + + # Execute the download with the fetched book info + return self._execute_download( + book_info, + cancel_flag, + progress_callback, + status_callback + ) + + except Exception as e: + if cancel_flag.is_set(): + logger.info(f"Download cancelled during error handling: {task.task_id}") + else: + logger.error(f"Error downloading book: {e}") + status_callback("error", str(e)) + return None + + def _execute_download( + self, + book_info: BookInfo, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, Optional[str]], None] + ) -> Optional[str]: + """ + Internal method to execute the download with fetched BookInfo. + + This contains the core download logic: cascade through sources, + handle bypass, move to final location. + """ + try: + logger.info(f"Starting download: {book_info.title}") + + # Prepare paths + full_name = book_info.get_filename() + book_name = full_name if config.USE_BOOK_TITLE else f"{book_info.id}.{book_info.format or 'bin'}" + book_path = TMP_DIR / book_name + + # Check cancellation before download + if cancel_flag.is_set(): + logger.info(f"Download cancelled before download call: {book_info.id}") + return None + + # Execute download via _download_book (handles cascade and bypass) + status_callback("resolving", "Finding download source...") + success_url = _download_book( + book_info, + book_path, + progress_callback, + cancel_flag, + status_callback + ) + + # Check for cancellation after download + if cancel_flag.is_set(): + logger.info(f"Download cancelled during download: {book_info.id}") + if book_path.exists(): + book_path.unlink() + return None + + if not success_url: + status_callback("error", "All download sources failed") + return None + + # Check cancellation before post-processing + if cancel_flag.is_set(): + logger.info(f"Download cancelled before post-processing: {book_info.id}") + if book_path.exists(): + book_path.unlink() + return None + + logger.debug(f"Post-processing download: {book_info.title}") + + # Run custom script if configured + if config.CUSTOM_SCRIPT: + logger.info(f"Running custom script: {config.CUSTOM_SCRIPT}") + subprocess.run([config.CUSTOM_SCRIPT, str(book_path)]) + + # Regenerate filename with fallback to successful download URL for format + full_name = book_info.get_filename(success_url) + book_name = full_name if config.USE_BOOK_TITLE else f"{book_info.id}.{book_info.format or 'bin'}" + + # Determine final directory based on content type + content = book_info.content + final_dir = DOWNLOAD_PATHS.get(content) if content and content in DOWNLOAD_PATHS else Path(config.INGEST_DIR) + os.makedirs(final_dir, exist_ok=True) + + intermediate_path = final_dir / f"{book_info.id}.crdownload" + final_path = final_dir / book_name + + # Handle file already exists - add suffix to avoid overwrite + if final_path.exists(): + base = final_path.stem + ext = final_path.suffix + counter = 1 + while final_path.exists(): + final_path = final_dir / f"{base}_{counter}{ext}" + counter += 1 + logger.info(f"File already exists, saving as: {final_path.name}") + + # Move file to final destination + if book_path.exists(): + logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}") + try: + shutil.move(str(book_path), str(intermediate_path)) + except Exception as e: + try: + logger.debug(f"Error moving book: {e}, will try copying instead") + shutil.copyfile(str(book_path), str(intermediate_path)) + os.remove(str(book_path)) + except Exception as e2: + logger.debug(f"Error copying book: {e2}") + raise + + # Final cancellation check before completing + if cancel_flag.is_set(): + logger.info(f"Download cancelled before final rename: {book_info.id}") + if intermediate_path.exists(): + intermediate_path.unlink() + return None + + os.rename(str(intermediate_path), str(final_path)) + logger.info(f"Download completed successfully: {book_info.title}") + + return str(final_path) + + except Exception as e: + if cancel_flag.is_set(): + logger.info(f"Download cancelled during error handling: {book_info.id}") + else: + logger.error(f"Error downloading book: {e}") + return None + + def cancel(self, task_id: str) -> bool: + """Cancel an in-progress download. + + Cancellation is handled via the cancel_flag passed to download(). + This method exists for the interface but actual cancellation + happens through the Event flag mechanism. + """ + # Cancellation is handled by the orchestrator via cancel_flag + return False diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ffffc885..81ad569e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,3 +1,4 @@ +# Local development - builds from source with debug enabled services: calibre-web-automated-book-downloader-dev: extends: @@ -10,5 +11,7 @@ services: environment: DEBUG: true volumes: - - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader - - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader + - ./.local/config:/config + - ./.local/ingest:/cwa-book-ingest + - ./.local/log:/var/log/cwa-book-downloader + - ./.local/tmp:/tmp/cwa-book-downloader diff --git a/docker-compose.extbp.dev.yml b/docker-compose.extbp.dev.yml index 85dd4a2c..13203e7a 100644 --- a/docker-compose.extbp.dev.yml +++ b/docker-compose.extbp.dev.yml @@ -1,6 +1,6 @@ +# Local development - External bypasser variant services: calibre-web-automated-book-downloader-extbp-dev: - container_name: cwa-bd-extbp-dev extends: file: ./docker-compose.extbp.yml service: calibre-web-automated-book-downloader-extbp @@ -10,25 +10,14 @@ services: target: cwa-bd-extbp environment: DEBUG: true - USE_DOH: true - CUSTOM_DNS: cloudflare - USE_CF_BYPASS: true # Enable Cloudflare bypass (default: true) - # External Cloudflare Bypass environment variables - EXT_BYPASSER_URL: "http://flaresolverr:8191" # URL of the external Cloudflare resolver service (used FlareSolverr) - EXT_BYPASSER_PATH: "/v1" # Path for external Cloudflare resolver API (default: /v1) - EXT_BYPASSER_TIMEOUT: 60000 # Timeout for external Cloudflare resolver requests (default: 60000) + EXT_BYPASSER_URL: http://flaresolverr:8191 + EXT_BYPASSER_PATH: /v1 + EXT_BYPASSER_TIMEOUT: 60000 volumes: - #- /tmp/cwa-book-downloader:/tmp/cwa-book-downloader - #- /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader - - ./deploy/ingest:/cwa-book-ingest - - ./deploy/log:/var/log/cwa-book-downloader - - ./deploy/tmp:/tmp/cwa-book-downloader + - ./.local/config:/config + - ./.local/ingest:/cwa-book-ingest + - ./.local/log:/var/log/cwa-book-downloader + - ./.local/tmp:/tmp/cwa-book-downloader - flaresolverr: # External Cloudflare resolver service - image: ghcr.io/flaresolverr/flaresolverr:v3.3.22 - container_name: flaresolverr - environment: - LOG_LEVEL: info - LOG_HTML: false - CAPTCHA_SOLVER: none - TZ: Europe/Rome + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest diff --git a/docker-compose.extbp.yml b/docker-compose.extbp.yml index 6893fb74..e0c992fb 100644 --- a/docker-compose.extbp.yml +++ b/docker-compose.extbp.yml @@ -1,27 +1,20 @@ +# Uses external Cloudflare bypasser (FlareSolverr/ByParr) instead of built-in Selenium services: calibre-web-automated-book-downloader-extbp: image: ghcr.io/calibrain/calibre-web-automated-book-downloader-extbp:latest environment: - FLASK_PORT: 8084 - LOG_LEVEL: info - BOOK_LANGUAGE: en - USE_BOOK_TITLE: true TZ: America/New_York - UID: 1000 - GID: 100 - # CWA_DB_PATH: /auth/app.db # Uncomment to enable authentication - # SESSION_COOKIE_SECURE: 'true' # Set to 'true' if accessing ONLY via HTTPS - # DEBUG: 'true' # Enable debug mode (debug button, verbose logging) EXT_BYPASSER_URL: http://flaresolverr:8191 + # UID: 1000 + # GID: 100 + # CWA_DB_PATH: /auth/app.db ports: - 8084:8084 restart: unless-stopped volumes: - # This is where the books will be downloaded to, usually it would be - # the same as whatever you gave in "calibre-web-automated" - /tmp/data/calibre-web/ingest:/cwa-book-ingest - # This is the location of CWA's app.db, which contains authentication - # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above) - #- /cwa/config/path/app.db:/auth/app.db:ro + - /path/to/config:/config + # - /cwa/config/path/app.db:/auth/app.db:ro + flaresolverr: image: ghcr.io/flaresolverr/flaresolverr:latest diff --git a/docker-compose.tor.dev.yml b/docker-compose.tor.dev.yml index 1eafee66..6419b369 100644 --- a/docker-compose.tor.dev.yml +++ b/docker-compose.tor.dev.yml @@ -1,3 +1,4 @@ +# Local development - Tor variant services: calibre-web-automated-book-downloader-tor-dev: extends: @@ -10,5 +11,7 @@ services: environment: DEBUG: true volumes: - - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader - - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader + - ./.local/config:/config + - ./.local/ingest:/cwa-book-ingest + - ./.local/log:/var/log/cwa-book-downloader + - ./.local/tmp:/tmp/cwa-book-downloader diff --git a/docker-compose.tor.yml b/docker-compose.tor.yml index 9e361e83..fbde64fc 100644 --- a/docker-compose.tor.yml +++ b/docker-compose.tor.yml @@ -1,16 +1,12 @@ +# Routes all traffic through Tor - requires NET_ADMIN capability services: calibre-web-automated-book-downloader-tor: image: ghcr.io/calibrain/calibre-web-automated-book-downloader-tor:latest environment: FLASK_PORT: 8084 - LOG_LEVEL: info - BOOK_LANGUAGE: en - USE_BOOK_TITLE: true TZ: America/New_York USING_TOR: true - # CWA_DB_PATH: /auth/app.db # Uncomment to enable authentication - # SESSION_COOKIE_SECURE: 'true' # Set to 'true' if accessing ONLY via HTTPS - # DEBUG: 'true' # Enable debug mode (debug button, verbose logging) + # CWA_DB_PATH: /auth/app.db cap_add: - NET_ADMIN - NET_RAW @@ -18,9 +14,6 @@ services: - 8084:8084 restart: unless-stopped volumes: - # This is where the books will be downloaded to, usually it would be - # the same as whatever you gave in "calibre-web-automated" - /tmp/data/calibre-web/ingest:/cwa-book-ingest - # This is the location of CWA's app.db, which contains authentication - # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above) - #- /cwa/config/path/app.db:/auth/app.db:ro + - /path/to/config:/config + # - /cwa/config/path/app.db:/auth/app.db:ro diff --git a/docker-compose.yml b/docker-compose.yml index 09fdd1c9..ed006963 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,15 @@ services: calibre-web-automated-book-downloader: image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest - # Uncomment to build the image from the Dockerfile for local testing changes. - # Remember to comment out the image line above. - #build: . container_name: calibre-web-automated-book-downloader environment: - FLASK_PORT: 8084 - LOG_LEVEL: info - BOOK_LANGUAGE: en - USE_BOOK_TITLE: true TZ: America/New_York - UID: 1000 - GID: 100 - # CWA_DB_PATH: /auth/app.db # Uncomment to enable authentication (also uncomment volume below) - # CALIBRE_WEB_URL: http://localhost:8080 # Uncomment and add your custom library URL to enable "Go To Library" button in the Web UI - # SESSION_COOKIE_SECURE: 'true' # Set to 'true' if accessing ONLY via HTTPS - # DEBUG: 'true' # Enable debug mode (debug button, verbose logging) - # Queue management settings - MAX_CONCURRENT_DOWNLOADS: 3 - DOWNLOAD_PROGRESS_UPDATE_INTERVAL: 5 + # UID: 1000 + # GID: 100 + # CWA_DB_PATH: /auth/app.db ports: - 8084:8084 restart: unless-stopped volumes: - # This is where the books will be downloaded to, usually it would be - # the same as whatever you gave in "calibre-web-automated" - - /tmp/data/calibre-web/ingest:/cwa-book-ingest - # This is the location of CWA's app.db, which contains authentication - # details. Uncomment to enable authentication (also uncomment CWA_DB_PATH above) - #- /cwa/config/path/app.db:/auth/app.db:ro + - /tmp/data/calibre-web/ingest:/cwa-book-ingest # This is where the books will be downloaded and ingested by your book management application + - /path/to/config:/config # Configuration files and database diff --git a/docs/plugin-settings.md b/docs/plugin-settings.md new file mode 100644 index 00000000..17beffbc --- /dev/null +++ b/docs/plugin-settings.md @@ -0,0 +1,628 @@ +# Plugin Settings Integration Guide + +This guide explains how to add configuration settings to plugins (Metadata Providers and Release Sources) so they appear in the Settings UI. + +## Overview + +The settings system uses a decorator-based registration pattern. Plugins register their settings when their module is imported, and the frontend dynamically renders the appropriate UI based on the schema provided by the backend. + +**Key features:** +- Settings are defined in Python and automatically rendered in the React frontend +- Values persist across container restarts via JSON config files +- Changes take effect immediately without restart (unless marked otherwise) + +## Quick Start + +Add settings to your plugin in 3 steps: + +```python +from cwa_book_downloader.core.settings_registry import ( + register_settings, + TextField, + PasswordField, + ActionButton, +) + +@register_settings( + name="my_plugin", # Unique identifier + display_name="My Plugin", # Shown in sidebar + icon="wrench", # Icon name + order=100, # Sort order (lower = higher in list) + group="metadata_providers" # Optional: group in sidebar +) +def my_plugin_settings(): + return [ + PasswordField( + key="MY_PLUGIN_API_KEY", + label="API Key", + description="Your API key from the provider", + required=True, + ), + ActionButton( + key="test_connection", + label="Test Connection", + style="primary", + callback=_test_connection, + ), + ] + +def _test_connection(): + # Perform connection test + return {"success": True, "message": "Connected successfully!"} +``` + +## Available Field Types + +### TextField + +Single-line text input for strings. + +```python +TextField( + key="MY_SETTING", # Config key + label="Setting Name", # Display label + description="Help text", # Optional description below field + default="", # Default value + placeholder="Enter value", # Placeholder text + max_length=100, # Optional max characters + required=False, # Is this field required? + requires_restart=False, # Does changing this need a restart? + show_when=None, # Conditional visibility (see below) + disabled_when=None, # Conditional disable (see below) +) +``` + +### PasswordField + +Masked input for sensitive values (API keys, passwords). Values are never echoed back to the frontend. + +```python +PasswordField( + key="API_KEY", + label="API Key", + description="Your secret API key", + placeholder="sk-...", + required=True, +) +``` + +### NumberField + +Numeric input with optional min/max constraints. + +```python +NumberField( + key="TIMEOUT", + label="Timeout (seconds)", + description="Connection timeout in seconds", + default=30, + min_value=5, + max_value=300, + step=1, # Increment step + required=False, +) +``` + +### CheckboxField + +Toggle switch for boolean values. + +```python +CheckboxField( + key="ENABLE_FEATURE", + label="Enable Feature", + description="Turn this feature on or off", + default=False, +) +``` + +### SelectField + +Dropdown for single-choice selection. + +```python +SelectField( + key="LOG_LEVEL", + label="Log Level", + description="Logging verbosity", + default="info", + options=[ + {"value": "debug", "label": "Debug"}, + {"value": "info", "label": "Info"}, + {"value": "warning", "label": "Warning"}, + {"value": "error", "label": "Error"}, + ], +) +``` + +### MultiSelectField + +Multi-choice selection from a list of options. + +```python +MultiSelectField( + key="SUPPORTED_FORMATS", + label="Supported Formats", + description="Select which formats to support", + default=["epub", "mobi"], + options=[ + {"value": "epub", "label": "EPUB"}, + {"value": "mobi", "label": "MOBI"}, + {"value": "pdf", "label": "PDF"}, + {"value": "azw3", "label": "AZW3"}, + ], +) +``` + +### ActionButton + +Button that executes a callback function. Does not store a value. + +```python +ActionButton( + key="test_connection", # Unique key for the action + label="Test Connection", # Button text + description="Test the API connection", + style="primary", # "default", "primary", or "danger" + callback=my_callback_fn, # Function to execute +) + +def my_callback_fn(): + """Callback must return dict with 'success' and 'message' keys.""" + try: + # Perform action + return {"success": True, "message": "Connection successful!"} + except Exception as e: + return {"success": False, "message": f"Failed: {str(e)}"} +``` + +### HeadingField + +Display-only section heading with optional link. Does not store a value. + +```python +HeadingField( + key="section_heading", # Unique key + title="Configuration", # Heading text + description="Configure the plugin settings below", + link_url="https://example.com/docs", # Optional link + link_text="View Documentation", # Link text +) +``` + +## Common Field Properties + +All field types support these common properties: + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `key` | `str` | Required | Unique identifier for this setting | +| `label` | `str` | Required | Display label in the UI | +| `description` | `str` | `""` | Help text shown below the field | +| `default` | `Any` | `None` | Default value if not set | +| `required` | `bool` | `False` | Whether the field must have a value | +| `disabled` | `bool` | `False` | Disable the field (greyed out) | +| `disabled_reason` | `str` | `""` | Explanation shown when disabled | +| `requires_restart` | `bool` | `False` | Whether changes require container restart | +| `show_when` | `dict` | `None` | Conditional visibility (see below) | +| `disabled_when` | `dict` | `None` | Conditional disable (see below) | + +## Conditional Visibility + +Fields can be shown/hidden based on other field values using `show_when`: + +```python +# Only show DNS servers field when custom DNS is selected +TextField( + key="CUSTOM_DNS_SERVERS", + label="DNS Servers", + description="Comma-separated DNS server IPs", + show_when={"field": "DNS_PROVIDER", "value": "manual"}, +) +``` + +The field will only be visible when the referenced field has the specified value. + +## Conditional Disable + +Fields can be enabled/disabled based on other field values using `disabled_when`: + +```python +# Disable timeout field when feature is disabled +NumberField( + key="FEATURE_TIMEOUT", + label="Timeout (seconds)", + description="Request timeout", + default=30, + disabled_when={ + "field": "FEATURE_ENABLED", + "value": False, + "reason": "Enable the feature first" + }, +) +``` + +The field will be greyed out with the specified reason when the condition is met. + +## Settings Groups + +Register a group to organize related settings tabs in the sidebar: + +```python +from cwa_book_downloader.core.settings_registry import register_group + +# Register a group (do this once, usually in a central config file) +register_group( + name="my_group", + display_name="My Group", + icon="folder", + order=50, +) + +# Then register settings to the group +@register_settings( + name="plugin_a", + display_name="Plugin A", + icon="puzzle", + order=51, + group="my_group", # Assigns to the group +) +def plugin_a_settings(): + return [...] +``` + +**Existing groups:** +- `direct_download` (order=20): For download-related settings +- `metadata_providers` (order=50): For metadata provider plugins + +## Value Resolution Priority + +Settings values are resolved in this order (highest priority first): + +1. **Config File** - Stored in `CONFIG_DIR/plugins/.json` +2. **Field Default** - Value specified in the field definition + +The `general` tab uses `CONFIG_DIR/settings.json` instead of the plugins subdirectory. + +## Reading Setting Values + +Use the `config` singleton to read setting values in your plugin code: + +```python +from cwa_book_downloader.core.config import config + +# Get a setting value with default fallback +api_key = config.get("MY_PLUGIN_API_KEY", "") +timeout = config.get("MY_PLUGIN_TIMEOUT", 30) + +# Or access as attributes (raises AttributeError if not found) +api_key = config.MY_PLUGIN_API_KEY + +# Check all cached settings +all_settings = config.get_all() +``` + +The config singleton: +- Automatically resolves values from config files with field defaults as fallback +- Caches values for performance +- Refreshes automatically when settings are updated via the UI + +## Complete Example: Metadata Provider + +Here's a complete example for a metadata provider plugin: + +```python +# cwa_book_downloader/metadata_providers/my_provider.py + +from cwa_book_downloader.metadata_providers.base import ( + MetadataProvider, + register_provider, +) +from cwa_book_downloader.core.settings_registry import ( + register_settings, + HeadingField, + TextField, + PasswordField, + CheckboxField, + ActionButton, +) +from cwa_book_downloader.core.config import config + + +def _test_connection(): + """Test API connection callback.""" + api_key = config.get("MY_PROVIDER_API_KEY", "") + if not api_key: + return {"success": False, "message": "API key not configured"} + + try: + # Perform actual connection test + # response = requests.get(...) + return {"success": True, "message": "Connected to My Provider API"} + except Exception as e: + return {"success": False, "message": f"Connection failed: {str(e)}"} + + +@register_settings( + name="my_provider", + display_name="My Provider", + icon="book", + order=53, + group="metadata_providers", +) +def my_provider_settings(): + """Define settings for this metadata provider.""" + return [ + HeadingField( + key="my_provider_heading", + title="My Provider", + description="A metadata provider for book information", + link_url="https://myprovider.com", + link_text="Visit My Provider", + ), + PasswordField( + key="MY_PROVIDER_API_KEY", + label="API Key", + description="Your My Provider API key", + placeholder="Enter your API key", + required=True, + ), + CheckboxField( + key="MY_PROVIDER_INCLUDE_COVERS", + label="Include Cover Images", + description="Fetch cover images when searching", + default=True, + ), + TextField( + key="MY_PROVIDER_BASE_URL", + label="API Base URL", + description="Override the default API endpoint", + default="https://api.myprovider.com/v1", + required=False, + ), + ActionButton( + key="test_connection", + label="Test Connection", + description="Verify your API key works", + style="primary", + callback=_test_connection, + ), + ] + + +@register_provider("my_provider") +class MyProvider(MetadataProvider): + """My Provider metadata implementation.""" + + name = "my_provider" + display_name = "My Provider" + requires_auth = True + + def __init__(self, api_key: str = None): + self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "") + self.base_url = config.get( + "MY_PROVIDER_BASE_URL", + "https://api.myprovider.com/v1" + ) + + def is_available(self) -> bool: + return bool(self.api_key) + + def search(self, query: str): + # Implementation... + pass + + def get_book(self, book_id: str): + # Implementation... + pass +``` + +## Complete Example: Release Source + +Here's a complete example for a release source plugin: + +```python +# cwa_book_downloader/release_sources/my_source.py + +from cwa_book_downloader.release_sources.base import ( + ReleaseSource, + DownloadHandler, + register_source, + register_handler, +) +from cwa_book_downloader.core.settings_registry import ( + register_settings, + HeadingField, + TextField, + NumberField, + CheckboxField, + SelectField, + ActionButton, +) +from cwa_book_downloader.core.config import config + + +def _test_source(): + """Test source availability callback.""" + base_url = config.get("MY_SOURCE_URL", "https://mysource.com") + try: + # Test connectivity + return {"success": True, "message": f"Source available at {base_url}"} + except Exception as e: + return {"success": False, "message": f"Source unavailable: {str(e)}"} + + +@register_settings( + name="my_source", + display_name="My Source", + icon="download", + order=25, + group="direct_download", +) +def my_source_settings(): + """Define settings for this release source.""" + return [ + HeadingField( + key="my_source_heading", + title="My Source Configuration", + description="Configure the My Source download provider", + ), + CheckboxField( + key="MY_SOURCE_ENABLED", + label="Enable My Source", + description="Include My Source in download fallback chain", + default=True, + ), + TextField( + key="MY_SOURCE_URL", + label="Source URL", + description="Base URL for the source", + default="https://mysource.com", + show_when={"field": "MY_SOURCE_ENABLED", "value": True}, + ), + NumberField( + key="MY_SOURCE_TIMEOUT", + label="Timeout (seconds)", + description="Request timeout", + default=30, + min_value=10, + max_value=120, + show_when={"field": "MY_SOURCE_ENABLED", "value": True}, + ), + SelectField( + key="MY_SOURCE_PRIORITY", + label="Priority", + description="Where in the fallback chain to try this source", + default="normal", + options=[ + {"value": "high", "label": "High (try first)"}, + {"value": "normal", "label": "Normal"}, + {"value": "low", "label": "Low (try last)"}, + ], + show_when={"field": "MY_SOURCE_ENABLED", "value": True}, + ), + ActionButton( + key="test_source", + label="Test Source", + description="Check if the source is accessible", + style="primary", + callback=_test_source, + ), + ] + + +@register_source("my_source") +class MySource(ReleaseSource): + """My Source release source implementation.""" + + name = "my_source" + display_name = "My Source" + + def __init__(self): + self.enabled = config.get("MY_SOURCE_ENABLED", True) + self.base_url = config.get("MY_SOURCE_URL", "https://mysource.com") + self.timeout = config.get("MY_SOURCE_TIMEOUT", 30) + + def is_available(self) -> bool: + return self.enabled + + def search(self, book): + # Implementation... + pass + + +@register_handler("my_source") +class MySourceHandler(DownloadHandler): + """Handler for downloading from My Source.""" + + name = "my_source" + + def download(self, release, output_path): + # Implementation... + pass +``` + +## Best Practices + +1. **Use descriptive keys**: Keys should be uppercase and prefixed with your plugin name (e.g., `MY_PLUGIN_API_KEY`) + +2. **Provide helpful descriptions**: Include enough detail in descriptions to help users understand what each setting does + +3. **Set sensible defaults**: Users should be able to get started without configuring everything + +4. **Use conditional visibility**: Hide advanced options behind enabling checkboxes to reduce UI clutter + +5. **Include a test button**: ActionButtons that test connections help users verify their configuration + +6. **Mark restart-required settings**: Use `requires_restart=True` for settings that can't be applied live + +7. **Group related settings**: Use HeadingField to visually separate sections, and put plugins in appropriate groups + +8. **Handle missing values gracefully**: Always provide fallbacks when reading settings in your code + +## API Reference + +### Backend Routes + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/settings` | Get all settings tabs, groups, and values | +| GET | `/api/settings/` | Get a specific settings tab | +| PUT | `/api/settings/` | Update settings for a tab | +| POST | `/api/settings//action/` | Execute an action button callback | + +### Response Format + +**GET /api/settings** +```json +{ + "groups": [ + {"name": "direct_download", "displayName": "Direct Download", "icon": "download", "order": 20} + ], + "tabs": [ + { + "name": "my_plugin", + "displayName": "My Plugin", + "icon": "book", + "order": 53, + "group": "metadata_providers", + "fields": [ + { + "type": "password", + "key": "MY_PLUGIN_API_KEY", + "label": "API Key", + "description": "Your API key", + "hasValue": true, + "value": "", + "required": true, + "disabled": false, + "requiresRestart": false + } + ] + } + ] +} +``` + +**PUT /api/settings/** +```json +// Request +{"MY_PLUGIN_API_KEY": "new-value", "MY_PLUGIN_TIMEOUT": 60} + +// Response +{ + "success": true, + "message": "Settings updated", + "updated": ["MY_PLUGIN_API_KEY", "MY_PLUGIN_TIMEOUT"], + "requiresRestart": false +} +``` + +**POST /api/settings//action/** +```json +// Response +{ + "success": true, + "message": "Connection successful!" +} +``` diff --git a/docs/release-sources-plugin-guide.md b/docs/release-sources-plugin-guide.md new file mode 100644 index 00000000..5fe85302 --- /dev/null +++ b/docs/release-sources-plugin-guide.md @@ -0,0 +1,1450 @@ +# Release Sources Plugin Development Guide + +This guide explains how to create custom release source plugins for the CWA Book Downloader. The plugin system allows you to add new sources for searching and downloading books while integrating seamlessly with the existing queue, progress reporting, and settings infrastructure. + +## Table of Contents + +1. [Architecture Overview](#architecture-overview) +2. [Core Concepts](#core-concepts) +3. [Quick Start](#quick-start) +4. [ReleaseSource Interface](#releasesource-interface) +5. [DownloadHandler Interface](#downloadhandler-interface) +6. [Data Models](#data-models) +7. [Registration System](#registration-system) +8. [Settings Integration](#settings-integration) +9. [UI Column Configuration](#ui-column-configuration) +10. [Progress & Status Reporting](#progress--status-reporting) +11. [Error Handling](#error-handling) +12. [Complete Example Plugin](#complete-example-plugin) +13. [Best Practices](#best-practices) + +--- + +## Architecture Overview + +The release sources system is built around two core interfaces: + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ ReleaseSource │────▶│ Release │────▶│ DownloadTask │ +│ (Search) │ │ (Result) │ │ (Queue) │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ + │ + ▼ + ┌─────────────────┐ + │ DownloadHandler │ + │ (Execute) │ + └─────────────────┘ +``` + +- **ReleaseSource**: Searches for releases based on book metadata (title, ISBN, author) +- **Release**: Standardized search result from any source +- **DownloadTask**: Source-agnostic task queued for download +- **DownloadHandler**: Executes the actual download with progress reporting + +This separation allows: +- Different search sources (Anna's Archive, Prowlarr, IRC, etc.) +- Different download protocols (HTTP, torrent, usenet, etc.) +- Shared queue and progress infrastructure + +--- + +## Core Concepts + +### Plugin Lifecycle + +1. **Registration** (import time): Decorators register your classes in the global registry +2. **Discovery**: `list_available_sources()` checks `is_available()` on each source +3. **Search**: User selects source, `search(book_metadata)` returns releases +4. **Queue**: User selects release, it becomes a `DownloadTask` in the queue +5. **Download**: Orchestrator calls `handler.download()` with callbacks +6. **Progress**: Handler reports via `progress_callback` and `status_callback` +7. **Completion**: Handler returns file path or `None` on failure + +### Source vs Handler + +A source has both a `ReleaseSource` (for searching) and a `DownloadHandler` (for downloading). They share the same `name` identifier: + +```python +@register_source("my_source") # Search registration +class MySource(ReleaseSource): ... + +@register_handler("my_source") # Download registration +class MyHandler(DownloadHandler): ... +``` + +--- + +## Quick Start + +Create a new file at `cwa_book_downloader/release_sources/my_source.py`: + +```python +from typing import Callable, List, Optional +from threading import Event + +from cwa_book_downloader.release_sources import ( + Release, + ReleaseSource, + DownloadHandler, + register_source, + register_handler, +) +from cwa_book_downloader.metadata_providers import BookMetadata +from cwa_book_downloader.core.models import DownloadTask + + +@register_source("my_source") +class MySource(ReleaseSource): + name = "my_source" + display_name = "My Source" + + def search(self, book: BookMetadata) -> List[Release]: + # Your search logic here + return [] + + def is_available(self) -> bool: + return True # Check if configured + + +@register_handler("my_source") +class MyHandler(DownloadHandler): + def download( + self, + task: DownloadTask, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, Optional[str]], None], + ) -> Optional[str]: + # Your download logic here + return None # Return file path on success + + def cancel(self, task_id: str) -> bool: + return False # Cancellation via cancel_flag +``` + +Register the import in `cwa_book_downloader/release_sources/__init__.py`: + +```python +# At the bottom of the file +from cwa_book_downloader.release_sources import my_source # noqa: F401, E402 +``` + +--- + +## ReleaseSource Interface + +The `ReleaseSource` abstract base class defines the search interface: + +```python +from abc import ABC, abstractmethod +from typing import List +from cwa_book_downloader.metadata_providers import BookMetadata +from cwa_book_downloader.release_sources import Release, ReleaseColumnConfig + +class ReleaseSource(ABC): + """Interface for searching a release source.""" + + name: str # Internal identifier: "direct_download", "prowlarr" + display_name: str # User-facing name: "Direct Download", "Prowlarr" + + @abstractmethod + def search(self, book: BookMetadata) -> List[Release]: + """Search for releases of a book. + + Args: + book: Book metadata including title, authors, ISBN, language + + Returns: + List of Release objects found from this source + """ + pass + + @abstractmethod + def is_available(self) -> bool: + """Check if this source is configured and reachable. + + Returns: + True if source can be used for searching + """ + pass + + @classmethod + def get_column_config(cls) -> ReleaseColumnConfig: + """Get the column configuration for this source's release list UI. + + Override to customize how releases are displayed in the modal. + Default implementation provides a basic layout. + """ + return _default_column_config() +``` + +### Search Method + +The `search` method receives complete book metadata and returns standardized releases: + +```python +def search(self, book: BookMetadata) -> List[Release]: + results = [] + + # Strategy 1: Search by ISBN (most accurate) + if book.isbn_13: + api_results = self.api.search_isbn(book.isbn_13) + for r in api_results: + results.append(Release( + source="my_source", + source_id=r.id, + title=r.title, + format=r.format, + size=r.size, + # ... more fields + )) + + # Strategy 2: Fallback to title + author search + if not results: + query = f"{book.title} {book.authors[0] if book.authors else ''}" + api_results = self.api.search_text(query) + for r in api_results: + results.append(Release(...)) + + return results +``` + +### Availability Check + +Return `True` only if the source is properly configured and reachable: + +```python +def is_available(self) -> bool: + # Check for required configuration + if not self.api_key: + return False + + # Optional: Test connection + try: + return self.api.ping() + except Exception: + return False +``` + +--- + +## DownloadHandler Interface + +The `DownloadHandler` abstract base class defines the download interface: + +```python +from abc import ABC, abstractmethod +from typing import Callable, Optional +from threading import Event +from cwa_book_downloader.core.models import DownloadTask + +class DownloadHandler(ABC): + """Interface for executing downloads from a source.""" + + @abstractmethod + def download( + self, + task: DownloadTask, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, Optional[str]], None], + ) -> Optional[str]: + """Execute download and return path to downloaded file. + + Args: + task: The download task with metadata and identifiers + cancel_flag: Threading Event - check .is_set() for cancellation + progress_callback: Call with progress 0-100 + status_callback: Call with (status, optional_message) + + Returns: + Absolute path to downloaded file, or None if failed/cancelled + """ + pass + + @abstractmethod + def cancel(self, task_id: str) -> bool: + """Cancel an in-progress download. + + For most implementations, cancellation is handled via cancel_flag. + This method is for external cancellation (e.g., torrent client). + + Returns: + True if cancellation was initiated + """ + pass +``` + +### Download Method Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `task` | `DownloadTask` | Contains `task_id`, `source`, `title`, `author`, `format`, `size`, `preview` | +| `cancel_flag` | `Event` | Check `cancel_flag.is_set()` periodically for cancellation | +| `progress_callback` | `Callable[[float], None]` | Call with progress 0-100 | +| `status_callback` | `Callable[[str, Optional[str]], None]` | Call with `(status, message)` | + +### Status Values + +| Status | Description | When to Use | +|--------|-------------|-------------| +| `"queued"` | Waiting in queue | Set by orchestrator, rarely needed in handler | +| `"resolving"` | Pre-download phase | Fetching metadata, extracting URLs, connecting | +| `"downloading"` | Active download | File transfer in progress | +| `"complete"` | Successfully finished | Set by orchestrator when handler returns a file path | +| `"error"` | Failed | Any unrecoverable error (handler should set this) | +| `"cancelled"` | User cancelled | Set by orchestrator when `cancel_flag` is triggered | + +**Note**: The orchestrator automatically sets `"complete"` when your handler returns a valid file path, and `"cancelled"` when cancellation is detected. Your handler should primarily use `"resolving"`, `"downloading"`, and `"error"`. + +### Typical Download Flow + +```python +def download(self, task, cancel_flag, progress_callback, status_callback): + try: + # Phase 1: Resolve download URL + status_callback("resolving", "Fetching download link...") + + if cancel_flag.is_set(): + return None + + download_url = self.api.get_download_url(task.task_id) + if not download_url: + status_callback("error", "No download URL available") + return None + + # Phase 2: Download file + status_callback("downloading", None) + + temp_path = Path(tempfile.mktemp(suffix=f".{task.format}")) + + response = requests.get(download_url, stream=True) + total_size = int(response.headers.get("content-length", 0)) + downloaded = 0 + + with open(temp_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if cancel_flag.is_set(): + temp_path.unlink(missing_ok=True) + return None + + f.write(chunk) + downloaded += len(chunk) + + if total_size > 0: + progress_callback((downloaded / total_size) * 100) + + # Phase 3: Post-process and move to final location + final_path = INGEST_DIR / f"{task.title}.{task.format}" + shutil.move(str(temp_path), str(final_path)) + + # Return the file path - orchestrator will set status to "complete" + return str(final_path) + + except Exception as e: + if not cancel_flag.is_set(): + status_callback("error", str(e)) + return None +``` + +--- + +## Data Models + +### Release + +Standardized search result returned by all sources: + +```python +@dataclass +class Release: + source: str # "direct_download", "prowlarr", "irc" + source_id: str # Unique ID within that source + title: str # Display title + + format: Optional[str] = None # File format: "epub", "mobi", "pdf" + size: Optional[str] = None # Human-readable: "5.2 MB" + size_bytes: Optional[int] = None # For sorting + + download_url: Optional[str] = None # Direct download URL if available + info_url: Optional[str] = None # Link to tracker/info page + + protocol: Optional[str] = None # "http", "torrent", "usenet" + indexer: Optional[str] = None # Display name: "Anna's Archive", "MyAnonamouse" + seeders: Optional[int] = None # For torrents + + extra: Dict = field(default_factory=dict) # Source-specific metadata +``` + +#### Using the `extra` Field + +Store source-specific data in `extra` that doesn't fit standard fields: + +```python +Release( + source="my_source", + source_id="abc123", + title="The Great Book", + format="epub", + size="2.5 MB", + extra={ + "author": "Author Name", + "language": "en", + "preview": "https://example.com/cover.jpg", + "quality": "high", + "publisher": "Publisher Name", + } +) +``` + +The frontend can access these via column config (e.g., `key="extra.language"`). + +### DownloadTask + +Source-agnostic task in the download queue: + +```python +@dataclass +class DownloadTask: + task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID) + source: str # Handler name: "direct_download" + title: str # Display title + + # Display info (from Release.extra or Release fields) + author: Optional[str] = None + format: Optional[str] = None + size: Optional[str] = None + preview: Optional[str] = None # Cover image URL + + # Runtime state (managed by orchestrator) + priority: int = 0 + added_time: float = field(default_factory=time.time) + progress: float = 0.0 + status: str = "queued" + status_message: Optional[str] = None + download_path: Optional[str] = None +``` + +### BookMetadata + +Book information passed to `search()`: + +```python +@dataclass +class BookMetadata: + provider: str # Metadata provider: "hardcover", "openlibrary" + provider_id: str # ID in provider's system + title: str + + provider_display_name: Optional[str] = None + authors: List[str] = field(default_factory=list) + isbn_10: Optional[str] = None + isbn_13: Optional[str] = None + cover_url: Optional[str] = None + description: Optional[str] = None + publisher: Optional[str] = None + publish_year: Optional[int] = None + language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr" + genres: List[str] = field(default_factory=list) + source_url: Optional[str] = None + display_fields: List[DisplayField] = field(default_factory=list) +``` + +--- + +## Registration System + +### Decorators + +```python +from cwa_book_downloader.release_sources import register_source, register_handler + +@register_source("my_source") +class MySource(ReleaseSource): + ... + +@register_handler("my_source") +class MyHandler(DownloadHandler): + ... +``` + +### Registry Functions + +```python +from cwa_book_downloader.release_sources import ( + get_source, + get_handler, + list_available_sources, +) + +# Get a source instance by name +source = get_source("my_source") +releases = source.search(book_metadata) + +# Get a handler instance by name +handler = get_handler("my_source") +file_path = handler.download(task, cancel_flag, progress_cb, status_cb) + +# List all sources where is_available() returns True +available = list_available_sources() +# Returns: [{"name": "direct_download", "display_name": "Direct Download"}, ...] +``` + +### Import Registration + +**Critical**: Your plugin must be imported at module load time. Add to `release_sources/__init__.py`: + +```python +# At the bottom of __init__.py +from cwa_book_downloader.release_sources import direct_download # noqa: F401, E402 +from cwa_book_downloader.release_sources import my_source # noqa: F401, E402 +``` + +The `noqa` comments suppress linter warnings about unused imports. + +--- + +## Settings Integration + +Register plugin settings using the settings registry decorator: + +```python +from cwa_book_downloader.core.settings_registry import ( + register_settings, + HeadingField, + TextField, + PasswordField, + NumberField, + CheckboxField, + SelectField, + ActionButton, + get_setting_value, + is_value_from_env, +) + +@register_settings( + name="my_source", # Unique identifier + display_name="My Source", # Shown in settings UI + icon="globe", # Icon name (optional) + order=60, # Sort order in sidebar + group="sources", # Grouping in sidebar +) +def my_source_settings(): + """Register settings for My Source plugin.""" + return [ + HeadingField( + key="heading", + title="My Source Configuration", + description="Configure connection to My Source API", + ), + + PasswordField( + key="MY_SOURCE_API_KEY", + label="API Key", + description="Your API key from mysource.com/settings", + required=True, + env_supported=True, # Can be set via MY_SOURCE_API_KEY env var + ), + + TextField( + key="MY_SOURCE_BASE_URL", + label="Base URL", + placeholder="https://api.mysource.com", + env_supported=True, + ), + + NumberField( + key="MY_SOURCE_TIMEOUT", + label="Request Timeout", + description="Seconds to wait for API responses", + default=30, + min_value=5, + max_value=120, + ), + + CheckboxField( + key="MY_SOURCE_ENABLED", + label="Enable My Source", + description="Include in release searches", + default=True, + ), + + ActionButton( + key="test_connection", + label="Test Connection", + callback=test_my_source_connection, + style="primary", + ), + ] + +def test_my_source_connection(): + """Test connection callback - returns result dict.""" + try: + # Get current API key + api_key = get_setting_value( + PasswordField(key="MY_SOURCE_API_KEY", label=""), + "my_source" + ) + + if not api_key: + return {"success": False, "message": "API key not configured"} + + # Test the connection + response = requests.get( + "https://api.mysource.com/ping", + headers={"Authorization": f"Bearer {api_key}"} + ) + + if response.status_code == 200: + return {"success": True, "message": "Connected successfully!"} + else: + return {"success": False, "message": f"API returned {response.status_code}"} + + except Exception as e: + return {"success": False, "message": str(e)} +``` + +### Field Types Reference + +| Field Type | Purpose | Key Properties | +|------------|---------|----------------| +| `HeadingField` | Section header | `title`, `description`, `link_url`, `link_text` | +| `TextField` | Single-line text | `placeholder`, `max_length` | +| `PasswordField` | Masked input | Not returned in GET unless changed | +| `NumberField` | Numeric input | `min_value`, `max_value`, `step`, `default` | +| `CheckboxField` | Boolean toggle | `default` | +| `SelectField` | Single dropdown | `options` (list or callable) | +| `MultiSelectField` | Multi-choice | `options` (list or callable) | +| `ActionButton` | Trigger callback | `callback`, `style` | + +### Reading Settings + +```python +from cwa_book_downloader.core.settings_registry import ( + get_setting_value, + is_value_from_env, + load_config_file, +) + +# In your source/handler class +def __init__(self): + # Load from config file + config = load_config_file("my_source") + self.api_key = config.get("MY_SOURCE_API_KEY") + self.base_url = config.get("MY_SOURCE_BASE_URL", "https://api.mysource.com") + +# Or use get_setting_value for ENV var priority +api_key = get_setting_value( + PasswordField(key="MY_SOURCE_API_KEY", label=""), + "my_source" +) +``` + +### Settings Priority + +Settings are resolved in this order (highest priority first): + +1. **Environment variable** (if `env_supported=True`) +2. **Config file** (`/config/plugins/{tab_name}.json`) +3. **Default value** (from field definition) + +When a value comes from an ENV var, the UI shows a "locked" badge and the field is read-only. + +--- + +## UI Column Configuration + +Customize how releases are displayed in the release modal by overriding `get_column_config()`: + +```python +from cwa_book_downloader.release_sources import ( + ReleaseColumnConfig, + ColumnSchema, + ColumnRenderType, + ColumnAlign, + ColumnColorHint, + LeadingCellConfig, + LeadingCellType, +) + +@classmethod +def get_column_config(cls) -> ReleaseColumnConfig: + return ReleaseColumnConfig( + columns=[ + ColumnSchema( + key="indexer", + label="Source", + render_type=ColumnRenderType.TEXT, + width="minmax(0, 1fr)", + ), + ColumnSchema( + key="extra.language", + label="Language", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="60px", + color_hint=ColumnColorHint(type="map", value="language"), + uppercase=True, + ), + ColumnSchema( + key="format", + label="Format", + render_type=ColumnRenderType.BADGE, + color_hint=ColumnColorHint(type="map", value="format"), + uppercase=True, + ), + ColumnSchema( + key="size", + label="Size", + render_type=ColumnRenderType.SIZE, + align=ColumnAlign.CENTER, + width="80px", + ), + ColumnSchema( + key="seeders", + label="Seeders", + render_type=ColumnRenderType.SEEDERS, + align=ColumnAlign.CENTER, + width="60px", + hide_mobile=True, + ), + ], + grid_template="minmax(0, 2fr) 60px 80px 80px 60px", + leading_cell=LeadingCellConfig( + type=LeadingCellType.THUMBNAIL, + key="extra.preview", + ), + ) +``` + +### Column Properties + +| Property | Type | Description | +|----------|------|-------------| +| `key` | `str` | Data path: `"format"`, `"extra.language"`, `"extra.seeders"` | +| `label` | `str` | Accessibility label | +| `render_type` | `ColumnRenderType` | `TEXT`, `BADGE`, `SIZE`, `NUMBER`, `SEEDERS` | +| `align` | `ColumnAlign` | `LEFT`, `CENTER`, `RIGHT` | +| `width` | `str` | CSS width: `"80px"`, `"minmax(0, 2fr)"` | +| `hide_mobile` | `bool` | Hide on small screens | +| `color_hint` | `ColumnColorHint` | For `BADGE` type coloring | +| `fallback` | `str` | Shown when data missing (default: `"-"`) | +| `uppercase` | `bool` | Force uppercase text | + +### Color Hints + +For `BADGE` render type, specify colors: + +```python +# Use frontend color map (defined in colorMaps.ts) +ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc. +ColumnColorHint(type="map", value="language") # Maps en→blue, de→yellow, etc. + +# Use static Tailwind class +ColumnColorHint(type="static", value="bg-purple-500/20 text-purple-400") +``` + +### Leading Cell Types + +Configure what appears before the title: + +```python +# Show cover thumbnail +LeadingCellConfig(type=LeadingCellType.THUMBNAIL, key="extra.preview") + +# Show colored badge +LeadingCellConfig( + type=LeadingCellType.BADGE, + key="protocol", + color_hint=ColumnColorHint(type="map", value="protocol"), +) + +# No leading cell +LeadingCellConfig(type=LeadingCellType.NONE) +``` + +--- + +## Progress & Status Reporting + +### Progress Callback + +Report download progress (0-100): + +```python +# In your download loop +for chunk in response.iter_content(8192): + downloaded += len(chunk) + progress_callback((downloaded / total_size) * 100) +``` + +Progress updates are throttled by the orchestrator before WebSocket broadcast: +- Start (0%) and completion (≥99%) always broadcast +- Otherwise every `DOWNLOAD_PROGRESS_UPDATE_INTERVAL` seconds +- On progress jumps >10% + +### Status Callback + +Report status changes with optional messages: + +```python +status_callback("resolving", "Connecting to server...") +status_callback("resolving", "Fetching download link...") +status_callback("downloading", None) # Message optional +status_callback("complete", None) +status_callback("error", "Connection timeout after 30s") +``` + +### Typical Status Sequence + +``` +queued → Initial state (set by orchestrator) +resolving → "Fetching book details..." +resolving → "Trying source 1..." +resolving → "Trying source 2..." (on retry) +downloading → (progress updates: 0%, 25%, 50%, 75%, 100%) +complete → Success +``` + +Or on failure: + +``` +queued → Initial state +resolving → "Connecting..." +error → "All download sources failed" +``` + +--- + +## Error Handling + +### In Search Methods + +Return empty list on errors, log for debugging: + +```python +def search(self, book: BookMetadata) -> List[Release]: + try: + results = self.api.search(book.title) + return [self._to_release(r) for r in results] + except ConnectionError as e: + logger.warning(f"Connection error searching {self.name}: {e}") + return [] + except AuthenticationError: + logger.error(f"{self.name} API key invalid") + return [] + except Exception as e: + logger.error(f"Unexpected error in {self.name}.search: {e}") + return [] +``` + +### In Download Methods + +Use `status_callback` to report errors, return `None`: + +```python +def download(self, task, cancel_flag, progress_callback, status_callback): + try: + # ... download logic ... + return file_path + + except Exception as e: + if not cancel_flag.is_set(): + logger.error(f"Download error for {task.task_id}: {e}") + status_callback("error", str(e)) + return None +``` + +### Handling Cancellation + +Check `cancel_flag.is_set()` at key points: + +```python +def download(self, task, cancel_flag, progress_callback, status_callback): + # Check before expensive operations + if cancel_flag.is_set(): + return None + + status_callback("resolving", "Fetching metadata...") + metadata = self.api.get_metadata(task.task_id) + + # Check before download + if cancel_flag.is_set(): + return None + + status_callback("downloading", None) + + # Check during download loop + for chunk in response.iter_content(8192): + if cancel_flag.is_set(): + # Clean up partial file + temp_file.unlink(missing_ok=True) + return None + + # ... process chunk ... + + return file_path +``` + +--- + +## Complete Example Plugin + +Here's a complete example plugin for a fictional "BookNet" API: + +```python +""" +BookNet Release Source Plugin + +A complete example plugin demonstrating all features of the release source system. +""" + +import requests +import tempfile +import shutil +from pathlib import Path +from typing import Callable, Dict, List, Optional +from threading import Event + +from cwa_book_downloader.release_sources import ( + Release, + ReleaseSource, + DownloadHandler, + register_source, + register_handler, + ReleaseColumnConfig, + ColumnSchema, + ColumnRenderType, + ColumnAlign, + ColumnColorHint, + LeadingCellConfig, + LeadingCellType, +) +from cwa_book_downloader.metadata_providers import BookMetadata +from cwa_book_downloader.core.models import DownloadTask +from cwa_book_downloader.core.logger import setup_logger +from cwa_book_downloader.core.settings_registry import ( + register_settings, + HeadingField, + TextField, + PasswordField, + NumberField, + CheckboxField, + ActionButton, + load_config_file, +) +from cwa_book_downloader.config.env import INGEST_DIR + +logger = setup_logger(__name__) + +# --------------------------------------------------------------------------- +# Settings Registration +# --------------------------------------------------------------------------- + +@register_settings( + name="booknet", + display_name="BookNet", + icon="book", + order=55, + group="sources", +) +def booknet_settings(): + """Register BookNet settings.""" + return [ + HeadingField( + key="heading", + title="BookNet API", + description="Connect to BookNet for additional book sources", + link_url="https://booknet.example.com/api-docs", + link_text="API Documentation", + ), + + PasswordField( + key="BOOKNET_API_KEY", + label="API Key", + description="Get your key from booknet.example.com/settings", + required=True, + env_supported=True, + ), + + TextField( + key="BOOKNET_BASE_URL", + label="Base URL", + placeholder="https://api.booknet.example.com", + env_supported=True, + ), + + NumberField( + key="BOOKNET_TIMEOUT", + label="Request Timeout", + description="Seconds to wait for API responses", + default=30, + min_value=5, + max_value=120, + ), + + CheckboxField( + key="BOOKNET_PREFER_EPUB", + label="Prefer EPUB format", + description="Sort EPUB results first when available", + default=True, + ), + + ActionButton( + key="test_connection", + label="Test Connection", + callback=_test_booknet_connection, + style="primary", + ), + ] + + +def _test_booknet_connection() -> Dict: + """Test BookNet API connection.""" + try: + config = load_config_file("booknet") + api_key = config.get("BOOKNET_API_KEY") + base_url = config.get("BOOKNET_BASE_URL", "https://api.booknet.example.com") + + if not api_key: + return {"success": False, "message": "API key not configured"} + + response = requests.get( + f"{base_url}/v1/ping", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=10, + ) + + if response.status_code == 200: + data = response.json() + return { + "success": True, + "message": f"Connected! {data.get('books_available', 0):,} books available" + } + elif response.status_code == 401: + return {"success": False, "message": "Invalid API key"} + else: + return {"success": False, "message": f"API error: {response.status_code}"} + + except requests.Timeout: + return {"success": False, "message": "Connection timeout"} + except Exception as e: + return {"success": False, "message": str(e)} + + +# --------------------------------------------------------------------------- +# Release Source Implementation +# --------------------------------------------------------------------------- + +@register_source("booknet") +class BookNetSource(ReleaseSource): + """Search BookNet for book releases.""" + + name = "booknet" + display_name = "BookNet" + + def __init__(self): + config = load_config_file("booknet") + self.api_key = config.get("BOOKNET_API_KEY") + self.base_url = config.get("BOOKNET_BASE_URL", "https://api.booknet.example.com") + self.timeout = config.get("BOOKNET_TIMEOUT", 30) + self.prefer_epub = config.get("BOOKNET_PREFER_EPUB", True) + + def search(self, book: BookMetadata) -> List[Release]: + """Search BookNet for releases.""" + if not self.is_available(): + return [] + + releases = [] + + try: + # Strategy 1: Search by ISBN (most accurate) + if book.isbn_13: + releases = self._search_isbn(book.isbn_13) + + # Strategy 2: Fallback to title + author + if not releases: + releases = self._search_text(book.title, book.authors) + + # Sort by preference + if self.prefer_epub: + releases.sort(key=lambda r: (0 if r.format == "epub" else 1)) + + return releases + + except requests.Timeout: + logger.warning("BookNet search timeout") + return [] + except Exception as e: + logger.error(f"BookNet search error: {e}") + return [] + + def _search_isbn(self, isbn: str) -> List[Release]: + """Search by ISBN.""" + response = requests.get( + f"{self.base_url}/v1/search", + params={"isbn": isbn}, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=self.timeout, + ) + response.raise_for_status() + return [self._to_release(r) for r in response.json().get("results", [])] + + def _search_text(self, title: str, authors: List[str]) -> List[Release]: + """Search by title and author.""" + query = title + if authors: + query += f" {authors[0]}" + + response = requests.get( + f"{self.base_url}/v1/search", + params={"q": query}, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=self.timeout, + ) + response.raise_for_status() + return [self._to_release(r) for r in response.json().get("results", [])] + + def _to_release(self, data: dict) -> Release: + """Convert API response to Release object.""" + return Release( + source="booknet", + source_id=data["id"], + title=data["title"], + format=data.get("format", "").lower(), + size=data.get("size_human"), + size_bytes=data.get("size_bytes"), + download_url=data.get("download_url"), + info_url=data.get("info_url"), + protocol="http", + indexer="BookNet", + extra={ + "author": data.get("author"), + "language": data.get("language", "en"), + "preview": data.get("cover_url"), + "quality": data.get("quality"), + "publisher": data.get("publisher"), + }, + ) + + def is_available(self) -> bool: + """Check if BookNet is configured.""" + return bool(self.api_key) + + @classmethod + def get_column_config(cls) -> ReleaseColumnConfig: + """Custom column layout for BookNet releases.""" + return ReleaseColumnConfig( + columns=[ + ColumnSchema( + key="extra.quality", + label="Quality", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="70px", + color_hint=ColumnColorHint(type="static", value="bg-purple-500/20 text-purple-400"), + ), + ColumnSchema( + key="extra.language", + label="Language", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="60px", + color_hint=ColumnColorHint(type="map", value="language"), + uppercase=True, + ), + ColumnSchema( + key="format", + label="Format", + render_type=ColumnRenderType.BADGE, + color_hint=ColumnColorHint(type="map", value="format"), + uppercase=True, + ), + ColumnSchema( + key="size", + label="Size", + render_type=ColumnRenderType.SIZE, + align=ColumnAlign.CENTER, + width="80px", + ), + ], + grid_template="minmax(0, 2fr) 70px 60px 80px 80px", + leading_cell=LeadingCellConfig( + type=LeadingCellType.THUMBNAIL, + key="extra.preview", + ), + ) + + +# --------------------------------------------------------------------------- +# Download Handler Implementation +# --------------------------------------------------------------------------- + +@register_handler("booknet") +class BookNetHandler(DownloadHandler): + """Handle downloads from BookNet.""" + + def __init__(self): + config = load_config_file("booknet") + self.api_key = config.get("BOOKNET_API_KEY") + self.base_url = config.get("BOOKNET_BASE_URL", "https://api.booknet.example.com") + self.timeout = config.get("BOOKNET_TIMEOUT", 30) + + def download( + self, + task: DownloadTask, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, Optional[str]], None], + ) -> Optional[str]: + """Execute download from BookNet.""" + try: + # Phase 1: Get download URL + status_callback("resolving", "Fetching download link...") + + if cancel_flag.is_set(): + return None + + download_info = self._get_download_info(task.task_id) + if not download_info: + status_callback("error", "Could not get download link") + return None + + download_url = download_info["url"] + expected_size = download_info.get("size_bytes", 0) + + # Phase 2: Download file + if cancel_flag.is_set(): + return None + + status_callback("downloading", None) + + temp_path = Path(tempfile.mktemp(suffix=f".{task.format or 'epub'}")) + + try: + downloaded_size = self._download_file( + download_url, + temp_path, + expected_size, + cancel_flag, + progress_callback, + ) + + if cancel_flag.is_set(): + temp_path.unlink(missing_ok=True) + return None + + # Validate download + if downloaded_size < 10 * 1024: # Less than 10KB + temp_path.unlink(missing_ok=True) + status_callback("error", f"File too small ({downloaded_size} bytes)") + return None + + # Phase 3: Move to final location + safe_title = "".join( + c if c.isalnum() or c in " .-_" else "_" + for c in task.title[:100] + ).strip() + + final_filename = f"{safe_title}.{task.format or 'epub'}" + final_path = Path(INGEST_DIR) / final_filename + + # Avoid overwriting existing files + counter = 1 + while final_path.exists(): + final_path = Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}" + counter += 1 + + shutil.move(str(temp_path), str(final_path)) + + # Return file path - orchestrator sets "complete" status + return str(final_path) + + except Exception as e: + temp_path.unlink(missing_ok=True) + raise + + except Exception as e: + if not cancel_flag.is_set(): + logger.error(f"BookNet download error: {e}") + status_callback("error", str(e)) + return None + + def _get_download_info(self, item_id: str) -> Optional[dict]: + """Get download URL from API.""" + response = requests.get( + f"{self.base_url}/v1/download/{item_id}", + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=self.timeout, + ) + + if response.status_code != 200: + return None + + return response.json() + + def _download_file( + self, + url: str, + dest_path: Path, + expected_size: int, + cancel_flag: Event, + progress_callback: Callable[[float], None], + ) -> int: + """Download file with progress reporting.""" + response = requests.get( + url, + headers={"Authorization": f"Bearer {self.api_key}"}, + stream=True, + timeout=self.timeout, + ) + response.raise_for_status() + + total_size = int(response.headers.get("content-length", expected_size)) + downloaded = 0 + + with open(dest_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if cancel_flag.is_set(): + return downloaded + + f.write(chunk) + downloaded += len(chunk) + + if total_size > 0: + progress_callback((downloaded / total_size) * 100) + + return downloaded + + def cancel(self, task_id: str) -> bool: + """Cancellation is handled via cancel_flag.""" + return False +``` + +--- + +## Best Practices + +### 1. Separation of Concerns + +- **ReleaseSource**: Only search logic, no download code +- **DownloadHandler**: Only download logic, no search code +- Keep settings registration separate from source/handler classes + +### 2. Graceful Degradation + +```python +def is_available(self) -> bool: + """Return False gracefully if not configured.""" + try: + return bool(self.api_key) and self._test_connection() + except Exception: + return False + +def search(self, book: BookMetadata) -> List[Release]: + """Return empty list on errors, don't raise.""" + try: + return self._do_search(book) + except Exception as e: + logger.warning(f"Search failed: {e}") + return [] +``` + +### 3. Cancellation Checks + +Check `cancel_flag.is_set()` before and during expensive operations: + +```python +def download(self, task, cancel_flag, progress_callback, status_callback): + # Before network calls + if cancel_flag.is_set(): + return None + + # During download loop + for chunk in response.iter_content(8192): + if cancel_flag.is_set(): + cleanup_partial_file() + return None +``` + +### 4. Progress Reporting + +Report progress frequently for good UX: + +```python +# In download loop +for chunk in response.iter_content(8192): + downloaded += len(chunk) + if total_size > 0: + progress_callback((downloaded / total_size) * 100) +``` + +### 5. Meaningful Status Messages + +```python +# Good: specific and actionable +status_callback("resolving", "Connecting to BookNet API...") +status_callback("resolving", "Fetching download link...") +status_callback("error", "API rate limit exceeded, try again in 60s") + +# Bad: vague +status_callback("resolving", "Working...") +status_callback("error", "Failed") +``` + +### 6. Clean Up on Failure + +```python +temp_path = Path(tempfile.mktemp()) +try: + # ... download to temp_path ... +except Exception: + temp_path.unlink(missing_ok=True) + raise +``` + +### 7. Settings with ENV Support + +For deployment flexibility, mark key settings as `env_supported=True`: + +```python +PasswordField( + key="MY_API_KEY", + label="API Key", + env_supported=True, # Can set via MY_API_KEY env var +) +``` + +### 8. Logging + +Use the project's logger for consistent output: + +```python +from cwa_book_downloader.core.logger import setup_logger + +logger = setup_logger(__name__) + +logger.debug("Detailed debug info") # Development only +logger.info("Normal operation info") # General progress +logger.warning("Recoverable issues") # Fallback used, retry happening +logger.error("Failures") # Unrecoverable errors +``` + +--- + +## Appendix: Import Checklist + +When creating a new plugin: + +1. Create `cwa_book_downloader/release_sources/my_plugin.py` +2. Implement `ReleaseSource` subclass with `@register_source("my_plugin")` +3. Implement `DownloadHandler` subclass with `@register_handler("my_plugin")` +4. Add settings with `@register_settings("my_plugin", ...)` +5. Add import to `cwa_book_downloader/release_sources/__init__.py`: + ```python + from cwa_book_downloader.release_sources import my_plugin # noqa: F401, E402 + ``` +6. Test `is_available()` returns `True` when configured +7. Test search returns valid `Release` objects +8. Test download handles cancellation and errors gracefully diff --git a/docs/url-search-parameters.md b/docs/url-search-parameters.md new file mode 100644 index 00000000..aa5b17fe --- /dev/null +++ b/docs/url-search-parameters.md @@ -0,0 +1,75 @@ +# URL Search Parameters + +You can trigger searches directly via URL by adding query parameters. This enables bookmarking searches and sharing links. + +## Basic Usage + +``` +http://your-server:8084/?q=harry+potter +``` + +## Supported Parameters + +| Parameter | Description | Example | +|-----------|-------------|---------| +| `q` or `query` | Main search query | `/?q=dune` | +| `author` | Filter by author name | `/?author=frank+herbert` | +| `title` | Filter by book title | `/?title=foundation` | +| `isbn` | Filter by ISBN | `/?isbn=978-0747532699` | +| `lang` | Filter by language (ISO 639-1 code) | `/?lang=en` | +| `format` | Filter by file format | `/?format=epub` | +| `content` | Filter by content type | `/?content=fiction` | +| `sort` | Sort order for results | `/?sort=newest` | + +## Multiple Values + +Some parameters support multiple values by repeating the parameter: + +``` +/?lang=en&lang=de&lang=fr +/?format=epub&format=mobi&format=azw3 +``` + +## Examples + +**Simple search:** +``` +/?q=lord+of+the+rings +``` + +**Search with author filter:** +``` +/?q=dune&author=frank+herbert +``` + +**Search with format and language:** +``` +/?q=harry+potter&format=epub&lang=en +``` + +**Author search with multiple formats:** +``` +/?author=stephen+king&format=epub&format=mobi +``` + +**Search with sort order:** +``` +/?q=science+fiction&sort=newest +``` + +## Search Mode Behavior + +### Direct Download Mode (default) + +All parameters are used to filter results from Anna's Archive. + +### Universal Mode + +Only `q` and `sort` are used. Other parameters (author, title, format, etc.) are silently ignored since metadata providers have their own search capabilities. + +## Notes + +- URL parameters are read once on page load +- The URL is not updated when you perform searches manually +- Spaces should be encoded as `+` or `%20` +- Invalid or unknown parameters are silently ignored diff --git a/entrypoint.sh b/entrypoint.sh index 4eab54be..045aed7c 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -109,7 +109,7 @@ make_writable /cwa-book-ingest # upgrades work reliably on customer machines. # Map app LOG_LEVEL (often DEBUG/INFO/...) to gunicorn's --log-level (lowercase). gunicorn_loglevel=$([ "$DEBUG" = "true" ] && echo debug || echo "${LOG_LEVEL:-info}" | tr '[:upper:]' '[:lower:]') -command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} app:app" +command="gunicorn --log-level ${gunicorn_loglevel} --access-logfile - --error-logfile - --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker --workers 1 -t 300 -b ${FLASK_HOST:-0.0.0.0}:${FLASK_PORT:-8084} cwa_book_downloader.main:app" # If DEBUG and not using an external bypass if [ "$DEBUG" = "true" ] && [ "$USING_EXTERNAL_BYPASSER" != "true" ]; then diff --git a/models.py b/models.py deleted file mode 100644 index bae92eb6..00000000 --- a/models.py +++ /dev/null @@ -1,432 +0,0 @@ -"""Data structures and models used across the application.""" - -from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple -from enum import Enum -from datetime import datetime, timedelta -from threading import Lock, Event -from pathlib import Path -import queue -import re -import time -from env import INGEST_DIR, STATUS_TIMEOUT - -class QueueStatus(str, Enum): - """Enum for possible book queue statuses.""" - QUEUED = "queued" - RESOLVING = "resolving" - DOWNLOADING = "downloading" - COMPLETE = "complete" - AVAILABLE = "available" - ERROR = "error" - DONE = "done" - CANCELLED = "cancelled" - -@dataclass -class QueueItem: - """Queue item with priority and metadata.""" - book_id: str - priority: int - added_time: float - - def __lt__(self, other): - """Compare items for priority queue (lower priority number = higher precedence).""" - if self.priority != other.priority: - return self.priority < other.priority - return self.added_time < other.added_time - -@dataclass -class BookInfo: - """Data class representing book information.""" - id: str - title: str - preview: Optional[str] = None - author: Optional[str] = None - publisher: Optional[str] = None - year: Optional[str] = None - language: Optional[str] = None - content: Optional[str] = None - format: Optional[str] = None - size: Optional[str] = None - info: Optional[Dict[str, List[str]]] = None - description: Optional[str] = None - download_urls: List[str] = field(default_factory=list) - download_path: Optional[str] = None - priority: int = 0 - progress: Optional[float] = None - status_message: Optional[str] = None # Detailed status message for UI display - added_time: Optional[float] = None # Timestamp when added to queue - - def get_filename(self, fallback_url: Optional[str] = None) -> str: - """Build sanitized filename: 'Author - Title (Year).format' - - Resolves format from self.format, download_urls, or fallback_url. - - Args: - fallback_url: URL to extract format from if not already known - - Returns: - Sanitized filename safe for filesystem use - """ - # Resolve format if needed - if not self.format: - for url in (self.download_urls[0] if self.download_urls else None, fallback_url): - if url: - ext = url.split(".")[-1].lower() - if ext and len(ext) <= 5 and ext.isalnum(): - self.format = ext - break - - # Build filename - parts = [] - if self.author: - parts.append(self.author) - parts.append(" - ") - parts.append(self.title) - if self.year: - parts.append(f" ({self.year})") - - filename = "".join(parts) - filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245] - - if self.format: - filename = f"{filename}.{self.format}" - - return filename - -class BookQueue: - """Thread-safe book queue manager with priority support and cancellation.""" - def __init__(self) -> None: - self._queue: queue.PriorityQueue[QueueItem] = queue.PriorityQueue() - self._lock = Lock() - self._status: dict[str, QueueStatus] = {} - self._book_data: dict[str, BookInfo] = {} - self._status_timestamps: dict[str, datetime] = {} # Track when each status was last updated - self._status_timeout = timedelta(seconds=STATUS_TIMEOUT) # 1 hour timeout - self._cancel_flags: dict[str, Event] = {} # Cancellation flags for active downloads - self._active_downloads: dict[str, bool] = {} # Track currently downloading books - - def add(self, book_id: str, book_data: BookInfo, priority: int = 0) -> None: - """Add a book to the queue with specified priority. - - Args: - book_id: Unique identifier for the book - book_data: Book information - priority: Priority level (lower number = higher priority) - """ - with self._lock: - # Don't add if already exists and not in error/done state - if book_id in self._status and self._status[book_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]: - return - - added_time = time.time() - book_data.priority = priority - book_data.added_time = added_time - queue_item = QueueItem(book_id, priority, added_time) - self._queue.put(queue_item) - self._book_data[book_id] = book_data - self._update_status(book_id, QueueStatus.QUEUED) - - def get_next(self) -> Optional[Tuple[str, Event]]: - """Get next book ID from queue with cancellation flag. - - Returns: - Tuple of (book_id, cancel_flag) or None if queue is empty - """ - # Use iterative approach to avoid stack overflow if many items are cancelled - while True: - try: - queue_item = self._queue.get_nowait() - book_id = queue_item.book_id - - with self._lock: - # Check if book was cancelled while in queue - if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED: - continue # Skip cancelled items, try next - - # Create cancellation flag for this download - cancel_flag = Event() - self._cancel_flags[book_id] = cancel_flag - self._active_downloads[book_id] = True - - return book_id, cancel_flag - except queue.Empty: - return None - - def _update_status(self, book_id: str, status: QueueStatus) -> None: - """Internal method to update status and timestamp.""" - self._status[book_id] = status - self._status_timestamps[book_id] = datetime.now() - - def update_status(self, book_id: str, status: QueueStatus) -> None: - """Update status of a book in the queue.""" - with self._lock: - self._update_status(book_id, status) - - # Clean up active download tracking when finished - if status in [QueueStatus.COMPLETE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]: - self._active_downloads.pop(book_id, None) - self._cancel_flags.pop(book_id, None) - - def update_download_path(self, book_id: str, download_path: str) -> None: - """Update the download path of a book in the queue.""" - with self._lock: - if book_id in self._book_data: - self._book_data[book_id].download_path = download_path - - def update_progress(self, book_id: str, progress: float) -> None: - """Update download progress for a book.""" - with self._lock: - if book_id in self._book_data: - self._book_data[book_id].progress = progress - - def update_status_message(self, book_id: str, message: str) -> None: - """Update detailed status message for a book.""" - with self._lock: - if book_id in self._book_data: - self._book_data[book_id].status_message = message - - def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]: - """Get current queue status.""" - self.refresh() - with self._lock: - result: Dict[QueueStatus, Dict[str, BookInfo]] = {status: {} for status in QueueStatus} - for book_id, status in self._status.items(): - if book_id in self._book_data: - result[status][book_id] = self._book_data[book_id] - return result - - def get_queue_order(self) -> List[Dict[str, any]]: - """Get current queue order for display.""" - with self._lock: - queue_items = [] - - # Get items from priority queue without removing them - temp_items = [] - while not self._queue.empty(): - try: - item = self._queue.get_nowait() - temp_items.append(item) - if item.book_id in self._book_data: - book_info = self._book_data[item.book_id] - queue_items.append({ - 'id': item.book_id, - 'title': book_info.title, - 'author': book_info.author, - 'priority': item.priority, - 'added_time': item.added_time, - 'status': self._status.get(item.book_id, QueueStatus.QUEUED) - }) - except queue.Empty: - break - - # Put items back in queue - for item in temp_items: - self._queue.put(item) - - return sorted(queue_items, key=lambda x: (x['priority'], x['added_time'])) - - def cancel_download(self, book_id: str) -> bool: - """Cancel a download or clear a completed/errored item. - - Args: - book_id: Book identifier to cancel or clear - - Returns: - bool: True if cancellation/clearing was successful - """ - with self._lock: - current_status = self._status.get(book_id) - - # Allow cancellation during any active state - if current_status in [QueueStatus.RESOLVING, QueueStatus.DOWNLOADING]: - # Signal active download to stop - if book_id in self._cancel_flags: - self._cancel_flags[book_id].set() - self._update_status(book_id, QueueStatus.CANCELLED) - return True - elif current_status == QueueStatus.QUEUED: - # Remove from queue and mark as cancelled - self._update_status(book_id, QueueStatus.CANCELLED) - return True - elif current_status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]: - # Clear completed/errored/cancelled items from tracking - self._status.pop(book_id, None) - self._status_timestamps.pop(book_id, None) - self._book_data.pop(book_id, None) - self._cancel_flags.pop(book_id, None) - self._active_downloads.pop(book_id, None) - return True - - return False - - def set_priority(self, book_id: str, new_priority: int) -> bool: - """Change the priority of a queued book. - - Args: - book_id: Book identifier - new_priority: New priority level (lower = higher priority) - - Returns: - bool: True if priority was successfully changed - """ - with self._lock: - if book_id not in self._status or self._status[book_id] != QueueStatus.QUEUED: - return False - - # Remove book from queue and re-add with new priority - temp_items = [] - found = False - - while not self._queue.empty(): - try: - item = self._queue.get_nowait() - if item.book_id == book_id: - # Create new item with updated priority - new_item = QueueItem(book_id, new_priority, item.added_time) - temp_items.append(new_item) - found = True - # Update book data priority - if book_id in self._book_data: - self._book_data[book_id].priority = new_priority - else: - temp_items.append(item) - except queue.Empty: - break - - # Put all items back - for item in temp_items: - self._queue.put(item) - - return found - - def reorder_queue(self, book_priorities: Dict[str, int]) -> bool: - """Bulk reorder queue by setting new priorities. - - Args: - book_priorities: Dict mapping book_id to new priority - - Returns: - bool: True if reordering was successful - """ - with self._lock: - # Extract all items from queue - all_items = [] - while not self._queue.empty(): - try: - item = self._queue.get_nowait() - # Update priority if specified - if item.book_id in book_priorities: - new_priority = book_priorities[item.book_id] - item = QueueItem(item.book_id, new_priority, item.added_time) - # Update book data priority - if item.book_id in self._book_data: - self._book_data[item.book_id].priority = new_priority - all_items.append(item) - except queue.Empty: - break - - # Put all items back with updated priorities - for item in all_items: - self._queue.put(item) - - return True - - def get_active_downloads(self) -> List[str]: - """Get list of currently active download book IDs.""" - with self._lock: - return list(self._active_downloads.keys()) - - def has_pending_work(self) -> bool: - """Check if there are any active downloads or queued items. - - This is useful for determining if the bypasser should stay active - even when the UI is closed. - - Returns: - bool: True if there are active downloads or queued items - """ - with self._lock: - # Check for active downloads - if self._active_downloads: - return True - - # Check for queued items (excluding cancelled ones) - for book_id, status in self._status.items(): - if status == QueueStatus.QUEUED: - return True - - return False - - def clear_completed(self) -> int: - """Remove all completed, errored, or cancelled books from tracking. - - Returns: - int: Number of books removed - """ - with self._lock: - to_remove = [] - for book_id, status in self._status.items(): - if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]: - to_remove.append(book_id) - - removed_count = len(to_remove) - for book_id in to_remove: - self._status.pop(book_id, None) - self._status_timestamps.pop(book_id, None) - self._book_data.pop(book_id, None) - self._cancel_flags.pop(book_id, None) - self._active_downloads.pop(book_id, None) - - return removed_count - - def refresh(self) -> None: - """Remove any books that are done downloading or have stale status.""" - with self._lock: - current_time = datetime.now() - - # Create a list of items to remove to avoid modifying dict during iteration - to_remove = [] - - for book_id, status in self._status.items(): - path = self._book_data[book_id].download_path - if path and not Path(path).exists(): - self._book_data[book_id].download_path = None - path = None - - # Check for completed downloads - if status == QueueStatus.AVAILABLE: - if not path: - self._update_status(book_id, QueueStatus.DONE) - - # Check for stale status entries - last_update = self._status_timestamps.get(book_id) - if last_update and (current_time - last_update) > self._status_timeout: - if status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.ERROR, QueueStatus.AVAILABLE, QueueStatus.CANCELLED]: - to_remove.append(book_id) - - # Remove stale entries - for book_id in to_remove: - del self._status[book_id] - del self._status_timestamps[book_id] - if book_id in self._book_data: - del self._book_data[book_id] - - def set_status_timeout(self, hours: int) -> None: - """Set the status timeout duration in hours.""" - with self._lock: - self._status_timeout = timedelta(hours=hours) - - -# Global instance of BookQueue -book_queue = BookQueue() - -@dataclass -class SearchFilters: - isbn: Optional[List[str]] = None - author: Optional[List[str]] = None - title: Optional[List[str]] = None - lang: Optional[List[str]] = None - sort: Optional[str] = None - content: Optional[List[str]] = None - format: Optional[List[str]] = None \ No newline at end of file diff --git a/readme.md b/readme.md index 6c6805ab..e5ba53a1 100644 --- a/readme.md +++ b/readme.md @@ -1,365 +1,240 @@ -# 📚 Calibre-Web-Automated-Book-Downloader +# 📚 Book Downloader +*calibre-web-automated-book-downloader* -Calibre-Web Automated Book Downloader +Book Downloader -An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library. +A unified web interface for searching and downloading books from multiple sources — all in one place. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to create a single hub for building your digital library. + +**Fully standalone** — no external dependencies required. Works great alongside library tools like [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) or [Booklore](https://github.com/booklore-app/booklore) for automatic import. ## ✨ Features -- 🌐 User-friendly web interface for book search and download -- 🔄 Automated download to your specified ingest folder -- 🔌 Seamless integration with Calibre-Web-Automated -- 📖 Support for multiple book formats (epub, mobi, azw3, fb2, djvu, cbz, cbr) -- 🛡️ Cloudflare bypass capability for reliable downloads -- 🐳 Docker-based deployment for quick setup +- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place +- **Real-Time Progress** - Unified download queue with live status updates across all sources +- **Two Search Modes**: + - **Direct Download** - Search and download from popular web sources + - **Universal Mode** - Search metadata providers (Hardcover, Open Library) for richer book discovery and multi-source downloads *(additional sources in development - coming soon!)* +- **Format Support** - EPUB, MOBI, AZW3, FB2, DJVU, CBZ, CBR and more +- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources +- **PWA Support** - Install as a mobile app for quick access +- **Docker Deployment** - Up and running in minutes ## 🖼️ Screenshots -![Main search interface Screenshot](README_images/search.png 'Main search interface') +**Home screen** +![Home screen](README_images/homescreen.png 'Home screen') -![Details modal Screenshot placeholder](README_images/details.png 'Details modal') +**Search results** +![Search results](README_images/search-results.png 'Search results') -![Download queue Screenshot placeholder](README_images/downloading.png 'Download queue') +**Multi-source downloads** +![Multi-source downloads](README_images/multi-source.png 'Multi-source downloads') + +**Download queue** +![Download queue](README_images/downloads.png 'Download queue') ## 🚀 Quick Start ### Prerequisites -- Docker -- Docker Compose -- A running instance of [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated) (recommended) +- Docker & Docker Compose -### Installation Steps - -1. Get the docker-compose.yml: +### Installation +1. Download the docker-compose file: ```bash - curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.yml + curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.yml ``` 2. Start the service: - ```bash docker compose up -d ``` -3. Access the web interface at `http://localhost:8084` +3. Open `http://localhost:8084` -## ⚙️ Configuration +That's it! Configure settings through the web interface as needed. -### Environment Variables - -#### Application Settings - -| Variable | Description | Default Value | -| ----------------- | ----------------------- | ------------------ | -| `FLASK_PORT` | Web interface port | `8084` | -| `FLASK_HOST` | Web interface binding | `0.0.0.0` | -| `DEBUG` | Debug mode toggle | `false` | -| `INGEST_DIR` | Book download directory | `/cwa-book-ingest` | -| `TZ` | Container timezone | `UTC` | -| `UID` | Runtime user ID | `1000` | -| `GID` | Runtime group ID | `100` | -| `CWA_DB_PATH` | Calibre-Web's database | None | -| `ENABLE_LOGGING` | Enable log file | `true` | -| `LOG_LEVEL` | Log level to use | `info` | -| `SESSION_COOKIE_SECURE` | Secure cookie enforcement - Use for HTTPS connections only | `false` | -| `CALIBRE_WEB_URL` | Custom WebUI library link | None | -| `BYPASS_WARMUP_ON_CONNECT` | Warm up Cloudflare bypasser when first client connects | `true` | - -If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password. - -Set `CALIBRE_WEB_URL` to your Calibre-Web / Booklore base URL. A ‘Go to library’ button will appear in the Web UI for quick access while downloading, and it also provides library access when CWA-BD is installed as a mobile PWA. - -If logging is enabled, log folder default location is `/var/log/cwa-book-downloader` -Available log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Higher levels show fewer messages. - -Note that if using TOR, the TZ will be calculated automatically based on IP. - -#### Download Settings - -| Variable | Description | Default Value | -| ---------------------- | --------------------------------------------------------- | --------------------------------- | -| `MAX_RETRY` | Maximum retry attempts | `3` | -| `DEFAULT_SLEEP` | Retry delay (seconds) | `5` | -| `MAIN_LOOP_SLEEP_TIME` | Processing loop delay (seconds) | `5` | -| `SUPPORTED_FORMATS` | Supported book formats | `epub,mobi,azw3,fb2,djvu,cbz,cbr` | -| `BOOK_LANGUAGE` | Preferred language for books | `en` | -| `AA_DONATOR_KEY` | Optional Donator key for Anna's Archive fast download API | `` | -| `USE_BOOK_TITLE` | Use book title as filename instead of ID | `false` | -| `PRIORITIZE_WELIB` | When downloading, download from WELIB first instead of AA | `false` | -| `ALLOW_USE_WELIB` | Allow usage of welib for downloading books if found there | `true` | - -If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc. - -Use the following environment variables to set specific folders in which to download -different content types (Book, Magazine, Comic, etc.): - -| Variable | Description | Default Value | -|---------------------------------|--------------------------------|---------------| -| `INGEST_DIR_BOOK_FICTION` | Book (fiction) folder name | `` | -| `INGEST_DIR_BOOK_NON_FICTION` | Book (non-fiction) folder name | `` | -| `INGEST_DIR_BOOK_UNKNOWN` | Book (unknown) folder name | `` | -| `INGEST_DIR_MAGAZINE` | Magazine folder name | `` | -| `INGEST_DIR_COMIC_BOOK` | Comic book folder name | `` | -| `INGEST_DIR_AUDIOBOOK` | Audiobook folder name | `` | -| `INGEST_DIR_STANDARDS_DOCUMENT` | Standards document folder name | `` | -| `INGEST_DIR_MUSICAL_SCORE` | Musical score folder name | `` | - -If no specific path is set for a content type the default is `INGEST_DIR`. -Remember to map the specified paths to where your instance of Calibre-Web-Automated (CWA) will find them, e.g.: -``` -volumes: - - /tmp/data/calibre-web/comicbook-ingest:/cwa-comicbook-ingest -``` -if `INGEST_DIR_COMIC_BOOK=/cwa-comicbook-ingest` and your CWA is configured to use `/tmp/data/calibre-web/comicbook-ingest` -for comic books. - - -#### AA - -| Variable | Description | Default Value | -| ---------------------- | --------------------------------------------------------- | --------------------------------- | -| `AA_BASE_URL` | Base URL of Annas-Archive (could be changed for a proxy) | `https://annas-archive.org` | -| `USE_CF_BYPASS` | Disable CF bypass and use alternative links instead | `true` | - -If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times. -If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books. - -#### Network Settings - -| Variable | Description | Default Value | -| ---------------------- | ------------------------------- | ----------------------- | -| `AA_ADDITIONAL_URLS` | Proxy URLs for AA (, separated) | `` | -| `HTTP_PROXY` | HTTP proxy URL | `` | -| `HTTPS_PROXY` | HTTPS proxy URL | `` | -| `CUSTOM_DNS` | DNS configuration | `auto` | -| `USE_DOH` | Use DNS over HTTPS | `false` | - -**Proxy Configuration** - -For proxy configuration, you can specify URLs in the following format: -```bash -# Basic proxy -HTTP_PROXY=http://proxy.example.com:8080 -HTTPS_PROXY=http://proxy.example.com:8080 - -# Proxy with authentication -HTTP_PROXY=http://username:password@proxy.example.com:8080 -HTTPS_PROXY=http://username:password@proxy.example.com:8080 -``` - -**DNS Configuration** - -The `CUSTOM_DNS` setting controls how DNS resolution works. By default, it is set to `auto` which provides automatic failover for reliable connectivity. - -**Auto Mode (Default)** - -When `CUSTOM_DNS=auto`, the application starts with your system's default DNS. If DNS resolution fails, it automatically rotates through alternative providers using DNS over HTTPS (DoH): - -1. System DNS (initial) -2. Cloudflare (1.1.1.1) -3. Google (8.8.8.8) -4. Quad9 (9.9.9.9) -5. OpenDNS (208.67.222.222) - -This automatic rotation helps bypass ISP-level blocks and DNS issues without any manual configuration. - -**Manual DNS Configuration** - -If you prefer to use a specific DNS configuration, you can override the auto behavior: - -1. **Preset DNS Providers**: Use one of these predefined options: - - `google` - Google DNS (8.8.8.8, 8.8.4.4) - - `quad9` - Quad9 DNS (9.9.9.9, 149.112.112.112) - - `cloudflare` - Cloudflare DNS (1.1.1.1, 1.0.0.1) - - `opendns` - OpenDNS (208.67.222.222, 208.67.220.220) - -2. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses - - Example: `127.0.0.53,127.0.1.53` (useful for PiHole) - - Supports both IPv4 and IPv6 addresses - -When using preset providers, you can optionally enable DNS over HTTPS with `USE_DOH=true`: -```bash -CUSTOM_DNS=cloudflare -USE_DOH=true -``` - -Note: When using custom IP addresses, the `USE_DOH` flag is ignored since DoH requires a known provider endpoint. - -#### Custom configuration - -| Variable | Description | Default Value | -| ---------------------- | ----------------------------------------------------------- | ----------------------- | -| `CUSTOM_SCRIPT` | Path to an executable script that tuns after each download | `` | - -If `CUSTOM_SCRIPT` is set, it will be executed after each successful download but before the file is moved to the ingest directory. This allows for custom processing like format conversion or validation. - -The script is called with the full path of the downloaded file as its argument. Important notes: -- The script must preserve the original filename for proper processing -- The file can be modified or even deleted if needed -- The file will be moved to `/cwa-book-ingest` after the script execution (if not deleted) - -You can specify these configuration in this format : -``` -environment: - - CUSTOM_SCRIPT=/scripts/process-book.sh - -volumes: - - local/scripts/custom_script.sh:/scripts/process-book.sh -``` - -### Volume Configuration +### Volume Setup ```yaml volumes: - - /your/local/path:/cwa-book-ingest - - /cwa/config/path/app.db:/auth/app.db:ro + - /your/config/path:/config # Config, database, and artwork cache directory + - /your/download/path:/cwa-book-ingest # Downloaded books ``` -**Note** - If your library volume is on a cifs share, you will get a "database locked" error until you add **nobrl** to your mount line in your fstab file. e.g. //192.168.1.1/Books /media/books cifs credentials=.smbcredentials,uid=1000,gid=1000,iocharset=utf8,**nobrl** - See https://github.com/crocodilestick/Calibre-Web-Automated/issues/64#issuecomment-2712769777 -Mount should align with your Calibre-Web-Automated ingest folder. +> **Tip**: Point the download volume to your CWA or Booklore ingest folder for automatic import. -## Variants: +> **Note**: CIFS shares require `nobrl` mount option to avoid database lock errors. -### 🧅 Tor Variant +## ⚙️ Configuration -This application also offers a variant that routes all its traffic through the Tor network. This can be useful for enhanced privacy or bypassing network restrictions. +### Search Modes -To use the Tor variant: +**Direct Download Mode** (default) +- Works out of the box, no setup required +- Searches a huge library of books directly +- Returns downloadable releases immediately -1. Get the Tor-specific docker-compose file: - ```bash - curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.tor.yml - ``` -2. Start the service using this file: - ```bash - docker compose -f docker-compose.tor.yml up -d - ``` +**Universal Mode** +- Cleaner search results via metadata providers (Hardcover, Open Library) +- Aggregates releases from multiple configured sources +- Requires manual setup (API keys, additional sources) -**Important Considerations for Tor:** +Set the mode via Settings or `SEARCH_MODE` environment variable. -* **Capabilities:** This variant requires the `NET_ADMIN` and `NET_RAW` Docker capabilities to configure `iptables` for transparent Tor proxying. -* **Timezone:** When running in Tor mode, the container will attempt to determine the timezone based on the Tor exit node's IP address and set it automatically. This will override the `TZ` environment variable if it is set. -* **Network Settings:** Custom DNS, DoH, and HTTP(S) proxy settings (`CUSTOM_DNS`, `USE_DOH`, `HTTP_PROXY`, `HTTPS_PROXY`) are ignored when using the Tor variant, as all traffic goes through Tor. +### Environment Variables -### External Cloudflare resolver variant +Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface. -This variant allows the application to use an external service to bypass Cloudflare protection, instead of relying on the built-in bypasser. This is useful if you already have a dedicated Cloudflare resolver (such as [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) or compatible services like [ByParr](https://github.com/ThePhaseless/Byparr)) running elsewhere. +| Variable | Description | Default | +|----------|-------------|---------| +| `FLASK_PORT` | Web interface port | `8084` | +| `INGEST_DIR` | Book download directory | `/cwa-book-ingest` | +| `TZ` | Container timezone | `UTC` | +| `UID` / `GID` | Runtime user/group ID | `1000` / `100` | +| `SEARCH_MODE` | `direct` or `universal` | `direct` | -#### How it works: +Some of the additional options available in Settings: +- **AA Donator Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads +- **Library Link** - Add a link to your Calibre-Web or Booklore instance in the UI header +- **Content Folders** - Route fiction, non-fiction, comics, etc. to separate directories +- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable +- **Format & Language** - Filter downloads by preferred formats and languages +- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc. -- When enabled, all requests that require Cloudflare bypass are sent to your external resolver service. -- The application communicates with the resolver using its API. +## 🐳 Docker Variants -#### Configuration - -| Variable | Description | Default Value | -| ---------------------- | ----------------------------------------------------------- | ----------------------- | -| `EXT_BYPASSER_URL` | The full URL of your external resolver (required) | | -| `EXT_BYPASSER_PATH` | API path for the resolver (usually `/v1`) | `/v1` | -| `EXT_BYPASSER_TIMEOUT` | Timeout for page loading (in milliseconds) | `60000` | - -#### Important - -This feature follows the same configuration of the built-in Cloudflare bypasser, so you should turn on the `USE_CF_BYPASS` configuration to enable it. - -#### To use the External Cloudflare resolver variant: - -1. Get the extbp-specific docker-compose file: - ```bash - curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/refs/heads/main/docker-compose.extbp.yml - ``` -2. Start the service using this file: - ```bash - docker compose -f docker-compose.extbp.yml up -d - ``` - -#### Compatibility: -This feature is designed to work with any resolver that implements the `FlareSolverr` API schema, including `ByParr` and similar projects. - -#### Internal vs External Bypasser - -The **internal bypasser** (default) is custom-designed for this application's specific needs. It handles session management, cookie persistence, and retry logic optimized for book downloading workflows. For most users, this provides the most reliable experience out of the box. - -The **external bypasser** is better suited if you: -- Already run FlareSolverr/ByParr for other services and want to consolidate -- Need to share bypass infrastructure across multiple applications -- Want to offload browser automation to a dedicated, more powerful container - -If you're unsure which to use, start with the default internal bypasser. - -## 🏗️ Architecture - -The application consists of a Flask backend with a React-based frontend: - -### Backend -- **Flask Application**: Python-based backend (`app.py`, `backend.py`) providing REST API and WebSocket support -- **Download Manager**: Handles book search, download requests, and queue management (`downloader.py`, `book_manager.py`) -- **Network Layer**: Cloudflare bypass and proxy support (`cloudflare_bypasser.py`, `network.py`) - -### Frontend -- **React + TypeScript**: Modern web interface built with Vite (`src/frontend`) -- **Real-time Updates**: WebSocket integration for live download status -- **Responsive UI**: TailwindCSS-based design for mobile and desktop - -For frontend development, use the provided Makefile: +### Standard ```bash -make install # Install dependencies -make dev # Start development server -make build # Build for production -``` -If you run the docker compose file, the frontend will be built and served automatically. But if you run the frontend dev server it will supercede the docker compose frontend. - -## 🏥 Health Monitoring - -Built-in health checks monitor: - -- Web interface availability -- Download service status -- Cloudflare bypass service connection - -Checks run every 30 seconds with a 30-second timeout and 3 retries. -You can enable by adding this to your compose : -``` -HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ - CMD curl -s http://localhost:8084/api/status || exit 1 +docker compose up -d ``` -## 📝 Logging +### Tor Variant +Routes all traffic through Tor for enhanced privacy: +```bash +curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.tor.yml +docker compose -f docker-compose.tor.yml up -d +``` -Logs are available in: +**Notes:** +- Requires `NET_ADMIN` and `NET_RAW` capabilities +- Timezone is auto-detected from Tor exit node +- Custom DNS/proxy settings are ignored -- Container: `/var/logs/cwa-book-downloader.log` -- Docker logs: Access via `docker logs` +### External Cloudflare Resolver +Use FlareSolverr or ByParr instead of the built-in bypasser: +```bash +curl -O https://raw.githubusercontent.com/calibrain/calibre-web-automated-book-downloader/main/docker-compose.extbp.yml +docker compose -f docker-compose.extbp.yml up -d +``` -## 🤝 Contributing +Configure the resolver URL in Settings under the Cloudflare tab. -Contributions are welcome! Feel free to submit a Pull Request. +**When to use external vs internal bypasser:** +- **External** is useful if you already run FlareSolverr for other services (saves resources) or if you rarely need bypassing +- **Internal** (default) is faster and more reliable for most users - it's optimized specifically for this application -## 📄 License +## 🔐 Authentication -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. +Authentication is optional but recommended for shared or exposed instances. Enable in Settings. -## ⚠️ Important Disclaimers +**Alternative**: If you're running Calibre-Web, you can reuse its user database by mounting it: + +```yaml +volumes: + - /path/to/calibre-web/app.db:/auth/app.db:ro +``` + +## Health Monitoring + +The application exposes a health endpoint at `/api/status`. Add a health check to your compose: + +```yaml +healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8084/api/status"] + interval: 30s + timeout: 30s + retries: 3 +``` + +## Logging + +Logs are available via: +- `docker logs ` +- `/var/log/cwa-book-downloader/` inside the container (when `ENABLE_LOGGING=true`) + +Log level is configurable via Settings or `LOG_LEVEL` environment variable. + +## Development + +```bash +# Frontend development +make install # Install dependencies +make dev # Start Vite dev server (localhost:5173) +make build # Production build +make typecheck # TypeScript checks + +# Backend (Docker) +make up # Start backend via docker-compose.dev.yml +make down # Stop services +make refresh # Rebuild and restart +``` + +The frontend dev server proxies to the backend on port 8084. + +### Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Web Interface │ +│ (React + TypeScript + Vite) │ +├─────────────────────────────────────────────────────────────┤ +│ Flask Backend │ +│ (REST API + WebSocket) │ +├───────────────────┬─────────────────────┬───────────────────┤ +│ Metadata Providers│ Download Queue │ Cloudflare │ +│ │ & Orchestrator │ Bypass │ +├───────────────────┼─────────────────────┼───────────────────┤ +│ • Hardcover │ • Task scheduling │ • Internal │ +│ • Open Library │ • Progress tracking │ • External │ +│ │ • Retry logic │ (FlareSolverr) │ +├───────────────────┴─────────────────────┴───────────────────┤ +│ Release Sources │ +├─────────────────────────────────────────────────────────────┤ +│ • Direct Download (Anna's Archive → Libgen → Welib) │ +├─────────────────────────────────────────────────────────────┤ +│ Network Layer │ +├─────────────────────────────────────────────────────────────┤ +│ • Auto DNS rotation • Mirror failover • Resume support │ +└─────────────────────────────────────────────────────────────┘ +``` + +The backend uses a plugin architecture. Metadata providers and release sources register via decorators and are automatically discovered. + +## Contributing + +Contributions are welcome! Please file issues or submit pull requests on GitHub. + +> **Note**: Additional release sources and download clients are under active development. Want to add support for your favorite source? Check out the plugin architecture above and submit a PR! + +## License + +MIT License - see [LICENSE](LICENSE) for details. + +## ⚠️ Disclaimers ### Copyright Notice -While this tool can access various sources including those that might contain copyrighted material (e.g., Anna's Archive), it is designed for legitimate use only. Users are responsible for: - +This tool can access various sources including those that might contain copyrighted material. Users are responsible for: - Ensuring they have the right to download requested materials - Respecting copyright laws and intellectual property rights - Using the tool in compliance with their local regulations -### Duplicate Downloads Warning +### Library Integration -Please note that the current version: +Downloads are written atomically (via intermediate `.crdownload` files) to prevent partial files from being ingested. However, if your library tool (CWA, Booklore, Calibre) is actively scanning or importing, there's a small chance of race conditions. If you experience database errors or import failures, try pausing your library's auto-import during bulk downloads. -- Does not check for existing files in the download directory -- Does not verify if books already exist in your Calibre database -- Exercise caution when requesting multiple books to avoid duplicates - -## 💬 Support - -For issues or questions, please file an issue on the GitHub repository. +## Support +For issues or questions, please [file an issue](https://github.com/calibrain/calibre-web-automated-book-downloader/issues) on GitHub. diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index 21773fac..9dac19fc 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -1,82 +1,130 @@ -import { useState, useEffect, useCallback, useRef, CSSProperties } from 'react'; -import { Navigate, Route, Routes, useNavigate } from 'react-router-dom'; +import { useState, useEffect, useCallback, useRef, useMemo, CSSProperties } from 'react'; +import { Navigate, Route, Routes } from 'react-router-dom'; import { Book, + Release, StatusData, - ButtonStateInfo, AppConfig, - LoginCredentials, - AdvancedFilterState, } from './types'; -import { searchBooks, getBookInfo, downloadBook, cancelDownload, clearCompleted, getConfig, login, logout, checkAuth, AuthenticationError } from './services/api'; +import { getBookInfo, getMetadataBookInfo, downloadBook, downloadRelease, cancelDownload, clearCompleted, getConfig } from './services/api'; import { useToast } from './hooks/useToast'; import { useRealtimeStatus } from './hooks/useRealtimeStatus'; +import { useAuth } from './hooks/useAuth'; +import { useSearch } from './hooks/useSearch'; +import { useUrlSearch } from './hooks/useUrlSearch'; +import { useDownloadTracking } from './hooks/useDownloadTracking'; import { Header } from './components/Header'; import { SearchSection } from './components/SearchSection'; import { AdvancedFilters } from './components/AdvancedFilters'; import { ResultsSection } from './components/ResultsSection'; import { DetailsModal } from './components/DetailsModal'; +import { ReleaseModal } from './components/ReleaseModal'; import { DownloadsSidebar } from './components/DownloadsSidebar'; import { ToastContainer } from './components/ToastContainer'; import { Footer } from './components/Footer'; import { LoginPage } from './pages/LoginPage'; +import { SettingsModal } from './components/settings'; +import { ConfigSetupBanner } from './components/ConfigSetupBanner'; import { DEFAULT_LANGUAGES, DEFAULT_SUPPORTED_FORMATS } from './data/languages'; -import { LANGUAGE_OPTION_DEFAULT } from './utils/languageFilters'; import { buildSearchQuery } from './utils/buildSearchQuery'; +import { SearchModeProvider } from './contexts/SearchModeContext'; import './styles.css'; -const DEFAULT_FORMAT_SELECTION = DEFAULT_SUPPORTED_FORMATS.filter(format => format !== 'pdf'); - function App() { - // Authentication state - const [isAuthenticated, setIsAuthenticated] = useState(false); - const [authRequired, setAuthRequired] = useState(true); - const [authChecked, setAuthChecked] = useState(false); - const [loginError, setLoginError] = useState(null); - const [isLoggingIn, setIsLoggingIn] = useState(false); - const navigate = useNavigate(); - - const [books, setBooks] = useState([]); - const [selectedBook, setSelectedBook] = useState(null); - const [isSearching, setIsSearching] = useState(false); - const [config, setConfig] = useState(null); - const [searchInput, setSearchInput] = useState(''); - const [showAdvanced, setShowAdvanced] = useState(false); - const [downloadsSidebarOpen, setDownloadsSidebarOpen] = useState(false); - const [lastSearchQuery, setLastSearchQuery] = useState(''); - const [advancedFilters, setAdvancedFilters] = useState({ - isbn: '', - author: '', - title: '', - lang: [LANGUAGE_OPTION_DEFAULT], - sort: '', - content: '', - formats: DEFAULT_FORMAT_SELECTION, - }); const { toasts, showToast, removeToast } = useToast(); - const updateAdvancedFilters = useCallback((updates: Partial) => { - setAdvancedFilters(prev => ({ ...prev, ...updates })); - }, []); - - // Determine WebSocket URL based on current location - // In production, use the same origin as the page; in dev, use localhost + + // WebSocket URL based on current location const wsUrl = window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1' ? 'http://localhost:8084' : window.location.origin; - - // Use realtime status with WebSocket and polling fallback - const { - status: currentStatus, + + // Realtime status with WebSocket and polling fallback + const { + status: currentStatus, isUsingWebSocket, - forceRefresh: fetchStatus + forceRefresh: fetchStatus } = useRealtimeStatus({ wsUrl, pollInterval: 5000, reconnectAttempts: 3, }); - - // Calculate status counts for header badges - const getStatusCounts = () => { + + // Download tracking for universal mode + const { + bookToReleaseMap, + trackRelease, + markBookCompleted, + clearTracking, + getButtonState, + getUniversalButtonState, + } = useDownloadTracking(currentStatus); + + // Authentication state and handlers + // Initialized first since search hook needs auth state + const { + isAuthenticated, + authRequired, + authChecked, + loginError, + isLoggingIn, + setIsAuthenticated, + handleLogin, + handleLogout, + } = useAuth({ + showToast, + }); + + // Search state and handlers + const { + books, + setBooks, + isSearching, + searchInput, + setSearchInput, + showAdvanced, + setShowAdvanced, + advancedFilters, + setAdvancedFilters, + updateAdvancedFilters, + handleSearch, + handleResetSearch, + handleSortChange, + resetSortFilter, + searchFieldValues, + updateSearchFieldValue, + } = useSearch({ + showToast, + setIsAuthenticated, + authRequired, + onSearchReset: clearTracking, + }); + + // Wire up logout callback to clear search state + const handleLogoutWithCleanup = useCallback(async () => { + await handleLogout(); + setBooks([]); + clearTracking(); + }, [handleLogout, setBooks, clearTracking]); + + // UI state + const [selectedBook, setSelectedBook] = useState(null); + const [releaseBook, setReleaseBook] = useState(null); + const [config, setConfig] = useState(null); + const [downloadsSidebarOpen, setDownloadsSidebarOpen] = useState(false); + const [settingsOpen, setSettingsOpen] = useState(false); + const [configBannerOpen, setConfigBannerOpen] = useState(false); + + // URL-based search: parse URL params for automatic search on page load + const urlSearchEnabled = isAuthenticated && config !== null; + const { parsedParams, wasProcessed } = useUrlSearch({ enabled: urlSearchEnabled }); + const urlSearchExecutedRef = useRef(false); + + // Track previous status and search mode for change detection + const prevStatusRef = useRef({}); + const prevSearchModeRef = useRef(undefined); + + // Calculate status counts for header badges (memoized) + const statusCounts = useMemo(() => { const ongoing = [ currentStatus.queued, currentStatus.resolving, @@ -90,9 +138,8 @@ function App() { const errored = currentStatus.error ? Object.keys(currentStatus.error).length : 0; return { ongoing, completed, errored }; - }; + }, [currentStatus]); - const statusCounts = getStatusCounts(); const activeCount = statusCounts.ongoing; // Compute visibility states @@ -133,6 +180,13 @@ function App() { if (prevDownloadingIds.has(bookId) || prevQueuedIds.has(bookId)) { const book = currComplete[bookId]; showToast(`${book.title || 'Book'} completed`, 'success'); + + // Track completed release IDs in session state for universal mode + Object.entries(bookToReleaseMap).forEach(([metadataBookId, releaseIds]) => { + if (releaseIds.includes(bookId)) { + markBookCompleted(metadataBookId); + } + }); } }); @@ -145,72 +199,7 @@ function App() { showToast(`${book.title || 'Book'}: ${errorMsg}`, 'error'); } }); - }, [showToast]); - - // Track previous status for change detection - const prevStatusRef = useRef({}); - - // Check authentication on mount - useEffect(() => { - const verifyAuth = async () => { - try { - const response = await checkAuth(); - const authenticated = response.authenticated || false; - const authIsRequired = response.auth_required !== false; // Default to true if undefined - - setAuthRequired(authIsRequired); - setIsAuthenticated(authenticated); - } catch (error) { - console.error('Auth check failed:', error); - // On error, assume auth is required and user is not authenticated - setAuthRequired(true); - setIsAuthenticated(false); - } finally { - setAuthChecked(true); - } - }; - verifyAuth(); - }, []); - - // Authentication handlers - const handleLogin = async (credentials: LoginCredentials) => { - setIsLoggingIn(true); - setLoginError(null); - try { - const response = await login(credentials); - if (response.success) { - setIsAuthenticated(true); - setLoginError(null); - navigate('/', { replace: true }); - } else { - setLoginError(response.error || 'Login failed'); - } - } catch (error) { - if (error instanceof Error) { - setLoginError(error.message || 'Login failed'); - } else { - setLoginError('Login failed'); - } - } finally { - setIsLoggingIn(false); - } - }; - - const handleLogout = async () => { - try { - await logout(); - setIsAuthenticated(false); - // Clear application state - setBooks([]); - setSelectedBook(null); - setSearchInput(''); - setLastSearchQuery(''); - navigate('/login', { replace: true }); - } catch (error) { - console.error('Logout failed:', error); - showToast('Logout failed', 'error'); - } - }; + }, [showToast, bookToReleaseMap, markBookCompleted]); // Detect status changes when currentStatus updates useEffect(() => { @@ -220,32 +209,119 @@ function App() { prevStatusRef.current = currentStatus; }, [currentStatus, detectChanges]); - // Fetch config on mount and when authentication changes - useEffect(() => { - const loadConfig = async () => { - try { - const cfg = await getConfig(); - setConfig(cfg); - // Update format selection to match supported formats from config - // This ensures PDF is auto-selected when added to SUPPORTED_FORMATS env var - if (cfg?.supported_formats) { + // Load config function + const loadConfig = useCallback(async (mode: 'initial' | 'settings-saved' = 'initial') => { + try { + const cfg = await getConfig(); + + // Check if search mode changed (only on settings save) + if (mode === 'settings-saved' && prevSearchModeRef.current !== cfg.search_mode) { + setBooks([]); + setSelectedBook(null); + resetSortFilter(); + clearTracking(); + } + + prevSearchModeRef.current = cfg.search_mode; + setConfig(cfg); + + if (cfg?.supported_formats) { + if (mode === 'initial') { setAdvancedFilters(prev => ({ ...prev, formats: cfg.supported_formats, })); + } else if (mode === 'settings-saved') { + setAdvancedFilters(prev => ({ + ...prev, + formats: prev.formats.filter(f => cfg.supported_formats.includes(f)), + })); } - } catch (error) { - console.error('Failed to load config:', error); - // Use defaults if config fails to load } - }; - // Only fetch config if authenticated (or auth is not required) - if (isAuthenticated) { - loadConfig(); + } catch (error) { + console.error('Failed to load config:', error); } - }, [isAuthenticated]); + }, [setBooks, setAdvancedFilters, resetSortFilter, clearTracking]); - // Log WebSocket connection status changes + // Fetch config when authenticated + useEffect(() => { + if (isAuthenticated) { + loadConfig('initial'); + } + }, [isAuthenticated, loadConfig]); + + // Execute URL-based search when params are present + useEffect(() => { + if ( + wasProcessed && + parsedParams?.hasSearchParams && + !urlSearchExecutedRef.current && + config + ) { + urlSearchExecutedRef.current = true; + + const searchMode = config.search_mode || 'direct'; + const bookLanguages = config.book_languages || []; + const defaultLanguageCodes = + config.default_language && config.default_language.length > 0 + ? config.default_language + : [bookLanguages[0]?.code || 'en']; + + // Populate search input from URL + if (parsedParams.searchInput) { + setSearchInput(parsedParams.searchInput); + } + + // Apply advanced filters from URL + if (Object.keys(parsedParams.advancedFilters).length > 0) { + setAdvancedFilters(prev => ({ + ...prev, + ...parsedParams.advancedFilters, + })); + + // Show advanced panel if we have filter values (not just query/sort) + const hasAdvancedValues = ['isbn', 'author', 'title', 'content'].some( + key => parsedParams.advancedFilters[key as keyof typeof parsedParams.advancedFilters] + ); + if (hasAdvancedValues) { + setShowAdvanced(true); + } + } + + // Build query and trigger search + const mergedFilters = { + ...advancedFilters, + ...parsedParams.advancedFilters, + }; + + const query = buildSearchQuery({ + searchInput: parsedParams.searchInput, + showAdvanced: true, + advancedFilters: mergedFilters as typeof advancedFilters, + bookLanguages, + defaultLanguage: defaultLanguageCodes, + searchMode, + }); + + handleSearch(query, config, searchFieldValues); + } + }, [ + wasProcessed, + parsedParams, + config, + advancedFilters, + searchFieldValues, + handleSearch, + setSearchInput, + setAdvancedFilters, + setShowAdvanced, + ]); + + const handleSettingsSaved = useCallback(() => { + loadConfig('settings-saved'); + }, [loadConfig]); + + // Log WebSocket connection status useEffect(() => { if (isUsingWebSocket) { console.log('✅ Using WebSocket for real-time updates'); @@ -254,67 +330,52 @@ function App() { } }, [isUsingWebSocket]); - // Fetch status immediately on startup + // Fetch status on startup useEffect(() => { fetchStatus(); }, [fetchStatus]); - // Search handler - const handleSearch = async (query: string) => { - if (!query) { - setBooks([]); - setLastSearchQuery(''); - return; - } - setIsSearching(true); - setLastSearchQuery(query); - try { - const results = await searchBooks(query); - setBooks(results); - if (results.length === 0) { - showToast('No results found', 'error'); + // Show book details + const handleShowDetails = async (id: string): Promise => { + const metadataBook = books.find(b => b.id === id && b.provider && b.provider_id); + + if (metadataBook) { + try { + const fullBook = await getMetadataBookInfo(metadataBook.provider!, metadataBook.provider_id!); + setSelectedBook({ + ...metadataBook, + description: fullBook.description || metadataBook.description, + }); + } catch (error) { + console.error('Failed to load book description, using search data:', error); + setSelectedBook(metadataBook); } - } catch (error) { - if (error instanceof AuthenticationError) { - setIsAuthenticated(false); - if (authRequired) { - navigate('/login', { replace: true }); - } - } else { - console.error('Search failed:', error); - setBooks([]); - const message = error instanceof Error ? error.message : 'Search failed'; - const friendly = message.includes("Anna's Archive") || message.includes('Network restricted') - ? message - : "Unable to reach Anna's Archive. Network may be restricted or mirrors blocked."; - showToast(friendly, 'error'); + } else { + try { + const book = await getBookInfo(id); + setSelectedBook(book); + } catch (error) { + console.error('Failed to load book details:', error); + showToast('Failed to load book details', 'error'); } - } finally { - setIsSearching(false); } }; - // Show book details - const handleShowDetails = async (id: string): Promise => { - try { - const book = await getBookInfo(id); - setSelectedBook(book); - } catch (error) { - console.error('Failed to load book details:', error); - showToast('Failed to load book details', 'error'); - } + // Handle "Find Downloads" from DetailsModal + const handleFindDownloads = (book: Book) => { + setSelectedBook(null); + setReleaseBook(book); }; // Download book const handleDownload = async (book: Book): Promise => { try { await downloadBook(book.id); - // Fetch status to update button states (detectChanges will show toast) await fetchStatus(); } catch (error) { console.error('Download failed:', error); showToast('Failed to queue download', 'error'); - throw error; // Re-throw so button components can reset their queuing state + throw error; } }; @@ -338,68 +399,51 @@ function App() { } }; - // Reset search state (clear books and search input) - const handleResetSearch = () => { - setBooks([]); - setSearchInput(''); - setShowAdvanced(false); - setLastSearchQuery(''); - // Use config's supported formats if available, otherwise fall back to default - const resetFormats = config?.supported_formats || DEFAULT_FORMAT_SELECTION; - setAdvancedFilters({ - isbn: '', - author: '', - title: '', - lang: [LANGUAGE_OPTION_DEFAULT], - sort: '', - content: '', - formats: resetFormats, - }); - }; - - const handleSortChange = (value: string) => { - updateAdvancedFilters({ sort: value }); - if (!lastSearchQuery) return; - - const params = new URLSearchParams(lastSearchQuery); - if (value) { - params.set('sort', value); + // Open release modal + const handleGetReleases = async (book: Book) => { + if (book.provider && book.provider_id) { + try { + const fullBook = await getMetadataBookInfo(book.provider, book.provider_id); + setReleaseBook({ + ...book, + description: fullBook.description || book.description, + }); + } catch (error) { + console.error('Failed to load book description, using search data:', error); + setReleaseBook(book); + } } else { - params.delete('sort'); + setReleaseBook(book); } - - const nextQuery = params.toString(); - if (!nextQuery) return; - handleSearch(nextQuery); }; - // Get button state for a book - memoized to ensure proper re-renders when status changes - const getButtonState = useCallback((bookId: string): ButtonStateInfo => { - // Check error first - if (currentStatus.error && currentStatus.error[bookId]) { - return { text: 'Failed', state: 'error' }; + // Handle download from ReleaseModal + const handleReleaseDownload = async (book: Book, release: Release) => { + try { + trackRelease(book.id, release.source_id); + + await downloadRelease({ + source: release.source, + source_id: release.source_id, + title: release.title, + format: release.format, + size: release.size, + size_bytes: release.size_bytes, + download_url: release.download_url, + protocol: release.protocol, + indexer: release.indexer, + seeders: release.seeders, + extra: release.extra, + preview: book.preview, // Pass book cover from metadata + author: book.author, // Pass author from metadata + }); + await fetchStatus(); + } catch (error) { + console.error('Release download failed:', error); + showToast('Failed to queue download', 'error'); + throw error; } - // Check completed - if (currentStatus.complete && currentStatus.complete[bookId]) { - return { text: 'Downloaded', state: 'complete' }; - } - // Check in-progress states - if (currentStatus.downloading && currentStatus.downloading[bookId]) { - const book = currentStatus.downloading[bookId]; - return { - text: 'Downloading', - state: 'downloading', - progress: book.progress - }; - } - if (currentStatus.resolving && currentStatus.resolving[bookId]) { - return { text: 'Resolving', state: 'resolving' }; - } - if (currentStatus.queued && currentStatus.queued[bookId]) { - return { text: 'Queued', state: 'queued' }; - } - return { text: 'Download', state: 'download' }; - }, [currentStatus]); + }; const bookLanguages = config?.book_languages || DEFAULT_LANGUAGES; const supportedFormats = config?.supported_formats || DEFAULT_SUPPORTED_FORMATS; @@ -408,21 +452,30 @@ function App() { ? config.default_language : [bookLanguages[0]?.code || 'en']; + const searchMode = config?.search_mode || 'direct'; + const mainAppContent = ( - <> -
+
setDownloadsSidebarOpen(true)} + onSettingsClick={() => { + if (config?.settings_enabled) { + setSettingsOpen(true); + } else { + setConfigBannerOpen(true); + } + }} statusCounts={statusCounts} - onLogoClick={handleResetSearch} + onLogoClick={() => handleResetSearch(config)} authRequired={authRequired} isAuthenticated={isAuthenticated} - onLogout={handleLogout} + onLogout={handleLogoutWithCleanup} onSearch={() => { const query = buildSearchQuery({ searchInput, @@ -430,15 +483,16 @@ function App() { advancedFilters, bookLanguages, defaultLanguage: defaultLanguageCodes, + searchMode, }); - handleSearch(query); + handleSearch(query, config, searchFieldValues); }} onAdvancedToggle={() => setShowAdvanced(!showAdvanced)} isLoading={isSearching} onShowToast={showToast} onRemoveToast={removeToast} /> - + { + const query = buildSearchQuery({ + searchInput, + showAdvanced, + advancedFilters, + bookLanguages, + defaultLanguage: defaultLanguageCodes, + searchMode, + }); + handleSearch(query, config, searchFieldValues); + }} /> - +
handleSearch(query, config, searchFieldValues)} isLoading={isSearching} isInitialState={isInitialState} bookLanguages={bookLanguages} @@ -461,8 +529,11 @@ function App() { onSearchInputChange={setSearchInput} showAdvanced={showAdvanced} onAdvancedToggle={() => setShowAdvanced(!showAdvanced)} - advancedFilters={advancedFilters} - onAdvancedFiltersChange={updateAdvancedFilters} + advancedFilters={advancedFilters} + onAdvancedFiltersChange={updateAdvancedFilters} + metadataSearchFields={config?.metadata_search_fields} + searchFieldValues={searchFieldValues} + onSearchFieldChange={updateSearchFieldValue} /> handleSortChange(value, config)} + metadataSortOptions={config?.metadata_sort_options} /> {selectedBook && ( @@ -480,20 +554,33 @@ function App() { book={selectedBook} onClose={() => setSelectedBook(null)} onDownload={handleDownload} + onFindDownloads={handleFindDownloads} buttonState={getButtonState(selectedBook.id)} /> )} + {releaseBook && ( + setReleaseBook(null)} + onDownload={handleReleaseDownload} + supportedFormats={supportedFormats} + defaultLanguages={defaultLanguageCodes} + bookLanguages={bookLanguages} + currentStatus={currentStatus} + defaultReleaseSource={config?.default_release_source} + /> + )} +
-
- - {/* Downloads Sidebar */} + setDownloadsSidebarOpen(false)} @@ -503,7 +590,29 @@ function App() { onCancel={handleCancel} activeCount={activeCount} /> - + + setSettingsOpen(false)} + onShowToast={showToast} + onSettingsSaved={handleSettingsSaved} + /> + + {/* Auto-show banner on startup for users without config */} + {config && ( + + )} + + {/* Controlled banner shown when clicking settings without config */} + setConfigBannerOpen(false)} + onContinue={() => { + setConfigBannerOpen(false); + setSettingsOpen(true); + }} + /> + ); const visuallyHiddenStyle: CSSProperties = { diff --git a/src/frontend/src/components/AdvancedFilters.tsx b/src/frontend/src/components/AdvancedFilters.tsx index 76f79133..6862be66 100644 --- a/src/frontend/src/components/AdvancedFilters.tsx +++ b/src/frontend/src/components/AdvancedFilters.tsx @@ -1,9 +1,11 @@ -import { ReactNode } from 'react'; -import { AdvancedFilterState, Language } from '../types'; +import { ReactNode, KeyboardEvent } from 'react'; +import { AdvancedFilterState, Language, MetadataSearchField } from '../types'; import { normalizeLanguageSelection } from '../utils/languageFilters'; +import { useSearchMode } from '../contexts/SearchModeContext'; import { LanguageMultiSelect } from './LanguageMultiSelect'; import { DropdownList } from './DropdownList'; import { CONTENT_OPTIONS } from '../data/filterOptions'; +import { SearchFieldRenderer } from './shared'; const FORMAT_TYPES = ['pdf', 'epub', 'mobi', 'azw3', 'fb2', 'djvu', 'cbz', 'cbr'] as const; @@ -16,6 +18,12 @@ interface AdvancedFiltersProps { onFiltersChange: (updates: Partial) => void; formClassName?: string; renderWrapper?: (form: ReactNode) => ReactNode; + // Universal mode props + metadataSearchFields?: MetadataSearchField[]; + searchFieldValues?: Record; + onSearchFieldChange?: (key: string, value: string | number | boolean) => void; + // Submit handler for Enter key + onSubmit?: () => void; } export const AdvancedFilters = ({ @@ -27,9 +35,21 @@ export const AdvancedFilters = ({ onFiltersChange, formClassName, renderWrapper, + metadataSearchFields = [], + searchFieldValues = {}, + onSearchFieldChange, + onSubmit, }: AdvancedFiltersProps) => { + const { searchMode } = useSearchMode(); const { isbn, author, title, lang, content, formats } = filters; + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Enter' && onSubmit) { + e.preventDefault(); + onSubmit(); + } + }; + const handleLangChange = (next: string[]) => { const normalized = normalizeLanguageSelection(next); onFiltersChange({ lang: normalized }); @@ -53,6 +73,52 @@ export const AdvancedFilters = ({ if (!visible) return null; + // Universal search mode: render dynamic provider fields + if (searchMode === 'universal') { + // If no fields defined for this provider, don't show the section + if (metadataSearchFields.length === 0) return null; + + const universalForm = ( +
+ {metadataSearchFields.map((field) => ( +
+ {field.type !== 'CheckboxSearchField' && ( + + )} + onSearchFieldChange?.(field.key, value)} + onSubmit={onSubmit} + /> + {field.description && ( +

{field.description}

+ )} +
+ ))} +
+ ); + + const wrappedUniversalForm = renderWrapper ? ( + renderWrapper(universalForm) + ) : ( +
+
{universalForm}
+
+ ); + + return wrappedUniversalForm; + } + + // Direct download mode: render existing hardcoded filters const form = (
{ onFiltersChange({ isbn: e.target.value }); }} + onKeyDown={handleKeyDown} />
@@ -91,6 +159,7 @@ export const AdvancedFilters = ({ type="text" placeholder="Author" autoComplete="off" + enterKeyHint="search" className="w-full px-3 py-2 rounded-md border" style={{ background: 'var(--bg-soft)', @@ -101,6 +170,7 @@ export const AdvancedFilters = ({ onChange={e => { onFiltersChange({ author: e.target.value }); }} + onKeyDown={handleKeyDown} />
@@ -112,6 +182,7 @@ export const AdvancedFilters = ({ type="text" placeholder="Title" autoComplete="off" + enterKeyHint="search" className="w-full px-3 py-2 rounded-md border" style={{ background: 'var(--bg-soft)', @@ -122,6 +193,7 @@ export const AdvancedFilters = ({ onChange={e => { onFiltersChange({ title: e.target.value }); }} + onKeyDown={handleKeyDown} />
Promise; + onGetReleases: (book: Book) => void; + isLoadingReleases?: boolean; + size?: ButtonSize; + variant?: ButtonVariant; + fullWidth?: boolean; + className?: string; + style?: CSSProperties; +} + +export function BookActionButton({ + book, + buttonState, + onDownload, + onGetReleases, + isLoadingReleases, + size, + variant = 'default', + fullWidth, + className, + style, +}: BookActionButtonProps) { + const { searchMode } = useSearchMode(); + + if (searchMode === 'universal') { + return ( + + ); + } + + return ( + onDownload(book)} + size={size} + variant={variant === 'default' ? 'primary' : 'icon'} + fullWidth={fullWidth} + className={className} + style={style} + ariaLabel={buttonState.text} + /> + ); +} diff --git a/src/frontend/src/components/BookDownloadButton.tsx b/src/frontend/src/components/BookDownloadButton.tsx index ed0af50f..de88c080 100644 --- a/src/frontend/src/components/BookDownloadButton.tsx +++ b/src/frontend/src/components/BookDownloadButton.tsx @@ -1,45 +1,6 @@ import { useEffect, useState, CSSProperties } from 'react'; import { ButtonStateInfo } from '../types'; - -interface CircularProgressProps { - progress?: number; - size?: number; - className?: string; -} - -const CircularProgress = ({ progress, size = 16, className }: CircularProgressProps) => { - const radius = (size - 2) / 2; - const circumference = 2 * Math.PI * radius; - const progressValue = progress ?? 0; - const strokeDashoffset = circumference - (progressValue / 100) * circumference; - const svgClassName = className ? `transform -rotate-90 ${className}` : 'transform -rotate-90'; - - return ( - - - - - ); -}; +import { CircularProgress } from './shared'; type ButtonSize = 'sm' | 'md'; type ButtonVariant = 'primary' | 'icon'; @@ -62,8 +23,8 @@ const sizeClasses: Record = { }; const iconVariantSizeClasses: Record = { - sm: 'p-1 sm:p-1.5', - md: 'p-1.5 sm:p-2', + sm: 'p-px m-0.5 sm:p-1 sm:m-0.5 aspect-square', + md: 'p-0.5 m-0.5 sm:p-1.5 sm:m-0.5 aspect-square', }; const primaryIconSizes: Record = { @@ -72,13 +33,13 @@ const primaryIconSizes: Record = { }; const iconVariantIconSizes: Record = { - sm: { mobile: 'w-3.5 h-3.5', desktop: 'w-4 h-4' }, - md: { mobile: 'w-4 h-4', desktop: 'w-5 h-5' }, + sm: { mobile: 'w-5 h-5', desktop: 'w-5 h-5' }, + md: { mobile: 'w-6 h-6', desktop: 'w-6 h-6' }, }; const iconVariantProgressSizes: Record = { - sm: { mobile: 14, desktop: 16 }, - md: { mobile: 16, desktop: 20 }, + sm: { mobile: 20, desktop: 20 }, + md: { mobile: 24, desktop: 24 }, }; export const BookDownloadButton = ({ @@ -190,26 +151,19 @@ export const BookDownloadButton = ({ if (showCircularProgress) { if (variant === 'icon') { - const sizes = iconVariantProgressSizes[size]; - return ( - <> - - - - ); + const progressSize = iconVariantProgressSizes[size].mobile; + return ; } return ; } if (showSpinner) { - const spinnerClass = - variant === 'icon' - ? size === 'sm' - ? 'w-3.5 h-3.5 sm:w-4 h-4' - : 'w-4 h-4 sm:w-5 h-5' - : size === 'sm' - ? 'w-3 h-3' - : 'w-4 h-4'; + if (variant === 'icon' && iconSizes) { + return ( +
+ ); + } + const spinnerClass = size === 'sm' ? 'w-3 h-3' : 'w-4 h-4'; return
; } diff --git a/src/frontend/src/components/BookGetButton.tsx b/src/frontend/src/components/BookGetButton.tsx new file mode 100644 index 00000000..c40f4244 --- /dev/null +++ b/src/frontend/src/components/BookGetButton.tsx @@ -0,0 +1,178 @@ +import { CSSProperties } from 'react'; +import { Book, ButtonStateInfo } from '../types'; +import { CircularProgress } from './shared'; + +type ButtonSize = 'sm' | 'md'; +type ButtonVariant = 'default' | 'icon'; + +interface BookGetButtonProps { + book: Book; + onGetReleases: (book: Book) => void; + buttonState?: ButtonStateInfo; + isLoading?: boolean; + size?: ButtonSize; + variant?: ButtonVariant; + fullWidth?: boolean; + className?: string; + style?: CSSProperties; +} + +const sizeClasses: Record = { + sm: 'px-2.5 py-1.5 text-xs', + md: 'px-4 py-2.5 text-sm', +}; + +const iconSizeClasses: Record = { + sm: 'p-1.5', + md: 'p-1.5 sm:p-2', +}; + +const iconSizes: Record = { + sm: 'w-3.5 h-3.5', + md: 'w-4 h-4', +}; + +const iconOnlySizes: Record = { + sm: 'w-4 h-4', + md: 'w-4 h-4 sm:w-5 sm:h-5', +}; + +export const BookGetButton = ({ + book, + onGetReleases, + buttonState, + isLoading = false, + size = 'md', + variant = 'default', + fullWidth = false, + className = '', + style, +}: BookGetButtonProps) => { + const isIconVariant = variant === 'icon'; + const widthClasses = fullWidth ? 'w-full' : ''; + const sizeClass = isIconVariant ? iconSizeClasses[size] : sizeClasses[size]; + const iconSize = isIconVariant ? iconOnlySizes[size] : iconSizes[size]; + + // Determine states based on buttonState + const isCompleted = buttonState?.state === 'complete'; + const hasError = buttonState?.state === 'error'; + const isInProgress = buttonState && ['queued', 'resolving', 'downloading'].includes(buttonState.state); + const showCircularProgress = buttonState?.state === 'downloading' && buttonState.progress !== undefined; + const showSpinner = (isInProgress && !showCircularProgress) || isLoading; + + // Disable button while loading metadata + const isDisabled = isLoading; + + // Determine button styling based on state + const getButtonClasses = () => { + if (isCompleted) { + return isIconVariant + ? 'bg-green-600 text-white' + : 'bg-green-600 hover:bg-green-700'; + } + if (hasError) { + return isIconVariant + ? 'bg-red-600 text-white opacity-75' + : 'bg-red-600 hover:bg-red-700'; + } + if (isLoading) { + // Show loading state (fetching metadata) + return isIconVariant + ? 'text-gray-400 dark:text-gray-500' + : 'bg-emerald-600/70'; + } + if (isInProgress) { + // Show progress state but keep it clickable + return isIconVariant + ? 'bg-sky-600 text-white' + : 'bg-sky-600 hover:bg-sky-700'; + } + // Default state - icon variant has no background + return isIconVariant + ? 'text-gray-600 dark:text-gray-200 hover-action' + : 'bg-emerald-600 hover:bg-emerald-700'; + }; + + const handleClick = () => { + if (isDisabled) return; + onGetReleases(book); + }; + + // Determine display text + const getDisplayText = () => { + if (isCompleted) return 'Downloaded'; + if (hasError) return 'Failed'; + if (isLoading) return 'Loading'; + if (buttonState?.state === 'downloading') return 'Downloading'; + if (buttonState?.state === 'resolving') return 'Resolving'; + if (buttonState?.state === 'queued') return 'Queued'; + return 'Get'; + }; + + // Render appropriate icon based on state + const renderIcon = () => { + if (isCompleted) { + return ( + + + + ); + } + + if (hasError) { + return ( + + + + ); + } + + if (showCircularProgress) { + const progressSize = isIconVariant ? (size === 'sm' ? 16 : 20) : (size === 'sm' ? 12 : 16); + return ; + } + + if (showSpinner) { + return ( +
+ ); + } + + // Default "+" icon for Get action + return ( + + + + ); + }; + + // Icon variant renders as a circular button without text + if (isIconVariant) { + return ( + + ); + } + + return ( + + ); +}; diff --git a/src/frontend/src/components/ConfigSetupBanner.tsx b/src/frontend/src/components/ConfigSetupBanner.tsx new file mode 100644 index 00000000..ce97d3e9 --- /dev/null +++ b/src/frontend/src/components/ConfigSetupBanner.tsx @@ -0,0 +1,176 @@ +import { useState, useEffect, useCallback } from 'react'; + +const STORAGE_KEY = 'cwa-config-banner-dismissed'; + +interface ConfigSetupBannerProps { + /** Whether to show the banner (controlled mode) */ + isOpen?: boolean; + /** Called when banner is closed */ + onClose?: () => void; + /** Called when "Continue to Settings" is clicked (only shown if provided) */ + onContinue?: () => void; + /** Auto-show mode: show banner if settings not enabled and not dismissed */ + settingsEnabled?: boolean; +} + +export const ConfigSetupBanner = ({ + isOpen: controlledOpen, + onClose, + onContinue, + settingsEnabled, +}: ConfigSetupBannerProps) => { + const [autoShowVisible, setAutoShowVisible] = useState(false); + const [isClosing, setIsClosing] = useState(false); + + // Auto-show mode: check localStorage on mount + useEffect(() => { + if (settingsEnabled !== undefined) { + const dismissed = localStorage.getItem(STORAGE_KEY); + setAutoShowVisible(!settingsEnabled && dismissed !== 'true'); + } + }, [settingsEnabled]); + + // Determine if we should show based on controlled or auto-show mode + const isControlledMode = controlledOpen !== undefined; + const isVisible = isControlledMode ? controlledOpen : autoShowVisible; + + const handleClose = useCallback(() => { + setIsClosing(true); + setTimeout(() => { + setIsClosing(false); + if (isControlledMode) { + onClose?.(); + } else { + // Auto-show mode: save to localStorage + localStorage.setItem(STORAGE_KEY, 'true'); + setAutoShowVisible(false); + } + }, 150); + }, [isControlledMode, onClose]); + + const handleContinue = useCallback(() => { + setIsClosing(true); + setTimeout(() => { + setIsClosing(false); + onContinue?.(); + }, 150); + }, [onContinue]); + + if (!isVisible && !isClosing) return null; + + // Determine which mode we're in for the footer buttons + const showContinueButton = !!onContinue; + + return ( +
+ {/* Backdrop */} +
+ + {/* Modal */} +
+ {/* Header */} +
+

+ {showContinueButton ? 'Config Volume Required' : 'New Feature: Settings Page'} +

+ +
+ + {/* Content */} +
+

+ {showContinueButton + ? 'To save settings, add a config volume to your Docker Compose file:' + : 'CWA Book Downloader now has a settings page! To enable it, add a config volume to your Docker Compose file:'} +

+ + {/* Code snippet */} +
+
+ docker-compose.yml +
+ 
+              
+                services:{'\n'}
+                {'  '}cwa-book-downloader:{'\n'}
+                {'    '}volumes:{'\n'}
+                {'      '}- /path/to/config:/config
+              
+
+
+ +

+ {showContinueButton + ? 'Without this volume, settings changes will not persist across container restarts.' + : 'This allows you to configure settings through the UI and persist them across container restarts.'} +

+
+ + {/* Footer */} +
+ {showContinueButton ? ( + <> + + + + ) : ( + + )} +
+
+
+ ); +}; diff --git a/src/frontend/src/components/DetailsModal.tsx b/src/frontend/src/components/DetailsModal.tsx index 59c8148d..cac914e6 100644 --- a/src/frontend/src/components/DetailsModal.tsx +++ b/src/frontend/src/components/DetailsModal.tsx @@ -1,47 +1,60 @@ -import { useState, useEffect } from 'react'; -import { Book, ButtonStateInfo } from '../types'; +import { useState, useEffect, useCallback } from 'react'; +import { Book, ButtonStateInfo, isMetadataBook } from '../types'; import { BookDownloadButton } from './BookDownloadButton'; interface DetailsModalProps { book: Book | null; onClose: () => void; onDownload: (book: Book) => Promise; + onFindDownloads?: (book: Book) => void; // For Universal mode buttonState: ButtonStateInfo; } -export const DetailsModal = ({ book, onClose, onDownload, buttonState }: DetailsModalProps) => { +export const DetailsModal = ({ book, onClose, onDownload, onFindDownloads, buttonState }: DetailsModalProps) => { const [isQueuing, setIsQueuing] = useState(false); + const [isClosing, setIsClosing] = useState(false); + + const handleClose = useCallback(() => { + setIsClosing(true); + setTimeout(() => { + onClose(); + setIsClosing(false); + }, 150); + }, [onClose]); // Clear queuing state and close modal once button state changes from download useEffect(() => { if (isQueuing && buttonState.state !== 'download') { setIsQueuing(false); // Close modal after status has updated - const timer = setTimeout(onClose, 500); + const timer = setTimeout(handleClose, 500); return () => clearTimeout(timer); } - }, [buttonState.state, isQueuing, onClose]); + }, [buttonState.state, isQueuing, handleClose]); // Handle ESC key to close modal useEffect(() => { const handleEscape = (e: KeyboardEvent) => { if (e.key === 'Escape') { - onClose(); + handleClose(); } }; document.addEventListener('keydown', handleEscape); return () => document.removeEventListener('keydown', handleEscape); - }, [onClose]); + }, [handleClose]); useEffect(() => { - const previousOverflow = document.body.style.overflow; - document.body.style.overflow = 'hidden'; - return () => { - document.body.style.overflow = previousOverflow; - }; - }, []); + if (book) { + const previousOverflow = document.body.style.overflow; + document.body.style.overflow = 'hidden'; + return () => { + document.body.style.overflow = previousOverflow; + }; + } + }, [book]); + if (!book && !isClosing) return null; if (!book) return null; const titleId = `book-details-title-${book.id}`; @@ -54,17 +67,41 @@ export const DetailsModal = ({ book, onClose, onDownload, buttonState }: Details } catch (error) { setIsQueuing(false); // Close on error - setTimeout(onClose, 300); + setTimeout(handleClose, 300); } }; + // Determine if this is a metadata book (Universal mode) vs a release (Direct Download) + const isMetadata = isMetadataBook(book); + const publisherInfo = { label: 'Publisher', value: book.publisher || '-' }; - const metadata = [ - { label: 'Year', value: book.year || '-' }, - { label: 'Language', value: book.language || '-' }, - { label: 'Format', value: book.format || '-' }, - { label: 'Size', value: book.size || '-' }, - ]; + + // Build metadata grid based on mode + // Universal mode: Year, Genres (no language, no publisher - often blank from providers) + // Direct Download mode: Year, Language, Format, Size + const metadata = isMetadata + ? [ + { label: 'Year', value: book.year || '-' }, + ...(book.genres && book.genres.length > 0 + ? [{ label: 'Genres', value: book.genres.slice(0, 3).join(', ') }] + : []), + ] + : [ + { label: 'Year', value: book.year || '-' }, + { label: 'Language', value: book.language || '-' }, + { label: 'Format', value: book.format || '-' }, + { label: 'Size', value: book.size || '-' }, + ]; + + // Extract rating and readers from display_fields for dedicated boxes (Universal mode) + const ratingField = isMetadata && book.display_fields?.find(f => f.icon === 'star'); + const readersField = isMetadata && book.display_fields?.find(f => f.icon === 'users'); + // Other display fields (pages, editions, etc.) shown inline + const otherDisplayFields = isMetadata && book.display_fields?.filter(f => f.icon !== 'star' && f.icon !== 'users'); + + // Use provider display name from backend, fall back to capitalized provider name + const providerDisplay = book.provider_display_name + || (book.provider ? book.provider.charAt(0).toUpperCase() + book.provider.slice(1) : ''); const artworkMaxHeight = 'calc(90vh - 220px)'; const artworkMaxWidth = 'min(45vw, 520px, calc((90vh - 220px) / 1.6))'; const additionalInfo = @@ -84,11 +121,11 @@ export const DetailsModal = ({ book, onClose, onDownload, buttonState }: Details
{ - if (e.target === e.currentTarget) onClose(); + if (e.target === e.currentTarget) handleClose(); }} >
)} -
+ {/* Metadata grid - adapts columns based on mode and available data */} +
{metadata.map(item => (

{item.label}

{item.value}

))} + + {/* Rating box - Universal mode only */} + {ratingField && ( +
+

{ratingField.label}

+

+ + + + {ratingField.value} +

+
+ )} + + {/* Readers box - Universal mode only */} + {readersField && ( +
+

{readersField.label}

+

+ + + + {readersField.value} +

+
+ )}
- {extendedInfoEntries.length > 0 && ( + {/* Other display fields (pages, editions) - Universal mode only */} + {otherDisplayFields && otherDisplayFields.length > 0 && ( +
+ {otherDisplayFields.map(field => ( + + {field.icon === 'book' && ( + + + + )} + {field.icon === 'editions' && ( + + + + )} + {field.label}: + {field.value} + + ))} +
+ )} + + {/* ISBN - Universal mode only */} + {isMetadata && (book.isbn_13 || book.isbn_10) && ( +
+

ISBN

+

{book.isbn_13 || book.isbn_10}

+
+ )} + + {/* Extended info (publisher, etc.) - Direct Download mode only */} + {!isMetadata && extendedInfoEntries.length > 0 && (
    {extendedInfoEntries.map(([key, value]) => ( @@ -181,15 +276,44 @@ export const DetailsModal = ({ book, onClose, onDownload, buttonState }: Details
-
- +
+ {/* Source link - Universal mode only */} + {isMetadata && book.source_url ? ( + + View on {providerDisplay} + + + + + ) : ( +
+ )} + {isMetadata ? ( + + ) : ( + + )}
diff --git a/src/frontend/src/components/DownloadsSidebar.tsx b/src/frontend/src/components/DownloadsSidebar.tsx index 1a501e0e..7c6d4272 100644 --- a/src/frontend/src/components/DownloadsSidebar.tsx +++ b/src/frontend/src/components/DownloadsSidebar.tsx @@ -33,9 +33,36 @@ if (!document.head.querySelector('style[data-wave-animation]')) { document.head.appendChild(styleSheet); } -// Helper to get book preview image -const getBookPreview = (book: Book): string => { - return book.preview || '/placeholder-book.png'; +// Book thumbnail component with fallback +const BookThumbnail = ({ preview, title }: { preview?: string; title?: string }) => { + if (!preview) { + return ( +
+ No Cover +
+ ); + } + + return ( + {title { + // Replace with placeholder on error + const target = e.target as HTMLImageElement; + const placeholder = document.createElement('div'); + placeholder.className = 'w-16 h-24 rounded-tl bg-gray-200 dark:bg-gray-700 flex items-center justify-center text-[8px] font-medium text-gray-500 dark:text-gray-400'; + placeholder.style.aspectRatio = '2/3'; + placeholder.textContent = 'No Cover'; + target.replaceWith(placeholder); + }} + /> + ); }; // Helper to get progress percentage based on status @@ -126,19 +153,14 @@ export const DownloadsSidebar = ({ const progress = getStatusProgress(statusName, book.progress); const progressBarColor = getProgressBarColor(statusName); - // Format progress text - use status_message if available, otherwise fall back to label + // Format progress text - use status_message from backend if available let progressText = book.status_message || statusStyle.label; - if (statusName === 'downloading' && book.progress && book.size) { + if (statusName === 'downloading' && !book.status_message && book.progress && book.size) { + // Fallback: calculate size progress only if backend didn't provide a message const sizeValue = parseFloat(book.size.replace(/[^\d.]/g, '')); - const sizeUnit = book.size.replace(/[\d.\s]/g, ''); // Extract unit as-is from backend + const sizeUnit = book.size.replace(/[\d.\s]/g, ''); const downloadedSize = (book.progress / 100) * sizeValue; - const sizeProgress = `${downloadedSize.toFixed(1)}${sizeUnit} / ${book.size}`; - // If there's attempt info in the status message, prepend it to the progress - if (book.status_message?.startsWith('Attempt')) { - progressText = `${book.status_message} - ${sizeProgress}`; - } else { - progressText = sizeProgress; - } + progressText = `${downloadedSize.toFixed(1)}${sizeUnit} / ${book.size}`; } else if (isCompleted) { progressText = 'Complete'; } else if (hasError) { @@ -171,22 +193,13 @@ export const DownloadsSidebar = ({
{/* Book Thumbnail - left side */}
- {book.title { - const target = e.target as HTMLImageElement; - target.src = '/placeholder-book.png'; - }} - /> +
{/* Book Info - right side */} -
+
{/* Title & Author - with safe area for cancel/clear button */} - - {/* Progress Bar - absolute positioned at bottom - always visible */} -
- {/* ml-16 clears the 64px thumbnail, gap-2 adds spacing */} -
- - {/* Wave animation overlay for in-progress states */} - {isInProgress && statusStyle.waveColor && ( - - )} - {progressText} - -
-
-
- {/* Animated wave effect for in-progress states */} - {isInProgress && progress < 100 && ( -
- )} -
+ {/* Progress Bar - at bottom */} +
+
+ {/* Animated wave effect for in-progress states */} + {isInProgress && progress < 100 && ( +
+ )}
diff --git a/src/frontend/src/components/Dropdown.tsx b/src/frontend/src/components/Dropdown.tsx index c4930c28..1cb3919c 100644 --- a/src/frontend/src/components/Dropdown.tsx +++ b/src/frontend/src/components/Dropdown.tsx @@ -100,14 +100,14 @@ export const Dropdown = ({ type="button" onClick={toggleOpen} disabled={disabled} - className={`w-full px-3 py-2 rounded-md border flex items-center justify-between text-left text-base focus:outline-none focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 ${buttonClassName}`} + className={`w-full px-3 py-2 rounded-md border flex items-center justify-between text-left focus:outline-none focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 ${buttonClassName}`} style={{ background: 'var(--bg-soft)', color: 'var(--text)', borderColor: 'var(--border-muted)', }} > - + {summary ?? Select an option} {placeholder}; + // For single select with empty string value, find and show the empty value option label + if (!multiple) { + const emptyOption = options.find(opt => opt.value === ''); + if (emptyOption) { + return emptyOption.label; + } + } + return {placeholder}; } if (!multiple) { @@ -102,7 +109,7 @@ export const DropdownList = ({ - Report a Bug + {/* Settings Button */} + {onSettingsClick && ( + + )} + {/* Debug Buttons */} {debug && ( <> @@ -318,16 +321,16 @@ export const Header = ({ method: 'GET', credentials: 'include', }); - + // Remove the loading toast if (loadingToastId) onRemoveToast?.(loadingToastId); - + if (!response.ok) { const errorData = await response.json().catch(() => ({})); onShowToast?.(`Debug download failed: ${errorData.error || response.statusText}`, 'error'); return; } - + // Get the filename from Content-Disposition header or use default const contentDisposition = response.headers.get('Content-Disposition'); let filename = 'debug.zip'; @@ -337,7 +340,7 @@ export const Header = ({ filename = filenameMatch[1].replace(/['"]/g, ''); } } - + // Create blob and trigger download const blob = await response.blob(); const url = window.URL.createObjectURL(blob); @@ -348,7 +351,7 @@ export const Header = ({ a.click(); window.URL.revokeObjectURL(url); a.remove(); - + onShowToast?.('Debug logs downloaded successfully', 'success'); } catch (error) { // Remove the loading toast on error too @@ -412,14 +415,14 @@ export const Header = ({
{/* Logo - visible on mobile only, aligned left */} {logoUrl && ( - Logo )} - +
@@ -427,14 +430,15 @@ export const Header = ({
{/* Logo - visible on desktop only, aligned with search */} {logoUrl && ( - Logo )}
); -}; +}); diff --git a/src/frontend/src/components/ReleaseCell.tsx b/src/frontend/src/components/ReleaseCell.tsx new file mode 100644 index 00000000..a41e4d1b --- /dev/null +++ b/src/frontend/src/components/ReleaseCell.tsx @@ -0,0 +1,167 @@ +import { ColumnSchema, ColumnColorHint, Release } from '../types'; +import { getFormatColor, getLanguageColor, getDownloadTypeColor, ColorStyle } from '../utils/colorMaps'; + +interface ReleaseCellProps { + column: ColumnSchema; + release: Release; + compact?: boolean; // When true, renders badges as plain text (for mobile info lines) +} + +/** + * Get a nested value from an object using dot-notation path. + * e.g., getNestedValue(obj, "extra.language") returns obj.extra.language + */ +const getNestedValue = (obj: Record, path: string): unknown => { + return path.split('.').reduce((current, key) => { + if (current && typeof current === 'object') { + return (current as Record)[key]; + } + return undefined; + }, obj as unknown); +}; + +const DEFAULT_COLOR_STYLE: ColorStyle = { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300' }; + +/** + * Get the color style for a value based on the color hint. + */ +const getColorStyle = (value: string, colorHint?: ColumnColorHint | null): ColorStyle => { + if (!colorHint) return DEFAULT_COLOR_STYLE; + + if (colorHint.type === 'static') { + // For static hints, assume it's a bg class and pair with default text + return { bg: colorHint.value, text: 'text-gray-700 dark:text-gray-300' }; + } + + if (colorHint.type === 'map') { + switch (colorHint.value) { + case 'format': + return getFormatColor(value); + case 'language': + return getLanguageColor(value); + case 'download_type': + return getDownloadTypeColor(value); + default: + return DEFAULT_COLOR_STYLE; + } + } + + return DEFAULT_COLOR_STYLE; +}; + +/** + * Generic cell renderer for release list columns. + * Renders different column types (text, badge, size, number, seeders) based on schema. + * When compact=true, badges render as plain text for use in mobile info lines. + */ +export const ReleaseCell = ({ column, release, compact = false }: ReleaseCellProps) => { + const rawValue = getNestedValue(release as unknown as Record, column.key); + const value = rawValue !== undefined && rawValue !== null + ? String(rawValue) + : column.fallback; + + const displayValue = column.uppercase ? value.toUpperCase() : value; + + // Alignment classes + const alignClass = { + left: 'text-left justify-start', + center: 'text-center justify-center', + right: 'text-right justify-end', + }[column.align]; + + // Render based on type + switch (column.render_type) { + case 'badge': { + // Compact mode: render as plain text (for mobile info lines) + if (compact) { + return {displayValue}; + } + const colorStyle = getColorStyle(value, column.color_hint); + return ( +
+ {value !== column.fallback ? ( + + {displayValue} + + ) : ( + {column.fallback} + )} +
+ ); + } + + case 'size': + if (compact) { + return {displayValue}; + } + return ( +
+ {displayValue} +
+ ); + + case 'peers': { + // Peers display: "S/L" string with badge colored by seeder count + // Color logic: 0 = red, 1-10 = yellow, 10+ = blue + const seeders = release.seeders; + const peersValue = value || column.fallback; + const isFallback = seeders == null || peersValue === column.fallback; + + // If no data, show plain text like badge type does + if (isFallback) { + if (compact) { + return {column.fallback}; + } + return ( +
+ {column.fallback} +
+ ); + } + + // Determine color based on seeder count + let badgeColors: string; + if (seeders >= 10) { + badgeColors = 'bg-blue-500/20 text-blue-700 dark:text-blue-300'; + } else if (seeders >= 1) { + badgeColors = 'bg-yellow-500/20 text-yellow-700 dark:text-yellow-300'; + } else { + badgeColors = 'bg-red-500/20 text-red-700 dark:text-red-300'; + } + + if (compact) { + return {peersValue}; + } + return ( +
+ + {peersValue} + +
+ ); + } + + case 'number': + if (compact) { + return {displayValue}; + } + return ( +
+ {displayValue} +
+ ); + + case 'text': + default: + if (compact) { + return {displayValue}; + } + return ( +
+ {displayValue} +
+ ); + } +}; + +export default ReleaseCell; diff --git a/src/frontend/src/components/ReleaseModal.tsx b/src/frontend/src/components/ReleaseModal.tsx new file mode 100644 index 00000000..09ea6662 --- /dev/null +++ b/src/frontend/src/components/ReleaseModal.tsx @@ -0,0 +1,1212 @@ +import { useEffect, useState, useCallback, useMemo, useRef } from 'react'; +import { Book, Release, ReleaseSource, ReleasesResponse, Language, StatusData, ButtonStateInfo, ColumnSchema, ReleaseColumnConfig, LeadingCellConfig, ColumnColorHint } from '../types'; +import { getReleases, getReleaseSources } from '../services/api'; +import { Dropdown } from './Dropdown'; +import { BookDownloadButton } from './BookDownloadButton'; +import { ReleaseCell } from './ReleaseCell'; +import { getFormatColor, getLanguageColor, getDownloadTypeColor, ColorStyle } from '../utils/colorMaps'; + +// Module-level cache for release search results +// Key format: `${provider}:${provider_id}:${source}` +// This persists across modal open/close cycles +const releaseCache = new Map(); + +const getCacheKey = (provider: string, providerId: string, source: string): string => + `${provider}:${providerId}:${source}`; + +// Optional: Clear cache entries older than 5 minutes to prevent stale data +const CACHE_TTL_MS = 5 * 60 * 1000; +const cacheTimestamps = new Map(); + +const getCachedReleases = (provider: string, providerId: string, source: string): ReleasesResponse | null => { + const key = getCacheKey(provider, providerId, source); + const timestamp = cacheTimestamps.get(key); + + // Check if cache entry exists and is not expired + if (timestamp && Date.now() - timestamp < CACHE_TTL_MS) { + return releaseCache.get(key) || null; + } + + // Clear expired entry + if (timestamp) { + releaseCache.delete(key); + cacheTimestamps.delete(key); + } + + return null; +}; + +const setCachedReleases = (provider: string, providerId: string, source: string, data: ReleasesResponse): void => { + const key = getCacheKey(provider, providerId, source); + releaseCache.set(key, data); + cacheTimestamps.set(key, Date.now()); +}; + +// Default column configuration (fallback when backend doesn't provide one) +const DEFAULT_COLUMN_CONFIG: ReleaseColumnConfig = { + columns: [ + { + key: 'extra.language', + label: 'Language', + render_type: 'badge', + align: 'center', + width: '60px', + hide_mobile: false, // Language shown on mobile + color_hint: { type: 'map', value: 'language' }, + fallback: '-', + uppercase: true, + }, + { + key: 'format', + label: 'Format', + render_type: 'badge', + align: 'center', + width: '80px', + hide_mobile: false, // Format shown on mobile + color_hint: { type: 'map', value: 'format' }, + fallback: '-', + uppercase: true, + }, + { + key: 'size', + label: 'Size', + render_type: 'size', + align: 'center', + width: '80px', + hide_mobile: false, // Size shown on mobile + fallback: '-', + uppercase: false, + }, + ], + grid_template: 'minmax(0,2fr) 60px 80px 80px', +}; + +interface ReleaseModalProps { + book: Book | null; + onClose: () => void; + onDownload: (book: Book, release: Release) => Promise; + supportedFormats: string[]; + defaultLanguages: string[]; + bookLanguages: Language[]; + currentStatus: StatusData; + defaultReleaseSource?: string; // Default tab to show (e.g., 'direct_download') +} + +// Color map lookup for dynamic color hints +const COLOR_MAP_HANDLERS: Record ColorStyle> = { + format: getFormatColor, + language: getLanguageColor, + download_type: getDownloadTypeColor, +}; + +const DEFAULT_COLOR_STYLE: ColorStyle = { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300' }; + +// Helper to get color style based on color hint +function getLeadingCellColor(value: string, colorHint?: ColumnColorHint): ColorStyle { + if (!colorHint) return DEFAULT_COLOR_STYLE; + + if (colorHint.type === 'map') { + const handler = COLOR_MAP_HANDLERS[colorHint.value]; + return handler ? handler(value) : DEFAULT_COLOR_STYLE; + } + + // Static color hint - assume it's a bg class + return { bg: colorHint.value, text: 'text-gray-700 dark:text-gray-300' }; +} + +// Helper to get nested value from object using dot notation path +function getNestedValue(obj: Record, path: string): unknown { + return path.split('.').reduce((current, key) => { + if (current && typeof current === 'object') { + return (current as Record)[key]; + } + return undefined; + }, obj as unknown); +} + +// Thumbnail component for release rows +const ReleaseThumbnail = ({ preview, title }: { preview?: string; title?: string }) => { + const [imageLoaded, setImageLoaded] = useState(false); + const [imageError, setImageError] = useState(false); + + if (!preview || imageError) { + return ( +
+ No Cover +
+ ); + } + + return ( +
+ {!imageLoaded && ( +
+ )} + {title setImageLoaded(true)} + onError={() => setImageError(true)} + style={{ opacity: imageLoaded ? 1 : 0, transition: 'opacity 0.2s ease-in-out' }} + /> +
+ ); +}; + +// Leading cell component - shows thumbnail, badge, or nothing based on config +const LeadingCell = ({ + config, + release +}: { + config?: LeadingCellConfig; + release: Release; +}) => { + // Default to thumbnail mode if no config + const cellType = config?.type || 'thumbnail'; + + if (cellType === 'none') { + return null; + } + + if (cellType === 'thumbnail') { + const key = config?.key || 'extra.preview'; + const preview = getNestedValue(release as unknown as Record, key) as string | undefined; + return ; + } + + // Badge type + if (cellType === 'badge' && config?.key) { + const value = getNestedValue(release as unknown as Record, config.key); + const displayValue = value ? String(value) : ''; + const colorStyle = getLeadingCellColor(displayValue, config.color_hint); + const text = config.uppercase ? displayValue.toUpperCase() : displayValue; + + return ( +
+ + {text} + +
+ ); + } + + // Fallback + return ; +}; + +// Release row component with dynamic columns +const ReleaseRow = ({ + release, + index, + onDownload, + buttonState, + columns, + gridTemplate, + leadingCell, +}: { + release: Release; + index: number; + onDownload: () => Promise; + buttonState: ButtonStateInfo; + columns: ColumnSchema[]; + gridTemplate: string; + leadingCell?: LeadingCellConfig; +}) => { + const author = release.extra?.author as string | undefined; + + // Filter columns visible on mobile + const mobileColumns = columns.filter((c) => !c.hide_mobile); + + // Determine if leading cell should be shown + // Default to showing thumbnail if no config provided, hide only if explicitly set to 'none' + const showLeadingCell = leadingCell?.type !== 'none'; + + // Build grid template based on whether leading cell is shown + const desktopGridTemplate = showLeadingCell + ? `auto ${gridTemplate} auto` + : `${gridTemplate} auto`; + + const mobileGridTemplate = showLeadingCell + ? 'auto 1fr auto' + : '1fr auto'; + + return ( +
+ {/* Desktop layout with dynamic grid */} +
+ {/* Leading cell: Thumbnail, Badge, or nothing */} + {showLeadingCell && } + + {/* Fixed: Title and author */} +
+

+ {release.info_url ? ( + e.stopPropagation()} + > + {release.title} + + ) : ( + release.title + )} +

+ {author && ( +

+ {author} +

+ )} +
+ + {/* Dynamic columns from schema */} + {columns.map((col) => ( + + ))} + + {/* Fixed: Action button */} + +
+ + {/* Mobile layout - author inline with title, info line below */} +
+ {/* Leading cell: Thumbnail, Badge, or nothing */} + {showLeadingCell && } + +
+ {/* Title and author on same line */} +

+ {release.info_url ? ( + e.stopPropagation()} + > + {release.title} + + ) : ( + {release.title} + )} + {author && ( + — {author} + )} +

+ {/* Plugin-provided info line (format, size, indexer, seeders, etc.) */} + {mobileColumns.length > 0 && ( +
+ {mobileColumns.map((col, idx) => ( + + {idx > 0 && ·} + + + ))} +
+ )} +
+ + +
+
+ ); +}; + +// Shimmer block with wave animation - same as DownloadsSidebar +const ShimmerBlock = ({ className }: { className: string }) => ( +
+
+
+); + +// Loading skeleton for releases - matches ReleaseRow layout +const ReleaseSkeleton = () => ( +
+ {[1, 2, 3, 4, 5].map((i) => ( +
+
+ {/* Thumbnail skeleton */} + + + {/* Title and author skeleton */} +
+ + +
+ + {/* Language badge skeleton - desktop */} +
+ +
+ + {/* Format badge skeleton - desktop */} +
+ +
+ + {/* Size skeleton - desktop */} +
+ +
+ + {/* Mobile info + action skeleton */} +
+ {/* Mobile: format + size inline */} +
+ + +
+ + {/* Action button skeleton */} + +
+
+
+ ))} +
+); + +// Empty state component +const EmptyState = ({ message }: { message: string }) => ( +
+
+ + + +
+

{message}

+
+); + +// Configure source CTA +const ConfigureSourceCTA = ({ sourceName }: { sourceName: string }) => ( +
+
+ + + + +
+

+ {sourceName} Not Configured +

+

+ Configure {sourceName} in Settings to enable searching this source. +

+
+); + +// Coming soon state component +const ComingSoonState = ({ sourceName }: { sourceName: string }) => ( +
+
+ + + +
+

+ {sourceName} Coming Soon +

+

+ Support for {sourceName} is currently in development and will be available in a future update. +

+
+); + +// Error state component +const ErrorState = ({ message }: { message: string }) => ( +
+
+ + + +
+

+ Error Loading Releases +

+

{message}

+
+); + +// Sources that are coming soon (show "Coming Soon" message) +const COMING_SOON_SOURCES: Record = { + prowlarr: 'Prowlarr', +}; + +// Known sources that require configuration (show "Configure" CTA) +const CONFIGURABLE_SOURCES: Record = { + // Add sources here that are implemented but need configuration +}; + +export const ReleaseModal = ({ + book, + onClose, + onDownload, + supportedFormats, + defaultLanguages, + bookLanguages, + currentStatus, + defaultReleaseSource, +}: ReleaseModalProps) => { + const [isClosing, setIsClosing] = useState(false); + + // Available sources from plugin registry + const [availableSources, setAvailableSources] = useState([]); + const [sourcesLoading, setSourcesLoading] = useState(true); + + // Active tab (source name) + const [activeTab, setActiveTab] = useState(''); + + // Track if book summary has scrolled out of view + const [showHeaderThumb, setShowHeaderThumb] = useState(false); + const scrollContainerRef = useRef(null); + const bookSummaryRef = useRef(null); + + // Releases data per source + const [releasesBySource, setReleasesBySource] = useState>({}); + const [loadingBySource, setLoadingBySource] = useState>({}); + const [errorBySource, setErrorBySource] = useState>({}); + + // Filters - initialized from config settings + // Empty string means "show all supported formats" (filtered by supportedFormats) + // A specific value means "show only that format" + const [formatFilter, setFormatFilter] = useState(''); + const [languageFilter, setLanguageFilter] = useState(''); + + // Description expansion + const [descriptionExpanded, setDescriptionExpanded] = useState(false); + + // Close handler with animation + const handleClose = useCallback(() => { + setIsClosing(true); + setTimeout(() => { + onClose(); + setIsClosing(false); + }, 150); + }, [onClose]); + + // Handle ESC key + useEffect(() => { + const handleEscape = (e: KeyboardEvent) => { + if (e.key === 'Escape') handleClose(); + }; + document.addEventListener('keydown', handleEscape); + return () => document.removeEventListener('keydown', handleEscape); + }, [handleClose]); + + // Body scroll lock + useEffect(() => { + if (book) { + const previousOverflow = document.body.style.overflow; + document.body.style.overflow = 'hidden'; + return () => { + document.body.style.overflow = previousOverflow; + }; + } + }, [book]); + + // Reset modal state when book changes to prevent stale data + useEffect(() => { + setDescriptionExpanded(false); + setReleasesBySource({}); + setLoadingBySource({}); + setErrorBySource({}); + setFormatFilter(''); + setLanguageFilter(''); + }, [book?.id]); + + // Track scroll to show/hide header thumbnail + useEffect(() => { + const scrollContainer = scrollContainerRef.current; + const bookSummary = bookSummaryRef.current; + if (!scrollContainer || !bookSummary) return; + + const handleScroll = () => { + const summaryRect = bookSummary.getBoundingClientRect(); + const containerRect = scrollContainer.getBoundingClientRect(); + // Show header thumb when the book summary section has scrolled past the top + setShowHeaderThumb(summaryRect.bottom < containerRect.top + 20); + }; + + scrollContainer.addEventListener('scroll', handleScroll, { passive: true }); + return () => scrollContainer.removeEventListener('scroll', handleScroll); + }, [book]); + + // Fetch available sources on mount + useEffect(() => { + if (!book) return; + + const fetchSources = async () => { + try { + setSourcesLoading(true); + const sources = await getReleaseSources(); + setAvailableSources(sources); + // Set active tab: use defaultReleaseSource if available, otherwise first source + if (sources.length > 0) { + const defaultSource = defaultReleaseSource && sources.some(s => s.name === defaultReleaseSource) + ? defaultReleaseSource + : sources[0].name; + setActiveTab(defaultSource); + } + } catch (err) { + console.error('Failed to fetch release sources:', err); + // Fallback: assume direct_download is available + setAvailableSources([{ name: 'direct_download', display_name: "Anna's Archive" }]); + setActiveTab('direct_download'); + } finally { + setSourcesLoading(false); + } + }; + + fetchSources(); + }, [book, defaultReleaseSource]); + + // Fetch releases when active tab changes (with caching) + useEffect(() => { + if (!book || !activeTab || !book.provider || !book.provider_id) return; + + // Extract to local variables for TypeScript narrowing + const provider = book.provider; + const bookId = book.provider_id; + + // Skip if already loaded in component state or currently loading + if (releasesBySource[activeTab] !== undefined || loadingBySource[activeTab]) return; + + // Check module-level cache first + const cached = getCachedReleases(provider, bookId, activeTab); + if (cached) { + setReleasesBySource((prev) => ({ ...prev, [activeTab]: cached })); + return; + } + + const fetchReleases = async () => { + setLoadingBySource((prev) => ({ ...prev, [activeTab]: true })); + setErrorBySource((prev) => ({ ...prev, [activeTab]: null })); + + try { + const response = await getReleases(provider, bookId, activeTab, book.title, book.author); + // Store in module-level cache + setCachedReleases(provider, bookId, activeTab, response); + setReleasesBySource((prev) => ({ ...prev, [activeTab]: response })); + } catch (err) { + const message = err instanceof Error ? err.message : 'Failed to fetch releases'; + setErrorBySource((prev) => ({ ...prev, [activeTab]: message })); + } finally { + setLoadingBySource((prev) => ({ ...prev, [activeTab]: false })); + } + }; + + fetchReleases(); + }, [book, activeTab, releasesBySource, loadingBySource]); + + // Build list of tabs to show + // Status: 'available' = working, 'coming_soon' = in development, 'not_configured' = needs setup + // Order: 1) Default source, 2) Other configured sources, 3) Unconfigured sources, 4) Coming soon + const allTabs = useMemo(() => { + type TabInfo = { name: string; displayName: string; status: 'available' | 'coming_soon' | 'not_configured' }; + + const configuredTabs: TabInfo[] = []; + const unconfiguredTabs: TabInfo[] = []; + const comingSoonTabs: TabInfo[] = []; + + // Add available/configured sources + availableSources.forEach((src) => { + configuredTabs.push({ name: src.name, displayName: src.display_name, status: 'available' }); + }); + + // Add configurable but not configured sources + Object.entries(CONFIGURABLE_SOURCES).forEach(([name, displayName]) => { + if (!availableSources.find((s) => s.name === name)) { + unconfiguredTabs.push({ name, displayName, status: 'not_configured' }); + } + }); + + // Add coming soon sources + Object.entries(COMING_SOON_SOURCES).forEach(([name, displayName]) => { + if (!availableSources.find((s) => s.name === name)) { + comingSoonTabs.push({ name, displayName, status: 'coming_soon' }); + } + }); + + // Sort configured tabs so default source appears first + if (defaultReleaseSource) { + configuredTabs.sort((a, b) => { + if (a.name === defaultReleaseSource) return -1; + if (b.name === defaultReleaseSource) return 1; + return 0; + }); + } + + // Combine in order: configured (with default first), unconfigured, coming soon + return [...configuredTabs, ...unconfiguredTabs, ...comingSoonTabs]; + }, [availableSources, defaultReleaseSource]); + + // Get unique formats from current releases for filter dropdown + // Only show formats that are in the supported list + const availableFormats = useMemo(() => { + const releases = releasesBySource[activeTab]?.releases || []; + const formats = new Set(); + const supportedLower = supportedFormats.map((f) => f.toLowerCase()); + + releases.forEach((r) => { + if (r.format) { + const fmt = r.format.toLowerCase(); + // Only include formats that are in the supported list + if (supportedLower.includes(fmt)) { + formats.add(fmt); + } + } + }); + return Array.from(formats).sort(); + }, [releasesBySource, activeTab, supportedFormats]); + + // Get unique languages from current releases for filter dropdown + const availableLanguages = useMemo(() => { + const releases = releasesBySource[activeTab]?.releases || []; + const languages = new Set(); + + releases.forEach((r) => { + // Check for language in release extra data or other fields + const lang = r.extra?.language as string | undefined; + if (lang) languages.add(lang.toLowerCase()); + }); + return Array.from(languages).sort(); + }, [releasesBySource, activeTab]); + + // Build select options for format filter + const formatOptions = useMemo(() => { + const options = [{ value: '', label: 'All Formats' }]; + availableFormats.forEach((fmt) => { + options.push({ value: fmt, label: fmt.toUpperCase() }); + }); + return options; + }, [availableFormats]); + + // Build select options for language filter + const languageOptions = useMemo(() => { + const defaultLabel = defaultLanguages.length > 0 ? 'Default Lang' : 'All Lang'; + const options = [{ value: '', label: defaultLabel }]; + availableLanguages.forEach((lang) => { + const langInfo = bookLanguages.find( + (l) => l.code.toLowerCase() === lang.toLowerCase() + ); + options.push({ value: lang, label: langInfo?.language || lang.toUpperCase() }); + }); + return options; + }, [availableLanguages, bookLanguages, defaultLanguages.length]); + + // Filter releases based on settings and user selection + const filteredReleases = useMemo(() => { + const releases = releasesBySource[activeTab]?.releases || []; + const supportedLower = supportedFormats.map((f) => f.toLowerCase()); + const defaultLangLower = defaultLanguages.map((l) => l.toLowerCase()); + + return releases.filter((r) => { + // Format filtering: always filter by supported formats + if (r.format) { + const fmt = r.format.toLowerCase(); + // If user selected a specific format, filter to that + if (formatFilter) { + if (fmt !== formatFilter.toLowerCase()) return false; + } else { + // Otherwise, only show supported formats + if (!supportedLower.includes(fmt)) return false; + } + } + + // Language filtering (if release has language info) + const releaseLang = r.extra?.language as string | undefined; + if (releaseLang && languageFilter) { + if (releaseLang.toLowerCase() !== languageFilter.toLowerCase()) { + return false; + } + } else if (releaseLang && !languageFilter && defaultLangLower.length > 0) { + // If no user filter but we have default languages, filter to those + // Skip this filter if defaultLanguages is empty or contains special "any" value + const hasAnyLanguage = defaultLangLower.some((l) => l === '' || l === 'any'); + if (!hasAnyLanguage && !defaultLangLower.includes(releaseLang.toLowerCase())) { + return false; + } + } + + return true; + }); + }, [releasesBySource, activeTab, formatFilter, languageFilter, supportedFormats, defaultLanguages]); + + // Get column config from response or use default + const columnConfig = useMemo((): ReleaseColumnConfig => { + const response = releasesBySource[activeTab]; + if (response?.column_config) { + return response.column_config; + } + return DEFAULT_COLUMN_CONFIG; + }, [releasesBySource, activeTab]); + + // Get button state for a release based on its source_id + const getButtonState = useCallback( + (releaseId: string): ButtonStateInfo => { + // Check error first + if (currentStatus.error && currentStatus.error[releaseId]) { + return { text: 'Failed', state: 'error' }; + } + // Check completed + if (currentStatus.complete && currentStatus.complete[releaseId]) { + return { text: 'Downloaded', state: 'complete' }; + } + // Check in-progress states + if (currentStatus.downloading && currentStatus.downloading[releaseId]) { + const book = currentStatus.downloading[releaseId]; + return { + text: 'Downloading', + state: 'downloading', + progress: book.progress, + }; + } + if (currentStatus.resolving && currentStatus.resolving[releaseId]) { + return { text: 'Resolving', state: 'resolving' }; + } + if (currentStatus.queued && currentStatus.queued[releaseId]) { + return { text: 'Queued', state: 'queued' }; + } + return { text: 'Download', state: 'download' }; + }, + [currentStatus] + ); + + // Handle download - close modal once download is successfully queued + const handleDownload = useCallback( + async (release: Release): Promise => { + if (book) { + try { + await onDownload(book, release); + // Close modal after successful queue + handleClose(); + } catch { + // Keep modal open on error so user can try again + } + } + }, + [book, onDownload, handleClose] + ); + + if (!book && !isClosing) return null; + if (!book) return null; + + const titleId = `release-modal-title-${book.id}`; + const providerDisplay = + book.provider_display_name || + (book.provider ? book.provider.charAt(0).toUpperCase() + book.provider.slice(1) : 'Unknown'); + + const currentTabLoading = loadingBySource[activeTab] ?? false; + const currentTabError = errorBySource[activeTab] ?? null; + const currentTabStatus = allTabs.find((t) => t.name === activeTab)?.status ?? 'not_configured'; + + return ( +
{ + if (e.target === e.currentTarget) handleClose(); + }} + > +
+
+ {/* Header */} +
+ {/* Animated thumbnail that appears when scrolling */} +
+
+ {book.preview ? ( + + ) : ( +
+ No cover +
+ )} +
+
+
+

+ Find Releases +

+

+ {book.title || 'Untitled'} +

+

+ {book.author || 'Unknown author'} +

+
+ +
+ + {/* Scrollable content */} +
+ {/* Book summary - scrolls with content */} +
+ {book.preview ? ( + Book cover + ) : ( +
+ No cover +
+ )} +
+ {/* Metadata row */} +
+ {book.year && {book.year}} + {book.display_fields?.find(f => f.icon === 'star') && ( + + + + + {book.display_fields.find(f => f.icon === 'star')?.value} + + )} + {book.display_fields?.find(f => f.icon === 'users') && ( + + + + + {book.display_fields.find(f => f.icon === 'users')?.value} + + )} + {book.display_fields?.find(f => f.icon === 'book') && ( + {book.display_fields.find(f => f.icon === 'book')?.value} pages + )} +
+ + {/* Description */} + {book.description && ( +
+

+ {book.description} + {descriptionExpanded && ( + <> + {' '} + + + )} +

+ {!descriptionExpanded && ( + + )} +
+ )} + + {/* Links row */} +
+ {(book.isbn_13 || book.isbn_10) && ( + + ISBN: {book.isbn_13 || book.isbn_10} + + )} + {book.source_url && ( + + View on {providerDisplay} + + + + + )} +
+
+
+ + {/* Source tabs + filters - sticky within scroll container */} +
+ {sourcesLoading ? ( +
+
+
+ ) : ( +
+ {/* Tabs - scrollable on narrow screens */} +
+
+ {allTabs.map((tab) => ( + + ))} +
+
+ + {/* Filter funnel button - stays fixed */} + {(availableFormats.length > 0 || availableLanguages.length > 0) && ( + { + const hasActiveFilter = formatFilter !== '' || languageFilter !== ''; + return ( + + ); + }} + > + {() => ( +
+ {availableFormats.length > 0 && ( +
+ + +
+ )} + {availableLanguages.length > 0 && ( +
+ + +
+ )} +
+ )} + + )} +
+ )} +
+ + {/* Release list content */} +
+ {currentTabStatus === 'coming_soon' ? ( + t.name === activeTab)?.displayName || activeTab} + /> + ) : currentTabStatus === 'not_configured' ? ( + t.name === activeTab)?.displayName || activeTab} + /> + ) : currentTabLoading ? ( + + ) : currentTabError ? ( + + ) : filteredReleases.length === 0 ? ( + + ) : ( +
+ {filteredReleases.map((release, index) => ( + handleDownload(release)} + buttonState={getButtonState(release.source_id)} + columns={columnConfig.columns} + gridTemplate={columnConfig.grid_template} + leadingCell={columnConfig.leading_cell} + /> + ))} +
+ )} +
+
+
+
+
+ ); +}; diff --git a/src/frontend/src/components/ResultsSection.tsx b/src/frontend/src/components/ResultsSection.tsx index 58b76d74..18524308 100644 --- a/src/frontend/src/components/ResultsSection.tsx +++ b/src/frontend/src/components/ResultsSection.tsx @@ -1,19 +1,30 @@ import { useState, useEffect } from 'react'; -import { Book, ButtonStateInfo } from '../types'; +import { Book, ButtonStateInfo, SortOption } from '../types'; +import { useSearchMode } from '../contexts/SearchModeContext'; import { CardView } from './resultsViews/CardView'; import { CompactView } from './resultsViews/CompactView'; import { ListView } from './resultsViews/ListView'; import { Dropdown } from './Dropdown'; import { SORT_OPTIONS } from '../data/filterOptions'; +// Grid layout classes by view mode +const GRID_CLASSES = { + mobile: 'grid-cols-1 items-start', + card: 'grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 items-stretch', + compact: 'grid-cols-1 sm:grid-cols-2 lg:grid-cols-2 xl:grid-cols-3 items-start', +} as const; + interface ResultsSectionProps { books: Book[]; visible: boolean; onDetails: (id: string) => Promise; onDownload: (book: Book) => Promise; + onGetReleases: (book: Book) => Promise; getButtonState: (bookId: string) => ButtonStateInfo; + getUniversalButtonState: (bookId: string) => ButtonStateInfo; sortValue: string; onSortChange: (value: string) => void; + metadataSortOptions?: SortOption[]; } export const ResultsSection = ({ @@ -21,10 +32,14 @@ export const ResultsSection = ({ visible, onDetails, onDownload, + onGetReleases, getButtonState, + getUniversalButtonState, sortValue, onSortChange, + metadataSortOptions, }: ResultsSectionProps) => { + const { searchMode } = useSearchMode(); const [viewMode, setViewMode] = useState<'card' | 'compact' | 'list'>(() => { const saved = localStorage.getItem('bookViewMode'); return saved === 'card' || saved === 'compact' || saved === 'list' ? saved : 'compact'; @@ -36,14 +51,25 @@ export const ResultsSection = ({ }, [viewMode]); // Track whether we're in desktop layout (sm breakpoint and above) + // Debounced to avoid excessive state updates during resize useEffect(() => { + let timeoutId: number; + const checkDesktop = () => { - setIsDesktop(window.innerWidth >= 640); // sm breakpoint + clearTimeout(timeoutId); + timeoutId = window.setTimeout(() => { + setIsDesktop(window.innerWidth >= 640); // sm breakpoint + }, 100); }; - - checkDesktop(); + + // Initial check without debounce + setIsDesktop(window.innerWidth >= 640); + window.addEventListener('resize', checkDesktop); - return () => window.removeEventListener('resize', checkDesktop); + return () => { + clearTimeout(timeoutId); + window.removeEventListener('resize', checkDesktop); + }; }, []); if (!visible) return null; @@ -51,7 +77,7 @@ export const ResultsSection = ({ return (
- + {/* View toggle buttons - Desktop: show all 3, Mobile: show Compact and List only */}
@@ -60,7 +86,9 @@ export const ResultsSection = ({ onClick={() => setViewMode('card')} className={`p-2 rounded-full transition-all duration-200 ${ viewMode === 'card' - ? 'text-white bg-sky-700 hover:bg-sky-800' + ? searchMode === 'universal' + ? 'text-white bg-emerald-600 hover:bg-emerald-700' + : 'text-white bg-sky-700 hover:bg-sky-800' : 'hover-action text-gray-900 dark:text-gray-100' }`} title="Card view" @@ -86,7 +114,9 @@ export const ResultsSection = ({ onClick={() => setViewMode('compact')} className={`p-2 rounded-full transition-all duration-200 ${ viewMode === 'compact' - ? 'text-white bg-sky-700 hover:bg-sky-800' + ? searchMode === 'universal' + ? 'text-white bg-emerald-600 hover:bg-emerald-700' + : 'text-white bg-sky-700 hover:bg-sky-800' : 'hover-action text-gray-900 dark:text-gray-100' }`} title="Compact view" @@ -110,7 +140,9 @@ export const ResultsSection = ({ onClick={() => setViewMode('list')} className={`p-2 rounded-full transition-all duration-200 ${ viewMode === 'list' - ? 'text-white bg-sky-700 hover:bg-sky-800' + ? searchMode === 'universal' + ? 'text-white bg-emerald-600 hover:bg-emerald-700' + : 'text-white bg-sky-700 hover:bg-sky-800' : 'hover-action text-gray-900 dark:text-gray-100' }`} title="List view" @@ -134,22 +166,19 @@ export const ResultsSection = ({
{viewMode === 'list' ? ( - + ) : (
{books.map((book, index) => { const shouldUseCardLayout = isDesktop && viewMode === 'card'; - const animationDelay = index * 50; + // Use appropriate button state function based on search mode + const buttonState = searchMode === 'universal' + ? getUniversalButtonState(book.id) + : getButtonState(book.id); return shouldUseCardLayout ? ( ) : ( @@ -166,7 +196,8 @@ export const ResultsSection = ({ book={book} onDetails={onDetails} onDownload={onDownload} - buttonState={getButtonState(book.id)} + onGetReleases={onGetReleases} + buttonState={buttonState} showDetailsButton={!isDesktop} animationDelay={animationDelay} /> @@ -184,10 +215,22 @@ export const ResultsSection = ({ interface SortControlProps { value: string; onChange: (value: string) => void; + metadataSortOptions?: SortOption[]; } -const SortControl = ({ value, onChange }: SortControlProps) => { - const selected = SORT_OPTIONS.find(option => option.value === value) ?? SORT_OPTIONS[0]; +// Default universal mode sort options (fallback if not provided by API) +const DEFAULT_UNIVERSAL_SORT_OPTIONS: SortOption[] = [ + { value: 'relevance', label: 'Most relevant' }, +]; + +const SortControl = ({ value, onChange, metadataSortOptions }: SortControlProps) => { + const { searchMode } = useSearchMode(); + // Use different sort options based on search mode + // For universal mode, use dynamic options from API (with fallback) + const sortOptions = searchMode === 'universal' + ? (metadataSortOptions && metadataSortOptions.length > 0 ? metadataSortOptions : DEFAULT_UNIVERSAL_SORT_OPTIONS) + : SORT_OPTIONS; + const selected = sortOptions.find(option => option.value === value) ?? sortOptions[0]; return ( { > {({ close }) => (
- {SORT_OPTIONS.map(option => { + {sortOptions.map(option => { const isSelected = option.value === selected.value; return ( )}
); -}; - - +}); diff --git a/src/frontend/src/components/SearchSection.tsx b/src/frontend/src/components/SearchSection.tsx index 55ad1296..2e83ed01 100644 --- a/src/frontend/src/components/SearchSection.tsx +++ b/src/frontend/src/components/SearchSection.tsx @@ -1,5 +1,6 @@ -import { AdvancedFilterState, Language } from '../types'; +import { AdvancedFilterState, Language, MetadataSearchField } from '../types'; import { buildSearchQuery } from '../utils/buildSearchQuery'; +import { useSearchMode } from '../contexts/SearchModeContext'; import { AdvancedFilters } from './AdvancedFilters'; import { SearchBar } from './SearchBar'; @@ -17,6 +18,10 @@ interface SearchSectionProps { onAdvancedToggle: () => void; advancedFilters: AdvancedFilterState; onAdvancedFiltersChange: (updates: Partial) => void; + // Universal mode props + metadataSearchFields?: MetadataSearchField[]; + searchFieldValues?: Record; + onSearchFieldChange?: (key: string, value: string | number | boolean) => void; } export const SearchSection = ({ @@ -33,7 +38,12 @@ export const SearchSection = ({ onAdvancedToggle, advancedFilters, onAdvancedFiltersChange, + metadataSearchFields, + searchFieldValues, + onSearchFieldChange, }: SearchSectionProps) => { + const { searchMode } = useSearchMode(); + const handleSearch = () => { const query = buildSearchQuery({ searchInput, @@ -41,6 +51,7 @@ export const SearchSection = ({ advancedFilters, bookLanguages, defaultLanguage, + searchMode, }); onSearch(query); }; @@ -79,6 +90,10 @@ export const SearchSection = ({ onFiltersChange={onAdvancedFiltersChange} formClassName="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4 px-2" renderWrapper={form => form} + metadataSearchFields={metadataSearchFields} + searchFieldValues={searchFieldValues} + onSearchFieldChange={onSearchFieldChange} + onSubmit={handleSearch} />
diff --git a/src/frontend/src/components/ToastContainer.tsx b/src/frontend/src/components/ToastContainer.tsx index 27e2128f..55d25b34 100644 --- a/src/frontend/src/components/ToastContainer.tsx +++ b/src/frontend/src/components/ToastContainer.tsx @@ -25,7 +25,7 @@ export const ToastContainer = ({ toasts }: ToastContainerProps) => { }; return ( -
+
{toasts.map(toast => (
(
@@ -10,12 +12,15 @@ interface CardViewProps { book: Book; onDetails: (id: string) => Promise; onDownload: (book: Book) => Promise; + onGetReleases: (book: Book) => Promise; buttonState: ButtonStateInfo; animationDelay?: number; } -export const CardView = ({ book, onDetails, onDownload, buttonState, animationDelay = 0 }: CardViewProps) => { +export const CardView = ({ book, onDetails, onDownload, onGetReleases, buttonState, animationDelay = 0 }: CardViewProps) => { + const { searchMode } = useSearchMode(); const [isLoadingDetails, setIsLoadingDetails] = useState(false); + const [isLoadingReleases, setIsLoadingReleases] = useState(false); const [imageLoaded, setImageLoaded] = useState(false); const [imageError, setImageError] = useState(false); const [isHovered, setIsHovered] = useState(false); @@ -29,6 +34,15 @@ export const CardView = ({ book, onDetails, onDownload, buttonState, animationDe } }; + const handleGetReleases = async (book: Book) => { + setIsLoadingReleases(true); + try { + await onGetReleases(book); + } finally { + setIsLoadingReleases(false); + } + }; + return (

{book.author || 'Unknown author'}

-
- {book.year || '-'} - • - {book.language || '-'} - • - {book.format || '-'} - {book.size && ( - <> - • - {book.size} - - )} -
+ {searchMode === 'universal' && book.display_fields && book.display_fields.length > 0 ? ( +
+ {book.year || '-'} + • + +
+ ) : ( +
+ {book.year || '-'} + • + {book.language || '-'} + • + {book.format || '-'} + {book.size && ( + <> + • + {book.size} + + )} +
+ )}
@@ -131,13 +153,24 @@ export const CardView = ({ book, onDetails, onDownload, buttonState, animationDe className={`details-spinner w-3 h-3 border-2 border-current border-t-transparent rounded-full ${isLoadingDetails ? '' : 'hidden'}`} /> - onDownload(book)} size="sm" className="flex-1" /> +
- onDownload(book)} + onDownload={onDownload} + onGetReleases={handleGetReleases} + isLoadingReleases={isLoadingReleases} className="hidden sm:flex rounded-none" fullWidth style={{ diff --git a/src/frontend/src/components/resultsViews/CompactView.tsx b/src/frontend/src/components/resultsViews/CompactView.tsx index 8960eeb2..bdcde4fc 100644 --- a/src/frontend/src/components/resultsViews/CompactView.tsx +++ b/src/frontend/src/components/resultsViews/CompactView.tsx @@ -1,6 +1,8 @@ import { useState } from 'react'; import { Book, ButtonStateInfo } from '../../types'; -import { BookDownloadButton } from '../BookDownloadButton'; +import { useSearchMode } from '../../contexts/SearchModeContext'; +import { BookActionButton } from '../BookActionButton'; +import { DisplayFieldBadges } from '../shared'; const SkeletonLoader = () => (
@@ -10,13 +12,16 @@ interface CompactViewProps { book: Book; onDetails: (id: string) => Promise; onDownload: (book: Book) => Promise; + onGetReleases: (book: Book) => Promise; buttonState: ButtonStateInfo; showDetailsButton?: boolean; animationDelay?: number; } -export const CompactView = ({ book, onDetails, onDownload, buttonState, showDetailsButton = false, animationDelay = 0 }: CompactViewProps) => { +export const CompactView = ({ book, onDetails, onDownload, onGetReleases, buttonState, showDetailsButton = false, animationDelay = 0 }: CompactViewProps) => { + const { searchMode } = useSearchMode(); const [isLoadingDetails, setIsLoadingDetails] = useState(false); + const [isLoadingReleases, setIsLoadingReleases] = useState(false); const [imageLoaded, setImageLoaded] = useState(false); const [imageError, setImageError] = useState(false); const [isHovered, setIsHovered] = useState(false); @@ -30,6 +35,15 @@ export const CompactView = ({ book, onDetails, onDownload, buttonState, showDeta } }; + const handleGetReleases = async (book: Book) => { + setIsLoadingReleases(true); + try { + await onGetReleases(book); + } finally { + setIsLoadingReleases(false); + } + }; + return (

{book.author || 'Unknown author'}

-
+
{book.year || '-'}
-
- {book.language || '-'} - • - {book.format || '-'} - {book.size && ( - <> - • - {book.size} - - )} -
+ {searchMode === 'universal' && book.display_fields && book.display_fields.length > 0 ? ( + + ) : ( +
+ {book.language || '-'} + • + {book.format || '-'} + {book.size && ( + <> + • + {book.size} + + )} +
+ )} {showDetailsButton ? (
@@ -133,10 +151,26 @@ export const CompactView = ({ book, onDetails, onDownload, buttonState, showDeta {isLoadingDetails ? 'Loading' : 'Details'} {isLoadingDetails &&
} - onDownload(book)} size="sm" className="flex-1" /> +
) : ( - onDownload(book)} size="sm" fullWidth /> + )}
diff --git a/src/frontend/src/components/resultsViews/ListView.tsx b/src/frontend/src/components/resultsViews/ListView.tsx index 5d250981..40e2109d 100644 --- a/src/frontend/src/components/resultsViews/ListView.tsx +++ b/src/frontend/src/components/resultsViews/ListView.tsx @@ -1,12 +1,17 @@ import { useState } from 'react'; import { Book, ButtonStateInfo } from '../../types'; -import { BookDownloadButton } from '../BookDownloadButton'; +import { useSearchMode } from '../../contexts/SearchModeContext'; +import { BookActionButton } from '../BookActionButton'; +import { DisplayFieldIcon, DisplayFieldBadge } from '../shared'; +import { getFormatColor, getLanguageColor } from '../../utils/colorMaps'; interface ListViewProps { books: Book[]; onDetails: (id: string) => Promise; onDownload: (book: Book) => Promise; + onGetReleases: (book: Book) => Promise; getButtonState: (bookId: string) => ButtonStateInfo; + getUniversalButtonState: (bookId: string) => ButtonStateInfo; } const ListViewThumbnail = ({ preview, title }: { preview?: string; title?: string }) => { @@ -42,51 +47,10 @@ const ListViewThumbnail = ({ preview, title }: { preview?: string; title?: strin ); }; -const getLanguageColor = (language?: string): string => { - if (!language || language === '-') return 'bg-gray-400 dark:bg-gray-600'; - const lang = language.toLowerCase(); - const colorMap: Record = { - en: 'bg-blue-500 dark:bg-blue-600', - english: 'bg-blue-500 dark:bg-blue-600', - es: 'bg-orange-500 dark:bg-orange-600', - spanish: 'bg-orange-500 dark:bg-orange-600', - fr: 'bg-purple-500 dark:bg-purple-600', - french: 'bg-purple-500 dark:bg-purple-600', - de: 'bg-yellow-500 dark:bg-yellow-600', - german: 'bg-yellow-500 dark:bg-yellow-600', - it: 'bg-green-500 dark:bg-green-600', - italian: 'bg-green-500 dark:bg-green-600', - pt: 'bg-teal-500 dark:bg-teal-600', - portuguese: 'bg-teal-500 dark:bg-teal-600', - ru: 'bg-red-500 dark:bg-red-600', - russian: 'bg-red-500 dark:bg-red-600', - ja: 'bg-pink-500 dark:bg-pink-600', - japanese: 'bg-pink-500 dark:bg-pink-600', - zh: 'bg-rose-500 dark:bg-rose-600', - chinese: 'bg-rose-500 dark:bg-rose-600', - }; - return colorMap[lang] || 'bg-indigo-500 dark:bg-indigo-600'; -}; - -const getFormatColor = (format?: string): string => { - if (!format || format === '-') return 'bg-gray-400 dark:bg-gray-600'; - const fmt = format.toLowerCase(); - const colorMap: Record = { - pdf: 'bg-red-500 dark:bg-red-600', - epub: 'bg-green-500 dark:bg-green-600', - mobi: 'bg-blue-500 dark:bg-blue-600', - azw3: 'bg-purple-500 dark:bg-purple-600', - txt: 'bg-gray-500 dark:bg-gray-600', - djvu: 'bg-orange-500 dark:bg-orange-600', - fb2: 'bg-teal-500 dark:bg-teal-600', - cbr: 'bg-yellow-500 dark:bg-yellow-600', - cbz: 'bg-amber-500 dark:bg-amber-600', - }; - return colorMap[fmt] || 'bg-cyan-500 dark:bg-cyan-600'; -}; - -export const ListView = ({ books, onDetails, onDownload, getButtonState }: ListViewProps) => { +export const ListView = ({ books, onDetails, onDownload, onGetReleases, getButtonState, getUniversalButtonState }: ListViewProps) => { + const { searchMode } = useSearchMode(); const [detailsLoadingId, setDetailsLoadingId] = useState(null); + const [releasesLoadingId, setReleasesLoadingId] = useState(null); if (books.length === 0) { return null; @@ -101,6 +65,15 @@ export const ListView = ({ books, onDetails, onDownload, getButtonState }: ListV } }; + const handleGetReleases = async (book: Book) => { + setReleasesLoadingId(book.id); + try { + await onGetReleases(book); + } finally { + setReleasesLoadingId((current) => (current === book.id ? null : current)); + } + }; + return (
{books.map((book, index) => { - const buttonState = getButtonState(book.id); + // Use appropriate button state function based on search mode + const buttonState = searchMode === 'universal' + ? getUniversalButtonState(book.id) + : getButtonState(book.id); const isLoadingDetails = detailsLoadingId === book.id; return ( @@ -127,7 +103,12 @@ export const ListView = ({ books, onDetails, onDownload, getButtonState }: ListV role="article" > {/* Mobile and Desktop: Single row layout */} -
+ {/* Universal mode uses separate columns for each display field, direct mode uses language/format/size */} +
{/* Thumbnail */}
@@ -144,10 +125,21 @@ export const ListView = ({ books, onDetails, onDownload, getButtonState }: ListV

- {/* Format and Size - Mobile only */} + {/* Mobile universal mode info */}
- {book.format || '-'} - {book.size && {book.size}} + {searchMode === 'universal' && book.display_fields && book.display_fields.length > 0 ? ( + book.display_fields.slice(0, 2).map((field, idx) => ( + + + {field.value} + + )) + ) : ( + <> + {book.format || '-'} + {book.size && {book.size}} + + )}
{/* Year - Desktop only */} @@ -155,33 +147,61 @@ export const ListView = ({ books, onDetails, onDownload, getButtonState }: ListV {book.year || '-'}
- {/* Language Badge - Desktop only */} -
- - {book.language || '-'} - -
+ {/* Universal mode: Display fields as separate columns - Desktop only */} + {searchMode === 'universal' && ( + <> + {/* First display field column */} +
+ {book.display_fields && book.display_fields[0] ? ( + + ) : ( + - + )} +
+ {/* Second display field column */} +
+ {book.display_fields && book.display_fields[1] ? ( + + ) : ( + - + )} +
+ + )} - {/* Format Badge - Desktop only */} -
- - {book.format || '-'} - -
+ {/* Direct mode: Language Badge - Desktop only */} + {searchMode !== 'universal' && ( +
+ + {book.language || '-'} + +
+ )} - {/* Size - Desktop only */} -
- {book.size || '-'} -
+ {/* Direct mode: Format Badge - Desktop only */} + {searchMode !== 'universal' && ( +
+ + {book.format || '-'} + +
+ )} + + {/* Direct mode: Size - Desktop only */} + {searchMode !== 'universal' && ( +
+ {book.size || '-'} +
+ )} {/* Action Buttons */} -
+
- onDownload(book)} + onDownload={onDownload} + onGetReleases={handleGetReleases} + isLoadingReleases={releasesLoadingId === book.id} variant="icon" size="md" - ariaLabel={buttonState.text} />
diff --git a/src/frontend/src/components/settings/SettingsContent.tsx b/src/frontend/src/components/settings/SettingsContent.tsx new file mode 100644 index 00000000..e04e1582 --- /dev/null +++ b/src/frontend/src/components/settings/SettingsContent.tsx @@ -0,0 +1,265 @@ +import { useEffect, useRef } from 'react'; +import { + SettingsTab, + SettingsField, + ActionResult, + TextFieldConfig, + PasswordFieldConfig, + NumberFieldConfig, + CheckboxFieldConfig, + SelectFieldConfig, + MultiSelectFieldConfig, + ActionButtonConfig, + HeadingFieldConfig, +} from '../../types/settings'; +import { FieldWrapper } from './shared'; +import { + TextField, + PasswordField, + NumberField, + CheckboxField, + SelectField, + MultiSelectField, + ActionButton, + HeadingField, +} from './fields'; + +interface SettingsContentProps { + tab: SettingsTab; + values: Record; + onChange: (key: string, value: unknown) => void; + onSave: () => Promise; + onAction: (key: string) => Promise; + isSaving: boolean; + hasChanges: boolean; +} + +// Check if a field should be visible based on showWhen condition +function isFieldVisible( + field: SettingsField, + values: Record +): boolean { + // HeadingField doesn't have showWhen + if (field.type === 'HeadingField') { + return true; + } + + const showWhen = field.showWhen; + if (!showWhen) return true; + + const currentValue = values[showWhen.field]; + + // Handle array of allowed values or single value + return Array.isArray(showWhen.value) + ? showWhen.value.includes(currentValue as string) + : currentValue === showWhen.value; +} + +// Check if a field should be disabled based on disabledWhen condition +// Returns { disabled: boolean, reason?: string } +function getDisabledState( + field: SettingsField, + values: Record +): { disabled: boolean; reason?: string } { + // HeadingField doesn't have disabledWhen + if (field.type === 'HeadingField') { + return { disabled: false }; + } + + // Check if value is locked by environment variable + if ('fromEnv' in field && field.fromEnv) { + return { disabled: true }; + } + + // Check static disabled first + if ('disabled' in field && field.disabled) { + return { + disabled: true, + reason: 'disabledReason' in field ? field.disabledReason : undefined, + }; + } + + // Check disabledWhen condition + if (!('disabledWhen' in field) || !field.disabledWhen) { + return { disabled: false }; + } + + const { field: conditionField, value: conditionValue, reason } = field.disabledWhen; + const currentValue = values[conditionField]; + + // Check if condition is met (handles both array and single value) + const isDisabled = Array.isArray(conditionValue) + ? conditionValue.includes(currentValue as string) + : currentValue === conditionValue; + + return { + disabled: isDisabled, + reason: isDisabled ? reason : undefined, + }; +} + +// Render the appropriate field component based on type +const renderField = ( + field: SettingsField, + value: unknown, + onChange: (value: unknown) => void, + onAction: () => Promise, + isDisabled: boolean +) => { + switch (field.type) { + case 'TextField': + return ( + + ); + case 'PasswordField': + return ( + + ); + case 'NumberField': + return ( + + ); + case 'CheckboxField': + return ( + + ); + case 'SelectField': + return ( + + ); + case 'MultiSelectField': + return ( + + ); + case 'ActionButton': + return ; + case 'HeadingField': + return ; + default: + return
Unknown field type
; + } +}; + +export const SettingsContent = ({ + tab, + values, + onChange, + onSave, + onAction, + isSaving, + hasChanges, +}: SettingsContentProps) => { + const scrollRef = useRef(null); + + // Reset scroll position when tab changes + useEffect(() => { + if (scrollRef.current) { + scrollRef.current.scrollTop = 0; + } + }, [tab.name]); + + return ( +
+ {/* Scrollable content area */} +
+
+ {tab.fields + .filter((field) => isFieldVisible(field, values)) + .map((field) => { + const disabledState = getDisabledState(field, values); + return ( + + {renderField( + field, + values[field.key], + (v) => onChange(field.key, v), + () => onAction(field.key), + disabledState.disabled + )} + + ); + })} +
+
+ + {/* Save button - only visible when there are changes */} + {hasChanges && ( +
+ +
+ )} +
+ ); +}; diff --git a/src/frontend/src/components/settings/SettingsHeader.tsx b/src/frontend/src/components/settings/SettingsHeader.tsx new file mode 100644 index 00000000..a2911e58 --- /dev/null +++ b/src/frontend/src/components/settings/SettingsHeader.tsx @@ -0,0 +1,58 @@ +interface SettingsHeaderProps { + title: string; + showBack?: boolean; + onBack?: () => void; + onClose: () => void; +} + +export const SettingsHeader = ({ + title, + showBack = false, + onBack, + onClose, +}: SettingsHeaderProps) => ( +
+ {showBack && ( + + )} +

{title}

+ +
+); diff --git a/src/frontend/src/components/settings/SettingsModal.tsx b/src/frontend/src/components/settings/SettingsModal.tsx new file mode 100644 index 00000000..9122bba0 --- /dev/null +++ b/src/frontend/src/components/settings/SettingsModal.tsx @@ -0,0 +1,321 @@ +import { useEffect, useState, useCallback } from 'react'; +import { useSettings } from '../../hooks/useSettings'; +import { SettingsHeader } from './SettingsHeader'; +import { SettingsSidebar } from './SettingsSidebar'; +import { SettingsContent } from './SettingsContent'; + +interface SettingsModalProps { + isOpen: boolean; + onClose: () => void; + onShowToast?: (message: string, type: 'success' | 'error' | 'info') => void; + onSettingsSaved?: () => void; +} + +export const SettingsModal = ({ isOpen, onClose, onShowToast, onSettingsSaved }: SettingsModalProps) => { + const { + tabs, + groups, + isLoading, + error, + selectedTab, + setSelectedTab, + values, + updateValue, + hasChanges, + saveTab, + executeAction, + isSaving, + } = useSettings(); + + // Track if we're showing detail view on mobile + const [isMobile, setIsMobile] = useState(false); + const [showMobileDetail, setShowMobileDetail] = useState(false); + const [isClosing, setIsClosing] = useState(false); + + // Check for mobile viewport + useEffect(() => { + const checkMobile = () => { + setIsMobile(window.innerWidth < 768); + }; + checkMobile(); + window.addEventListener('resize', checkMobile); + return () => window.removeEventListener('resize', checkMobile); + }, []); + + const handleClose = useCallback(() => { + setIsClosing(true); + setTimeout(() => { + onClose(); + setIsClosing(false); + }, 150); + }, [onClose]); + + // Handle ESC key + useEffect(() => { + if (!isOpen) return; + + const handleEscape = (e: KeyboardEvent) => { + if (e.key === 'Escape') { + if (isMobile && showMobileDetail) { + setShowMobileDetail(false); + } else { + handleClose(); + } + } + }; + + document.addEventListener('keydown', handleEscape); + return () => document.removeEventListener('keydown', handleEscape); + }, [isOpen, isMobile, showMobileDetail, handleClose]); + + // Prevent body scroll when open + useEffect(() => { + if (isOpen) { + const previousOverflow = document.body.style.overflow; + document.body.style.overflow = 'hidden'; + return () => { + document.body.style.overflow = previousOverflow; + }; + } + }, [isOpen]); + + // Reset mobile detail view when modal opens + useEffect(() => { + if (isOpen) { + setShowMobileDetail(false); + setIsClosing(false); + } + }, [isOpen]); + + // On desktop, select first tab when modal first opens (only if no tab selected) + useEffect(() => { + if (isOpen && !isMobile && tabs.length > 0 && !selectedTab) { + setSelectedTab(tabs[0].name); + } + }, [isOpen, isMobile, tabs, selectedTab, setSelectedTab]); + + const handleSelectTab = useCallback( + (tabName: string) => { + setSelectedTab(tabName); + if (isMobile) { + setShowMobileDetail(true); + } + }, + [isMobile, setSelectedTab] + ); + + const handleBack = useCallback(() => { + setShowMobileDetail(false); + }, []); + + const handleSave = useCallback(async () => { + if (!selectedTab) return; + const result = await saveTab(selectedTab); + if (result.success) { + onShowToast?.(result.message, 'success'); + // Notify parent that settings were saved so it can refresh config + onSettingsSaved?.(); + // Show additional toast if some settings require restart + if (result.requiresRestart) { + setTimeout(() => { + onShowToast?.('Some settings require a container restart to take effect', 'info'); + }, 500); + } + } else { + onShowToast?.(result.message, 'error'); + } + }, [selectedTab, saveTab, onShowToast, onSettingsSaved]); + + const handleAction = useCallback( + async (actionKey: string) => { + if (!selectedTab) { + return { success: false, message: 'No tab selected' }; + } + return executeAction(selectedTab, actionKey); + }, + [selectedTab, executeAction] + ); + + if (!isOpen && !isClosing) return null; + + const currentTab = tabs.find((t) => t.name === selectedTab); + const currentTabDisplayName = currentTab?.displayName || 'Settings'; + + // Loading state + if (isLoading) { + return ( +
+
+
+
+ + + + + Loading settings... +
+
+
+ ); + } + + // Error state + if (error) { + return ( +
+
+
+
+
+ + + +
+

{error}

+ +
+
+
+ ); + } + + // Mobile layout + if (isMobile) { + return ( +
+ {!showMobileDetail ? ( + // Category list view + <> + + + + ) : ( + // Detail view + <> + + {currentTab && ( + updateValue(currentTab.name, key, value)} + onSave={handleSave} + onAction={handleAction} + isSaving={isSaving} + hasChanges={hasChanges(currentTab.name)} + /> + )} + + )} +
+ ); + } + + // Desktop layout + return ( +
+ {/* Backdrop */} +
+ + {/* Modal */} +
+ + +
+ + + {currentTab ? ( + updateValue(currentTab.name, key, value)} + onSave={handleSave} + onAction={handleAction} + isSaving={isSaving} + hasChanges={hasChanges(currentTab.name)} + /> + ) : ( +
+ Select a category to configure +
+ )} +
+
+
+ ); +}; diff --git a/src/frontend/src/components/settings/SettingsSidebar.tsx b/src/frontend/src/components/settings/SettingsSidebar.tsx new file mode 100644 index 00000000..bf43225f --- /dev/null +++ b/src/frontend/src/components/settings/SettingsSidebar.tsx @@ -0,0 +1,319 @@ +import { useState } from 'react'; +import { SettingsTab, SettingsGroup } from '../../types/settings'; + +interface SettingsSidebarProps { + tabs: SettingsTab[]; + groups: SettingsGroup[]; + selectedTab: string | null; + onSelectTab: (tabName: string) => void; + mode: 'sidebar' | 'list'; +} + +// Map icon names to SVG paths +const getIcon = (iconName?: string) => { + switch (iconName) { + case 'settings': + case 'cog': + return ( + + + + + ); + case 'folder': + return ( + + + + ); + case 'shield': + return ( + + + + ); + case 'globe': + return ( + + + + ); + case 'download': + return ( + + + + ); + case 'book': + return ( + + + + ); + case 'library': + return ( + + + + ); + case 'beaker': + case 'wrench': + return ( + + + + ); + default: + return ( + + + + ); + } +}; + +// Chevron icon for expandable groups +const ChevronIcon = ({ expanded }: { expanded: boolean }) => ( + + + +); + +// Represents either a tab, a group, or a section header in the sorted list +type SidebarItem = + | { type: 'tab'; tab: SettingsTab; order: number } + | { type: 'group'; group: SettingsGroup; tabs: SettingsTab[]; order: number } + | { type: 'section'; label: string; order: number }; + +// Section headers for organizing the sidebar +// Can trigger before a group (beforeGroup) or before a tab (beforeTab) +const SECTION_HEADERS: { beforeGroup?: string; beforeTab?: string; label: string }[] = [ + { beforeGroup: 'direct_download', label: 'Release Sources' }, + { beforeTab: 'hardcover', label: 'Metadata Providers' }, +]; + +export const SettingsSidebar = ({ + tabs, + groups, + selectedTab, + onSelectTab, + mode, +}: SettingsSidebarProps) => { + // Track which groups are expanded (all closed by default) + const [expandedGroups, setExpandedGroups] = useState>( + () => new Set() + ); + + const toggleGroup = (groupName: string) => { + setExpandedGroups((prev) => { + const next = new Set(prev); + if (next.has(groupName)) { + next.delete(groupName); + } else { + next.add(groupName); + } + return next; + }); + }; + + // Build grouped tabs map + const groupedTabs = new Map(); + tabs.forEach((tab) => { + if (tab.group) { + const existing = groupedTabs.get(tab.group) || []; + existing.push(tab); + groupedTabs.set(tab.group, existing); + } + }); + + // Build a unified sorted list of tabs and groups + const sidebarItems: SidebarItem[] = []; + + // Add ungrouped tabs (with section headers where needed) + tabs.forEach((tab) => { + if (!tab.group) { + // Check if this tab needs a section header before it + const sectionHeader = SECTION_HEADERS.find((s) => s.beforeTab === tab.name); + if (sectionHeader) { + sidebarItems.push({ type: 'section', label: sectionHeader.label, order: tab.order - 0.5 }); + } + sidebarItems.push({ type: 'tab', tab, order: tab.order }); + } + }); + + // Add groups (with their tabs) and section headers + groups.forEach((group) => { + const groupTabList = groupedTabs.get(group.name) || []; + if (groupTabList.length > 0) { + // Check if this group needs a section header before it + const sectionHeader = SECTION_HEADERS.find((s) => s.beforeGroup === group.name); + if (sectionHeader) { + // Insert section header just before this group (order - 0.5 to sort before) + sidebarItems.push({ type: 'section', label: sectionHeader.label, order: group.order - 0.5 }); + } + sidebarItems.push({ type: 'group', group, tabs: groupTabList, order: group.order }); + } + }); + + // Sort by order + sidebarItems.sort((a, b) => a.order - b.order); + + if (mode === 'list') { + // Mobile: Clean list style with inset dividers + return ( +
+ {sidebarItems.map((item, itemIndex) => { + if (item.type === 'section') { + return ( +
+ + {item.label} + +
+ ); + } + + if (item.type === 'tab') { + return ( +
+ + {itemIndex < sidebarItems.length - 1 && ( +
+ )} +
+ ); + } + + // Group + const isExpanded = expandedGroups.has(item.group.name); + return ( +
+ + + {isExpanded && ( +
+ {item.tabs.map((tab, index) => ( +
+ + {index < item.tabs.length - 1 && ( +
+ )} +
+ ))} +
+ )} + + {itemIndex < sidebarItems.length - 1 && ( +
+ )} +
+ ); + })} +
+ ); + } + + // Desktop: Sidebar navigation + return ( + + ); +}; diff --git a/src/frontend/src/components/settings/fields/ActionButton.tsx b/src/frontend/src/components/settings/fields/ActionButton.tsx new file mode 100644 index 00000000..b06b04ca --- /dev/null +++ b/src/frontend/src/components/settings/fields/ActionButton.tsx @@ -0,0 +1,96 @@ +import { useState } from 'react'; +import { ActionButtonConfig, ActionResult } from '../../../types/settings'; + +interface ActionButtonProps { + field: ActionButtonConfig; + onAction: () => Promise; + disabled?: boolean; +} + +export const ActionButton = ({ field, onAction, disabled }: ActionButtonProps) => { + const [isLoading, setIsLoading] = useState(false); + const [result, setResult] = useState(null); + const isDisabled = disabled ?? field.disabled ?? isLoading; + + const handleClick = async () => { + if (isDisabled) return; + setIsLoading(true); + setResult(null); + try { + const res = await onAction(); + setResult(res); + } catch (err) { + setResult({ + success: false, + message: err instanceof Error ? err.message : 'Action failed', + }); + } finally { + setIsLoading(false); + } + }; + + const styleClasses = { + default: + 'bg-[var(--bg-soft)] border border-[var(--border-muted)] hover:bg-[var(--hover-surface)]', + primary: 'bg-sky-600 text-white hover:bg-sky-700', + danger: 'bg-red-600 text-white hover:bg-red-700', + }; + + return ( +
+
+ + {field.description && ( + {field.description} + )} +
+ + {result && ( +
+ {result.message} +
+ )} + + {field.disabled && field.disabledReason && ( +

{field.disabledReason}

+ )} +
+ ); +}; diff --git a/src/frontend/src/components/settings/fields/CheckboxField.tsx b/src/frontend/src/components/settings/fields/CheckboxField.tsx new file mode 100644 index 00000000..fc6ac9f9 --- /dev/null +++ b/src/frontend/src/components/settings/fields/CheckboxField.tsx @@ -0,0 +1,33 @@ +import { CheckboxFieldConfig } from '../../../types/settings'; + +interface CheckboxFieldProps { + field: CheckboxFieldConfig; + value: boolean; + onChange: (value: boolean) => void; + disabled?: boolean; // Override for dynamic disabled state +} + +export const CheckboxField = ({ field: _field, value, onChange, disabled }: CheckboxFieldProps) => { + // disabled prop is already computed by SettingsContent.getDisabledState() + const isDisabled = disabled ?? false; + + return ( + + ); +}; diff --git a/src/frontend/src/components/settings/fields/HeadingField.tsx b/src/frontend/src/components/settings/fields/HeadingField.tsx new file mode 100644 index 00000000..838e62cf --- /dev/null +++ b/src/frontend/src/components/settings/fields/HeadingField.tsx @@ -0,0 +1,29 @@ +import { HeadingFieldConfig } from '../../../types/settings'; + +interface HeadingFieldProps { + field: HeadingFieldConfig; +} + +export const HeadingField = ({ field }: HeadingFieldProps) => ( +
+

{field.title}

+ {field.description && ( +

+ {field.description} + {field.linkUrl && ( + <> + {' '} + + {field.linkText || field.linkUrl} + + + )} +

+ )} +
+); diff --git a/src/frontend/src/components/settings/fields/MultiSelectField.tsx b/src/frontend/src/components/settings/fields/MultiSelectField.tsx new file mode 100644 index 00000000..5e4a6c50 --- /dev/null +++ b/src/frontend/src/components/settings/fields/MultiSelectField.tsx @@ -0,0 +1,198 @@ +import { useState, useRef, useEffect } from 'react'; +import { MultiSelectFieldConfig } from '../../../types/settings'; + +interface MultiSelectFieldProps { + field: MultiSelectFieldConfig; + value: string[]; + onChange: (value: string[]) => void; + disabled?: boolean; +} + +// Threshold for when to enable collapsible behavior +const COLLAPSE_THRESHOLD_OPTIONS = 12; +// Approximate height for ~4 rows of pills (pills are ~32px + 8px gap) +const COLLAPSED_HEIGHT = 156; + +/** + * Sort options with selected items first, preserving relative order within each group + */ +const sortOptionsWithSelectedFirst = ( + options: MultiSelectFieldConfig['options'], + selectedValues: string[] +): MultiSelectFieldConfig['options'] => { + const selectedSet = new Set(selectedValues); + const selectedOptions = options.filter((opt) => selectedSet.has(opt.value)); + const unselectedOptions = options.filter((opt) => !selectedSet.has(opt.value)); + return [...selectedOptions, ...unselectedOptions]; +}; + +export const MultiSelectField = ({ field, value, onChange, disabled }: MultiSelectFieldProps) => { + const selected = value ?? []; + // disabled prop is already computed by SettingsContent.getDisabledState() + const isDisabled = disabled ?? false; + const [isExpanded, setIsExpanded] = useState(false); + // Initialize based on option count to avoid flash of expanded content + const [needsCollapse, setNeedsCollapse] = useState( + () => field.options.length > COLLAPSE_THRESHOLD_OPTIONS + ); + const containerRef = useRef(null); + + // Track the last value we set via onChange to detect external changes + const lastInternalValueRef = useRef(null); + + // Sorted options - initialized with selected items first, updated only on external changes + const [sortedOptions, setSortedOptions] = useState(() => + sortOptionsWithSelectedFirst(field.options, selected) + ); + + // Detect external value changes (like after save or initial load) and re-sort + useEffect(() => { + // If the value changed and it's not from our own onChange call, re-sort + const lastInternal = lastInternalValueRef.current; + const isExternalChange = + lastInternal === null || // Initial mount + lastInternal.length !== selected.length || + !lastInternal.every((v) => selected.includes(v)); + + // Only re-sort if the change wasn't triggered by user interaction + if (isExternalChange && lastInternal !== null) { + // Check if this is truly external (values differ in a way that suggests a save/reset) + const wasInternalToggle = + Math.abs(lastInternal.length - selected.length) === 1 && + (lastInternal.every((v) => selected.includes(v)) || + selected.every((v) => lastInternal.includes(v))); + + if (!wasInternalToggle) { + setSortedOptions(sortOptionsWithSelectedFirst(field.options, selected)); + } + } + }, [selected, field.options]); + + // Update sortedOptions when field.options changes (e.g., different field) + useEffect(() => { + setSortedOptions(sortOptionsWithSelectedFirst(field.options, selected)); + }, [field.key]); + + // Verify collapse need after render (handles edge cases where few options still fit) + useEffect(() => { + if (containerRef.current) { + if (field.options.length > COLLAPSE_THRESHOLD_OPTIONS) { + const scrollHeight = containerRef.current.scrollHeight; + setNeedsCollapse(scrollHeight > COLLAPSED_HEIGHT + 20); + } else { + setNeedsCollapse(false); + } + } + }, [field.options.length]); + + const toggleOption = (optValue: string) => { + if (isDisabled) return; + let newValue: string[]; + if (selected.includes(optValue)) { + newValue = selected.filter((v) => v !== optValue); + } else { + newValue = [...selected, optValue]; + } + // Track this as an internal change so we don't re-sort + lastInternalValueRef.current = newValue; + onChange(newValue); + }; + + const isCollapsible = needsCollapse; + const isCollapsed = isCollapsible && !isExpanded; + + return ( +
+ {/* Container with optional max-height constraint */} +
+
+ {sortedOptions.map((opt) => { + const isSelected = selected.includes(opt.value); + return ( + + ); + })} +
+ + {/* Gradient fade overlay when collapsed */} + {isCollapsed && ( +
+ )} +
+ + {/* Expand/Collapse toggle - outside the relative container */} + {isCollapsible && ( + + )} +
+ ); +}; diff --git a/src/frontend/src/components/settings/fields/NumberField.tsx b/src/frontend/src/components/settings/fields/NumberField.tsx new file mode 100644 index 00000000..101a5d6a --- /dev/null +++ b/src/frontend/src/components/settings/fields/NumberField.tsx @@ -0,0 +1,30 @@ +import { NumberFieldConfig } from '../../../types/settings'; + +interface NumberFieldProps { + field: NumberFieldConfig; + value: number; + onChange: (value: number) => void; + disabled?: boolean; +} + +export const NumberField = ({ field, value, onChange, disabled }: NumberFieldProps) => { + // disabled prop is already computed by SettingsContent.getDisabledState() + const isDisabled = disabled ?? false; + + return ( + onChange(parseFloat(e.target.value) || 0)} + min={field.min} + max={field.max} + step={field.step ?? 1} + disabled={isDisabled} + className="w-full px-3 py-2 rounded-lg border border-[var(--border-muted)] + bg-[var(--bg-soft)] text-sm + focus:outline-none focus:ring-2 focus:ring-sky-500/50 focus:border-sky-500 + disabled:opacity-60 disabled:cursor-not-allowed + transition-colors" + /> + ); +}; diff --git a/src/frontend/src/components/settings/fields/PasswordField.tsx b/src/frontend/src/components/settings/fields/PasswordField.tsx new file mode 100644 index 00000000..ed5e587a --- /dev/null +++ b/src/frontend/src/components/settings/fields/PasswordField.tsx @@ -0,0 +1,79 @@ +import { useState } from 'react'; +import { PasswordFieldConfig } from '../../../types/settings'; + +interface PasswordFieldProps { + field: PasswordFieldConfig; + value: string; + onChange: (value: string) => void; + disabled?: boolean; +} + +export const PasswordField = ({ field, value, onChange, disabled }: PasswordFieldProps) => { + const [showPassword, setShowPassword] = useState(false); + // disabled prop is already computed by SettingsContent.getDisabledState() + const isDisabled = disabled ?? false; + + return ( +
+ onChange(e.target.value)} + placeholder={field.placeholder} + disabled={isDisabled} + className="w-full px-3 py-2 pr-10 rounded-lg border border-[var(--border-muted)] + bg-[var(--bg-soft)] text-sm + focus:outline-none focus:ring-2 focus:ring-sky-500/50 focus:border-sky-500 + disabled:opacity-60 disabled:cursor-not-allowed + transition-colors" + /> + +
+ ); +}; diff --git a/src/frontend/src/components/settings/fields/SelectField.tsx b/src/frontend/src/components/settings/fields/SelectField.tsx new file mode 100644 index 00000000..412b0d79 --- /dev/null +++ b/src/frontend/src/components/settings/fields/SelectField.tsx @@ -0,0 +1,43 @@ +import { SelectFieldConfig } from '../../../types/settings'; + +interface SelectFieldProps { + field: SelectFieldConfig; + value: string; + onChange: (value: string) => void; + disabled?: boolean; +} + +export const SelectField = ({ field, value, onChange, disabled }: SelectFieldProps) => { + // disabled prop is already computed by SettingsContent.getDisabledState() + const isDisabled = disabled ?? false; + + return ( + + ); +}; diff --git a/src/frontend/src/components/settings/fields/TextField.tsx b/src/frontend/src/components/settings/fields/TextField.tsx new file mode 100644 index 00000000..c0e9a67f --- /dev/null +++ b/src/frontend/src/components/settings/fields/TextField.tsx @@ -0,0 +1,29 @@ +import { TextFieldConfig } from '../../../types/settings'; + +interface TextFieldProps { + field: TextFieldConfig; + value: string; + onChange: (value: string) => void; + disabled?: boolean; +} + +export const TextField = ({ field, value, onChange, disabled }: TextFieldProps) => { + // disabled prop is already computed by SettingsContent.getDisabledState() + const isDisabled = disabled ?? false; + + return ( + onChange(e.target.value)} + placeholder={field.placeholder} + maxLength={field.maxLength} + disabled={isDisabled} + className="w-full px-3 py-2 rounded-lg border border-[var(--border-muted)] + bg-[var(--bg-soft)] text-sm + focus:outline-none focus:ring-2 focus:ring-sky-500/50 focus:border-sky-500 + disabled:opacity-60 disabled:cursor-not-allowed + transition-colors" + /> + ); +}; diff --git a/src/frontend/src/components/settings/fields/index.ts b/src/frontend/src/components/settings/fields/index.ts new file mode 100644 index 00000000..484ad69c --- /dev/null +++ b/src/frontend/src/components/settings/fields/index.ts @@ -0,0 +1,8 @@ +export { TextField } from './TextField'; +export { PasswordField } from './PasswordField'; +export { NumberField } from './NumberField'; +export { CheckboxField } from './CheckboxField'; +export { SelectField } from './SelectField'; +export { MultiSelectField } from './MultiSelectField'; +export { ActionButton } from './ActionButton'; +export { HeadingField } from './HeadingField'; diff --git a/src/frontend/src/components/settings/index.ts b/src/frontend/src/components/settings/index.ts new file mode 100644 index 00000000..c7833e55 --- /dev/null +++ b/src/frontend/src/components/settings/index.ts @@ -0,0 +1,4 @@ +export { SettingsModal } from './SettingsModal'; +export { SettingsHeader } from './SettingsHeader'; +export { SettingsSidebar } from './SettingsSidebar'; +export { SettingsContent } from './SettingsContent'; diff --git a/src/frontend/src/components/settings/shared/EnvLockBadge.tsx b/src/frontend/src/components/settings/shared/EnvLockBadge.tsx new file mode 100644 index 00000000..b993f38a --- /dev/null +++ b/src/frontend/src/components/settings/shared/EnvLockBadge.tsx @@ -0,0 +1,28 @@ +interface EnvLockBadgeProps { + className?: string; +} + +export const EnvLockBadge = ({ className = '' }: EnvLockBadgeProps) => ( + + + + + ENV + +); diff --git a/src/frontend/src/components/settings/shared/FieldWrapper.tsx b/src/frontend/src/components/settings/shared/FieldWrapper.tsx new file mode 100644 index 00000000..5882d477 --- /dev/null +++ b/src/frontend/src/components/settings/shared/FieldWrapper.tsx @@ -0,0 +1,106 @@ +import { ReactNode } from 'react'; +import { SettingsField } from '../../../types/settings'; +import { EnvLockBadge } from './EnvLockBadge'; + +interface FieldWrapperProps { + field: SettingsField; + children: ReactNode; + // Optional overrides for dynamic disabled state (from disabledWhen) + disabledOverride?: boolean; + disabledReasonOverride?: string; +} + +// Badge shown when a field is disabled +const DisabledBadge = ({ reason }: { reason?: string }) => ( + + + + + Unavailable + +); + +// Badge shown when changing a field requires a container restart +const RestartRequiredBadge = () => ( + + + + + Restart + +); + +export const FieldWrapper = ({ + field, + children, + disabledOverride, + disabledReasonOverride, +}: FieldWrapperProps) => { + // Action buttons and headings handle their own layout + if (field.type === 'ActionButton' || field.type === 'HeadingField') { + return <>{children}; + } + + // At this point, field is a regular input field with standard properties + // Use overrides if provided (from disabledWhen), otherwise use field's static values + const isDisabled = disabledOverride ?? field.disabled; + const disabledReason = disabledReasonOverride ?? field.disabledReason; + const requiresRestart = field.requiresRestart; + + // ENV-locked fields should only dim the control, not the label/description + const isFullyDimmed = isDisabled && !field.fromEnv; + + return ( +
+
+ + {field.fromEnv && } + {requiresRestart && !isDisabled && !field.fromEnv && } + {isDisabled && !field.fromEnv && } +
+ + {children} + + {field.description && ( +

{field.description}

+ )} + + {isDisabled && disabledReason && ( +

{disabledReason}

+ )} +
+ ); +}; diff --git a/src/frontend/src/components/settings/shared/index.ts b/src/frontend/src/components/settings/shared/index.ts new file mode 100644 index 00000000..8116ed81 --- /dev/null +++ b/src/frontend/src/components/settings/shared/index.ts @@ -0,0 +1,2 @@ +export { EnvLockBadge } from './EnvLockBadge'; +export { FieldWrapper } from './FieldWrapper'; diff --git a/src/frontend/src/components/shared/CircularProgress.tsx b/src/frontend/src/components/shared/CircularProgress.tsx new file mode 100644 index 00000000..7b79e93c --- /dev/null +++ b/src/frontend/src/components/shared/CircularProgress.tsx @@ -0,0 +1,39 @@ +interface CircularProgressProps { + progress?: number; + size?: number; + className?: string; +} + +export const CircularProgress = ({ progress, size = 16, className }: CircularProgressProps) => { + const radius = (size - 2) / 2; + const circumference = 2 * Math.PI * radius; + const progressValue = progress ?? 0; + const strokeDashoffset = circumference - (progressValue / 100) * circumference; + const svgClassName = className ? `transform -rotate-90 ${className}` : 'transform -rotate-90'; + + return ( + + + + + ); +}; diff --git a/src/frontend/src/components/shared/DisplayFieldIcon.tsx b/src/frontend/src/components/shared/DisplayFieldIcon.tsx new file mode 100644 index 00000000..21620b86 --- /dev/null +++ b/src/frontend/src/components/shared/DisplayFieldIcon.tsx @@ -0,0 +1,68 @@ +import { DisplayField } from '../../types'; + +interface DisplayFieldIconProps { + icon?: string; + className?: string; +} + +export function DisplayFieldIcon({ icon, className = 'w-3 h-3' }: DisplayFieldIconProps) { + switch (icon) { + case 'star': + return ( + + + + ); + case 'book': + return ( + + + + ); + case 'users': + return ( + + + + ); + case 'editions': + return ( + + + + ); + default: + return null; + } +} + +interface DisplayFieldBadgeProps { + field: DisplayField; + className?: string; +} + +export function DisplayFieldBadge({ field, className = '' }: DisplayFieldBadgeProps) { + return ( + + + {field.value} + + ); +} + +interface DisplayFieldBadgesProps { + fields?: DisplayField[]; + className?: string; +} + +export function DisplayFieldBadges({ fields, className = '' }: DisplayFieldBadgesProps) { + if (!fields || fields.length === 0) return null; + + return ( +
+ {fields.map((field, idx) => ( + + ))} +
+ ); +} diff --git a/src/frontend/src/components/shared/SearchFieldRenderer.tsx b/src/frontend/src/components/shared/SearchFieldRenderer.tsx new file mode 100644 index 00000000..186703ef --- /dev/null +++ b/src/frontend/src/components/shared/SearchFieldRenderer.tsx @@ -0,0 +1,104 @@ +import { KeyboardEvent } from 'react'; +import { MetadataSearchField } from '../../types'; +import { DropdownList } from '../DropdownList'; + +interface SearchFieldRendererProps { + field: MetadataSearchField; + value: string | number | boolean; + onChange: (value: string | number | boolean) => void; + onSubmit?: () => void; +} + +const baseInputClass = + 'w-full px-3 py-2 rounded-md border border-[var(--border-muted)] ' + + 'bg-[var(--bg-soft)] text-sm ' + + 'focus:outline-none focus:ring-2 focus:ring-emerald-500/50 focus:border-emerald-500 ' + + 'transition-colors'; + +export const SearchFieldRenderer = ({ field, value, onChange, onSubmit }: SearchFieldRendererProps) => { + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Enter' && onSubmit) { + e.preventDefault(); + onSubmit(); + } + }; + switch (field.type) { + case 'TextSearchField': + return ( + onChange(e.target.value)} + onKeyDown={handleKeyDown} + placeholder={field.placeholder} + autoComplete="off" + enterKeyHint="search" + className={baseInputClass} + style={{ + background: 'var(--bg-soft)', + color: 'var(--text)', + borderColor: 'var(--border-muted)', + }} + /> + ); + + case 'NumberSearchField': { + const handleNumberChange = (e: React.ChangeEvent) => { + const raw = e.target.value; + if (!raw) { + onChange(''); + return; + } + const num = parseInt(raw, 10); + if (!isNaN(num)) { + onChange(num); + } + }; + return ( + + ); + } + + case 'SelectSearchField': + return ( + onChange(Array.isArray(v) ? v[0] ?? '' : v)} + placeholder="All" + /> + ); + + case 'CheckboxSearchField': + return ( + + ); + + default: + return null; + } +}; diff --git a/src/frontend/src/components/shared/index.ts b/src/frontend/src/components/shared/index.ts new file mode 100644 index 00000000..8fc67388 --- /dev/null +++ b/src/frontend/src/components/shared/index.ts @@ -0,0 +1,3 @@ +export { DisplayFieldIcon, DisplayFieldBadge, DisplayFieldBadges } from './DisplayFieldIcon'; +export { CircularProgress } from './CircularProgress'; +export { SearchFieldRenderer } from './SearchFieldRenderer'; diff --git a/src/frontend/src/contexts/SearchModeContext.tsx b/src/frontend/src/contexts/SearchModeContext.tsx new file mode 100644 index 00000000..afa36ae8 --- /dev/null +++ b/src/frontend/src/contexts/SearchModeContext.tsx @@ -0,0 +1,30 @@ +import { createContext, useContext, ReactNode } from 'react'; +import { SearchMode } from '../types'; + +interface SearchModeContextValue { + searchMode: SearchMode; + isUniversalMode: boolean; +} + +const SearchModeContext = createContext(null); + +export function useSearchMode(): SearchModeContextValue { + const ctx = useContext(SearchModeContext); + if (!ctx) { + throw new Error('useSearchMode must be used within SearchModeProvider'); + } + return ctx; +} + +interface SearchModeProviderProps { + searchMode: SearchMode; + children: ReactNode; +} + +export function SearchModeProvider({ searchMode, children }: SearchModeProviderProps) { + return ( + + {children} + + ); +} diff --git a/src/frontend/src/data/filterOptions.ts b/src/frontend/src/data/filterOptions.ts index b84e584e..73f52ddb 100644 --- a/src/frontend/src/data/filterOptions.ts +++ b/src/frontend/src/data/filterOptions.ts @@ -1,3 +1,4 @@ +// Direct download mode sort options (Anna's Archive) export const SORT_OPTIONS = [ { value: '', label: 'Most relevant' }, { value: 'newest', label: 'Newest (publication year)' }, @@ -8,6 +9,10 @@ export const SORT_OPTIONS = [ { value: 'oldest_added', label: 'Oldest (open sourced)' }, ]; +// Note: Metadata mode sort options are now dynamic per provider +// They come from the /api/config endpoint as metadata_sort_options + +// Direct download mode content type options (Anna's Archive) export const CONTENT_OPTIONS = [ { value: '', label: 'All' }, { value: 'book_nonfiction', label: 'Book (non-fiction)' }, diff --git a/src/frontend/src/hooks/useAuth.ts b/src/frontend/src/hooks/useAuth.ts new file mode 100644 index 00000000..514bcb1d --- /dev/null +++ b/src/frontend/src/hooks/useAuth.ts @@ -0,0 +1,98 @@ +import { useState, useEffect, useCallback } from 'react'; +import { useNavigate } from 'react-router-dom'; +import { LoginCredentials } from '../types'; +import { login, logout, checkAuth } from '../services/api'; + +interface UseAuthOptions { + onLogoutSuccess?: () => void; + showToast?: (message: string, type: 'info' | 'success' | 'error') => void; +} + +interface UseAuthReturn { + isAuthenticated: boolean; + authRequired: boolean; + authChecked: boolean; + loginError: string | null; + isLoggingIn: boolean; + setIsAuthenticated: (value: boolean) => void; + handleLogin: (credentials: LoginCredentials) => Promise; + handleLogout: () => Promise; +} + +export function useAuth(options: UseAuthOptions = {}): UseAuthReturn { + const { onLogoutSuccess, showToast } = options; + const navigate = useNavigate(); + + const [isAuthenticated, setIsAuthenticated] = useState(false); + const [authRequired, setAuthRequired] = useState(true); + const [authChecked, setAuthChecked] = useState(false); + const [loginError, setLoginError] = useState(null); + const [isLoggingIn, setIsLoggingIn] = useState(false); + + // Check authentication on mount + useEffect(() => { + const verifyAuth = async () => { + try { + const response = await checkAuth(); + const authenticated = response.authenticated || false; + const authIsRequired = response.auth_required !== false; + + setAuthRequired(authIsRequired); + setIsAuthenticated(authenticated); + } catch (error) { + console.error('Auth check failed:', error); + setAuthRequired(true); + setIsAuthenticated(false); + } finally { + setAuthChecked(true); + } + }; + verifyAuth(); + }, []); + + const handleLogin = useCallback(async (credentials: LoginCredentials) => { + setIsLoggingIn(true); + setLoginError(null); + try { + const response = await login(credentials); + if (response.success) { + setIsAuthenticated(true); + setLoginError(null); + navigate('/', { replace: true }); + } else { + setLoginError(response.error || 'Login failed'); + } + } catch (error) { + if (error instanceof Error) { + setLoginError(error.message || 'Login failed'); + } else { + setLoginError('Login failed'); + } + } finally { + setIsLoggingIn(false); + } + }, [navigate]); + + const handleLogout = useCallback(async () => { + try { + await logout(); + setIsAuthenticated(false); + onLogoutSuccess?.(); + navigate('/login', { replace: true }); + } catch (error) { + console.error('Logout failed:', error); + showToast?.('Logout failed', 'error'); + } + }, [navigate, onLogoutSuccess, showToast]); + + return { + isAuthenticated, + authRequired, + authChecked, + loginError, + isLoggingIn, + setIsAuthenticated, + handleLogin, + handleLogout, + }; +} diff --git a/src/frontend/src/hooks/useDownloadTracking.ts b/src/frontend/src/hooks/useDownloadTracking.ts new file mode 100644 index 00000000..4ff8f6db --- /dev/null +++ b/src/frontend/src/hooks/useDownloadTracking.ts @@ -0,0 +1,124 @@ +import { useState, useCallback } from 'react'; +import { StatusData, ButtonStateInfo } from '../types'; + +interface UseDownloadTrackingReturn { + bookToReleaseMap: Record; + sessionCompletedBookIds: Set; + trackRelease: (bookId: string, releaseId: string) => void; + markBookCompleted: (bookId: string) => void; + clearTracking: () => void; + getButtonState: (bookId: string) => ButtonStateInfo; + getUniversalButtonState: (bookId: string) => ButtonStateInfo; +} + +export function useDownloadTracking(currentStatus: StatusData): UseDownloadTrackingReturn { + // Track mapping of metadata book IDs to release source IDs for universal mode + const [bookToReleaseMap, setBookToReleaseMap] = useState>({}); + // Session-only tracking of completed book IDs (survives clearCompleted, resets on refresh) + const [sessionCompletedBookIds, setSessionCompletedBookIds] = useState>(new Set()); + + const trackRelease = useCallback((bookId: string, releaseId: string) => { + setBookToReleaseMap(prev => ({ + ...prev, + [bookId]: [...(prev[bookId] || []), releaseId], + })); + }, []); + + const markBookCompleted = useCallback((bookId: string) => { + setSessionCompletedBookIds(prev => new Set([...prev, bookId])); + }, []); + + const clearTracking = useCallback(() => { + setBookToReleaseMap({}); + setSessionCompletedBookIds(new Set()); + }, []); + + // Get button state for a book in direct mode + const getButtonState = useCallback((bookId: string): ButtonStateInfo => { + if (currentStatus.error && currentStatus.error[bookId]) { + return { text: 'Failed', state: 'error' }; + } + if (currentStatus.complete && currentStatus.complete[bookId]) { + return { text: 'Downloaded', state: 'complete' }; + } + if (currentStatus.downloading && currentStatus.downloading[bookId]) { + const book = currentStatus.downloading[bookId]; + return { + text: 'Downloading', + state: 'downloading', + progress: book.progress, + }; + } + if (currentStatus.resolving && currentStatus.resolving[bookId]) { + return { text: 'Resolving', state: 'resolving' }; + } + if (currentStatus.queued && currentStatus.queued[bookId]) { + return { text: 'Queued', state: 'queued' }; + } + return { text: 'Download', state: 'download' }; + }, [currentStatus]); + + // Get button state for a metadata book in universal mode + const getUniversalButtonState = useCallback((bookId: string): ButtonStateInfo => { + const releaseIds = bookToReleaseMap[bookId] || []; + + // No releases downloaded yet - check session state + if (releaseIds.length === 0) { + if (sessionCompletedBookIds.has(bookId)) { + return { text: 'Downloaded', state: 'complete' }; + } + return { text: 'Get', state: 'download' }; + } + + // Check each release ID and find the most relevant state + let bestState: ButtonStateInfo = { text: 'Get', state: 'download' }; + let foundActiveState = false; + + for (const releaseId of releaseIds) { + if (currentStatus.downloading && currentStatus.downloading[releaseId]) { + const downloadingBook = currentStatus.downloading[releaseId]; + return { + text: 'Downloading', + state: 'downloading', + progress: downloadingBook.progress, + }; + } + + if (currentStatus.resolving && currentStatus.resolving[releaseId]) { + return { text: 'Resolving', state: 'resolving' }; + } + + if (currentStatus.queued && currentStatus.queued[releaseId]) { + return { text: 'Queued', state: 'queued' }; + } + + if (!foundActiveState) { + if (currentStatus.complete && currentStatus.complete[releaseId]) { + bestState = { text: 'Downloaded', state: 'complete' }; + foundActiveState = true; + } else if (currentStatus.error && currentStatus.error[releaseId]) { + if (bestState.state === 'download') { + bestState = { text: 'Failed', state: 'error' }; + } + } + } + } + + // Check session tracking if no state found + if (bestState.state === 'download' && sessionCompletedBookIds.has(bookId)) { + return { text: 'Downloaded', state: 'complete' }; + } + + return bestState; + }, [currentStatus, bookToReleaseMap, sessionCompletedBookIds]); + + return { + bookToReleaseMap, + sessionCompletedBookIds, + trackRelease, + markBookCompleted, + clearTracking, + getButtonState, + getUniversalButtonState, + }; +} diff --git a/src/frontend/src/hooks/useSearch.ts b/src/frontend/src/hooks/useSearch.ts new file mode 100644 index 00000000..894fc10e --- /dev/null +++ b/src/frontend/src/hooks/useSearch.ts @@ -0,0 +1,236 @@ +import { useState, useCallback } from 'react'; +import { useNavigate } from 'react-router-dom'; +import { Book, AppConfig, AdvancedFilterState } from '../types'; +import { searchBooks, searchMetadata, AuthenticationError } from '../services/api'; +import { LANGUAGE_OPTION_DEFAULT } from '../utils/languageFilters'; +import { DEFAULT_SUPPORTED_FORMATS } from '../data/languages'; + +const DEFAULT_FORMAT_SELECTION = DEFAULT_SUPPORTED_FORMATS.filter(format => format !== 'pdf'); + +interface UseSearchOptions { + showToast: (message: string, type: 'info' | 'success' | 'error') => void; + setIsAuthenticated: (value: boolean) => void; + authRequired: boolean; + onSearchReset?: () => void; +} + +// Search field values for universal mode (provider-specific fields) +type SearchFieldValues = Record; + +interface UseSearchReturn { + books: Book[]; + setBooks: (books: Book[]) => void; + isSearching: boolean; + lastSearchQuery: string; + searchInput: string; + setSearchInput: (value: string) => void; + showAdvanced: boolean; + setShowAdvanced: (value: boolean) => void; + advancedFilters: AdvancedFilterState; + setAdvancedFilters: React.Dispatch>; + updateAdvancedFilters: (updates: Partial) => void; + handleSearch: (query: string, config: AppConfig | null, fieldValues?: Record) => Promise; + handleResetSearch: (config: AppConfig | null) => void; + handleSortChange: (value: string, config: AppConfig | null) => void; + resetSortFilter: () => void; + // Universal mode search field values + searchFieldValues: SearchFieldValues; + updateSearchFieldValue: (key: string, value: string | number | boolean) => void; +} + +export function useSearch(options: UseSearchOptions): UseSearchReturn { + const { showToast, setIsAuthenticated, authRequired, onSearchReset } = options; + const navigate = useNavigate(); + + const [books, setBooks] = useState([]); + const [isSearching, setIsSearching] = useState(false); + const [lastSearchQuery, setLastSearchQuery] = useState(''); + const [searchInput, setSearchInput] = useState(''); + const [showAdvanced, setShowAdvanced] = useState(false); + const [advancedFilters, setAdvancedFilters] = useState({ + isbn: '', + author: '', + title: '', + lang: [LANGUAGE_OPTION_DEFAULT], + sort: '', + content: '', + formats: DEFAULT_FORMAT_SELECTION, + }); + + // Universal mode: provider-specific search field values + const [searchFieldValues, setSearchFieldValues] = useState({}); + + const updateAdvancedFilters = useCallback((updates: Partial) => { + setAdvancedFilters(prev => ({ ...prev, ...updates })); + }, []); + + const updateSearchFieldValue = useCallback((key: string, value: string | number | boolean) => { + console.log('[useSearch] updateSearchFieldValue:', key, '=', value); + setSearchFieldValues(prev => { + const next = { ...prev, [key]: value }; + console.log('[useSearch] searchFieldValues updated:', next); + return next; + }); + }, []); + + const resetSortFilter = useCallback(() => { + setAdvancedFilters(prev => ({ ...prev, sort: '' })); + }, []); + + const handleSearch = useCallback(async ( + query: string, + config: AppConfig | null, + fieldValues?: Record + ) => { + const searchMode = config?.search_mode || 'direct'; + + // In universal mode, check if we have either a query or field values + if (searchMode === 'universal') { + const params = new URLSearchParams(query); + const searchQuery = params.get('query') || ''; + const sort = params.get('sort') || 'relevance'; + // Use explicitly passed fieldValues if provided, otherwise fall back to state + const effectiveFieldValues = fieldValues ?? searchFieldValues; + const hasFieldValues = Object.values(effectiveFieldValues).some(v => v !== '' && v !== false); + + // Debug logging + console.log('[useSearch] Universal mode search:', { + query, + searchQuery, + sort, + searchFieldValues, + fieldValues, + effectiveFieldValues, + hasFieldValues, + }); + + if (!searchQuery && !hasFieldValues) { + console.log('[useSearch] Early return: no query and no field values'); + setBooks([]); + setLastSearchQuery(''); + return; + } + + setIsSearching(true); + setLastSearchQuery(query); + + try { + const results = await searchMetadata(searchQuery, 20, sort, effectiveFieldValues); + if (results.length > 0) { + setBooks(results); + } else { + showToast('No results found', 'error'); + } + } catch (error) { + if (error instanceof AuthenticationError) { + setIsAuthenticated(false); + if (authRequired) { + navigate('/login', { replace: true }); + } + } else { + console.error('Search failed:', error); + // API now returns user-friendly error messages directly + const message = error instanceof Error ? error.message : 'Search failed'; + showToast(message, 'error'); + } + } finally { + setIsSearching(false); + } + return; + } + + // Direct mode: require a query + if (!query) { + setBooks([]); + setLastSearchQuery(''); + return; + } + setIsSearching(true); + setLastSearchQuery(query); + + try { + const results = await searchBooks(query); + + if (results.length > 0) { + setBooks(results); + } else { + showToast('No results found', 'error'); + } + } catch (error) { + if (error instanceof AuthenticationError) { + setIsAuthenticated(false); + if (authRequired) { + navigate('/login', { replace: true }); + } + } else { + console.error('Search failed:', error); + const message = error instanceof Error ? error.message : 'Search failed'; + const friendly = message.includes("Anna's Archive") || message.includes('Network restricted') + ? message + : "Unable to reach Anna's Archive. Network may be restricted or mirrors blocked."; + showToast(friendly, 'error'); + } + } finally { + setIsSearching(false); + } + }, [showToast, setIsAuthenticated, authRequired, navigate, searchFieldValues]); + + const handleResetSearch = useCallback((config: AppConfig | null) => { + setBooks([]); + setSearchInput(''); + setShowAdvanced(false); + setLastSearchQuery(''); + onSearchReset?.(); + + const resetFormats = config?.supported_formats || DEFAULT_FORMAT_SELECTION; + setAdvancedFilters({ + isbn: '', + author: '', + title: '', + lang: [LANGUAGE_OPTION_DEFAULT], + sort: '', + content: '', + formats: resetFormats, + }); + + // Reset universal mode search field values + setSearchFieldValues({}); + }, [onSearchReset]); + + const handleSortChange = useCallback((value: string, config: AppConfig | null) => { + updateAdvancedFilters({ sort: value }); + if (!lastSearchQuery) return; + + const params = new URLSearchParams(lastSearchQuery); + if (value) { + params.set('sort', value); + } else { + params.delete('sort'); + } + + const nextQuery = params.toString(); + if (!nextQuery) return; + handleSearch(nextQuery, config); + }, [lastSearchQuery, updateAdvancedFilters, handleSearch]); + + return { + books, + setBooks, + isSearching, + lastSearchQuery, + searchInput, + setSearchInput, + showAdvanced, + setShowAdvanced, + advancedFilters, + setAdvancedFilters, + updateAdvancedFilters, + handleSearch, + handleResetSearch, + handleSortChange, + resetSortFilter, + // Universal mode search field values + searchFieldValues, + updateSearchFieldValue, + }; +} diff --git a/src/frontend/src/hooks/useSettings.ts b/src/frontend/src/hooks/useSettings.ts new file mode 100644 index 00000000..f812452e --- /dev/null +++ b/src/frontend/src/hooks/useSettings.ts @@ -0,0 +1,264 @@ +import { useState, useCallback, useEffect } from 'react'; +import { getSettings, updateSettings, executeSettingsAction } from '../services/api'; +import { + SettingsTab, + SettingsGroup, + SettingsField, + SelectFieldConfig, + ActionResult, + UpdateResult, +} from '../types/settings'; + +// Client-side only theme field that gets injected into the general tab +const THEME_FIELD: SelectFieldConfig = { + type: 'SelectField', + key: '_THEME', + label: 'Theme', + description: 'Choose your preferred color scheme.', + value: 'auto', // Placeholder, actual value comes from localStorage + options: [ + { value: 'light', label: 'Light' }, + { value: 'dark', label: 'Dark' }, + { value: 'auto', label: 'Auto (System)' }, + ], +}; + +// Apply theme to document +function applyTheme(theme: string): void { + const effectiveTheme = theme === 'auto' + ? (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light') + : theme; + document.documentElement.setAttribute('data-theme', effectiveTheme); +} + +// Extract value from a field based on its type +function getFieldValue(field: SettingsField): unknown { + // These field types have no value property + if (field.type === 'ActionButton' || field.type === 'HeadingField') { + return undefined; + } + // All other fields have a value property + return field.value ?? ''; +} + +interface UseSettingsReturn { + tabs: SettingsTab[]; + groups: SettingsGroup[]; + isLoading: boolean; + error: string | null; + selectedTab: string | null; + setSelectedTab: (tab: string | null) => void; + values: Record>; + updateValue: (tabName: string, key: string, value: unknown) => void; + hasChanges: (tabName: string) => boolean; + saveTab: (tabName: string) => Promise; + executeAction: (tabName: string, actionKey: string) => Promise; + isSaving: boolean; + refetch: () => Promise; +} + +export function useSettings(): UseSettingsReturn { + const [tabs, setTabs] = useState([]); + const [groups, setGroups] = useState([]); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [selectedTab, setSelectedTab] = useState(null); + const [values, setValues] = useState>>({}); + const [originalValues, setOriginalValues] = useState>>({}); + const [isSaving, setIsSaving] = useState(false); + + const fetchSettings = useCallback(async (silent = false) => { + if (!silent) { + setIsLoading(true); + } + setError(null); + try { + const response = await getSettings(); + + // Inject theme field into the general tab at the beginning + const tabsWithTheme = response.tabs.map((tab) => { + if (tab.name === 'general') { + return { + ...tab, + fields: [THEME_FIELD, ...tab.fields], + }; + } + return tab; + }); + setTabs(tabsWithTheme); + setGroups(response.groups || []); + + // Initialize values from fetched data + const initialValues: Record> = {}; + tabsWithTheme.forEach((tab) => { + initialValues[tab.name] = {}; + tab.fields.forEach((field) => { + if (field.type !== 'ActionButton') { + // Special handling for theme field - get from localStorage + if (field.key === '_THEME') { + initialValues[tab.name][field.key] = localStorage.getItem('preferred-theme') || 'auto'; + } else { + initialValues[tab.name][field.key] = getFieldValue(field); + } + } + }); + }); + setValues(initialValues); + setOriginalValues(JSON.parse(JSON.stringify(initialValues))); + + // Select first tab by default if none selected + if (tabsWithTheme.length > 0) { + setSelectedTab((current) => current ?? tabsWithTheme[0].name); + } + } catch (err) { + console.error('Failed to fetch settings:', err); + setError(err instanceof Error ? err.message : 'Failed to load settings'); + } finally { + if (!silent) { + setIsLoading(false); + } + } + }, []); + + useEffect(() => { + fetchSettings(); + }, [fetchSettings]); + + const updateValue = useCallback((tabName: string, key: string, value: unknown) => { + // Apply theme immediately when changed (no save button needed) + if (key === '_THEME' && typeof value === 'string') { + localStorage.setItem('preferred-theme', value); + applyTheme(value); + // Also update original value so it doesn't show as pending change + setOriginalValues((prev) => ({ + ...prev, + [tabName]: { + ...prev[tabName], + [key]: value, + }, + })); + } + + setValues((prev) => ({ + ...prev, + [tabName]: { + ...prev[tabName], + [key]: value, + }, + })); + }, []); + + const hasChanges = useCallback( + (tabName: string) => { + const current = values[tabName]; + const original = originalValues[tabName]; + if (!current || !original) return false; + + const tab = tabs.find((t) => t.name === tabName); + if (!tab) return false; + + for (const field of tab.fields) { + if (field.type === 'ActionButton' || field.type === 'HeadingField') continue; + + const currentValue = current[field.key]; + const originalValue = original[field.key]; + + // Compare values - works for all field types including password + if (JSON.stringify(currentValue) !== JSON.stringify(originalValue)) { + return true; + } + } + + return false; + }, + [values, originalValues, tabs] + ); + + const saveTab = useCallback( + async (tabName: string): Promise => { + setIsSaving(true); + try { + const tabValues = values[tabName] || {}; + const originalTabValues = originalValues[tabName] || {}; + + // Only send values that actually changed + const tab = tabs.find((t) => t.name === tabName); + const valuesToSave: Record = {}; + + if (tab) { + for (const field of tab.fields) { + if (field.type === 'ActionButton' || field.type === 'HeadingField') continue; + if (field.fromEnv) continue; // Skip env-locked fields + if (field.key === '_THEME') continue; // Skip client-side only theme field + + const value = tabValues[field.key]; + const originalValue = originalTabValues[field.key]; + + // Skip empty password fields + if (field.type === 'PasswordField' && (!value || value === '')) { + continue; + } + + // Only include if value actually changed + if (JSON.stringify(value) !== JSON.stringify(originalValue)) { + valuesToSave[field.key] = value; + } + } + } + + const result = await updateSettings(tabName, valuesToSave); + + if (result.success) { + // Refetch all settings silently to pick up any backend-triggered changes + // (e.g., enabling a metadata provider auto-updates METADATA_PROVIDER) + await fetchSettings(true); + } + + return result; + } catch (err) { + console.error('Failed to save settings tab:', tabName, err); + return { + success: false, + message: err instanceof Error ? err.message : 'Failed to save settings', + updated: [], + }; + } finally { + setIsSaving(false); + } + }, + [values, originalValues, tabs] + ); + + const executeAction = useCallback( + async (tabName: string, actionKey: string): Promise => { + try { + // Pass current form values so action can use unsaved values + const currentValues = values[tabName] || {}; + return await executeSettingsAction(tabName, actionKey, currentValues); + } catch (err) { + console.error('Action execution failed:', tabName, actionKey, err); + return { + success: false, + message: err instanceof Error ? err.message : 'Action failed', + }; + } + }, + [values] + ); + + return { + tabs, + groups, + isLoading, + error, + selectedTab, + setSelectedTab, + values, + updateValue, + hasChanges, + saveTab, + executeAction, + isSaving, + refetch: fetchSettings, + }; +} diff --git a/src/frontend/src/hooks/useUrlSearch.ts b/src/frontend/src/hooks/useUrlSearch.ts new file mode 100644 index 00000000..d07ed9c9 --- /dev/null +++ b/src/frontend/src/hooks/useUrlSearch.ts @@ -0,0 +1,54 @@ +import { useEffect, useRef } from 'react'; +import { useSearchParams } from 'react-router-dom'; +import { parseUrlSearchParams, ParsedUrlSearch } from '../utils/parseUrlSearchParams'; + +interface UseUrlSearchOptions { + /** Only process URL params after auth check and config are loaded */ + enabled: boolean; +} + +interface UseUrlSearchReturn { + /** Parsed URL parameters, or null if none found */ + parsedParams: ParsedUrlSearch | null; + /** Whether URL has been processed (regardless of whether params existed) */ + wasProcessed: boolean; +} + +/** + * Hook to parse URL search parameters on initial page load. + * + * This is a read-only operation - URL params are parsed once when enabled, + * and the URL is not updated when users perform searches. + * + * @example + * // In App.tsx: + * const { parsedParams, wasProcessed } = useUrlSearch({ + * enabled: isAuthenticated && config !== null, + * }); + * + * useEffect(() => { + * if (wasProcessed && parsedParams?.hasSearchParams) { + * // Trigger search with parsed params + * } + * }, [wasProcessed, parsedParams]); + */ +export function useUrlSearch({ enabled }: UseUrlSearchOptions): UseUrlSearchReturn { + const [searchParams] = useSearchParams(); + const processedRef = useRef(false); + const parsedRef = useRef(null); + + useEffect(() => { + if (enabled && !processedRef.current) { + const parsed = parseUrlSearchParams(searchParams); + if (parsed.hasSearchParams) { + parsedRef.current = parsed; + } + processedRef.current = true; + } + }, [enabled, searchParams]); + + return { + parsedParams: parsedRef.current, + wasProcessed: processedRef.current, + }; +} diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index 24aefe7f..dcb9c3fe 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -1,10 +1,13 @@ -import { Book, StatusData, AppConfig, LoginCredentials, AuthResponse } from '../types'; +import { Book, StatusData, AppConfig, LoginCredentials, AuthResponse, ReleaseSource, ReleasesResponse } from '../types'; +import { SettingsResponse, ActionResult, UpdateResult } from '../types/settings'; +import { MetadataBookData, transformMetadataToBook } from '../utils/bookTransformers'; const API_BASE = '/api'; // API endpoints const API = { search: `${API_BASE}/search`, + metadataSearch: `${API_BASE}/metadata/search`, info: `${API_BASE}/info`, download: `${API_BASE}/download`, status: `${API_BASE}/status`, @@ -14,7 +17,8 @@ const API = { config: `${API_BASE}/config`, login: `${API_BASE}/auth/login`, logout: `${API_BASE}/auth/logout`, - authCheck: `${API_BASE}/auth/check` + authCheck: `${API_BASE}/auth/check`, + settings: `${API_BASE}/settings`, }; // Custom error class for authentication failures @@ -41,18 +45,22 @@ async function fetchJSON(url: string, opts: RequestInit = {}): Promise { let errorMessage = `${res.status} ${res.statusText}`; try { const errorData = await res.json(); - if (errorData.error) { + // Prefer user-friendly 'message' field, fall back to 'error' + if (errorData.message) { + errorMessage = errorData.message; + } else if (errorData.error) { errorMessage = errorData.error; } } catch (e) { - // If we can't parse JSON, use the default error message + // Log parse failure for debugging - server may have returned non-JSON (e.g., HTML error page) + console.warn(`Failed to parse error response from ${url}:`, e instanceof Error ? e.message : e); } - + // Throw appropriate error based on status code if (res.status === 401) { throw new AuthenticationError(errorMessage); } - + throw new Error(errorMessage); } @@ -65,14 +73,92 @@ export const searchBooks = async (query: string): Promise => { return fetchJSON(`${API.search}?${query}`); }; +// Metadata search response type (internal) +interface MetadataSearchResponse { + books: MetadataBookData[]; + provider: string; + query: string; +} + +// Search metadata providers and normalize to Book format +export const searchMetadata = async ( + query: string, + limit: number = 20, + sort: string = 'relevance', + fields: Record = {} +): Promise => { + const hasFields = Object.values(fields).some(v => v !== '' && v !== false); + + // Debug logging + console.log('[api.searchMetadata] Called with:', { query, limit, sort, fields, hasFields }); + + if (!query && !hasFields) { + console.log('[api.searchMetadata] Early return: no query and no fields'); + return []; + } + + const params = new URLSearchParams(); + if (query) { + params.set('query', query); + } + params.set('limit', String(limit)); + params.set('sort', sort); + + // Add custom search field values + Object.entries(fields).forEach(([key, value]) => { + if (value !== '' && value !== false) { + params.set(key, String(value)); + } + }); + + const requestUrl = `${API.metadataSearch}?${params.toString()}`; + console.log('[api.searchMetadata] Request URL:', requestUrl); + + const response = await fetchJSON(requestUrl); + console.log('[api.searchMetadata] Response:', response); + + return response.books.map(transformMetadataToBook); +}; + export const getBookInfo = async (id: string): Promise => { return fetchJSON(`${API.info}?id=${encodeURIComponent(id)}`); }; +// Get full book details from a metadata provider +export const getMetadataBookInfo = async (provider: string, bookId: string): Promise => { + const response = await fetchJSON( + `${API_BASE}/metadata/book/${encodeURIComponent(provider)}/${encodeURIComponent(bookId)}` + ); + + return transformMetadataToBook(response); +}; + export const downloadBook = async (id: string): Promise => { await fetchJSON(`${API.download}?id=${encodeURIComponent(id)}`); }; +// Download a specific release (from ReleaseModal) +export const downloadRelease = async (release: { + source: string; + source_id: string; + title: string; + format?: string; + size?: string; + size_bytes?: number; + download_url?: string; + protocol?: string; + indexer?: string; + seeders?: number; + extra?: Record; + preview?: string; // Book cover from metadata provider + author?: string; // Author from metadata provider +}): Promise => { + await fetchJSON(`${API_BASE}/releases/download`, { + method: 'POST', + body: JSON.stringify(release), + }); +}; + export const getStatus = async (): Promise => { return fetchJSON(API.status); }; @@ -106,3 +192,60 @@ export const logout = async (): Promise => { export const checkAuth = async (): Promise => { return fetchJSON(API.authCheck); }; + +// Settings API functions +export const getSettings = async (): Promise => { + return fetchJSON(API.settings); +}; + +export const updateSettings = async ( + tabName: string, + values: Record +): Promise => { + return fetchJSON(`${API.settings}/${tabName}`, { + method: 'PUT', + body: JSON.stringify(values), + }); +}; + +export const executeSettingsAction = async ( + tabName: string, + actionKey: string, + currentValues?: Record +): Promise => { + return fetchJSON(`${API.settings}/${tabName}/action/${actionKey}`, { + method: 'POST', + body: currentValues ? JSON.stringify(currentValues) : undefined, + }); +}; + +// Release source API functions + +// Get available release sources from plugin registry +export const getReleaseSources = async (): Promise => { + return fetchJSON(`${API_BASE}/release-sources`); +}; + +// Search for releases of a book +export const getReleases = async ( + provider: string, + bookId: string, + source?: string, + title?: string, + author?: string +): Promise => { + const params = new URLSearchParams({ + provider, + book_id: bookId, + }); + if (source) { + params.set('source', source); + } + if (title) { + params.set('title', title); + } + if (author) { + params.set('author', author); + } + return fetchJSON(`${API_BASE}/releases?${params.toString()}`); +}; diff --git a/src/frontend/src/styles.css b/src/frontend/src/styles.css index 278f0ca6..f5372f6a 100644 --- a/src/frontend/src/styles.css +++ b/src/frontend/src/styles.css @@ -3,6 +3,17 @@ @tailwind components; @tailwind utilities; +/* Custom utilities */ +@layer utilities { + .scrollbar-hide { + -ms-overflow-style: none; + scrollbar-width: none; + } + .scrollbar-hide::-webkit-scrollbar { + display: none; + } +} + /* Disable transitions on initial load to prevent flash */ html.preload *, html.preload *::before, @@ -247,7 +258,27 @@ footer { } /* Responsive Design */ -@media (max-width: 768px) { +@media (max-width: 639px) { + .modal-overlay { + padding: 0; + align-items: stretch; + /* Extend background into iOS safe areas */ + background: var(--bg-soft); + } + + .details-container { + max-width: none; + max-height: none; + height: 100%; + /* Add safe area padding for iOS notch/home indicator */ + padding-top: env(safe-area-inset-top); + padding-bottom: env(safe-area-inset-bottom); + padding-left: env(safe-area-inset-left); + padding-right: env(safe-area-inset-right); + } +} + +@media (min-width: 640px) and (max-width: 768px) { .modal-overlay { padding: 1rem; } @@ -545,3 +576,52 @@ button:disabled { background-size: 200% 100%; animation: shimmer 2s ease-in-out infinite; } + +/* Settings Modal Animations */ +@keyframes settings-modal-enter { + from { + opacity: 0; + transform: scale(0.95); + } + to { + opacity: 1; + transform: scale(1); + } +} + +@keyframes settings-modal-exit { + from { + opacity: 1; + transform: scale(1); + } + to { + opacity: 0; + transform: scale(0.95); + } +} + +@keyframes fade-in { + from { opacity: 0; } + to { opacity: 1; } +} + +@keyframes fade-out { + from { opacity: 1; } + to { opacity: 0; } +} + +.settings-modal-enter { + animation: settings-modal-enter 0.2s ease-out; +} + +.settings-modal-exit { + animation: settings-modal-exit 0.15s ease-in; +} + +.animate-fade-in { + animation: fade-in 0.2s ease-out; +} + +.animate-fade-out { + animation: fade-out 0.15s ease-in; +} diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts index 8c2a06d4..40d47038 100644 --- a/src/frontend/src/types/index.ts +++ b/src/frontend/src/types/index.ts @@ -1,3 +1,18 @@ +// Search mode constants and type +export const SEARCH_MODE = { + DIRECT: 'direct', + UNIVERSAL: 'universal', +} as const; + +export type SearchMode = typeof SEARCH_MODE[keyof typeof SEARCH_MODE]; + +// Display field for metadata cards (provider-specific info like ratings, pages, etc.) +export interface DisplayField { + label: string; // e.g., "Rating", "Pages", "Readers" + value: string; // e.g., "4.5", "496", "8,041" + icon?: string; // Icon name: "star", "book", "users", "editions" +} + // Book data types export interface Book { id: string; @@ -15,6 +30,17 @@ export interface Book { progress?: number; status_message?: string; // Detailed status message (e.g., "Trying Libgen (2/5)") added_time?: number; // Timestamp when added to queue + source?: string; // Release source handler (e.g., "direct_download", "prowlarr") + source_display_name?: string; // Human-readable source name (e.g., "Direct Download") + // Metadata provider fields (used in universal search mode) + provider?: string; // e.g., 'hardcover', 'openlibrary' + provider_display_name?: string; // e.g., 'Hardcover', 'Open Library' + provider_id?: string; // ID in provider's system + isbn_10?: string; + isbn_13?: string; + genres?: string[]; + source_url?: string; // Link to book on provider's site + display_fields?: DisplayField[]; // Provider-specific display data } // Status response types @@ -63,6 +89,54 @@ export interface Toast { type: 'success' | 'error' | 'info'; } +// Sort option for dropdowns +export interface SortOption { + value: string; + label: string; +} + +// Search field types (mirror backend search field types) +export type SearchFieldType = + | 'TextSearchField' + | 'NumberSearchField' + | 'SelectSearchField' + | 'CheckboxSearchField'; + +interface SearchFieldBase { + key: string; + label: string; + type: SearchFieldType; + placeholder?: string; + description?: string; +} + +export interface TextSearchField extends SearchFieldBase { + type: 'TextSearchField'; +} + +export interface NumberSearchField extends SearchFieldBase { + type: 'NumberSearchField'; + min?: number; + max?: number; + step?: number; +} + +export interface SelectSearchField extends SearchFieldBase { + type: 'SelectSearchField'; + options: SortOption[]; +} + +export interface CheckboxSearchField extends SearchFieldBase { + type: 'CheckboxSearchField'; + default?: boolean; +} + +export type MetadataSearchField = + | TextSearchField + | NumberSearchField + | SelectSearchField + | CheckboxSearchField; + // App configuration export interface AppConfig { calibre_web_url: string; @@ -72,6 +146,11 @@ export interface AppConfig { book_languages: Language[]; default_language: string[]; supported_formats: string[]; + search_mode: SearchMode; + metadata_sort_options: SortOption[]; + metadata_search_fields: MetadataSearchField[]; + default_release_source?: string; // Default tab in ReleaseModal (e.g., 'direct_download') + settings_enabled: boolean; // Whether config directory is mounted and writable } // Authentication types @@ -87,3 +166,92 @@ export interface AuthResponse { auth_required?: boolean; error?: string; } + +// Type guard to check if a book is from a metadata provider +// Returns true and narrows type to include required provider fields +export const isMetadataBook = (book: Book): book is Book & { + provider: string; + provider_id: string; +} => { + return !!book.provider && !!book.provider_id; +}; + +// Release source types (from plugin system) +export interface ReleaseSource { + name: string; // e.g., 'direct_download', 'prowlarr' + display_name: string; // e.g., 'Direct Download', 'Prowlarr' +} + +// Column schema types for plugin-driven release list UI +export type ColumnRenderType = 'text' | 'badge' | 'size' | 'number' | 'peers'; +export type ColumnAlign = 'left' | 'center' | 'right'; + +export interface ColumnColorHint { + type: 'map' | 'static'; // 'map' uses colorMaps.ts, 'static' is a fixed Tailwind class + value: string; // Map name ('format', 'language') or Tailwind class +} + +export interface ColumnSchema { + key: string; // Data path (e.g., 'format', 'extra.language') + label: string; // Accessibility label + render_type: ColumnRenderType; + align: ColumnAlign; + width: string; // CSS width (e.g., '80px') + hide_mobile: boolean; // Hide on small screens + color_hint?: ColumnColorHint | null; + fallback: string; // Value when data is missing + uppercase: boolean; // Force uppercase display +} + +// Leading cell config - what to show in the left-most position of each row +export type LeadingCellType = 'thumbnail' | 'badge' | 'none'; + +export interface LeadingCellConfig { + type: LeadingCellType; + key?: string; // Field path for data (e.g., 'extra.preview' or 'extra.download_type') + color_hint?: ColumnColorHint; // For badge type - maps values to colors + uppercase?: boolean; // Force uppercase for badge text +} + +export interface ReleaseColumnConfig { + columns: ColumnSchema[]; + grid_template: string; // CSS grid-template-columns for dynamic section + leading_cell?: LeadingCellConfig; // Defaults to thumbnail from extra.preview +} + +// A downloadable release from any source +export interface Release { + source: string; // Source plugin name + source_id: string; // ID within that source + title: string; + format?: string; // epub, pdf, mobi, etc. + language?: string; // ISO 639-1 code (e.g., "en", "de", "fr") + size?: string; // Human-readable size + size_bytes?: number; // Size in bytes for sorting + download_url?: string; + info_url?: string; // Link to release info page (e.g., tracker page) - makes title clickable + protocol?: 'http' | 'torrent' | 'nzb' | 'dcc'; + indexer?: string; // Display name for the source/indexer + seeders?: number; // For torrents + peers?: string; // For torrents: "seeders/leechers" display string + extra?: Record; // Source-specific metadata +} + +// Response from /api/releases endpoint +export interface ReleasesResponse { + releases: Release[]; + book: { + provider: string; + provider_id: string; + title: string; + authors?: string[]; + isbn_10?: string; + isbn_13?: string; + cover_url?: string; + publish_year?: number; + language?: string; + }; + sources_searched: string[]; + errors?: string[]; + column_config?: ReleaseColumnConfig | null; // Plugin-driven column configuration +} diff --git a/src/frontend/src/types/settings.ts b/src/frontend/src/types/settings.ts new file mode 100644 index 00000000..69461cb1 --- /dev/null +++ b/src/frontend/src/types/settings.ts @@ -0,0 +1,148 @@ +// Settings field types matching backend settings_registry.py + +export type FieldType = + | 'TextField' + | 'PasswordField' + | 'NumberField' + | 'CheckboxField' + | 'SelectField' + | 'MultiSelectField' + | 'ActionButton' + | 'HeadingField'; + +export interface SelectOption { + value: string; + label: string; +} + +// Conditional visibility configuration +export interface ShowWhenCondition { + field: string; // The field key to check + value: string | string[]; // The value(s) that make this field visible +} + +// Conditional disable configuration +export interface DisabledWhenCondition { + field: string; // The field key to check + value: string | string[] | boolean; // The value(s) that disable this field + reason?: string; // Explanation shown when disabled +} + +// Base field interface - common properties +export interface BaseField { + key: string; + label: string; + type: FieldType; + description?: string; + required?: boolean; + fromEnv?: boolean; // True if value is set via environment variable + disabled?: boolean; // True if field is disabled/greyed out + disabledReason?: string; // Explanation shown when field is disabled + showWhen?: ShowWhenCondition; // Conditional visibility based on another field's value + disabledWhen?: DisabledWhenCondition; // Conditional disable based on another field's value + requiresRestart?: boolean; // True if changing this setting requires a container restart +} + +// Specific field interfaces +export interface TextFieldConfig extends BaseField { + type: 'TextField'; + value: string; + placeholder?: string; + maxLength?: number; +} + +export interface PasswordFieldConfig extends BaseField { + type: 'PasswordField'; + value: string; + placeholder?: string; +} + +export interface NumberFieldConfig extends BaseField { + type: 'NumberField'; + value: number; + min?: number; + max?: number; + step?: number; +} + +export interface CheckboxFieldConfig extends BaseField { + type: 'CheckboxField'; + value: boolean; +} + +export interface SelectFieldConfig extends BaseField { + type: 'SelectField'; + value: string; + options: SelectOption[]; +} + +export interface MultiSelectFieldConfig extends BaseField { + type: 'MultiSelectField'; + value: string[]; + options: SelectOption[]; +} + +export interface ActionButtonConfig extends BaseField { + type: 'ActionButton'; + style: 'default' | 'primary' | 'danger'; +} + +export interface HeadingFieldConfig { + key: string; + type: 'HeadingField'; + title: string; + description?: string; + linkUrl?: string; + linkText?: string; +} + +// Union type for all fields +export type SettingsField = + | TextFieldConfig + | PasswordFieldConfig + | NumberFieldConfig + | CheckboxFieldConfig + | SelectFieldConfig + | MultiSelectFieldConfig + | ActionButtonConfig + | HeadingFieldConfig; + +// Settings tab structure +export interface SettingsTab { + name: string; // Internal identifier + displayName: string; // UI display name + icon?: string; // Icon name + order: number; // Sort order + group?: string; // Group this tab belongs to + fields: SettingsField[]; +} + +// Settings group structure +export interface SettingsGroup { + name: string; // Internal identifier + displayName: string; // UI display name + icon?: string; // Icon name + order: number; // Sort order +} + +// API response types +export interface SettingsResponse { + tabs: SettingsTab[]; + groups: SettingsGroup[]; +} + +export interface ActionResult { + success: boolean; + message: string; +} + +export interface UpdateResult { + success: boolean; + message: string; + updated: string[]; // Keys that were updated + requiresRestart?: boolean; // True if any updated setting requires a restart + restartRequiredFor?: string[]; // Keys of settings that require restart +} + +// Form values type - maps field keys to their values +export type SettingsValues = Record>; diff --git a/src/frontend/src/utils/bookTransformers.ts b/src/frontend/src/utils/bookTransformers.ts new file mode 100644 index 00000000..330691ff --- /dev/null +++ b/src/frontend/src/utils/bookTransformers.ts @@ -0,0 +1,57 @@ +import { Book } from '../types'; + +/** + * Raw metadata book data from the API (provider responses). + * Used by both search and single-book endpoints. + */ +export interface MetadataBookData { + provider: string; + provider_display_name?: string; + provider_id: string; + title: string; + authors?: string[]; + isbn_10?: string; + isbn_13?: string; + cover_url?: string; + description?: string; + publisher?: string; + publish_year?: number; + language?: string; + genres?: string[]; + source_url?: string; + display_fields?: Array<{ + label: string; + value: string; + icon?: string; + }>; +} + +/** + * Transform raw metadata book data to the frontend Book format. + * Handles ID generation, author joining, and info object construction. + */ +export function transformMetadataToBook(data: MetadataBookData): Book { + return { + id: `${data.provider}:${data.provider_id}`, + title: data.title, + author: data.authors?.join(', ') || 'Unknown', + year: data.publish_year?.toString(), + language: data.language, + preview: data.cover_url, + publisher: data.publisher, + description: data.description, + provider: data.provider, + provider_display_name: data.provider_display_name, + provider_id: data.provider_id, + isbn_10: data.isbn_10, + isbn_13: data.isbn_13, + genres: data.genres, + source_url: data.source_url, + display_fields: data.display_fields, + info: { + ...(data.isbn_13 && { ISBN: data.isbn_13 }), + ...(data.isbn_10 && !data.isbn_13 && { ISBN: data.isbn_10 }), + ...(data.genres && data.genres.length > 0 && { Genres: data.genres }), + }, + }; +} diff --git a/src/frontend/src/utils/buildSearchQuery.ts b/src/frontend/src/utils/buildSearchQuery.ts index f89a2eba..41ec40a1 100644 --- a/src/frontend/src/utils/buildSearchQuery.ts +++ b/src/frontend/src/utils/buildSearchQuery.ts @@ -7,6 +7,7 @@ interface BuildSearchQueryOptions { advancedFilters: AdvancedFilterState; bookLanguages: Language[]; defaultLanguage: string[]; + searchMode?: 'direct' | 'universal'; } export const buildSearchQuery = ({ @@ -15,6 +16,7 @@ export const buildSearchQuery = ({ advancedFilters, bookLanguages, defaultLanguage, + searchMode = 'direct', }: BuildSearchQueryOptions): string => { const queryParts: string[] = []; @@ -23,6 +25,16 @@ export const buildSearchQuery = ({ queryParts.push(`query=${encodeURIComponent(basic)}`); } + // In universal mode, only include query and sort + // Provider-specific fields are handled separately via searchFieldValues + if (searchMode === 'universal') { + if (advancedFilters.sort) { + queryParts.push(`sort=${encodeURIComponent(advancedFilters.sort)}`); + } + return queryParts.join('&'); + } + + // Direct mode: include all Anna's Archive filters if (showAdvanced) { const { isbn, author, title, content, formats, lang } = advancedFilters; diff --git a/src/frontend/src/utils/colorMaps.ts b/src/frontend/src/utils/colorMaps.ts new file mode 100644 index 00000000..08d7cd61 --- /dev/null +++ b/src/frontend/src/utils/colorMaps.ts @@ -0,0 +1,68 @@ +// Color styles with transparent backgrounds and contrasting text (matches DownloadsSidebar style) +interface ColorStyle { + bg: string; + text: string; +} + +const FORMAT_COLORS: Record = { + pdf: { bg: 'bg-red-500/20', text: 'text-red-700 dark:text-red-300' }, + epub: { bg: 'bg-green-500/20', text: 'text-green-700 dark:text-green-300' }, + mobi: { bg: 'bg-blue-500/20', text: 'text-blue-700 dark:text-blue-300' }, + azw3: { bg: 'bg-purple-500/20', text: 'text-purple-700 dark:text-purple-300' }, + txt: { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300' }, + djvu: { bg: 'bg-orange-500/20', text: 'text-orange-700 dark:text-orange-300' }, + fb2: { bg: 'bg-teal-500/20', text: 'text-teal-700 dark:text-teal-300' }, + cbr: { bg: 'bg-yellow-500/20', text: 'text-yellow-700 dark:text-yellow-300' }, + cbz: { bg: 'bg-amber-500/20', text: 'text-amber-700 dark:text-amber-300' }, +}; + +const LANGUAGE_COLORS: Record = { + en: { bg: 'bg-blue-500/20', text: 'text-blue-700 dark:text-blue-300' }, + english: { bg: 'bg-blue-500/20', text: 'text-blue-700 dark:text-blue-300' }, + es: { bg: 'bg-orange-500/20', text: 'text-orange-700 dark:text-orange-300' }, + spanish: { bg: 'bg-orange-500/20', text: 'text-orange-700 dark:text-orange-300' }, + fr: { bg: 'bg-purple-500/20', text: 'text-purple-700 dark:text-purple-300' }, + french: { bg: 'bg-purple-500/20', text: 'text-purple-700 dark:text-purple-300' }, + de: { bg: 'bg-yellow-500/20', text: 'text-yellow-700 dark:text-yellow-300' }, + german: { bg: 'bg-yellow-500/20', text: 'text-yellow-700 dark:text-yellow-300' }, + it: { bg: 'bg-green-500/20', text: 'text-green-700 dark:text-green-300' }, + italian: { bg: 'bg-green-500/20', text: 'text-green-700 dark:text-green-300' }, + pt: { bg: 'bg-teal-500/20', text: 'text-teal-700 dark:text-teal-300' }, + portuguese: { bg: 'bg-teal-500/20', text: 'text-teal-700 dark:text-teal-300' }, + ru: { bg: 'bg-red-500/20', text: 'text-red-700 dark:text-red-300' }, + russian: { bg: 'bg-red-500/20', text: 'text-red-700 dark:text-red-300' }, + ja: { bg: 'bg-pink-500/20', text: 'text-pink-700 dark:text-pink-300' }, + japanese: { bg: 'bg-pink-500/20', text: 'text-pink-700 dark:text-pink-300' }, + zh: { bg: 'bg-rose-500/20', text: 'text-rose-700 dark:text-rose-300' }, + chinese: { bg: 'bg-rose-500/20', text: 'text-rose-700 dark:text-rose-300' }, +}; + +const DOWNLOAD_TYPE_COLORS: Record = { + torrent: { bg: 'bg-orange-500/20', text: 'text-orange-700 dark:text-orange-300' }, + usenet: { bg: 'bg-sky-500/20', text: 'text-sky-700 dark:text-sky-300' }, + nzb: { bg: 'bg-sky-500/20', text: 'text-sky-700 dark:text-sky-300' }, + ddl: { bg: 'bg-emerald-500/20', text: 'text-emerald-700 dark:text-emerald-300' }, + direct: { bg: 'bg-emerald-500/20', text: 'text-emerald-700 dark:text-emerald-300' }, +}; + +const DEFAULT_FORMAT_COLOR: ColorStyle = { bg: 'bg-cyan-500/20', text: 'text-cyan-700 dark:text-cyan-300' }; +const DEFAULT_LANGUAGE_COLOR: ColorStyle = { bg: 'bg-indigo-500/20', text: 'text-indigo-700 dark:text-indigo-300' }; +const DEFAULT_DOWNLOAD_TYPE_COLOR: ColorStyle = { bg: 'bg-violet-500/20', text: 'text-violet-700 dark:text-violet-300' }; +const FALLBACK_COLOR: ColorStyle = { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300' }; + +export function getFormatColor(format?: string): ColorStyle { + if (!format || format === '-') return FALLBACK_COLOR; + return FORMAT_COLORS[format.toLowerCase()] || DEFAULT_FORMAT_COLOR; +} + +export function getLanguageColor(language?: string): ColorStyle { + if (!language || language === '-') return FALLBACK_COLOR; + return LANGUAGE_COLORS[language.toLowerCase()] || DEFAULT_LANGUAGE_COLOR; +} + +export function getDownloadTypeColor(downloadType?: string): ColorStyle { + if (!downloadType || downloadType === '-') return FALLBACK_COLOR; + return DOWNLOAD_TYPE_COLORS[downloadType.toLowerCase()] || DEFAULT_DOWNLOAD_TYPE_COLOR; +} + +export type { ColorStyle }; diff --git a/src/frontend/src/utils/parseUrlSearchParams.ts b/src/frontend/src/utils/parseUrlSearchParams.ts new file mode 100644 index 00000000..9f64dff8 --- /dev/null +++ b/src/frontend/src/utils/parseUrlSearchParams.ts @@ -0,0 +1,80 @@ +import { AdvancedFilterState } from '../types'; + +/** + * Parsed search parameters from URL + */ +export interface ParsedUrlSearch { + searchInput: string; + advancedFilters: Partial; + hasSearchParams: boolean; +} + +/** + * Parse URL search parameters into search state. + * + * Supports both Direct Download and Universal mode parameters. + * In Universal mode, only query and sort are used (others are parsed but + * ignored by buildSearchQuery). + * + * @example + * // Direct mode: /?q=harry+potter&author=rowling&format=epub&lang=en + * // Universal mode: /?q=dune&sort=popularity + */ +export function parseUrlSearchParams(searchParams: URLSearchParams): ParsedUrlSearch { + const result: ParsedUrlSearch = { + searchInput: '', + advancedFilters: {}, + hasSearchParams: false, + }; + + // Parse main query (support both 'q' and 'query' for flexibility) + const query = searchParams.get('q') || searchParams.get('query') || ''; + if (query) { + result.searchInput = query; + result.hasSearchParams = true; + } + + // Parse single-value filters + const isbn = searchParams.get('isbn'); + const author = searchParams.get('author'); + const title = searchParams.get('title'); + const sort = searchParams.get('sort'); + const content = searchParams.get('content'); + + if (isbn) { + result.advancedFilters.isbn = isbn; + result.hasSearchParams = true; + } + if (author) { + result.advancedFilters.author = author; + result.hasSearchParams = true; + } + if (title) { + result.advancedFilters.title = title; + result.hasSearchParams = true; + } + if (sort) { + result.advancedFilters.sort = sort; + result.hasSearchParams = true; + } + if (content) { + result.advancedFilters.content = content; + result.hasSearchParams = true; + } + + // Parse multi-value filters (can appear multiple times in URL) + // e.g., /?lang=en&lang=de or /?format=epub&format=mobi + const langValues = searchParams.getAll('lang').filter(Boolean); + const formatValues = searchParams.getAll('format').filter(Boolean); + + if (langValues.length > 0) { + result.advancedFilters.lang = langValues; + result.hasSearchParams = true; + } + if (formatValues.length > 0) { + result.advancedFilters.formats = formatValues; + result.hasSearchParams = true; + } + + return result; +} diff --git a/tor.sh b/tor.sh index 04e7ff6f..b8a551a2 100644 --- a/tor.sh +++ b/tor.sh @@ -11,8 +11,6 @@ echo "Log file: $LOG_FILE" set +x set -e -#!/bin/bash - # Check if EXT_BYPASSER_URL is defined if [ -n "$EXT_BYPASSER_URL" ]; then echo "Extracting hostname and ip from bypasser into /etc/hosts" @@ -160,14 +158,26 @@ echo "[*] Starting Tor via Supervisor..." # Wait a bit to ensure Tor has bootstrapped echo "[*] Waiting for Tor to finish bootstrapping... (up to 5 minutes)" -timeout 300 bash -c ' - while ! grep -q "Bootstrapped 100%" <(tail -n 20 -F /var/log/tor/notices.log 2>/dev/null); do - printf "\r\033[KCurrent log: %s" "$(tail -n 1 /var/log/tor/notices.log 2>/dev/null)" +BOOTSTRAP_TIMEOUT=300 +BOOTSTRAP_START=$(date +%s) +while true; do + if grep -q "Bootstrapped 100%" /var/log/tor/notices.log 2>/dev/null; then + echo "" + echo "[✓] Tor bootstrap complete." + break + fi + + ELAPSED=$(($(date +%s) - BOOTSTRAP_START)) + if [ $ELAPSED -ge $BOOTSTRAP_TIMEOUT ]; then + echo "" + echo "[✗] Tor bootstrap timed out after ${BOOTSTRAP_TIMEOUT}s" + exit 1 + fi + + CURRENT_LOG=$(tail -n 1 /var/log/tor/notices.log 2>/dev/null) + printf "\r\033[K[%ds] %s" "$ELAPSED" "$CURRENT_LOG" sleep 1 - done - # Print a newline when finished. - echo "" -' +done echo "[✓] Tor is ready." @@ -241,5 +251,71 @@ else echo "[*] Falling back to container's default timezone: $TZ" fi +# Start a background health check process to monitor Tor +echo "[*] Starting Tor health check monitor..." +( + check_count=0 + first_check=true + + while true; do + if [ "$first_check" = true ]; then + sleep 60 + first_check=false + else + sleep 300 + fi + + check_count=$((check_count + 1)) + echo "[*] Tor health check #$check_count at $(date)" + + # Check Tor service + if ! service tor status > /dev/null 2>&1; then + echo "[!] $(date): Tor service not running, restarting..." + service tor restart + sleep 10 + continue + fi + + # Test DNS + if ! timeout 10 nslookup google.com 127.0.0.1 > /dev/null 2>&1; then + echo "[!] $(date): DNS resolution failed, reloading Tor..." + service tor reload + sleep 5 + if timeout 10 nslookup google.com 127.0.0.1 > /dev/null 2>&1; then + echo "[✓] $(date): DNS resolution restored" + else + echo "[✗] $(date): DNS still failing after reload, restarting Tor..." + service tor restart + sleep 10 + continue + fi + fi + + # Test TCP connectivity + if ! timeout 15 curl -s --max-time 10 https://check.torproject.org/api/ip > /dev/null 2>&1; then + echo "[!] $(date): TCP connectivity test failed, rotating circuits..." + pkill -HUP tor || true + sleep 5 + if ! timeout 15 curl -s --max-time 10 https://check.torproject.org/api/ip > /dev/null 2>&1; then + echo "[✗] $(date): TCP still failing after rotation, restarting Tor..." + service tor restart + sleep 10 + continue + else + echo "[✓] $(date): TCP connectivity restored after circuit rotation" + fi + else + echo "[✓] $(date): Health check passed (DNS + TCP OK)" + fi + + # Rotate circuits + echo "[*] $(date): Rotating Tor circuits..." + pkill -HUP tor || true + done +) >> $LOG_FILE 2>&1 & + +TOR_MONITOR_PID=$! +echo "[✓] Tor health check monitor started in background (PID: $TOR_MONITOR_PID)" + # Run the entrypoint script echo "[*] End of tor script"