Update Readme with new changes (#344)

Fix for #340

app.py

@@ -1,24 +1,29 @@
 """Flask web application for book download service with URL rewrite support."""
 
+import io
 import logging
-import io, re, os
+import os
 import sqlite3
 import time
 from datetime import datetime, timedelta
 from functools import wraps
-from flask import Flask, request, jsonify, send_file, send_from_directory, session
+from typing import Any, Dict, Tuple, Union
+
+from flask import Flask, jsonify, request, send_file, send_from_directory, session
 from flask_cors import CORS
 from flask_socketio import SocketIO, emit
 from werkzeug.middleware.proxy_fix import ProxyFix
 from werkzeug.security import check_password_hash
 from werkzeug.wrappers import Response
-import typing
 
-from logger import setup_logger
-from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE, SUPPORTED_FORMATS
-from env import FLASK_HOST, FLASK_PORT, CWA_DB_PATH, DEBUG, USING_EXTERNAL_BYPASSER, BUILD_VERSION, RELEASE_VERSION, CALIBRE_WEB_URL
 import backend
-
+from book_manager import SearchUnavailable
+from config import BOOK_LANGUAGE, SUPPORTED_FORMATS, _SUPPORTED_BOOK_LANGUAGE
+from env import (
+    BUILD_VERSION, CALIBRE_WEB_URL, CWA_DB_PATH, DEBUG, FLASK_HOST, FLASK_PORT,
+    RELEASE_VERSION, USING_EXTERNAL_BYPASSER,
+)
+from logger import setup_logger
 from models import SearchFilters
 from websocket_manager import ws_manager
 
@@ -58,7 +63,7 @@ logger.info(f"Flask-SocketIO initialized with async_mode='{async_mode}'")
 
 # Rate limiting for login attempts
 # Structure: {username: {'count': int, 'lockout_until': datetime}}
-failed_login_attempts: typing.Dict[str, typing.Dict[str, typing.Any]] = {}
+failed_login_attempts: Dict[str, Dict[str, Any]] = {}
 MAX_LOGIN_ATTEMPTS = 10
 LOCKOUT_DURATION_MINUTES = 30
 
@@ -124,15 +129,45 @@ if DEBUG:
         }
     })
 
-# Custom log filter to exclude routine status endpoint polling
+# Custom log filter to exclude routine status endpoint polling and WebSocket noise
 class StatusEndpointFilter(logging.Filter):
-    """Filter out routine status endpoint requests to reduce log noise."""
+    """Filter out routine status endpoint requests and WebSocket upgrade errors to reduce log noise."""
     def filter(self, record):
-        # Exclude GET /api/status requests
         if hasattr(record, 'getMessage'):
             message = record.getMessage()
+            # Exclude GET /api/status requests (polling noise)
             if 'GET /api/status' in message:
                 return False
+            # Exclude WebSocket upgrade errors (benign - falls back to polling)
+            if 'write() before start_response' in message:
+                return False
+            # Exclude the Error on request line that precedes WebSocket errors
+            if 'Error on request:' in message and record.levelno == logging.ERROR:
+                return False
+        return True
+
+
+class WebSocketErrorFilter(logging.Filter):
+    """Filter out WebSocket upgrade errors that occur in Werkzeug dev server.
+    
+    These errors are benign - Flask-SocketIO automatically falls back to polling transport.
+    The error occurs because Werkzeug's built-in server doesn't fully support WebSocket upgrades.
+    """
+    def filter(self, record):
+        # Filter out the AssertionError traceback for WebSocket upgrades
+        if record.levelno == logging.ERROR:
+            message = record.getMessage() if hasattr(record, 'getMessage') else str(record.msg)
+            # Filter out the full traceback that includes the WebSocket assertion error
+            if 'write() before start_response' in message:
+                return False
+            # Also filter the "Error on request" header that precedes it
+            if hasattr(record, 'exc_info') and record.exc_info:
+                exc_type = record.exc_info[0]
+                if exc_type and exc_type.__name__ == 'AssertionError':
+                    # Check if it's the WebSocket-related assertion
+                    exc_value = record.exc_info[1]
+                    if exc_value and 'write() before start_response' in str(exc_value):
+                        return False
         return True
 
 # Flask logger
@@ -142,8 +177,9 @@ app.logger.setLevel(logger.level)
 werkzeug_logger = logging.getLogger('werkzeug')
 werkzeug_logger.handlers = logger.handlers
 werkzeug_logger.setLevel(logger.level)
-# Add filter to suppress routine status endpoint polling logs
+# Add filters to suppress routine status endpoint polling logs and WebSocket upgrade errors
 werkzeug_logger.addFilter(StatusEndpointFilter())
+werkzeug_logger.addFilter(WebSocketErrorFilter())
 
 # Set up authentication defaults
 # The secret key will reset every time we restart, which will
@@ -210,32 +246,38 @@ def logo() -> Response:
 
 @app.route('/favicon.ico')
 @app.route('/favico<path:_>')
-def favicon(_ : typing.Any = None) -> Response:
+def favicon(_: Any = None) -> Response:
     """
     Serve favicon from built frontend assets.
     """
     return send_from_directory(os.path.join(app.root_path, 'frontend-dist'),
         'favicon.ico', mimetype='image/vnd.microsoft.icon')
 
-from typing import Union, Tuple
+# Register bypasser warmup callback for when first WebSocket client connects
+# and shutdown callback for when all clients disconnect
+if not USING_EXTERNAL_BYPASSER:
+    from cloudflare_bypasser import warmup as bypasser_warmup, shutdown_if_idle as bypasser_shutdown
+    ws_manager.register_on_first_connect(bypasser_warmup)
+    ws_manager.register_on_all_disconnect(bypasser_shutdown)
+    logger.info("Registered Cloudflare bypasser warmup/shutdown on WebSocket connect/disconnect")
 
 if DEBUG:
     import subprocess
-    import time
     if USING_EXTERNAL_BYPASSER:
-        STOP_GUI = lambda: None  # No-op for external bypasser
+        STOP_GUI = lambda: None
     else:
         from cloudflare_bypasser import _reset_driver as STOP_GUI
-    @app.route('/debug', methods=['GET'])
+    @app.route('/api/debug', methods=['GET'])
     @login_required
     def debug() -> Union[Response, Tuple[Response, int]]:
         """
-        This will run the /app/debug.sh script, which will generate a debug zip with all the logs
+        This will run the /app/genDebug.sh script, which will generate a debug zip with all the logs
         The file will be named /tmp/cwa-book-downloader-debug.zip
         And then return it to the user
         """
         try:
             # Run the debug script
+            logger.info("Debug endpoint called, stopping GUI and generating debug info...")
             STOP_GUI()
             time.sleep(1)
             result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True)
@@ -244,9 +286,10 @@ if DEBUG:
             logger.info(f"Debug script executed: {result.stdout}")
             debug_file_path = result.stdout.strip().split('\n')[-1]
             if not os.path.exists(debug_file_path):
-                logger.error("Debug zip file not found after running debug script")
+                logger.error(f"Debug zip file not found at: {debug_file_path}")
                 return jsonify({"error": "Failed to generate debug information"}), 500
                 
+            logger.info(f"Sending debug file: {debug_file_path}")
             # Return the file to the user
             return send_file(
                 debug_file_path,
@@ -307,6 +350,9 @@ def api_search() -> Union[Response, Tuple[Response, int]]:
     try:
         books = backend.search_books(query, filters)
         return jsonify(books)
+    except SearchUnavailable as e:
+        logger.warning(f"Search unavailable: {e}")
+        return jsonify({"error": str(e)}), 503
     except Exception as e:
         logger.error_trace(f"Search error: {e}")
         return jsonify({"error": str(e)}), 500
@@ -431,15 +477,12 @@ def api_local_download() -> Union[Response, Tuple[Response, int]]:
         if file_data is None:
             # Book data not found or not available
             return jsonify({"error": "File not found"}), 404
-        # Santize the file name
-        file_name = book_info.title
-        file_name = re.sub(r'[\\/:*?"<>|]', '_', file_name.strip())[:245]
-        file_extension = book_info.format
+        file_name = book_info.get_filename()
         # Prepare the file for sending to the client
         data = io.BytesIO(file_data)
         return send_file(
             data,
-            download_name=f"{file_name}.{file_extension}",
+            download_name=file_name,
             as_attachment=True
         )
 
@@ -580,7 +623,7 @@ def api_clear_completed() -> Union[Response, Tuple[Response, int]]:
         removed_count = backend.clear_completed()
         
         # Broadcast status update after clearing
-        if ws_manager and ws_manager.is_enabled():
+        if ws_manager:
             ws_manager.broadcast_status_update(backend.queue_status())
         
         return jsonify({"status": "cleared", "removed_count": removed_count})
@@ -787,6 +830,10 @@ def catch_all(path: str) -> Response:
 def handle_connect():
     """Handle client connection."""
     logger.info("WebSocket client connected")
+    
+    # Track the connection (triggers warmup callbacks on first connect)
+    ws_manager.client_connected()
+    
     # Send initial status to the newly connected client
     try:
         status = backend.queue_status()
@@ -798,6 +845,9 @@ def handle_connect():
 def handle_disconnect():
     """Handle client disconnection."""  
     logger.info("WebSocket client disconnected")
+    
+    # Track the disconnection
+    ws_manager.client_disconnected()
 
 @socketio.on('request_status')
 def handle_status_request():

backend.py

@@ -1,34 +1,41 @@
 """Backend logic for the book download application."""
 
-import threading, time
-import shutil
-from pathlib import Path
-from typing import Dict, List, Optional, Any, Tuple
-import subprocess
 import os
-from concurrent.futures import ThreadPoolExecutor, Future
-from threading import Event
+import random
+import shutil
+import subprocess
+import threading
+import time
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from threading import Event, Lock
+from typing import Any, Dict, List, Optional, Tuple
 
-from logger import setup_logger
-from config import CUSTOM_SCRIPT
-from env import (INGEST_DIR, DOWNLOAD_PATHS, TMP_DIR, MAIN_LOOP_SLEEP_TIME, USE_BOOK_TITLE,
-                 MAX_CONCURRENT_DOWNLOADS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL)
-from models import book_queue, BookInfo, QueueStatus, SearchFilters
 import book_manager
+from book_manager import SearchUnavailable
+from config import CUSTOM_SCRIPT
+from env import (
+    DOWNLOAD_PATHS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL, INGEST_DIR,
+    MAIN_LOOP_SLEEP_TIME, MAX_CONCURRENT_DOWNLOADS, TMP_DIR, USE_BOOK_TITLE,
+)
+from logger import setup_logger
+from models import BookInfo, QueueStatus, SearchFilters, book_queue
 
 logger = setup_logger(__name__)
 
-# Import WebSocket manager (will be initialized by app.py)
+# WebSocket manager (initialized by app.py)
 try:
     from websocket_manager import ws_manager
 except ImportError:
-    logger.warning("WebSocket manager not available")
     ws_manager = None
 
-def _sanitize_filename(filename: str) -> str:
-    """Sanitize a filename by replacing spaces with underscores and removing invalid characters."""
-    keepcharacters = (' ','.','_')
-    return "".join(c for c in filename if c.isalnum() or c in keepcharacters).rstrip()
+# Progress update throttling - track last broadcast time per book
+_progress_last_broadcast: Dict[str, float] = {}
+_progress_lock = Lock()
+
+# Stall detection - track last activity time per download
+_last_activity: Dict[str, float] = {}
+STALL_TIMEOUT = 300  # 5 minutes without progress/status update = stalled
 
 def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
     """Search for books matching the query.
@@ -43,6 +50,9 @@ def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
     try:
         books = book_manager.search_books(query, filters)
         return [_book_info_to_dict(book) for book in books]
+    except SearchUnavailable as e:
+        logger.warning(f"Search unavailable: {e}")
+        raise
     except Exception as e:
         logger.error_trace(f"Error searching books: {e}")
         return []
@@ -79,7 +89,7 @@ def queue_book(book_id: str, priority: int = 0) -> bool:
         logger.info(f"Book queued with priority {priority}: {book_info.title}")
         
         # Broadcast status update via WebSocket
-        if ws_manager and ws_manager.is_enabled():
+        if ws_manager:
             ws_manager.broadcast_status_update(queue_status())
         
         return True
@@ -162,14 +172,12 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option
         book_info = book_queue._book_data[book_id]
         logger.info(f"Starting download: {book_info.title}")
 
-        if USE_BOOK_TITLE:
-            book_name = _sanitize_filename(book_info.title)
-        else:
-            book_name = book_id
-        # If format is not set, use the format of the first download URL
-        if book_info.format == "":
-            book_info.format = book_info.download_urls[0].split(".")[-1]
-        book_name += f".{book_info.format}"
+        if not book_info.download_urls:
+            raise ValueError(f"No download URLs available for {book_id}")
+
+        # get_filename() resolves format as side effect
+        full_name = book_info.get_filename()
+        book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}"
         book_path = TMP_DIR / book_name
 
         # Check cancellation before download
@@ -178,7 +186,11 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option
             return None
         
         progress_callback = lambda progress: update_download_progress(book_id, progress)
-        status_callback = lambda status: update_download_status(book_id, status)
+        status_callback = lambda status, message=None: update_download_status(book_id, status, message)
+        
+        # Set status to resolving immediately when processing starts
+        update_download_status(book_id, "resolving")
+        
         success_download_url = book_manager.download_book(book_info, book_path, progress_callback, cancel_flag, status_callback)
         
         # Stop progress updates
@@ -201,29 +213,30 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option
                 book_path.unlink()
             return None
 
-        # Update status to verifying
-        book_queue.update_status(book_id, QueueStatus.VERIFYING)
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
-        logger.info(f"Verifying download: {book_info.title}")
+        logger.debug(f"Post-processing download: {book_info.title}")
 
         if CUSTOM_SCRIPT:
             logger.info(f"Running custom script: {CUSTOM_SCRIPT}")
             subprocess.run([CUSTOM_SCRIPT, book_path])
-            
-        # Update status to ingesting
-        book_queue.update_status(book_id, QueueStatus.INGESTING)
-        if ws_manager and ws_manager.is_enabled():
-            ws_manager.broadcast_status_update(queue_status())
         
-        if success_download_url and book_info.format == "":
-            book_info.format = success_download_url.split(".")[-1]
-            book_name += f".{book_info.format}"
+        # Regenerate filename with fallback to successful download URL for format
+        full_name = book_info.get_filename(success_download_url)
+        book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}"
 
         final_dir = _prepare_download_folder(book_info)
         intermediate_path = final_dir / f"{book_id}.crdownload"
         final_path = final_dir / book_name
         
+        # Handle file already exists - add suffix to avoid overwrite
+        if final_path.exists():
+            base = final_path.stem
+            ext = final_path.suffix
+            counter = 1
+            while final_path.exists():
+                final_path = final_dir / f"{base}_{counter}{ext}"
+                counter += 1
+            logger.info(f"File already exists, saving as: {final_path.name}")
+        
         if os.path.exists(book_path):
             logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}")
             try:
@@ -256,23 +269,60 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option
         return None
 
 def update_download_progress(book_id: str, progress: float) -> None:
-    """Update download progress."""
-    book_queue.update_progress(book_id, progress)
-    
-    # Broadcast progress via WebSocket
-    if ws_manager and ws_manager.is_enabled():
-        ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
+    """Update download progress with throttled WebSocket broadcasts.
 
-def update_download_status(book_id: str, status: str) -> None:
-    """Update download status."""
+    Progress is always stored in the queue, but WebSocket broadcasts are
+    throttled to avoid flooding clients with updates. Broadcasts occur:
+    - At most once per DOWNLOAD_PROGRESS_UPDATE_INTERVAL seconds
+    - Always at 0% (start) and 100% (complete)
+    - On significant progress jumps (>10%)
+    """
+    book_queue.update_progress(book_id, progress)
+
+    # Track activity for stall detection
+    with _progress_lock:
+        _last_activity[book_id] = time.time()
+    
+    # Broadcast progress via WebSocket with throttling
+    if ws_manager:
+        current_time = time.time()
+        should_broadcast = False
+        
+        with _progress_lock:
+            last_broadcast = _progress_last_broadcast.get(book_id, 0)
+            last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0)
+            time_elapsed = current_time - last_broadcast
+            
+            # Always broadcast at start (0%) or completion (>=99%)
+            if progress <= 1 or progress >= 99:
+                should_broadcast = True
+            # Broadcast if enough time has passed (convert interval from seconds)
+            elif time_elapsed >= DOWNLOAD_PROGRESS_UPDATE_INTERVAL:
+                should_broadcast = True
+            # Broadcast on significant progress jumps (>10%)
+            elif progress - last_progress >= 10:
+                should_broadcast = True
+            
+            if should_broadcast:
+                _progress_last_broadcast[book_id] = current_time
+                _progress_last_broadcast[f"{book_id}_progress"] = progress
+        
+        if should_broadcast:
+            ws_manager.broadcast_download_progress(book_id, progress, 'downloading')
+
+def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None:
+    """Update download status with optional detailed message.
+    
+    Args:
+        book_id: Book identifier
+        status: Status string (e.g., 'resolving', 'downloading')
+        message: Optional detailed status message for UI display
+    """
     # Map string status to QueueStatus enum
     status_map = {
         'queued': QueueStatus.QUEUED,
         'resolving': QueueStatus.RESOLVING,
-        'bypassing': QueueStatus.BYPASSING,
         'downloading': QueueStatus.DOWNLOADING,
-        'verifying': QueueStatus.VERIFYING,
-        'ingesting': QueueStatus.INGESTING,
         'complete': QueueStatus.COMPLETE,
         'available': QueueStatus.AVAILABLE,
         'error': QueueStatus.ERROR,
@@ -283,9 +333,17 @@ def update_download_status(book_id: str, status: str) -> None:
     queue_status_enum = status_map.get(status.lower())
     if queue_status_enum:
         book_queue.update_status(book_id, queue_status_enum)
-        
+
+        # Track activity for stall detection
+        with _progress_lock:
+            _last_activity[book_id] = time.time()
+
+        # Update status message if provided (empty string clears the message)
+        if message is not None:
+            book_queue.update_status_message(book_id, message)
+
         # Broadcast status update via WebSocket
-        if ws_manager and ws_manager.is_enabled():
+        if ws_manager:
             ws_manager.broadcast_status_update(queue_status())
 
 def cancel_download(book_id: str) -> bool:
@@ -340,17 +398,27 @@ def clear_completed() -> int:
     """Clear all completed downloads from tracking."""
     return book_queue.clear_completed()
 
+def _cleanup_progress_tracking(book_id: str) -> None:
+    """Clean up progress tracking data for a completed/cancelled download."""
+    with _progress_lock:
+        _progress_last_broadcast.pop(book_id, None)
+        _progress_last_broadcast.pop(f"{book_id}_progress", None)
+        _last_activity.pop(book_id, None)
+
 def _process_single_download(book_id: str, cancel_flag: Event) -> None:
     """Process a single download job."""
     try:
         # Status will be updated through callbacks during download process
-        # (resolving -> bypassing -> downloading -> verifying -> ingesting -> complete)
+        # (resolving -> downloading -> complete)
         download_path = _download_book_with_cancellation(book_id, cancel_flag)
         
+        # Clean up progress tracking
+        _cleanup_progress_tracking(book_id)
+        
         if cancel_flag.is_set():
             book_queue.update_status(book_id, QueueStatus.CANCELLED)
             # Broadcast cancellation
-            if ws_manager and ws_manager.is_enabled():
+            if ws_manager:
                 ws_manager.broadcast_status_update(queue_status())
             return
             
@@ -363,23 +431,26 @@ def _process_single_download(book_id: str, cancel_flag: Event) -> None:
         book_queue.update_status(book_id, new_status)
         
         # Broadcast final status (completed or error)
-        if ws_manager and ws_manager.is_enabled():
+        if ws_manager:
             ws_manager.broadcast_status_update(queue_status())
         
-        logger.info(
-            f"Book {book_id} download {'successful' if download_path else 'failed'}"
-        )
         
     except Exception as e:
+        # Clean up progress tracking even on error
+        _cleanup_progress_tracking(book_id)
+        
         if not cancel_flag.is_set():
             logger.error_trace(f"Error in download processing: {e}")
             book_queue.update_status(book_id, QueueStatus.ERROR)
+            # Set error message if not already set by download_book()
+            if book_id in book_queue._book_data and not book_queue._book_data[book_id].status_message:
+                book_queue.update_status_message(book_id, f"Download failed: {type(e).__name__}: {str(e)}")
         else:
             logger.info(f"Download cancelled: {book_id}")
             book_queue.update_status(book_id, QueueStatus.CANCELLED)
         
         # Broadcast error/cancelled status
-        if ws_manager and ws_manager.is_enabled():
+        if ws_manager:
             ws_manager.broadcast_status_update(queue_status())
 
 def concurrent_download_loop() -> None:
@@ -398,16 +469,32 @@ def concurrent_download_loop() -> None:
                     future.result()  # This will raise any exceptions from the worker
                 except Exception as e:
                     logger.error_trace(f"Future exception for {book_id}: {e}")
-            
+
+            # Check for stalled downloads (no activity in STALL_TIMEOUT seconds)
+            current_time = time.time()
+            with _progress_lock:
+                for future, book_id in list(active_futures.items()):
+                    last_active = _last_activity.get(book_id, current_time)
+                    if current_time - last_active > STALL_TIMEOUT:
+                        logger.warning(f"Download stalled for {book_id}, cancelling")
+                        book_queue.cancel_download(book_id)
+                        book_queue.update_status_message(book_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)")
+
             # Start new downloads if we have capacity
             while len(active_futures) < MAX_CONCURRENT_DOWNLOADS:
                 next_download = book_queue.get_next()
                 if not next_download:
                     break
-                    
+
+                # Stagger concurrent downloads to avoid rate limiting on shared download servers
+                # Only delay if other downloads are already active
+                if active_futures:
+                    stagger_delay = random.uniform(2, 5)
+                    logger.debug(f"Staggering download start by {stagger_delay:.1f}s")
+                    time.sleep(stagger_delay)
+
                 book_id, cancel_flag = next_download
-                logger.info(f"Starting concurrent download: {book_id}")
-                
+
                 # Submit download job to thread pool
                 future = executor.submit(_process_single_download, book_id, cancel_flag)
                 active_futures[future] = book_id

book_manager.py

@@ -1,19 +1,37 @@
 """Book download manager handling search and retrieval operations."""
 
-import time, json, os, re
+import itertools
+import json
+import re
+import time
 from pathlib import Path
-from urllib.parse import quote
-from typing import List, Optional, Dict, Union, Callable
 from threading import Event
-from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet
+from typing import Callable, Dict, List, Optional
+from urllib.parse import quote
+
+from bs4 import BeautifulSoup, NavigableString, Tag
 
 import downloader
+import network
+from config import BOOK_LANGUAGE, SUPPORTED_FORMATS
+from env import AA_DONATOR_KEY, ALLOW_USE_WELIB, DEBUG_SKIP_SOURCES, DOWNLOAD_PATHS, PRIORITIZE_WELIB, USE_CF_BYPASS
 from logger import setup_logger
-from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL
-from env import AA_DONATOR_KEY, USE_CF_BYPASS, PRIORITIZE_WELIB, ALLOW_USE_WELIB, DOWNLOAD_PATHS
 from models import BookInfo, SearchFilters
+
 logger = setup_logger(__name__)
 
+# Round-robin counter for AA slow download source rotation
+# Distributes concurrent downloads across different partner mirrors
+_aa_slow_rotation = itertools.count()
+
+if DEBUG_SKIP_SOURCES:
+    logger.warning("DEBUG_SKIP_SOURCES active: skipping sources %s", DEBUG_SKIP_SOURCES)
+
+
+class SearchUnavailable(Exception):
+    """Raised when Anna's Archive cannot be reached via any mirror/DNS."""
+    pass
+
 
 
 def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
@@ -62,8 +80,10 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
                 )
                 index += 1
 
+    selector = network.AAMirrorSelector()
+
     url = (
-        f"{AA_BASE_URL}"
+        f"{network.get_aa_base_url()}"
         f"/search?index=&page=1&display=table"
         f"&acc=aa_download&acc=external_download"
         f"&ext={'&ext='.join(formats_to_use)}"
@@ -71,13 +91,14 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
         f"{filters_query}"
     )
 
-    html = downloader.html_get_page(url)
+    html = downloader.html_get_page(url, selector=selector)
     if not html:
-        raise Exception("Failed to fetch search results")
+        # Network/mirror exhaustion path bubbles up so API can notify clients
+        raise SearchUnavailable("Unable to reach Anna's Archive. Network restricted or mirrors are blocked.")
 
     if "No files found." in html:
         logger.info(f"No books found for query: {query}")
-        raise Exception("No books found. Please try another query.")
+        return []
 
     soup = BeautifulSoup(html, "html.parser")
     tbody: Tag | NavigableString | None = soup.find("table")
@@ -143,8 +164,9 @@ def get_book_info(book_id: str) -> BookInfo:
     Returns:
         BookInfo: Detailed book information
     """
-    url = f"{AA_BASE_URL}/md5/{book_id}"
-    html = downloader.html_get_page(url)
+    url = f"{network.get_aa_base_url()}/md5/{book_id}"
+    selector = network.AAMirrorSelector()
+    html = downloader.html_get_page(url, selector=selector)
 
     if not html:
         raise Exception(f"Failed to fetch book info for ID: {book_id}")
@@ -174,58 +196,75 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
     data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div")
     divs = list(data.children)
 
-    every_url = soup.find_all("a")
-    slow_urls_no_waitlist = set()
-    slow_urls_with_waitlist = set()
-    external_urls_libgen = set()
-    external_urls_z_lib = set()
-    external_urls_welib = set()
+    # Collect download URLs by source type (lists preserve page order, dedup inline)
+    slow_urls_no_waitlist: list[str] = []
+    slow_urls_with_waitlist: list[str] = []
+    external_urls_libgen: list[str] = []
+    external_urls_z_lib: list[str] = []
 
-    for url in every_url:
+    def _append_unique(lst: list[str], href: str) -> None:
+        if href and href not in lst:
+            lst.append(href)
+
+    for anchor in soup.find_all("a"):
         try:
-            if url.text.strip().lower().startswith("slow partner server"):
-                if (
-                    url.next is not None
-                    and url.next.next is not None
-                    and "waitlist" in url.next.next.strip().lower()
-                ):
-                    internal_text = url.next.next.strip().lower()
-                    if "no waitlist" in internal_text:
-                        slow_urls_no_waitlist.add(url["href"])
-                    else:
-                        slow_urls_with_waitlist.add(url["href"])
-            elif (
-                url.next is not None
-                and url.next.next is not None
-                and "click “GET” at the top" in url.next.next.text.strip()
-            ):
-                libgen_url = url["href"]
-                # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ?
-                libgen_url = re.sub(r'libgen\.(lc|is|bz|st)', 'libgen.gl', url["href"])
+            text = anchor.text.strip().lower()
+            href = anchor.get("href", "")
+            next_text = ""
+            if anchor.next and anchor.next.next:
+                next_text = getattr(anchor.next.next, 'text', str(anchor.next.next)).strip().lower()
 
-                external_urls_libgen.add(libgen_url)
-            elif url.text.strip().lower().startswith("z-lib"):
-                if ".onion/" not in url["href"]:
-                    external_urls_z_lib.add(url["href"])
+            if text.startswith("slow partner server") and "waitlist" in next_text:
+                if "no waitlist" in next_text:
+                    _append_unique(slow_urls_no_waitlist, href)
+                else:
+                    _append_unique(slow_urls_with_waitlist, href)
+            elif 'libgen.li' in href:
+                # Normalize libgen domains
+                libgen_url = re.sub(r'libgen\.(li|lc|is|bz|st)', 'libgen.gl', href)
+                _append_unique(external_urls_libgen, libgen_url)
+            elif text.startswith("z-lib") and ".onion/" not in href:
+                _append_unique(external_urls_z_lib, href)
         except:
             pass
 
-    external_urls_welib = _get_download_urls_from_welib(book_id) if USE_CF_BYPASS else set()
+    logger.debug(
+        "Source inventory for %s -> aa_no_wait=%d, aa_wait=%d, libgen=%d, zlib=%d",
+        book_id,
+        len(slow_urls_no_waitlist),
+        len(slow_urls_with_waitlist),
+        len(external_urls_libgen),
+        len(external_urls_z_lib),
+    )
 
     urls = []
-    urls += list(external_urls_welib) if PRIORITIZE_WELIB else []
-    urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else []
-    urls += list(external_urls_libgen)
-    urls += list(external_urls_welib) if not PRIORITIZE_WELIB else []
-    urls += list(slow_urls_with_waitlist)  if USE_CF_BYPASS else []
-    urls += list(external_urls_z_lib)
+
+    # Priority: reliable sources first, then external fallbacks
+    # 1. AA slow (no waitlist) - instant but can be slow
+    # 2. Libgen - instant, external
+    # 3. AA slow (waitlist) - has countdown timer but faster once started
+    # Note: Z-Library disabled - download tokens are session-bound
+    urls += slow_urls_no_waitlist if USE_CF_BYPASS else []
+    urls += external_urls_libgen
+    urls += slow_urls_with_waitlist if USE_CF_BYPASS else []
 
     for i in range(len(urls)):
-        urls[i] = downloader.get_absolute_url(AA_BASE_URL, urls[i])
+        urls[i] = downloader.get_absolute_url(network.get_aa_base_url(), urls[i])
 
     # Remove empty urls
     urls = [url for url in urls if url != ""]
 
+    # Tag AA slow URLs with detailed source type for skip/retry tracking
+    base_url = network.get_aa_base_url()
+    for rel_url in slow_urls_no_waitlist:
+        abs_url = downloader.get_absolute_url(base_url, rel_url)
+        if abs_url:
+            _url_source_types[abs_url] = "aa-slow-nowait"
+    for rel_url in slow_urls_with_waitlist:
+        abs_url = downloader.get_absolute_url(base_url, rel_url)
+        if abs_url:
+            _url_source_types[abs_url] = "aa-slow-wait"
+
     # Filter out divs that are not text
     original_divs = divs
     divs = [div for div in divs if div.text.strip() != ""]
@@ -241,7 +280,8 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
             if format == "" and f.strip().lower() in SUPPORTED_FORMATS:
                 format = f.strip().lower()
             if size == "" and any(u in f.strip().lower() for u in ["mb", "kb", "gb"]):
-                size = f.strip().lower()
+                # Preserve original case but uppercase the unit (e.g., "5.2 mb" -> "5.2 MB")
+                size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE)
             if content == "":
                 for ct in DOWNLOAD_PATHS.keys():
                     if ct in f.strip().lower():
@@ -253,7 +293,8 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
                 if format == "" and stripped and " " not in stripped:
                     format = stripped
                 if size == "" and "." in stripped:
-                    size = stripped
+                    # Uppercase any size units
+                    size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE)
     
     book_title = _find_in_divs(divs, "🔍")[0].strip("🔍").strip()
 
@@ -265,8 +306,8 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
         preview=preview,
         title=book_title,
         content=content,
-        publisher=_find_in_divs(divs, "icon-[mdi--company]", isClass=True)[0],
-        author=_find_in_divs(divs, "icon-[mdi--user-edit]", isClass=True)[0],
+        publisher=_find_in_divs(divs, "icon-[mdi--company]", is_class=True)[0],
+        author=_find_in_divs(divs, "icon-[mdi--user-edit]", is_class=True)[0],
         format=format,
         size=size,
         description=description,
@@ -289,32 +330,85 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
 
     return book_info
 
-def _find_in_divs(divs: List[str], text: str, isClass: bool = False) -> List[str]:
-    divs_found = []
+def _find_in_divs(divs: List, text: str, is_class: bool = False) -> List[str]:
+    """Find divs containing text or having a specific class."""
+    results = []
     for div in divs:
-        if isClass:
-            if div.find(class_ = text):
-                divs_found.append(div.text.strip())
-        else:
-            if text in div.text.strip():
-                divs_found.append(div.text.strip())
-    return divs_found
+        if is_class:
+            if div.find(class_=text):
+                results.append(div.text.strip())
+        elif text in div.text.strip():
+            results.append(div.text.strip())
+    return results
 
-def _get_download_urls_from_welib(book_id: str) -> set[str]:
-    if ALLOW_USE_WELIB == False:
-        return set()
-    """Get download urls from welib.org."""
-    url = f"https://welib.org/md5/{book_id}"
-    logger.info(f"Getting download urls from welib.org for {book_id}. While this uses the bypasser, it will not start downloading them yet.")
-    html = downloader.html_get_page(url, use_bypasser=True)
-    if not html:
+# Download source definitions: (log_label, friendly_name, url_patterns)
+_DOWNLOAD_SOURCES = [
+    ("welib", "Welib", ["welib.org"]),
+    ("aa-fast", "Anna's Archive (Fast)", ["/dyn/api/fast_download"]),
+    ("aa-slow-wait", "Anna's Archive (Waitlist)", []),  # Matched via _url_source_types
+    ("aa-slow-nowait", "Anna's Archive", []),  # Matched via _url_source_types
+    ("aa-slow", "Anna's Archive", ["/slow_download/", "annas-"]),  # Fallback for untagged AA URLs
+    ("libgen", "Libgen", ["libgen"]),
+    ("zlib", "Z-Library", ["z-lib", "zlibrary"]),
+]
+
+# Track detailed source types for AA slow URLs (populated during get_book_info)
+_url_source_types: dict[str, str] = {}
+
+
+def _get_source_info(link: str) -> tuple[str, str]:
+    """Get source label and friendly name for a download link.
+
+    Args:
+        link: Download URL
+
+    Returns:
+        Tuple of (log_label, friendly_name)
+    """
+    # Check detailed source type mapping first (for AA slow distinction)
+    if link in _url_source_types:
+        detailed_label = _url_source_types[link]
+        for log_label, friendly_name, _ in _DOWNLOAD_SOURCES:
+            if log_label == detailed_label:
+                return log_label, friendly_name
+
+    for log_label, friendly_name, patterns in _DOWNLOAD_SOURCES:
+        if patterns and any(pattern in link for pattern in patterns):
+            return log_label, friendly_name
+    return "unknown", "Mirror"
+
+
+def _label_source(link: str) -> str:
+    """Get lightweight source tag for logging/metrics."""
+    return _get_source_info(link)[0]
+
+
+def _friendly_source_name(link: str) -> str:
+    """Get user-friendly name for a download source."""
+    return _get_source_info(link)[1]
+
+def _get_download_urls_from_welib(book_id: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> list[str]:
+    """Get download URLs from welib.org (bypasser required)."""
+    if not ALLOW_USE_WELIB:
         return []
+    url = f"https://welib.org/md5/{book_id}"
+    logger.info(f"Fetching welib.org download URLs for {book_id}")
+    try:
+        html = downloader.html_get_page(url, use_bypasser=True, selector=selector or network.AAMirrorSelector(), cancel_flag=cancel_flag)
+    except Exception as exc:
+        logger.error_trace(f"Welib fetch failed for {book_id}: {exc}")
+        return []
+    if not html:
+        logger.warning(f"Welib page empty for {book_id}")
+        return []
+    
     soup = BeautifulSoup(html, "html.parser")
-    download_links = soup.find_all("a", href=True)
-    download_links = [link["href"] for link in download_links]
-    download_links = [link for link in download_links if "/slow_download/" in link]
-    download_links = [downloader.get_absolute_url(url, link) for link in download_links]
-    return set(download_links)
+    links = [
+        downloader.get_absolute_url(url, a["href"])
+        for a in soup.find_all("a", href=True)
+        if "/slow_download/" in a["href"]
+    ]
+    return list(dict.fromkeys(links))  # Dedupe while preserving order
 
 def _get_next_value_div(label_div: Tag) -> Optional[Tag]:
     """Find the next sibling div that holds the value for a metadata label."""
@@ -400,7 +494,14 @@ def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]:
     }
 
 
-def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str], None]] = None) -> Optional[str]:
+# After N consecutive failures of the same source type, skip remaining sources of that type
+SOURCE_FAILURE_THRESHOLD = 4
+
+# Minimum valid file size in bytes (10KB) - anything smaller is likely an error page
+MIN_VALID_FILE_SIZE = 10 * 1024
+
+
+def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None) -> Optional[str]:
     """Download a book from available sources.
 
     Args:
@@ -408,88 +509,277 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio
         title: Book title for logging
         progress_callback: Optional callback for download progress updates
         cancel_flag: Optional cancellation flag
-        status_callback: Optional callback for status updates
+        status_callback: Optional callback for status updates (status, message)
 
     Returns:
         str: Download URL if successful, None otherwise
     """
 
+    selector = network.AAMirrorSelector()
+
     if len(book_info.download_urls) == 0:
         book_info = get_book_info(book_info.id)
-    download_links = book_info.download_urls
+    download_links = list(book_info.download_urls)
 
     # If AA_DONATOR_KEY is set, use the fast download URL. Else try other sources.
     if AA_DONATOR_KEY != "":
         download_links.insert(
             0,
-            f"{AA_BASE_URL}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}",
+            f"{network.get_aa_base_url()}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}",
         )
 
-    for link in download_links:
-        try:
-            # Update status to resolving before attempting download URL fetch
-            if status_callback:
-                status_callback("resolving")
+    # Preserve order but drop duplicates to avoid retrying the same host
+    download_links = list(dict.fromkeys(download_links))
 
-            download_url = _get_download_url(link, book_info.title, cancel_flag, status_callback)
-            if download_url != "":
-                # Update status to downloading before starting actual download
-                if status_callback:
-                    status_callback("downloading")
+    # Round-robin rotation for AA slow download URLs to distribute load across mirrors
+    # This prevents all concurrent downloads from hitting the same partner server first
+    # Rotate aa-slow-nowait and aa-slow-wait independently to preserve priority ordering
+    rotation_value = next(_aa_slow_rotation)
 
-                logger.info(f"Downloading `{book_info.title}` from `{download_url}`")
+    def _rotate_category_in_place(links: list, source_type: str) -> int:
+        """Rotate URLs of a specific source type within the list, preserving their positions."""
+        indices = [i for i, u in enumerate(links) if _url_source_types.get(u) == source_type]
+        if len(indices) <= 1:
+            return 0
+        rotation = rotation_value % len(indices)
+        if rotation == 0:
+            return 0
+        # Extract values, rotate, put back
+        values = [links[i] for i in indices]
+        rotated = values[rotation:] + values[:rotation]
+        for idx, val in zip(indices, rotated):
+            links[idx] = val
+        return rotation
 
-                data = downloader.download_url(download_url, book_info.size or "", progress_callback, cancel_flag)
-                if not data:
-                    raise Exception("No data received")
+    nowait_rotation = _rotate_category_in_place(download_links, "aa-slow-nowait")
+    wait_rotation = _rotate_category_in_place(download_links, "aa-slow-wait")
 
-                logger.info(f"Download finished. Writing to {book_path}")
-                with open(book_path, "wb") as f:
-                    f.write(data.getbuffer())
-                logger.info(f"Writing `{book_info.title}` successfully")
-                return download_url
+    if nowait_rotation or wait_rotation:
+        logger.info(f"AA source rotation: nowait={nowait_rotation}, wait={wait_rotation}")
 
-        except Exception as e:
-            logger.error_trace(f"Failed to download from {link}: {e}")
+    links_queue = download_links
+    
+    # Fetch welib URLs upfront when prioritized
+    welib_fallback_loaded = "welib" in DEBUG_SKIP_SOURCES  # Skip welib entirely if in debug skip list
+    if USE_CF_BYPASS and PRIORITIZE_WELIB and ALLOW_USE_WELIB and not welib_fallback_loaded:
+        logger.info("Fetching welib.org download URLs (PRIORITIZE_WELIB enabled)")
+        if status_callback:
+            status_callback("resolving", "Fetching welib sources...")
+        welib_links = _get_download_urls_from_welib(book_info.id, selector=selector, cancel_flag=cancel_flag)
+        if welib_links:
+            links_queue = welib_links + [l for l in links_queue if l not in welib_links]
+        welib_fallback_loaded = True
+    
+    total_sources = len(links_queue)
+    
+    # Handle case where no download sources are available
+    if total_sources == 0:
+        logger.warning(f"No download sources available for: {book_info.title}")
+        if status_callback:
+            status_callback("error", "No download sources found")
+        return None
+    
+    # Track consecutive failures per source type to skip after threshold
+    source_failures: dict[str, int] = {}
+    # Iterate with index so we can append welib links later
+    idx = 0
+    while idx < len(links_queue):
+        link = links_queue[idx]
+        source_label = _label_source(link)
+        friendly_name = _friendly_source_name(link)
+
+        # Debug: skip sources for testing fallback chains
+        if source_label in DEBUG_SKIP_SOURCES:
+            logger.info("DEBUG_SKIP_SOURCES: skipping %s (%s)", source_label, link)
+            idx += 1
             continue
 
+        # Skip source types that have failed too many times
+        if source_failures.get(source_label, 0) >= SOURCE_FAILURE_THRESHOLD:
+            logger.info("Skipping %s - source type '%s' failed %d times", link, source_label, SOURCE_FAILURE_THRESHOLD)
+            idx += 1
+            continue
+        
+        try:
+            current_pos = idx + 1
+            # Update total if we added more sources
+            total_sources = len(links_queue)
+            
+            logger.info("Trying download source [%s]: %s (%d/%d)", source_label, link, current_pos, total_sources)
+            
+            # Build source context for status messages (e.g., "Welib (1/12)")
+            source_context = f"{friendly_name} (Server #{current_pos})"
+            
+            # Update status with simple message showing which source we're trying
+            if status_callback:
+                status_callback("resolving", f"Trying {source_context}")
+
+            download_url = _get_download_url(link, book_info.title, cancel_flag, status_callback, selector, source_context)
+            if download_url == "":
+                raise Exception("No download URL resolved")
+
+            logger.info("Resolved download URL [%s]: %s", source_label, download_url)
+
+            # Pass source page as referer (required by some sites)
+            data = downloader.download_url(download_url, book_info.size or "", progress_callback, cancel_flag, selector, status_callback, referer=link)
+            if not data:
+                raise Exception("No data received from download")
+            
+            # Validate file size - reject suspiciously small files
+            file_size = data.tell()
+            if file_size < MIN_VALID_FILE_SIZE:
+                logger.warning(f"Downloaded file too small ({file_size} bytes), likely an error page")
+                raise Exception(f"File too small ({file_size} bytes)")
+
+            logger.debug(f"Download finished ({file_size} bytes). Writing to {book_path}")
+            data.seek(0)  # Reset buffer position before writing
+            with open(book_path, "wb") as f:
+                f.write(data.getbuffer())
+            return download_url
+
+        except Exception as e:
+            logger.warning(f"Failed to download from {link} (source={source_label}): {e}")
+            source_failures[source_label] = source_failures.get(source_label, 0) + 1
+            idx += 1
+            # If we exhausted primary links and haven't loaded welib yet, fetch them lazily
+            if (
+                idx >= len(links_queue)
+                and not welib_fallback_loaded
+                and USE_CF_BYPASS
+                and ALLOW_USE_WELIB
+            ):
+                welib_selector = selector  # reuse AA mirror selector for consistency
+                welib_links = _get_download_urls_from_welib(book_info.id, selector=welib_selector, cancel_flag=cancel_flag)
+                welib_fallback_loaded = True
+                if welib_links:
+                    new_links = [wl for wl in welib_links if wl not in links_queue]
+                    if new_links:
+                        logger.info("Adding welib fallback links (%d)", len(new_links))
+                        links_queue.extend(new_links)
+                        # continue loop to try newly added links
+            continue
+
+    # All sources exhausted - report final error to UI
+    if status_callback:
+        status_callback("error", f"All {len(links_queue)} sources failed")
+    
     return None
 
 
-def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str], None]] = None) -> str:
-    """Extract actual download URL from various source pages."""
+def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None, selector: Optional[network.AAMirrorSelector] = None, source_context: Optional[str] = None) -> str:
+    """Extract actual download URL from various source pages.
+    
+    Args:
+        link: URL to extract download link from
+        title: Book title for logging
+        cancel_flag: Optional cancellation flag
+        status_callback: Optional callback for status updates
+        selector: Optional AA mirror selector
+        source_context: Optional context string like "Welib (1/12)" for status messages
+    """
+    sel = selector or network.AAMirrorSelector()
 
+    # AA fast download API (JSON response)
+    if link.startswith(f"{network.get_aa_base_url()}/dyn/api/fast_download.json"):
+        page = downloader.html_get_page(link, selector=sel, cancel_flag=cancel_flag)
+        return downloader.get_absolute_url(link, json.loads(page).get("download_url", ""))
+
+    html = downloader.html_get_page(link, selector=sel, cancel_flag=cancel_flag)
+    if not html:
+        return ""
+
+    soup = BeautifulSoup(html, "html.parser")
     url = ""
 
-    if link.startswith(f"{AA_BASE_URL}/dyn/api/fast_download.json"):
-        page = downloader.html_get_page(link, status_callback=status_callback)
-        url = json.loads(page).get("download_url")
+    # Z-Library
+    if link.startswith("https://z-lib."):
+        dl = soup.find("a", href=True, class_="addDownloadedBook")
+        url = dl["href"] if dl else ""
+
+    # AA slow download / partner servers
+    elif "/slow_download/" in link:
+        url = _extract_slow_download_url(soup, link, title, cancel_flag, status_callback, sel, source_context)
+
+    # Libgen (GET button)
     else:
-        html = downloader.html_get_page(link, status_callback=status_callback)
-
-        if html == "":
-            return ""
-
-        soup = BeautifulSoup(html, "html.parser")
-
-        if link.startswith("https://z-lib."):
-            download_link = soup.find_all("a", href=True, class_="addDownloadedBook")
-            if download_link:
-                url = download_link[0]["href"]
-        elif "/slow_download/" in link:
-            download_links = soup.find_all("a", href=True, string="📚 Download now")
-            if not download_links:
-                countdown = soup.find_all("span", class_="js-partner-countdown")
-                if countdown:
-                    sleep_time = int(countdown[0].text)
-                    logger.info(f"Waiting {sleep_time}s for {title}")
-                    if cancel_flag is not None and cancel_flag.wait(timeout=sleep_time):
-                        logger.info(f"Cancelled wait for {title}")
-                        return ""
-                    url = _get_download_url(link, title, cancel_flag, status_callback)
-            else:
-                url = download_links[0]["href"]
-        else:
-            url = soup.find_all("a", string="GET")[0]["href"]
+        get_btn = soup.find("a", string="GET")
+        url = get_btn["href"] if get_btn else ""
 
     return downloader.get_absolute_url(link, url)
+
+
+def _extract_slow_download_url(soup: BeautifulSoup, link: str, title: str, cancel_flag: Optional[Event], status_callback, selector, source_context: Optional[str] = None) -> str:
+    """Extract download URL from AA slow download pages."""
+    # Try "Download now" button variations
+    dl_link = soup.find("a", href=True, string="📚 Download now")
+    if not dl_link:
+        dl_link = soup.find("a", href=True, string=lambda s: s and "Download now" in s)
+    if dl_link:
+        return dl_link["href"]
+
+    # Try finding URL in gray background span (AA's copy URL format)
+    # The URL appears as plain text in <span class="bg-gray-200 ...">http://...</span>
+    for span in soup.find_all("span", class_=lambda c: c and "bg-gray-200" in c):
+        text = span.get_text(strip=True)
+        if text.startswith("http://") or text.startswith("https://"):
+            return text
+
+    # Try "copy this URL" pattern (legacy)
+    copy_text = soup.find(string=lambda s: s and "copy this url" in s.lower())
+    if copy_text and copy_text.parent:
+        parent = copy_text.parent
+        next_link = parent.find_next("a", href=True)
+        if next_link and next_link.get("href"):
+            return next_link["href"]
+        code_elem = parent.find_next("code")
+        if code_elem:
+            return code_elem.get_text(strip=True)
+        for sibling in parent.find_next_siblings():
+            text = sibling.get_text(strip=True) if hasattr(sibling, 'get_text') else str(sibling).strip()
+            if text.startswith("http"):
+                return text
+
+    # Check for countdown timer (waitlist)
+    countdown = soup.find("span", class_="js-partner-countdown")
+    if countdown:
+        # Cap countdown at 10 minutes to prevent malformed HTML from blocking indefinitely
+        MAX_COUNTDOWN_SECONDS = 600
+        try:
+            raw_countdown = int(countdown.text)
+        except (ValueError, TypeError):
+            logger.warning(f"Invalid countdown value '{countdown.text}', skipping wait")
+            raw_countdown = 0
+        sleep_time = min(raw_countdown, MAX_COUNTDOWN_SECONDS)
+        if raw_countdown > MAX_COUNTDOWN_SECONDS:
+            logger.warning(f"Countdown {raw_countdown}s exceeds max, capping at {MAX_COUNTDOWN_SECONDS}s")
+        logger.info(f"Waiting {sleep_time}s for {title}")
+        
+        # Live countdown with status updates
+        remaining = sleep_time
+        while remaining > 0:
+            # Format countdown message with source context
+            if source_context:
+                wait_msg = f"{source_context} - Waiting {remaining}s"
+            else:
+                wait_msg = f"Waiting {remaining}s"
+            
+            if status_callback:
+                status_callback("resolving", wait_msg)
+            
+            # Wait 1 second (or until cancelled)
+            if cancel_flag and cancel_flag.wait(timeout=1):
+                logger.info(f"Cancelled wait for {title}")
+                return ""
+            
+            remaining -= 1
+        
+        # After countdown, update status and re-fetch
+        if status_callback and source_context:
+            status_callback("resolving", f"{source_context} - Fetching...")
+        
+        return _get_download_url(link, title, cancel_flag, status_callback, selector, source_context)
+
+    # Debug fallback
+    link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]]
+    logger.warning(f"No download URL found. First 10 links: {link_texts}")
+    return ""

cloudflare_bypasser_external.py

@@ -1,6 +1,17 @@
 from logger import setup_logger
-from typing import Optional
+from threading import Event
+from typing import Optional, TYPE_CHECKING
 import requests
+import time
+import random
+
+if TYPE_CHECKING:
+    import network
+
+
+class BypassCancelledException(Exception):
+    """Raised when a bypass operation is cancelled."""
+    pass
 
 try:
     from env import EXT_BYPASSER_PATH, EXT_BYPASSER_TIMEOUT, EXT_BYPASSER_URL
@@ -9,26 +20,140 @@ except ImportError:
 
 logger = setup_logger(__name__)
 
+# Connection timeout (seconds) - how long to wait for external bypasser to accept connection
+CONNECT_TIMEOUT = 10
+# Maximum read timeout cap (seconds) - hard limit regardless of EXT_BYPASSER_TIMEOUT
+MAX_READ_TIMEOUT = 120
+# Buffer added to bypasser's configured timeout (seconds) - accounts for processing overhead
+READ_TIMEOUT_BUFFER = 15
+# Retry settings for bypasser failures
+MAX_RETRY = 5
+BACKOFF_BASE = 1.0
+BACKOFF_CAP = 10.0
 
-def get_bypassed_page(url: str) -> Optional[str]:
-    """Fetch HTML content from a URL using an External Cloudflare Resolver.
 
+def _fetch_via_bypasser(target_url: str) -> Optional[str]:
+    """Make a single request to the external bypasser service.
+    
     Args:
-        url: Target URL
+        target_url: The URL to fetch through the bypasser
+        
     Returns:
-        str: HTML content if successful, None otherwise
+        HTML content if successful, None otherwise
     """
     if not EXT_BYPASSER_URL or not EXT_BYPASSER_PATH:
-        logger.error("Wrong External Bypass configuration. Please check your environment configuration.")
+        logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.")
         return None
-    ext_url = f"{EXT_BYPASSER_URL}{EXT_BYPASSER_PATH}"
+    
+    bypasser_endpoint = f"{EXT_BYPASSER_URL}{EXT_BYPASSER_PATH}"
     headers = {"Content-Type": "application/json"}
-    data = {
+    payload = {
         "cmd": "request.get",
-        "url": url,
+        "url": target_url,
         "maxTimeout": EXT_BYPASSER_TIMEOUT
     }
-    response = requests.post(ext_url, headers=headers, json=data)
-    response.raise_for_status()
-    logger.debug(f"External Bypass response for '{url}': {response.json()['status']} - {response.json()['message']}")
-    return response.json()['solution']['response']
+    
+    # Calculate read timeout: bypasser timeout (ms → s) + buffer, capped at max
+    read_timeout = min((EXT_BYPASSER_TIMEOUT / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT)
+    
+    try:
+        response = requests.post(
+            bypasser_endpoint, 
+            headers=headers, 
+            json=payload, 
+            timeout=(CONNECT_TIMEOUT, read_timeout)
+        )
+        response.raise_for_status()
+        result = response.json()
+        
+        status = result.get('status', 'unknown')
+        message = result.get('message', '')
+        logger.debug(f"External bypasser response for '{target_url}': {status} - {message}")
+        
+        # Check for error status (bypasser returns status="error" with solution=null on failure)
+        if status != 'ok':
+            logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}")
+            return None
+        
+        solution = result.get('solution')
+        if not solution:
+            logger.warning(f"External bypasser returned empty solution for '{target_url}'")
+            return None
+        
+        html = solution.get('response', '')
+        if not html:
+            logger.warning(f"External bypasser returned empty response for '{target_url}'")
+            return None
+        
+        return html
+        
+    except requests.exceptions.Timeout:
+        logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)")
+        return None
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"External bypasser request failed for '{target_url}': {e}")
+        return None
+    except (KeyError, TypeError, ValueError) as e:
+        logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}")
+        return None
+
+
+def get_bypassed_page(url: str, selector: Optional["network.AAMirrorSelector"] = None, cancel_flag: Optional[Event] = None) -> Optional[str]:
+    """Fetch HTML content from a URL using an external Cloudflare bypasser service.
+
+    Retries with exponential backoff and mirror/DNS rotation on failure.
+
+    Args:
+        url: Target URL to fetch
+        selector: Mirror selector for AA URL rewriting and rotation
+        cancel_flag: Optional threading Event to signal cancellation
+
+    Returns:
+        HTML content if successful, None otherwise
+
+    Raises:
+        BypassCancelledException: If cancel_flag is set during operation
+    """
+    import network
+    sel = selector or network.AAMirrorSelector()
+
+    for attempt in range(1, MAX_RETRY + 1):
+        # Check for cancellation before each attempt
+        if cancel_flag and cancel_flag.is_set():
+            logger.info("External bypasser cancelled by user")
+            raise BypassCancelledException("Bypass cancelled")
+
+        attempt_url = sel.rewrite(url)
+        result = _fetch_via_bypasser(attempt_url)
+        if result:
+            return result
+
+        if attempt == MAX_RETRY:
+            break
+
+        # Check for cancellation before backoff wait
+        if cancel_flag and cancel_flag.is_set():
+            logger.info("External bypasser cancelled during retry")
+            raise BypassCancelledException("Bypass cancelled")
+
+        # Backoff with jitter before retry, checking cancellation during wait
+        delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random()
+        logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s")
+
+        # Check cancellation during delay (check every second)
+        for _ in range(int(delay)):
+            if cancel_flag and cancel_flag.is_set():
+                logger.info("External bypasser cancelled during backoff")
+                raise BypassCancelledException("Bypass cancelled")
+            time.sleep(1)
+        # Sleep remaining fraction
+        remaining = delay - int(delay)
+        if remaining > 0:
+            time.sleep(remaining)
+
+        # Rotate mirror/DNS for next attempt
+        new_base, action = sel.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            logger.info(f"Rotated {action} for retry")
+
+    return None

config.py

@@ -35,22 +35,34 @@ logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
 # Network settings
 _custom_dns = env._CUSTOM_DNS.lower().strip()
 _doh_server = ""
-if _custom_dns == "google":
+
+if _custom_dns == "auto" or _custom_dns == "":
+    # Auto mode - DNS provider rotation handled by network.py
+    # Starts with system DNS, switches to providers from DNS_PROVIDERS on failure
+    CUSTOM_DNS = []
+    _doh_server = ""
+    logger.info("CUSTOM_DNS: auto (starts with system DNS, rotates on failure)")
+elif _custom_dns == "google":
     CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860:0000:0000:0000:0000:8888", "2001:4860:4860:0000:0000:0000:0000:8844"]
-    _doh_server = "https://dns.google/dns-query"
+    _doh_server = "https://dns.google/resolve"
+    logger.info(f"CUSTOM_DNS: google {CUSTOM_DNS}")
 elif _custom_dns == "quad9":
     CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:00fe:0000:0000:0000:0000:0000:00fe", "2620:00fe:0000:0000:0000:0000:0000:0009"]
     _doh_server = "https://dns.quad9.net/dns-query"
+    logger.info(f"CUSTOM_DNS: quad9 {CUSTOM_DNS}")
 elif _custom_dns == "cloudflare":
     CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700:0000:0000:0000:0000:1111", "2606:4700:4700:0000:0000:0000:0000:1001"]
     _doh_server = "https://cloudflare-dns.com/dns-query"
+    logger.info(f"CUSTOM_DNS: cloudflare {CUSTOM_DNS}")
 elif _custom_dns == "opendns":
     CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:0119:0035:0000:0000:0000:0000:0035", "2620:0119:0053:0000:0000:0000:0000:0053"]
     _doh_server = "https://doh.opendns.com/dns-query"
+    logger.info(f"CUSTOM_DNS: opendns {CUSTOM_DNS}")
 else:
+    # Custom DNS IPs provided by user
     _custom_dns_ip = _custom_dns.split(",")
     CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()]
-logger.info(f"CUSTOM_DNS: {CUSTOM_DNS}")
+    logger.info(f"CUSTOM_DNS: custom {CUSTOM_DNS}")
 DOH_SERVER = _doh_server
 if env.USE_DOH:
     DOH_SERVER = _doh_server
@@ -58,6 +70,15 @@ else:
     DOH_SERVER = ""
 logger.info(f"DOH_SERVER: {DOH_SERVER}")
 
+# Warn about external bypasser DNS limitations
+if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS:
+    logger.warning(
+        "Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
+        "not this application's custom DNS settings. If you experience DNS-related blocks, "
+        "configure DNS at the Docker/system level for your FlareSolverr container, "
+        "or consider using the internal bypasser which integrates with the app's DNS system."
+    )
+
 # Proxy settings
 PROXIES = {}
 if env.HTTP_PROXY:

docker-compose.dev.yml

@@ -9,8 +9,6 @@ services:
       target: cwa-bd
     environment:
       DEBUG: true
-      USE_DOH: true
-      CUSTOM_DNS: cloudflare
     volumes:
       - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader
       - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader

downloader.py

@@ -1,138 +1,357 @@
 """Network operations manager for the book downloader application."""
 
-import network
-network.init()
-import requests
+import random
 import time
 from io import BytesIO
-from typing import Optional
-from urllib.parse import urlparse
-from tqdm import tqdm
-from typing import Callable
 from threading import Event
-from logger import setup_logger
+from typing import Callable, Optional
+from urllib.parse import urlparse
+
+import requests
+from tqdm import tqdm
+
+import network
 from config import PROXIES
-from env import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS, USING_EXTERNAL_BYPASSER
+from env import DEFAULT_SLEEP, MAX_RETRY, USE_CF_BYPASS, USING_EXTERNAL_BYPASSER
+from logger import setup_logger
+
+# Import bypasser if enabled
 if USE_CF_BYPASS:
     if USING_EXTERNAL_BYPASSER:
         from cloudflare_bypasser_external import get_bypassed_page
+        # External bypasser doesn't share cookies
+        get_cf_cookies_for_domain = lambda domain: {}
     else:
-        from cloudflare_bypasser import get_bypassed_page
+        from cloudflare_bypasser import get_bypassed_page, get_cf_cookies_for_domain
 
 logger = setup_logger(__name__)
 
+# Network settings
+REQUEST_TIMEOUT = (5, 10)  # (connect, read)
+MAX_DOWNLOAD_RETRIES = 2
+MAX_RESUME_ATTEMPTS = 3
+RETRYABLE_CODES = (429, 500, 502, 503, 504)
+CONNECTION_ERRORS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout,
+                     requests.exceptions.SSLError, requests.exceptions.ChunkedEncodingError)
+DOWNLOAD_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+}
 
-def html_get_page(url: str, retry: int = MAX_RETRY, use_bypasser: bool = False, status_callback: Optional[Callable[[str], None]] = None) -> str:
-    """Fetch HTML content from a URL with retry mechanism.
-    
-    Args:
-        url: Target URL
-        retry: Number of retry attempts
-        use_bypasser: Whether to use Cloudflare bypasser
-        status_callback: Optional callback for status updates
-        
-    Returns:
-        str: HTML content if successful, None otherwise
-    """
-    response = None
+
+def parse_size_string(size: str) -> Optional[float]:
+    """Parse a human-readable size string (e.g., '10.5 MB') into bytes."""
+    if not size:
+        return None
     try:
-        logger.debug(f"html_get_page: {url}, retry: {retry}, use_bypasser: {use_bypasser}")
-        if use_bypasser and USE_CF_BYPASS:
-            if status_callback:
-                status_callback("bypassing")
-            logger.info(f"GET Using Cloudflare Bypasser for: {url}")
-            return get_bypassed_page(url)
-        else:
-            logger.info(f"GET: {url}")
-            response = requests.get(url, proxies=PROXIES)
-            response.raise_for_status()
-            logger.debug(f"Success getting: {url}")
-            time.sleep(1)
-        return str(response.text)
-        
-    except Exception as e:
-        if retry == 0:
-            logger.error_trace(f"Failed to fetch page: {url}, error: {e}")
-            return ""
-        
-        if use_bypasser and USE_CF_BYPASS:
-            logger.warning(f"Exception while using cloudflare bypass for URL: {url}")
-            logger.warning(f"Exception: {e}")
-            logger.warning(f"Response: {response}")
-        elif response is not None and response.status_code == 404:
-            logger.warning(f"404 error for URL: {url}")
-            return ""
-        elif response is not None and response.status_code == 403:
-            logger.warning(f"403 detected for URL: {url}. Should retry using cloudflare bypass.")
-            return html_get_page(url, retry - 1, True, status_callback)
-            
-        sleep_time = DEFAULT_SLEEP * (MAX_RETRY - retry + 1)
-        logger.warning(
-            f"Retrying GET {url} in {sleep_time} seconds due to error: {e}"
-        )
-        time.sleep(sleep_time)
-        return html_get_page(url, retry - 1, use_bypasser, status_callback)
-
-def download_url(link: str, size: str = "", progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None) -> Optional[BytesIO]:
-    """Download content from URL into a BytesIO buffer.
-    
-    Args:
-        link: URL to download from
-        
-    Returns:
-        BytesIO: Buffer containing downloaded content if successful
-    """
-    try:
-        logger.info(f"Downloading from: {link}")
-        response = requests.get(link, stream=True, proxies=PROXIES)
-        response.raise_for_status()
-
-        total_size : float = 0.0
-        try:
-            # we assume size is in MB
-            total_size = float(size.strip().replace(" ", "").replace(",", ".").upper()[:-2].strip()) * 1024 * 1024
-        except:
-            total_size = float(response.headers.get('content-length', 0))
-        
-        buffer = BytesIO()
-
-        # Initialize the progress bar with your guess
-        pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading')
-        for chunk in response.iter_content(chunk_size=1000):
-            buffer.write(chunk)
-            pbar.update(len(chunk))
-            if progress_callback is not None:
-                progress_callback(pbar.n * 100.0 / total_size)
-            if cancel_flag is not None and cancel_flag.is_set():
-                logger.info(f"Download cancelled: {link}")
-                return None
-            
-        pbar.close()
-        if buffer.tell() * 0.1 < total_size * 0.9:
-            # Check the content of the buffer if its HTML or binary
-            if response.headers.get('content-type', '').startswith('text/html'):
-                logger.warn(f"Failed to download content for {link}. Found HTML content instead.")
-                return None
-        return buffer
-    except requests.exceptions.RequestException as e:
-        logger.error_trace(f"Failed to download from {link}: {e}")
+        normalized = size.strip().replace(" ", "").replace(",", ".").upper()
+        multipliers = {"GB": 1024**3, "MB": 1024**2, "KB": 1024}
+        for suffix, mult in multipliers.items():
+            if normalized.endswith(suffix):
+                return float(normalized[:-2]) * mult
+        return float(normalized)
+    except (ValueError, IndexError):
         return None
 
-def get_absolute_url(base_url: str, url: str) -> str:
-    """Get absolute URL from relative URL and base URL.
+def _backoff_delay(attempt: int, base: float = 0.25, cap: float = 3.0) -> float:
+    """Exponential backoff with jitter."""
+    return min(cap, base * (2 ** (attempt - 1))) + random.random() * base
+
+
+def _get_status_code(e: Exception) -> Optional[int]:
+    """Extract HTTP status code from an exception, or None if not applicable."""
+    if isinstance(e, requests.exceptions.HTTPError) and e.response is not None:
+        return e.response.status_code
+    return None
+
+def _is_retryable_error(e: Exception) -> bool:
+    """Check if error is retryable (connection error or retryable HTTP status)."""
+    if isinstance(e, CONNECTION_ERRORS):
+        return True
+    status = _get_status_code(e)
+    return status in RETRYABLE_CODES if status else False
+
+
+def _try_rotation(original_url: str, current_url: str, selector: network.AAMirrorSelector) -> Optional[str]:
+    """Try mirror/DNS rotation. Returns new URL or None."""
+    if current_url.startswith(network.get_aa_base_url()):
+        new_base, action = selector.next_mirror_or_rotate_dns()
+        if action in ("mirror", "dns") and new_base:
+            new_url = selector.rewrite(original_url)
+            logger.info(f"[{action}] switching to: {new_url}")
+            return new_url
+    elif network.should_rotate_dns_for_url(current_url) and network.rotate_dns_provider():
+        logger.info(f"[dns-rotate] retrying: {original_url}")
+        return original_url
+    return None
+
+
+def html_get_page(
+    url: str,
+    retry: int = MAX_RETRY,
+    use_bypasser: bool = False,
+    selector: Optional[network.AAMirrorSelector] = None,
+    cancel_flag: Optional[Event] = None,
+) -> str:
+    """Fetch HTML content from a URL with retry mechanism."""
+    selector = selector or network.AAMirrorSelector()
+    original_url = url
+    current_url = selector.rewrite(original_url)
+    use_bypasser_now = use_bypasser
+
+    for attempt in range(1, retry + 1):
+        # Check for cancellation before each attempt
+        if cancel_flag and cancel_flag.is_set():
+            logger.info(f"html_get_page cancelled before attempt {attempt}")
+            return ""
+
+        try:
+            if use_bypasser_now and USE_CF_BYPASS:
+                logger.info(f"GET (bypasser): {current_url}")
+                try:
+                    result = get_bypassed_page(current_url, selector, cancel_flag)
+                    return result or ""
+                except Exception as e:
+                    logger.warning(f"Bypasser error: {type(e).__name__}: {e}")
+                    return ""
+
+            logger.info(f"GET: {current_url}")
+            # Try with CF cookies if available (from previous bypass)
+            cookies = {}
+            if USE_CF_BYPASS:
+                parsed = urlparse(current_url)
+                cookies = get_cf_cookies_for_domain(parsed.hostname or "")
+            response = requests.get(current_url, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies)
+            response.raise_for_status()
+            time.sleep(1)
+            return response.text
+
+        except Exception as e:
+            status = _get_status_code(e)
+
+            # 403 = Cloudflare/DDoS-Guard protection
+            if status == 403:
+                if USE_CF_BYPASS and not use_bypasser_now:
+                    # Before switching to bypasser, check if cookies have become available
+                    # (another concurrent download may have completed bypass and extracted cookies)
+                    parsed = urlparse(current_url)
+                    fresh_cookies = get_cf_cookies_for_domain(parsed.hostname or "")
+                    if fresh_cookies and not cookies:
+                        # Cookies are now available - retry with cookies before using bypasser
+                        logger.debug(f"403 but cookies now available - retrying with cookies: {current_url}")
+                        continue
+                    logger.info(f"403 detected; switching to bypasser: {current_url}")
+                    use_bypasser_now = True
+                    continue
+                logger.warning(f"403 error, giving up: {current_url}")
+                return ""
+
+            # 404 = Not found
+            if status == 404:
+                logger.warning(f"404 error: {current_url}")
+                return ""
+
+            # Try mirror/DNS rotation on retryable errors
+            if _is_retryable_error(e):
+                new_url = _try_rotation(original_url, current_url, selector)
+                if new_url:
+                    current_url = new_url
+                    continue
+
+            # Retry with backoff
+            if attempt < retry:
+                logger.warning(f"Retry {attempt}/{retry} for {current_url}: {type(e).__name__}: {e}")
+                time.sleep(_backoff_delay(attempt))
+            else:
+                logger.error(f"Giving up after {retry} attempts: {current_url}")
+
+    return ""
+
+
+def download_url(
+    link: str,
+    size: str = "",
+    progress_callback: Optional[Callable[[float], None]] = None,
+    cancel_flag: Optional[Event] = None,
+    _selector: Optional[network.AAMirrorSelector] = None,
+    status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
+    referer: Optional[str] = None,
+) -> Optional[BytesIO]:
+    """Download content from URL with automatic retry and resume support."""
+    selector = _selector or network.AAMirrorSelector()
+    current_url = selector.rewrite(link)
+
+    # Build headers with optional referer
+    headers = DOWNLOAD_HEADERS.copy()
+    if referer:
+        headers['Referer'] = referer
+    total_size = parse_size_string(size) or 0
+
+    attempt = 0
+
+    while attempt < MAX_DOWNLOAD_RETRIES:
+        if cancel_flag and cancel_flag.is_set():
+            return None
+
+        buffer = BytesIO()
+        bytes_downloaded = 0
+
+        try:
+            if attempt > 0 and status_callback:
+                status_callback("resolving", f"Connecting (Attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
+
+            logger.info(f"Downloading: {current_url} (attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})")
+            # Try with CF cookies if available
+            cookies = {}
+            if USE_CF_BYPASS:
+                parsed = urlparse(current_url)
+                cookies = get_cf_cookies_for_domain(parsed.hostname or "")
+                if cookies:
+                    logger.debug(f"Using {len(cookies)} cookies for {parsed.hostname}: {list(cookies.keys())}")
+            response = requests.get(current_url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers)
+            response.raise_for_status()
+
+            if status_callback:
+                status_callback("downloading", "")
+
+            total_size = total_size or float(response.headers.get('content-length', 0))
+            pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading')
+
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    buffer.write(chunk)
+                    bytes_downloaded += len(chunk)
+                    pbar.update(len(chunk))
+                    if progress_callback and total_size > 0:
+                        progress_callback(bytes_downloaded * 100.0 / total_size)
+                    if cancel_flag and cancel_flag.is_set():
+                        pbar.close()
+                        return None
+            pbar.close()
+
+            # Validate - check we didn't get HTML instead of file
+            if total_size > 0 and bytes_downloaded < total_size * 0.9:
+                if response.headers.get('content-type', '').startswith('text/html'):
+                    logger.warning(f"Received HTML instead of file: {current_url}")
+                    return None
+
+            logger.debug(f"Download completed: {bytes_downloaded} bytes")
+            return buffer
+
+        except requests.exceptions.RequestException as e:
+            status = _get_status_code(e)
+            retryable = _is_retryable_error(e)
+
+            # Non-retryable errors
+            if status in (403, 404):
+                logger.warning(f"Download failed ({status}): {current_url}")
+                return None
+
+            # Rate limited - skip to next source immediately
+            # (waiting doesn't help with concurrent downloads hitting the same server)
+            if status == 429:
+                logger.info(f"Rate limited (429) - trying next source")
+                if status_callback:
+                    status_callback("resolving", "Server busy, trying next...")
+                return None
+
+            # Timeout - don't retry, server likely overloaded
+            if isinstance(e, requests.exceptions.Timeout):
+                logger.warning(f"Timeout: {current_url} - skipping to next source")
+                if status_callback:
+                    status_callback("resolving", "Server timed out, trying next...")
+                return None
+
+            # Try to resume if we got some data
+            if bytes_downloaded > 0 and retryable:
+                resumed = _try_resume(current_url, buffer, bytes_downloaded, total_size, progress_callback, cancel_flag, headers)
+                if resumed:
+                    return resumed
+
+            # Try mirror/DNS rotation if nothing downloaded yet
+            if bytes_downloaded == 0 and retryable:
+                new_url = _try_rotation(link, current_url, selector)
+                if new_url:
+                    current_url = new_url
+                    attempt += 1
+                    continue
+
+            logger.warning(f"Download error: {type(e).__name__}: {e}")
+            if attempt < MAX_DOWNLOAD_RETRIES - 1:
+                time.sleep(_backoff_delay(attempt + 1))
+            attempt += 1
+
+    logger.error(f"Download failed after {MAX_DOWNLOAD_RETRIES} attempts: {link}")
+    return None
+
+
+def _try_resume(
+    url: str,
+    buffer: BytesIO,
+    start_byte: int,
+    total_size: float,
+    progress_callback: Optional[Callable[[float], None]],
+    cancel_flag: Optional[Event],
+    base_headers: Optional[dict] = None,
+) -> Optional[BytesIO]:
+    """Try to resume an interrupted download."""
+    for attempt in range(MAX_RESUME_ATTEMPTS):
+        logger.info(f"Resuming from {start_byte} bytes (attempt {attempt + 1}/{MAX_RESUME_ATTEMPTS})")
+        time.sleep(_backoff_delay(attempt + 1, base=0.5, cap=5.0))
+
+        try:
+            # Try with CF cookies if available
+            cookies = {}
+            if USE_CF_BYPASS:
+                parsed = urlparse(url)
+                cookies = get_cf_cookies_for_domain(parsed.hostname or "")
+            resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'}
+            response = requests.get(
+                url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT,
+                headers=resume_headers, cookies=cookies
+            )
+            
+            # Check resume support
+            if response.status_code == 200:  # Server doesn't support resume
+                logger.info("Server doesn't support resume")
+                return None
+            if response.status_code == 416:  # Range not satisfiable
+                logger.warning("Range not satisfiable")
+                return None
+            if response.status_code != 206:
+                response.raise_for_status()
+            
+            pbar = tqdm(total=total_size, initial=start_byte, unit='B', unit_scale=True, desc='Resuming')
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    buffer.write(chunk)
+                    start_byte += len(chunk)
+                    pbar.update(len(chunk))
+                    if progress_callback and total_size > 0:
+                        progress_callback(start_byte * 100.0 / total_size)
+                    if cancel_flag and cancel_flag.is_set():
+                        pbar.close()
+                        return None
+            pbar.close()
+            
+            logger.info(f"Resume completed: {start_byte} bytes")
+            return buffer
+            
+        except requests.exceptions.RequestException as e:
+            logger.debug(f"Resume attempt {attempt + 1} failed: {e}")
     
-    Args:
-        base_url: Base URL
-        url: Relative URL
-    """
-    if url.strip() == "":
-        return ""
-    if url.strip("#") == "":
-        return ""
-    if url.startswith("http"):
-        return url
-    parsed_url = urlparse(url)
-    parsed_base = urlparse(base_url)
-    if parsed_url.netloc == "" or parsed_url.scheme == "":
-        parsed_url = parsed_url._replace(netloc=parsed_base.netloc, scheme=parsed_base.scheme)
-    return parsed_url.geturl()
+    logger.warning(f"Resume failed after {MAX_RESUME_ATTEMPTS} attempts")
+    return None
+
+
+def get_absolute_url(base_url: str, url: str) -> str:
+    """Convert a relative URL to absolute using the base URL."""
+    url = url.strip()
+    if not url or url == "#" or url.startswith("http"):
+        return url if url.startswith("http") else ""
+    parsed = urlparse(url)
+    base = urlparse(base_url)
+    if not parsed.netloc or not parsed.scheme:
+        parsed = parsed._replace(netloc=base.netloc, scheme=base.scheme)
+    return parsed.geturl()

env.py

@@ -50,6 +50,10 @@ _CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
 FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
 FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
 DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
+# Debug: skip specific download sources for testing fallback chains
+# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib
+_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
+DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip())
 PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
 ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true"))
 
@@ -65,11 +69,12 @@ else:
 ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
 MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
 MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
-DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "5"))
+DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "1"))
 DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
-_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "").strip()
+_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "auto").strip()
 USE_DOH = string_to_bool(os.getenv("USE_DOH", "false"))
 BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
+BYPASS_WARMUP_ON_CONNECT = string_to_bool(os.getenv("BYPASS_WARMUP_ON_CONNECT", "true"))
 
 # Logging settings
 LOG_FILE = LOG_DIR / "cwa-book-downloader.log"

genDebug.sh

@@ -124,15 +124,15 @@ fi
 # Add environment variables (redacting sensitive info)
 env | grep -v -E "(AA_DONATOR_KEY)" | sort > "$LOG_DIR/environment.txt"
 
-echo "--- HTTPBin ---" > $LOG_DIR/network_info.txt
+echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt
 curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt
-ehco ""
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt
 curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt
-ehco ""
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt
 curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt
-ehco ""
+echo "" >> $LOG_DIR/network_info.txt
 echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt
 curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt
 

logger.py

@@ -23,16 +23,18 @@ class CustomLogger(logging.Logger):
         self.warning(msg, *args, exc_info=True, **kwargs)
     
     def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log an info message with full stack trace."""
-        self.log_resource_usage()
+        """Log an info message (stack trace only if exception active)."""
         kwargs.pop('exc_info', None)
-        self.info(msg, *args, exc_info=True, **kwargs)
-    
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.info(msg, *args, exc_info=has_exception, **kwargs)
+
     def debug_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None:
-        """Log a debug message with full stack trace."""
-        self.log_resource_usage()
+        """Log a debug message (stack trace only if exception active)."""
         kwargs.pop('exc_info', None)
-        self.debug(msg, *args, exc_info=True, **kwargs)
+        # Only include exc_info if there's actually an exception
+        has_exception = sys.exc_info()[0] is not None
+        self.debug(msg, *args, exc_info=has_exception, **kwargs)
     
     def log_resource_usage(self):
         import psutil

models.py

@@ -7,6 +7,7 @@ from datetime import datetime, timedelta
 from threading import Lock, Event
 from pathlib import Path
 import queue
+import re
 import time
 from env import INGEST_DIR, STATUS_TIMEOUT
 
@@ -14,10 +15,7 @@ class QueueStatus(str, Enum):
     """Enum for possible book queue statuses."""
     QUEUED = "queued"
     RESOLVING = "resolving"
-    BYPASSING = "bypassing"
     DOWNLOADING = "downloading"
-    VERIFYING = "verifying"
-    INGESTING = "ingesting"
     COMPLETE = "complete"
     AVAILABLE = "available"
     ERROR = "error"
@@ -56,6 +54,45 @@ class BookInfo:
     download_path: Optional[str] = None
     priority: int = 0
     progress: Optional[float] = None
+    status_message: Optional[str] = None  # Detailed status message for UI display
+    added_time: Optional[float] = None  # Timestamp when added to queue
+
+    def get_filename(self, fallback_url: Optional[str] = None) -> str:
+        """Build sanitized filename: 'Author - Title (Year).format'
+
+        Resolves format from self.format, download_urls, or fallback_url.
+
+        Args:
+            fallback_url: URL to extract format from if not already known
+
+        Returns:
+            Sanitized filename safe for filesystem use
+        """
+        # Resolve format if needed
+        if not self.format:
+            for url in (self.download_urls[0] if self.download_urls else None, fallback_url):
+                if url:
+                    ext = url.split(".")[-1].lower()
+                    if ext and len(ext) <= 5 and ext.isalnum():
+                        self.format = ext
+                        break
+
+        # Build filename
+        parts = []
+        if self.author:
+            parts.append(self.author)
+            parts.append(" - ")
+        parts.append(self.title)
+        if self.year:
+            parts.append(f" ({self.year})")
+
+        filename = "".join(parts)
+        filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245]
+
+        if self.format:
+            filename = f"{filename}.{self.format}"
+
+        return filename
 
 class BookQueue:
     """Thread-safe book queue manager with priority support and cancellation."""
@@ -81,36 +118,40 @@ class BookQueue:
             # Don't add if already exists and not in error/done state
             if book_id in self._status and self._status[book_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]:
                 return
-                
+
+            added_time = time.time()
             book_data.priority = priority
-            queue_item = QueueItem(book_id, priority, time.time())
+            book_data.added_time = added_time
+            queue_item = QueueItem(book_id, priority, added_time)
             self._queue.put(queue_item)
             self._book_data[book_id] = book_data
             self._update_status(book_id, QueueStatus.QUEUED)
     
     def get_next(self) -> Optional[Tuple[str, Event]]:
         """Get next book ID from queue with cancellation flag.
-        
+
         Returns:
             Tuple of (book_id, cancel_flag) or None if queue is empty
         """
-        try:
-            queue_item = self._queue.get_nowait()
-            book_id = queue_item.book_id
-            
-            with self._lock:
-                # Check if book was cancelled while in queue
-                if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED:
-                    return self.get_next()  # Recursively get next non-cancelled item
-                
-                # Create cancellation flag for this download
-                cancel_flag = Event()
-                self._cancel_flags[book_id] = cancel_flag
-                self._active_downloads[book_id] = True
-                
-            return book_id, cancel_flag
-        except queue.Empty:
-            return None
+        # Use iterative approach to avoid stack overflow if many items are cancelled
+        while True:
+            try:
+                queue_item = self._queue.get_nowait()
+                book_id = queue_item.book_id
+
+                with self._lock:
+                    # Check if book was cancelled while in queue
+                    if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED:
+                        continue  # Skip cancelled items, try next
+
+                    # Create cancellation flag for this download
+                    cancel_flag = Event()
+                    self._cancel_flags[book_id] = cancel_flag
+                    self._active_downloads[book_id] = True
+
+                return book_id, cancel_flag
+            except queue.Empty:
+                return None
             
     def _update_status(self, book_id: str, status: QueueStatus) -> None:
         """Internal method to update status and timestamp."""
@@ -138,6 +179,12 @@ class BookQueue:
         with self._lock:
             if book_id in self._book_data:
                 self._book_data[book_id].progress = progress
+    
+    def update_status_message(self, book_id: str, message: str) -> None:
+        """Update detailed status message for a book."""
+        with self._lock:
+            if book_id in self._book_data:
+                self._book_data[book_id].status_message = message
             
     def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]:
         """Get current queue status."""
@@ -180,19 +227,19 @@ class BookQueue:
             return sorted(queue_items, key=lambda x: (x['priority'], x['added_time']))
             
     def cancel_download(self, book_id: str) -> bool:
-        """Cancel a download and mark it as cancelled.
-        
+        """Cancel a download or clear a completed/errored item.
+
         Args:
-            book_id: Book identifier to cancel
-            
+            book_id: Book identifier to cancel or clear
+
         Returns:
-            bool: True if cancellation was successful
+            bool: True if cancellation/clearing was successful
         """
         with self._lock:
             current_status = self._status.get(book_id)
-            
+
             # Allow cancellation during any active state
-            if current_status in [QueueStatus.RESOLVING, QueueStatus.BYPASSING, QueueStatus.DOWNLOADING, QueueStatus.VERIFYING, QueueStatus.INGESTING]:
+            if current_status in [QueueStatus.RESOLVING, QueueStatus.DOWNLOADING]:
                 # Signal active download to stop
                 if book_id in self._cancel_flags:
                     self._cancel_flags[book_id].set()
@@ -202,7 +249,15 @@ class BookQueue:
                 # Remove from queue and mark as cancelled
                 self._update_status(book_id, QueueStatus.CANCELLED)
                 return True
-            
+            elif current_status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]:
+                # Clear completed/errored/cancelled items from tracking
+                self._status.pop(book_id, None)
+                self._status_timestamps.pop(book_id, None)
+                self._book_data.pop(book_id, None)
+                self._cancel_flags.pop(book_id, None)
+                self._active_downloads.pop(book_id, None)
+                return True
+
             return False
             
     def set_priority(self, book_id: str, new_priority: int) -> bool:
@@ -281,6 +336,27 @@ class BookQueue:
         """Get list of currently active download book IDs."""
         with self._lock:
             return list(self._active_downloads.keys())
+    
+    def has_pending_work(self) -> bool:
+        """Check if there are any active downloads or queued items.
+        
+        This is useful for determining if the bypasser should stay active
+        even when the UI is closed.
+        
+        Returns:
+            bool: True if there are active downloads or queued items
+        """
+        with self._lock:
+            # Check for active downloads
+            if self._active_downloads:
+                return True
+            
+            # Check for queued items (excluding cancelled ones)
+            for book_id, status in self._status.items():
+                if status == QueueStatus.QUEUED:
+                    return True
+            
+            return False
             
     def clear_completed(self) -> int:
         """Remove all completed, errored, or cancelled books from tracking.

network.py

@@ -13,9 +13,147 @@ import ipaddress
 from logger import setup_logger
 from config import PROXIES, AA_BASE_URL, CUSTOM_DNS, AA_AVAILABLE_URLS, DOH_SERVER
 import config
+import env
+from datetime import datetime, timedelta
+
+# Try to use gevent locks if available (for gevent worker compatibility)
+# Fall back to threading locks for non-gevent environments
+try:
+    from gevent.lock import RLock as _RLock
+    _using_gevent_locks = True
+except ImportError:
+    from threading import RLock as _RLock
+    _using_gevent_locks = False
 
 logger = setup_logger(__name__)
 
+# In-memory state (no disk persistence)
+STATE_TTL_DAYS = 30
+_initialized = False
+_dns_initialized = False
+_aa_initialized = False
+state: dict[str, Any] = {}
+
+# Locks for greenlet-safe initialization and DNS switching
+# Use RLock (reentrant lock) since init() calls init_dns() and init_aa()
+_init_lock = _RLock()
+_dns_switch_lock = _RLock()
+
+# DNS rotation callbacks - called when DNS provider switches in auto mode
+# Callbacks receive (provider_name: str, servers: List[str], doh_url: str)
+_dns_rotation_callbacks: List[Callable[[str, List[str], str], None]] = []
+_dns_callback_lock = _RLock()
+
+
+def register_dns_rotation_callback(callback: Callable[[str, List[str], str], None]) -> None:
+    """Register a callback to be called when DNS provider rotates.
+
+    The callback receives (provider_name, servers, doh_url) as arguments.
+    Use this to restart components that cache DNS resolution (e.g., Chrome).
+    """
+    with _dns_callback_lock:
+        if callback not in _dns_rotation_callbacks:
+            _dns_rotation_callbacks.append(callback)
+            logger.debug(f"Registered DNS rotation callback: {callback.__name__}")
+
+
+def unregister_dns_rotation_callback(callback: Callable[[str, List[str], str], None]) -> None:
+    """Unregister a previously registered DNS rotation callback."""
+    with _dns_callback_lock:
+        if callback in _dns_rotation_callbacks:
+            _dns_rotation_callbacks.remove(callback)
+            logger.debug(f"Unregistered DNS rotation callback: {callback.__name__}")
+
+
+def _notify_dns_rotation(provider_name: str, servers: List[str], doh_url: str) -> None:
+    """Notify all registered callbacks about DNS rotation."""
+    with _dns_callback_lock:
+        callbacks = _dns_rotation_callbacks.copy()
+
+    for callback in callbacks:
+        try:
+            logger.debug(f"Calling DNS rotation callback: {callback.__name__}")
+            callback(provider_name, servers, doh_url)
+        except Exception as e:
+            logger.warning(f"DNS rotation callback {callback.__name__} failed: {e}")
+
+def _agent_debug_log(code: str, source: str, reason: str, meta: Optional[dict] = None) -> None:
+    """Lightweight debug hook for automated runs; safe no-op on failure."""
+    try:
+        logger.debug(f"[agent] code={code} source={source} reason={reason} meta={meta or {}}")
+    except Exception as exc:
+        # Avoid raising inside debug logger
+        logger.debug(f"[agent] log failed: {exc}")
+
+def _load_state():
+    """Return current in-memory network state (no disk persistence)."""
+    if state.get('chosen_at'):
+        chosen = datetime.fromisoformat(state['chosen_at'])
+        if datetime.now() - chosen > timedelta(days=STATE_TTL_DAYS):
+            state.clear()
+    return state
+
+def _save_state(aa_url=None, dns_provider=None):
+    """Update in-memory network state (no disk persistence)."""
+    if aa_url:
+        state['aa_base_url'] = aa_url
+    if dns_provider:
+        state['dns_provider'] = dns_provider
+    state['chosen_at'] = datetime.now().isoformat()
+
+# AA URL failover state
+_current_aa_url_index = 0
+_aa_urls = AA_AVAILABLE_URLS.copy()
+
+def _ensure_initialized() -> None:
+    """Lazy guard so runtime setup happens once and late calls still work."""
+    global _initialized
+    if _initialized:
+        return
+    with _init_lock:
+        # Double-check after acquiring lock
+        if not _initialized:
+            init()
+
+# DNS provider definitions: (name, servers, doh_url)
+# Note: Google uses /resolve endpoint for JSON API, others use /dns-query
+DNS_PROVIDERS = [
+    ("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"),
+    ("google", ["8.8.8.8", "8.8.4.4"], "https://dns.google/resolve"),
+    ("quad9", ["9.9.9.9", "149.112.112.112"], "https://dns.quad9.net/dns-query"),
+    ("opendns", ["208.67.222.222", "208.67.220.220"], "https://doh.opendns.com/dns-query"),
+]
+
+# Domain patterns that should trigger DNS rotation on failure
+DNS_ROTATION_DOMAINS = [
+    "annas-archive",
+]
+
+
+def should_rotate_dns_for_url(url: str) -> bool:
+    """Check if a URL matches a known source domain for DNS rotation."""
+    url_lower = url.lower()
+    return any(domain in url_lower for domain in DNS_ROTATION_DOMAINS)
+
+
+# DNS state
+_current_dns_index = -1  # -1 = system DNS
+_dns_exhausted_logged = False
+
+
+def _is_auto_dns_mode() -> bool:
+    """Check if DNS is in auto-rotation mode."""
+    return env._CUSTOM_DNS.lower().strip() == "auto" and not env.USING_TOR
+
+
+def _current_dns_label() -> str:
+    """Readable label for the active DNS choice."""
+    if _current_dns_index >= 0:
+        return DNS_PROVIDERS[_current_dns_index][0]
+    if CUSTOM_DNS:
+        return f"custom {CUSTOM_DNS}"
+    return "system"
+
 # Common helper functions for DNS resolution
 def _decode_host(host: Union[str, bytes, None]) -> str:
     """Convert host to string, handling bytes and None cases."""
@@ -34,7 +172,6 @@ def _decode_port(port: Union[str, bytes, int, None]) -> int:
     return int(port)
 
 def _is_local_address(host_str: str) -> bool:
-    """Check if an address is local and should bypass custom DNS."""
     """Check if an address is local or private and should bypass custom DNS."""
     # Localhost checks
     if (host_str == 'localhost' or 
@@ -67,17 +204,34 @@ def _is_ip_address(host_str: str) -> bool:
     except ValueError:
         return False
 
+def _aa_hostnames() -> List[str]:
+    """Return hostname portions for all configured AA URLs."""
+    return [
+        parsed.hostname for parsed in (urllib.parse.urlparse(url) for url in _aa_urls)
+        if parsed.hostname
+    ]
+
+def _is_aa_hostname(host_str: str) -> bool:
+    """Check if a hostname matches any configured AA mirror host."""
+    return any(host_str.endswith(hostname) for hostname in _aa_hostnames())
+
 # Store the original getaddrinfo function
 original_getaddrinfo = socket.getaddrinfo
 
 class DoHResolver:
-    """DNS over HTTPS resolver implementation."""
+    """DNS over HTTPS resolver implementation with caching."""
+    
+    # Cache TTL in seconds (5 minutes)
+    CACHE_TTL = 300
+    
     def __init__(self, provider_url: str, hostname: str, ip: str):
         """Initialize DoH resolver with specified provider."""
         self.base_url = provider_url.lower().strip()
         self.hostname = hostname  # Store the hostname for hostname-based skipping
         self.ip = ip              # Store IP for direct connections
         self.session = requests.Session()
+        # DNS cache: {(hostname, record_type): (ip_list, timestamp)}
+        self._cache: dict[tuple[str, str], tuple[List[str], datetime]] = {}
         
         # Different headers based on provider
         if 'google' in self.base_url:
@@ -89,6 +243,24 @@ class DoHResolver:
                 'Accept': 'application/dns-json',
             })
     
+    def _get_cached(self, hostname: str, record_type: str) -> Optional[List[str]]:
+        """Get cached DNS result if still valid."""
+        key = (hostname, record_type)
+        if key in self._cache:
+            ips, timestamp = self._cache[key]
+            if datetime.now() - timestamp < timedelta(seconds=self.CACHE_TTL):
+                logger.debug(f"DoH cache hit for {hostname}: {ips}")
+                return ips
+            else:
+                # Cache expired, remove it
+                del self._cache[key]
+        return None
+    
+    def _set_cached(self, hostname: str, record_type: str, ips: List[str]) -> None:
+        """Cache DNS result."""
+        if ips:  # Only cache non-empty results
+            self._cache[(hostname, record_type)] = (ips, datetime.now())
+    
     def resolve(self, hostname: str, record_type: str) -> List[str]:
         """Resolve a hostname using DoH.
         
@@ -113,6 +285,11 @@ class DoHResolver:
         if hostname == self.hostname:
             logger.debug(f"Skipping DoH resolution for DoH server itself: {hostname}")
             return [self.ip]
+        
+        # Check cache first
+        cached = self._get_cached(hostname, record_type)
+        if cached is not None:
+            return cached
             
         try:
             params = {
@@ -124,7 +301,7 @@ class DoHResolver:
                 self.base_url,
                 params=params,
                 proxies=PROXIES,
-                timeout=5
+                timeout=10  # Increased from 5s to handle slow network conditions
             )
             response.raise_for_status()
             
@@ -136,35 +313,31 @@ class DoHResolver:
             # Extract IP addresses from the response    
             answers = [answer['data'] for answer in data['Answer'] 
                     if answer.get('type') == (28 if record_type == 'AAAA' else 1)]
-            logger.debug(f"Resolved {hostname} to {len(answers)} addresses using DoH: {answers}")
+            
+            # Cache the result
+            self._set_cached(hostname, record_type, answers)
+            
+            # Don't log here - the caller (custom_getaddrinfo) will log the final result
             return answers
             
         except Exception as e:
             logger.warning(f"DoH resolution failed for {hostname}: {e}")
             return []
 
-def create_custom_resolver():
-    """Create a custom DNS resolver using the configured DNS servers."""
+def create_custom_resolver(servers: Optional[List[str]] = None):
+    """Create a custom DNS resolver using the specified or configured DNS servers."""
     custom_resolver = dns.resolver.Resolver()
-    custom_resolver.nameservers = CUSTOM_DNS
+    custom_resolver.nameservers = servers if servers is not None else CUSTOM_DNS
     return custom_resolver
 
 def resolve_with_custom_dns(resolver, hostname: str, record_type: str) -> List[str]:
-    """Resolve hostname using custom DNS resolver.
-    
-    Args:
-        resolver: The DNS resolver to use
-        hostname: The hostname to resolve
-        record_type: The DNS record type (A or AAAA)
-        
-    Returns:
-        List of resolved IP addresses
-    """
+    """Resolve hostname using custom DNS resolver."""
     try:
         answers = resolver.resolve(hostname, record_type)
         return [str(answer) for answer in answers]
-    except Exception as e:
-        logger.debug(f"{record_type} resolution failed for {hostname}: {e}")
+    except Exception:
+        # Don't log here - let the caller handle it to prevent spam
+        # Don't trigger DNS switch here either - caller handles it
         return []
 
 def create_custom_getaddrinfo(
@@ -193,42 +366,63 @@ def create_custom_getaddrinfo(
         host_str = _decode_host(host)
         port_int = _decode_port(port)
         
+        def _log_results(source: str, provider_label: str, res: Sequence[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]], is_bypass: bool = False) -> None:
+            """Emit a unified resolver log with the IPs returned.
+            
+            Args:
+                source: Description of resolver source
+                provider_label: Label for the DNS provider
+                res: Resolution results
+                is_bypass: If True, log at DEBUG level (for local/IP addresses)
+            """
+            # Skip logging entirely for localhost to reduce noise
+            if host_str in ('localhost', '127.0.0.1', '::1'):
+                return
+            try:
+                ips = [entry[4][0] for entry in res if len(entry) >= 5 and entry[4]]
+                msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}"
+                if is_bypass:
+                    logger.debug(msg)
+                else:
+                    logger.info(msg)
+            except Exception:
+                pass  # Silently ignore logging failures
+        
         # Skip custom resolution for IP addresses, local addresses, or if skip check passes
         if _is_ip_address(host_str) or _is_local_address(host_str) or (skip_check and skip_check(host_str)):
-            logger.debug(f"Using system DNS for IP address or local/private address: {host_str}")
-            return original_getaddrinfo(host, port, family, type, proto, flags)
+            # Quietly bypass custom resolution for IP/local targets
+            res = original_getaddrinfo(host, port, family, type, proto, flags)
+            _log_results("system resolver (bypass)", "system", res, is_bypass=True)
+            return res
         
         results: list[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]] = []
         
         try:
-            # Try IPv6 first if family allows it
-            if family == 0 or family == socket.AF_INET6:
-                logger.debug(f"Resolving IPv6 address for {host_str}")
-                ipv6_answers = resolve_ipv6(host_str)
-                for answer in ipv6_answers:
-                    results.append((socket.AF_INET6, cast(SocketKind, type), proto, '', (answer, port_int, 0, 0)))
-                if ipv6_answers:
-                    logger.debug(f"Found {len(ipv6_answers)} IPv6 addresses for {host_str}")
-            
-            # Then try IPv4
+            # Try IPv4 (IPv6 disabled to avoid noisy AAAA failures)
             if family == 0 or family == socket.AF_INET:
-                logger.debug(f"Resolving IPv4 address for {host_str}")
                 ipv4_answers = resolve_ipv4(host_str)
                 for answer in ipv4_answers:
                     results.append((socket.AF_INET, cast(SocketKind, type), proto, '', (answer, port_int)))
-                if ipv4_answers:
-                    logger.debug(f"Found {len(ipv4_answers)} IPv4 addresses for {host_str}")
             
             if results:
-                logger.debug(f"Resolved {host_str} to {len(results)} addresses")
+                _log_results("custom resolver", _current_dns_label(), results)
                 return results
                 
         except Exception as e:
             logger.warning(f"Custom DNS resolution failed for {host_str}: {e}, falling back to system DNS")
+            # Trigger DNS switch on failure (if auto mode)
+            if _is_auto_dns_mode() and not _is_local_address(host_str) and not _is_ip_address(host_str):
+                # Only switch if we haven't exhausted all providers
+                if _current_dns_index < len(DNS_PROVIDERS):
+                    logger.info(f"Requesting DNS provider switch after custom resolver failure for {host_str}")
+                    switch_dns_provider()
         
         # Fall back to system DNS if custom resolution fails
+        logger.info(f"Custom DNS returned no addresses for {host_str}; falling back to system resolver")
         try:
-            return original_getaddrinfo(host, port, family, type, proto, flags)
+            res = original_getaddrinfo(host, port, family, type, proto, flags)
+            _log_results("system resolver (fallback)", "system", res)
+            return res
         except Exception as e:
             logger.error(f"System DNS resolution also failed for {host_str}: {e}")
             # Last resort: Try to connect to the hostname directly
@@ -240,11 +434,45 @@ def create_custom_getaddrinfo(
     
     return custom_getaddrinfo
 
-def init_doh_resolver(doh_server: str = DOH_SERVER):
-    """Initialize DNS over HTTPS resolver.
+def create_system_failover_getaddrinfo():
+    """Wrap system getaddrinfo to trigger DNS provider switch on failure."""
+    _switch_logged: set[str] = set()
+    
+    def system_failover_getaddrinfo(
+        host: Union[str, bytes, None],
+        port: Union[str, bytes, int, None],
+        family: int = 0,
+        type: int = 0,
+        proto: int = 0,
+        flags: int = 0
+    ) -> Sequence[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]]:
+        host_str = _decode_host(host)
+        try:
+            return original_getaddrinfo(host, port, family, type, proto, flags)
+        except Exception as e:
+            if host_str not in _switch_logged:
+                logger.warning(f"System DNS resolution failed for {host_str}: {e}")
+            
+            # Trigger DNS switch only in auto mode for non-local targets
+            if _is_auto_dns_mode() and not _is_ip_address(host_str) and not _is_local_address(host_str):
+                if _current_dns_index + 1 < len(DNS_PROVIDERS):
+                    if host_str not in _switch_logged:
+                        logger.info(f"Switching DNS provider after system DNS failure for {host_str}")
+                        _switch_logged.add(host_str)
+                    if switch_dns_provider():
+                        return socket.getaddrinfo(host, port, family, type, proto, flags)
+            raise
+    
+    return system_failover_getaddrinfo
+
+def _init_doh_resolver_internal(doh_server: str) -> DoHResolver:
+    """Internal: Initialize DNS over HTTPS resolver with specified server.
     
     Args:
         doh_server: The DoH server URL
+        
+    Returns:
+        Configured DoHResolver instance
     """
     # Pre-resolve the DoH server hostname to prevent recursion
     url = urllib.parse.urlparse(doh_server)
@@ -292,9 +520,14 @@ def init_doh_resolver(doh_server: str = DOH_SERVER):
     logger.info("DoH resolver successfully configured and activated")
     return doh_resolver
 
-def init_custom_resolver():
-    """Initialize custom DNS resolver using configured DNS servers."""
-    custom_resolver = create_custom_resolver()
+
+def _init_custom_resolver_internal(servers: List[str]):
+    """Internal: Initialize custom DNS resolver with specified servers.
+    
+    Args:
+        servers: List of DNS server IPs to use
+    """
+    custom_resolver = create_custom_resolver(servers)
     
     # Create resolver functions
     def resolve_ipv4(hostname: str) -> List[str]:
@@ -309,32 +542,286 @@ def init_custom_resolver():
     logger.info("Custom DNS resolver successfully configured and activated")
     return custom_resolver
 
-# Initialize DNS resolvers based on configuration
+
+def init_doh_resolver(doh_server: str = ""):
+    """Initialize DNS over HTTPS resolver."""
+    server = doh_server or DOH_SERVER
+    if not server:
+        return None
+    return _init_doh_resolver_internal(server)
+
+
+def init_custom_resolver():
+    """Initialize custom DNS resolver using configured DNS servers."""
+    if not CUSTOM_DNS:
+        return None
+    return _init_custom_resolver_internal(CUSTOM_DNS)
+
+def switch_dns_provider() -> bool:
+    """Switch to next DNS provider (auto mode only)."""
+    global CUSTOM_DNS, DOH_SERVER, _current_dns_index, _dns_exhausted_logged
+
+    if not _is_auto_dns_mode():
+        return False
+
+    with _dns_switch_lock:
+        if _current_dns_index + 1 >= len(DNS_PROVIDERS):
+            if not _dns_exhausted_logged:
+                logger.warning("All DNS providers exhausted, staying with current")
+                _dns_exhausted_logged = True
+            return False
+
+        _current_dns_index += 1
+        name, servers, doh = DNS_PROVIDERS[_current_dns_index]
+        CUSTOM_DNS = servers
+        DOH_SERVER = doh
+        config.CUSTOM_DNS = servers
+        config.DOH_SERVER = doh
+
+        logger.warning(f"Switched DNS provider to: {name} (using DoH)")
+        _save_state(dns_provider=name)
+        init_dns_resolvers()
+
+        # Notify listeners (e.g., Chrome bypasser) to restart with new DNS
+        _notify_dns_rotation(name, servers, doh)
+        return True
+
+
+def rotate_dns_provider() -> bool:
+    """Rotate DNS provider (auto mode only), cycling back if exhausted."""
+    global _current_dns_index, _dns_exhausted_logged
+    
+    if not _is_auto_dns_mode():
+        return False
+    
+    if _current_dns_index + 1 >= len(DNS_PROVIDERS):
+        logger.warning("DNS rotation: cycling back to first provider")
+        _current_dns_index = -1
+        _dns_exhausted_logged = False
+    
+    return switch_dns_provider()
+
+def rotate_dns_and_reset_aa() -> bool:
+    """
+    Switch DNS provider (auto mode) and reset AA URL list to the first entry.
+    Returns True if DNS switched; False if no providers left or not in auto mode.
+    
+    Note: This function can be called during initialization, so we must NOT call
+    _ensure_initialized() here to avoid recursive init loops.
+    """
+    if not rotate_dns_provider():
+        return False
+    # Reset AA URL to first available auto option if using auto AA
+    global AA_BASE_URL, _current_aa_url_index
+    if AA_BASE_URL == "auto" or AA_BASE_URL in _aa_urls:
+        _current_aa_url_index = 0
+        AA_BASE_URL = _aa_urls[0]
+        config.AA_BASE_URL = AA_BASE_URL
+        logger.info(f"After DNS switch, resetting AA URL to: {AA_BASE_URL}")
+        _save_state(aa_url=AA_BASE_URL)
+    return True
+
 def init_dns_resolvers():
     """Initialize DNS resolvers based on configuration."""
-    if len(CUSTOM_DNS) > 0:
+    global CUSTOM_DNS, DOH_SERVER
+    
+    if _is_auto_dns_mode():
+        if _current_dns_index >= 0:
+            name, servers, doh = DNS_PROVIDERS[_current_dns_index]
+            CUSTOM_DNS = servers
+            DOH_SERVER = doh
+            config.CUSTOM_DNS = servers
+            config.DOH_SERVER = doh
+            logger.info(f"Using DNS provider: {name} (DoH enabled)")
+        else:
+            CUSTOM_DNS = []
+            DOH_SERVER = ""
+            config.CUSTOM_DNS = []
+            config.DOH_SERVER = ""
+            logger.info("Using system DNS (auto mode - will switch on failure)")
+            socket.getaddrinfo = cast(Any, create_system_failover_getaddrinfo())
+            return
+    
+    if CUSTOM_DNS:
         init_custom_resolver()
         if DOH_SERVER:
-            init_doh_resolver()
+            init_doh_resolver(DOH_SERVER)
 
-# Initialize DNS resolvers
-init_dns_resolvers()
 
-# Check available AA_BASE_URLs if set to auto
-if AA_BASE_URL == "auto":
-    logger.info(f"AA_BASE_URL: auto, checking available urls {AA_AVAILABLE_URLS}")
-    for url in AA_AVAILABLE_URLS:
-        try:
-            response = requests.get(url, proxies=PROXIES)
-            if response.status_code == 200:
-                AA_BASE_URL = url
-                break
-        except Exception as e:
-            logger.error_trace(f"Error checking {url}: {e}")
+def _initialize_dns_state() -> None:
+    """Restore persisted DNS choice or start fresh."""
+    global _current_dns_index
+    
+    if _is_auto_dns_mode():
+        persisted = state.get('dns_provider') if state else None
+        if persisted:
+            for i, (name, _, _) in enumerate(DNS_PROVIDERS):
+                if name == persisted:
+                    _current_dns_index = i
+                    logger.info(f"Restored DNS provider from state: {name}")
+                    return
+        _current_dns_index = -1
+
+def _initialize_aa_state() -> None:
+    """Restore or probe AA URL state."""
+    global AA_BASE_URL, _current_aa_url_index
     if AA_BASE_URL == "auto":
-        AA_BASE_URL = AA_AVAILABLE_URLS[0]
-config.AA_BASE_URL = AA_BASE_URL
-logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
+        if state.get('aa_base_url') and state['aa_base_url'] in _aa_urls:
+            _current_aa_url_index = _aa_urls.index(state['aa_base_url'])
+            AA_BASE_URL = state['aa_base_url']
+        else:
+            logger.info(f"AA_BASE_URL: auto, checking available urls {_aa_urls}")
+            for i, url in enumerate(_aa_urls):
+                try:
+                    response = requests.get(url, proxies=PROXIES, timeout=3)
+                    if response.status_code == 200:
+                        _current_aa_url_index = i
+                        AA_BASE_URL = url
+                        _save_state(aa_url=AA_BASE_URL)
+                        break
+                except Exception:
+                    pass
+            if AA_BASE_URL == "auto":
+                AA_BASE_URL = _aa_urls[0]
+                _current_aa_url_index = 0
+    elif AA_BASE_URL not in _aa_urls:
+        logger.info(f"AA_BASE_URL set to custom value {AA_BASE_URL}; skipping auto-switch")
+    else:
+        _current_aa_url_index = _aa_urls.index(AA_BASE_URL)
+
+    config.AA_BASE_URL = AA_BASE_URL
+    logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
+
+def init_dns(force: bool = False) -> None:
+    """Initialize DNS state and resolvers."""
+    global state, _dns_initialized
+    if _dns_initialized and not force:
+        return
+    with _init_lock:
+        # Double-check after acquiring lock
+        if _dns_initialized and not force:
+            return
+        # Set flag BEFORE doing work to prevent recursive calls during init
+        _dns_initialized = True
+        try:
+            logger.debug(f"Initializing DNS (using {'gevent' if _using_gevent_locks else 'threading'} locks)")
+            state = _load_state()
+            _initialize_dns_state()
+            init_dns_resolvers()
+        except Exception:
+            _dns_initialized = False
+            raise
+
+def init_aa(force: bool = False) -> None:
+    """Initialize AA mirror selection."""
+    global state, _aa_initialized
+    if _aa_initialized and not force:
+        return
+    with _init_lock:
+        # Double-check after acquiring lock
+        if _aa_initialized and not force:
+            return
+        # Set flag BEFORE doing work to prevent recursive calls during init
+        _aa_initialized = True
+        try:
+            state = _load_state()
+            _initialize_aa_state()
+        except Exception:
+            _aa_initialized = False
+            raise
+
+def init(force: bool = False) -> None:
+    """
+    Initialize network state (DNS resolvers and AA mirror selection).
+
+    Called lazily on first network operation. Safe to call repeatedly;
+    later calls no-op unless force=True.
+    """
+    global _initialized
+    if _initialized and not force:
+        return
+    with _init_lock:
+        # Double-check after acquiring lock
+        if _initialized and not force:
+            return
+        # Set flag BEFORE doing work to prevent recursive calls during init
+        # (e.g., DNS failover handlers calling back into init)
+        _initialized = True
+        try:
+            init_dns(force=force)
+            init_aa(force=force)
+        except Exception:
+            # Reset flag on failure so retry is possible
+            _initialized = False
+            raise
+
+def get_aa_base_url():
+    """Get current AA base URL."""
+    _ensure_initialized()
+    return AA_BASE_URL
+
+def get_available_aa_urls():
+    """Get list of configured AA URLs (copy)."""
+    _ensure_initialized()
+    return _aa_urls.copy()
+
+def set_aa_url_index(new_index: int) -> bool:
+    """Set AA base URL by index in available list; returns True if applied."""
+    _ensure_initialized()
+    global AA_BASE_URL, _current_aa_url_index
+    if new_index < 0 or new_index >= len(_aa_urls):
+        return False
+    _current_aa_url_index = new_index
+    AA_BASE_URL = _aa_urls[_current_aa_url_index]
+    config.AA_BASE_URL = AA_BASE_URL
+    logger.info(f"Set AA URL to: {AA_BASE_URL}")
+    _save_state(aa_url=AA_BASE_URL)
+    return True
+
+class AAMirrorSelector:
+    """
+    Small helper to keep AA mirror switching consistent across call sites.
+    Tracks attempts per DNS cycle and rewrites URLs safely.
+    """
+    def __init__(self) -> None:
+        self._ensure_fresh_state(reset_attempts=True)
+
+    def _ensure_fresh_state(self, reset_attempts: bool = False) -> None:
+        _ensure_initialized()
+        self.aa_urls = get_available_aa_urls()
+        self._index = self._safe_index(get_aa_base_url())
+        self.current_base = self.aa_urls[self._index] if self.aa_urls else ""
+        if reset_attempts:
+            self.attempts_this_dns = 0
+
+    def _safe_index(self, base: str) -> int:
+        if base in self.aa_urls:
+            return self.aa_urls.index(base)
+        return 0
+
+    def rewrite(self, url: str) -> str:
+        """Replace any known AA base in url with current_base."""
+        for base in self.aa_urls:
+            if url.startswith(base):
+                return url.replace(base, self.current_base, 1)
+        return url
+
+    def next_mirror_or_rotate_dns(self, allow_dns: bool = True) -> tuple[Optional[str], str]:
+        """
+        Advance to next mirror; if exhausted and allowed, rotate DNS and reset to first.
+        Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'.
+        """
+        self.attempts_this_dns += 1
+        if self.attempts_this_dns >= len(self.aa_urls):
+            if allow_dns and rotate_dns_and_reset_aa():
+                self._ensure_fresh_state(reset_attempts=True)
+                return self.current_base, "dns"
+            return None, "exhausted"
+
+        next_index = (self._index + 1) % len(self.aa_urls)
+        set_aa_url_index(next_index)
+        self._ensure_fresh_state(reset_attempts=False)
+        return self.current_base, "mirror"
 
 # Configure urllib opener with appropriate headers
 opener = urllib.request.build_opener()
@@ -344,7 +831,3 @@ opener.addheaders = [
         'Chrome/129.0.0.0 Safari/537.3')
 ]
 urllib.request.install_opener(opener)
-
-# Need an empty function to be called by downloader.py
-def init():
-    pass

readme.md

@@ -1,6 +1,6 @@
 # 📚 Calibre-Web-Automated-Book-Downloader
 
-![Calibre-Web Automated Book Downloader](src/frontend/public/logo.png 'Calibre-Web Automated Book Downloader')
+<img src="src/frontend/public/logo.png" alt="Calibre-Web Automated Book Downloader" width="200">
 
 An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated). This project streamlines the process of downloading books and preparing them for integration into your Calibre library.
 
@@ -65,6 +65,7 @@ An intuitive web interface for searching and requesting book downloads, designed
 | `LOG_LEVEL`       | Log level to use        | `info`             |
 | `SESSION_COOKIE_SECURE` | Secure cookie enforcement - Use for HTTPS connections only | `false` |
 | `CALIBRE_WEB_URL` | Custom WebUI library link | None |
+| `BYPASS_WARMUP_ON_CONNECT` | Warm up Cloudflare bypasser when first client connects | `true` |
 
 If you wish to enable authentication, you must set `CWA_DB_PATH` to point to Calibre-Web's `app.db`, in order to match the username and password.
 
@@ -132,9 +133,11 @@ If disabling the cloudflare bypass, you will be using alternative download hosts
 | `AA_ADDITIONAL_URLS`   | Proxy URLs for AA (, separated) | ``                      |
 | `HTTP_PROXY`           | HTTP proxy URL                  | ``                      |
 | `HTTPS_PROXY`          | HTTPS proxy URL                 | ``                      |
-| `CUSTOM_DNS`           | Custom DNS IP                   | ``                      |
+| `CUSTOM_DNS`           | DNS configuration               | `auto`                  |
 | `USE_DOH`              | Use DNS over HTTPS              | `false`                 |
 
+**Proxy Configuration**
+
 For proxy configuration, you can specify URLs in the following format:
 ```bash
 # Basic proxy
@@ -146,31 +149,44 @@ HTTP_PROXY=http://username:password@proxy.example.com:8080
 HTTPS_PROXY=http://username:password@proxy.example.com:8080
 ```
 
+**DNS Configuration**
 
-The `CUSTOM_DNS` setting supports two formats:
+The `CUSTOM_DNS` setting controls how DNS resolution works. By default, it is set to `auto` which provides automatic failover for reliable connectivity.
 
-1. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
+**Auto Mode (Default)**
+
+When `CUSTOM_DNS=auto`, the application starts with your system's default DNS. If DNS resolution fails, it automatically rotates through alternative providers using DNS over HTTPS (DoH):
+
+1. System DNS (initial)
+2. Cloudflare (1.1.1.1)
+3. Google (8.8.8.8)
+4. Quad9 (9.9.9.9)
+5. OpenDNS (208.67.222.222)
+
+This automatic rotation helps bypass ISP-level blocks and DNS issues without any manual configuration.
+
+**Manual DNS Configuration**
+
+If you prefer to use a specific DNS configuration, you can override the auto behavior:
+
+1. **Preset DNS Providers**: Use one of these predefined options:
+   - `google` - Google DNS (8.8.8.8, 8.8.4.4)
+   - `quad9` - Quad9 DNS (9.9.9.9, 149.112.112.112)
+   - `cloudflare` - Cloudflare DNS (1.1.1.1, 1.0.0.1)
+   - `opendns` - OpenDNS (208.67.222.222, 208.67.220.220)
+
+2. **Custom DNS Servers**: A comma-separated list of DNS server IP addresses
    - Example: `127.0.0.53,127.0.1.53` (useful for PiHole)
-   - Supports both IPv4 and IPv6 addresses in the same string
+   - Supports both IPv4 and IPv6 addresses
 
-2. **Preset DNS Providers**: Use one of these predefined options:
-   - `google` - Google DNS
-   - `quad9` - Quad9 DNS
-   - `cloudflare` - Cloudflare DNS
-   - `opendns` - OpenDNS
-
-For users experiencing ISP-level website blocks (such as Virgin Media in the UK), using alternative DNS providers like Cloudflare may help bypass these restrictions
-
-If a `CUSTOM_DNS` is specified from the preset providers, you can also set a `USE_DOH=true` to force using DNS over HTTPS,
-which might also help in certain network situations. Note that only `google`, `quad9`, `cloudflare` and `opendns` are 
-supported for now, and any other value in `CUSTOM_DNS` will make the `USE_DOH` flag ignored.
-
-Try something like this :
+When using preset providers, you can optionally enable DNS over HTTPS with `USE_DOH=true`:
 ```bash
 CUSTOM_DNS=cloudflare
 USE_DOH=true
 ```
 
+Note: When using custom IP addresses, the `USE_DOH` flag is ignored since DoH requires a known provider endpoint.
+
 #### Custom configuration
 
 | Variable               | Description                                                 | Default Value           |
@@ -235,7 +251,6 @@ This variant allows the application to use an external service to bypass Cloudfl
 
 - When enabled, all requests that require Cloudflare bypass are sent to your external resolver service.
 - The application communicates with the resolver using its API.
-- This approach can improve reliability and performance, especially if your external resolver is optimized or shared across multiple applications.
 
 #### Configuration
 
@@ -263,11 +278,16 @@ This feature follows the same configuration of the built-in Cloudflare bypasser,
 #### Compatibility:
 This feature is designed to work with any resolver that implements the `FlareSolverr` API schema, including `ByParr` and similar projects.
 
-#### Benefits:
+#### Internal vs External Bypasser
 
-- Centralizes Cloudflare bypass logic for easier maintenance.
-- Can leverage more powerful or distributed resolver infrastructure.
-- Reduces load on the main application container.
+The **internal bypasser** (default) is custom-designed for this application's specific needs. It handles session management, cookie persistence, and retry logic optimized for book downloading workflows. For most users, this provides the most reliable experience out of the box.
+
+The **external bypasser** is better suited if you:
+- Already run FlareSolverr/ByParr for other services and want to consolidate
+- Need to share bypass infrastructure across multiple applications
+- Want to offload browser automation to a dedicated, more powerful container
+
+If you're unsure which to use, start with the default internal bypasser.
 
 ## 🏗️ Architecture
 

src/frontend/src/App.tsx

@@ -53,7 +53,7 @@ function App() {
     content: '',
     formats: DEFAULT_FORMAT_SELECTION,
   });
-  const { toasts, showToast } = useToast();
+  const { toasts, showToast, removeToast } = useToast();
   const updateAdvancedFilters = useCallback((updates: Partial<AdvancedFilterState>) => {
     setAdvancedFilters(prev => ({ ...prev, ...updates }));
   }, []);
@@ -80,18 +80,12 @@ function App() {
     const ongoing = [
       currentStatus.queued,
       currentStatus.resolving,
-      currentStatus.bypassing,
       currentStatus.downloading,
-      currentStatus.verifying,
-      currentStatus.ingesting,
     ].reduce((sum, status) => sum + (status ? Object.keys(status).length : 0), 0);
 
-    const completed = [
-      currentStatus.completed,
-      currentStatus.complete,
-      currentStatus.available,
-      currentStatus.done,
-    ].reduce((sum, status) => sum + (status ? Object.keys(status).length : 0), 0);
+    const completed = currentStatus.complete
+      ? Object.keys(currentStatus.complete).length
+      : 0;
 
     const errored = currentStatus.error ? Object.keys(currentStatus.error).length : 0;
 
@@ -131,21 +125,24 @@ function App() {
 
     // Check for completed items
     const prevDownloadingIds = new Set(Object.keys(prevDownloading));
+    const prevResolvingIds = new Set(Object.keys(prev.resolving || {}));
     const prevQueuedIds = new Set(Object.keys(prevQueued));
-    const currAvailable = curr.available || {};
-    const currDone = curr.done || {};
+    const currComplete = curr.complete || {};
 
-    Object.keys(currAvailable).forEach(bookId => {
+    Object.keys(currComplete).forEach(bookId => {
       if (prevDownloadingIds.has(bookId) || prevQueuedIds.has(bookId)) {
-        const book = currAvailable[bookId];
+        const book = currComplete[bookId];
         showToast(`${book.title || 'Book'} completed`, 'success');
       }
     });
 
-    Object.keys(currDone).forEach(bookId => {
-      if (prevDownloadingIds.has(bookId) || prevQueuedIds.has(bookId)) {
-        const book = currDone[bookId];
-        showToast(`${book.title || 'Book'} completed`, 'success');
+    // Check for failed items
+    const currError = curr.error || {};
+    Object.keys(currError).forEach(bookId => {
+      if (prevDownloadingIds.has(bookId) || prevResolvingIds.has(bookId) || prevQueuedIds.has(bookId)) {
+        const book = currError[bookId];
+        const errorMsg = book.status_message || 'Download failed';
+        showToast(`${book.title || 'Book'}: ${errorMsg}`, 'error');
       }
     });
   }, [showToast]);
@@ -229,6 +226,14 @@ function App() {
       try {
         const cfg = await getConfig();
         setConfig(cfg);
+        // Update format selection to match supported formats from config
+        // This ensures PDF is auto-selected when added to SUPPORTED_FORMATS env var
+        if (cfg?.supported_formats) {
+          setAdvancedFilters(prev => ({
+            ...prev,
+            formats: cfg.supported_formats,
+          }));
+        }
       } catch (error) {
         console.error('Failed to load config:', error);
         // Use defaults if config fails to load
@@ -278,6 +283,11 @@ function App() {
       } else {
         console.error('Search failed:', error);
         setBooks([]);
+        const message = error instanceof Error ? error.message : 'Search failed';
+        const friendly = message.includes("Anna's Archive") || message.includes('Network restricted')
+          ? message
+          : "Unable to reach Anna's Archive. Network may be restricted or mirrors blocked.";
+        showToast(friendly, 'error');
       }
     } finally {
       setIsSearching(false);
@@ -304,6 +314,7 @@ function App() {
     } catch (error) {
       console.error('Download failed:', error);
       showToast('Failed to queue download', 'error');
+      throw error; // Re-throw so button components can reset their queuing state
     }
   };
 
@@ -333,6 +344,8 @@ function App() {
     setSearchInput('');
     setShowAdvanced(false);
     setLastSearchQuery('');
+    // Use config's supported formats if available, otherwise fall back to default
+    const resetFormats = config?.supported_formats || DEFAULT_FORMAT_SELECTION;
     setAdvancedFilters({
       isbn: '',
       author: '',
@@ -340,7 +353,7 @@ function App() {
       lang: [LANGUAGE_OPTION_DEFAULT],
       sort: '',
       content: '',
-      formats: DEFAULT_FORMAT_SELECTION,
+      formats: resetFormats,
     });
   };
 
@@ -366,26 +379,11 @@ function App() {
     if (currentStatus.error && currentStatus.error[bookId]) {
       return { text: 'Failed', state: 'error' };
     }
-    // Check completed states
-    if (currentStatus.completed && currentStatus.completed[bookId]) {
-      return { text: 'Downloaded', state: 'completed' };
-    }
+    // Check completed
     if (currentStatus.complete && currentStatus.complete[bookId]) {
-      return { text: 'Downloaded', state: 'completed' };
-    }
-    if (currentStatus.available && currentStatus.available[bookId]) {
-      return { text: 'Downloaded', state: 'completed' };
-    }
-    if (currentStatus.done && currentStatus.done[bookId]) {
-      return { text: 'Downloaded', state: 'completed' };
-    }
-    // Check in-progress states with detailed status
-    if (currentStatus.ingesting && currentStatus.ingesting[bookId]) {
-      return { text: 'Ingesting', state: 'ingesting' };
-    }
-    if (currentStatus.verifying && currentStatus.verifying[bookId]) {
-      return { text: 'Verifying', state: 'verifying' };
+      return { text: 'Downloaded', state: 'complete' };
     }
+    // Check in-progress states
     if (currentStatus.downloading && currentStatus.downloading[bookId]) {
       const book = currentStatus.downloading[bookId];
       return {
@@ -394,9 +392,6 @@ function App() {
         progress: book.progress
       };
     }
-    if (currentStatus.bypassing && currentStatus.bypassing[bookId]) {
-      return { text: 'Bypassing Cloudflare...', state: 'bypassing' };
-    }
     if (currentStatus.resolving && currentStatus.resolving[bookId]) {
       return { text: 'Resolving', state: 'resolving' };
     }
@@ -440,6 +435,8 @@ function App() {
         }}
         onAdvancedToggle={() => setShowAdvanced(!showAdvanced)}
         isLoading={isSearching}
+        onShowToast={showToast}
+        onRemoveToast={removeToast}
       />
       
       <AdvancedFilters

src/frontend/src/components/BookDownloadButton.tsx

@@ -100,11 +100,9 @@ export const BookDownloadButton = ({
     }
   }, [buttonState.state, isQueuing]);
 
-  const isCompleted = buttonState.state === 'completed';
+  const isCompleted = buttonState.state === 'complete';
   const hasError = buttonState.state === 'error';
-  const isInProgress = ['queued', 'resolving', 'bypassing', 'downloading', 'verifying', 'ingesting'].includes(
-    buttonState.state,
-  );
+  const isInProgress = ['queued', 'resolving', 'downloading'].includes(buttonState.state);
   const isDisabled = buttonState.state !== 'download' || isQueuing || isCompleted;
   const displayText = isQueuing ? 'Queuing...' : buttonState.text;
   const showCircularProgress = buttonState.state === 'downloading' && buttonState.progress !== undefined;

src/frontend/src/components/DownloadsSidebar.tsx

@@ -11,33 +11,21 @@ interface DownloadsSidebarProps {
   activeCount: number;
 }
 
-const STATUS_STYLES: Record<string, { bg: string; text: string; label: string }> = {
-  queued: { bg: 'bg-amber-500/10', text: 'text-amber-600', label: 'Queued' },
-  resolving: { bg: 'bg-indigo-500/10', text: 'text-indigo-600', label: 'Resolving' },
-  bypassing: { bg: 'bg-purple-500/10', text: 'text-purple-600', label: 'Bypassing Cloudflare...' },
-  downloading: { bg: 'bg-blue-500/10', text: 'text-blue-600', label: 'Downloading' },
-  verifying: { bg: 'bg-cyan-500/10', text: 'text-cyan-600', label: 'Verifying' },
-  ingesting: { bg: 'bg-teal-500/10', text: 'text-teal-600', label: 'Ingesting' },
-  complete: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Complete' },
-  completed: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Completed' },
-  available: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Available' },
-  done: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Done' },
-  error: { bg: 'bg-red-500/10', text: 'text-red-600', label: 'Error' },
-  cancelled: { bg: 'bg-gray-500/10', text: 'text-gray-600', label: 'Cancelled' },
-};
-
-// Helper to format file size
-const formatSize = (sizeStr?: string): string => {
-  if (!sizeStr) return '';
-  return sizeStr;
+const STATUS_STYLES: Record<string, { bg: string; text: string; label: string; waveColor: string }> = {
+  queued: { bg: 'bg-amber-500/20', text: 'text-amber-700 dark:text-amber-300', label: 'Queued', waveColor: 'rgba(217, 119, 6, 0.3)' },
+  resolving: { bg: 'bg-indigo-500/20', text: 'text-indigo-700 dark:text-indigo-300', label: 'Resolving', waveColor: 'rgba(79, 70, 229, 0.3)' },
+  downloading: { bg: 'bg-sky-500/20', text: 'text-sky-700 dark:text-sky-300', label: 'Downloading', waveColor: 'rgba(2, 132, 199, 0.3)' },
+  complete: { bg: 'bg-green-500/20', text: 'text-green-700 dark:text-green-300', label: 'Complete', waveColor: '' },
+  error: { bg: 'bg-red-500/20', text: 'text-red-700 dark:text-red-300', label: 'Error', waveColor: '' },
+  cancelled: { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300', label: 'Cancelled', waveColor: '' },
 };
 
 // Add keyframe animation for wave effect
 const styleSheet = document.createElement('style');
 styleSheet.textContent = `
   @keyframes wave {
-    0% { background-position: -200% 0; }
-    100% { background-position: 200% 0; }
+    0% { background-position: 200% 0; }
+    100% { background-position: -200% 0; }
   }
 `;
 if (!document.head.querySelector('style[data-wave-animation]')) {
@@ -56,24 +44,14 @@ const getStatusProgress = (statusName: string, bookProgress?: number): number =>
     case 'queued':
       return 5;
     case 'resolving':
-      return 10;
-    case 'bypassing':
       return 15;
     case 'downloading':
-      // Map actual progress (0-100) to 20-90 range
+      // Map actual progress (0-100) to 20-100 range
       if (typeof bookProgress === 'number') {
-        return 20 + (bookProgress * 0.7);
+        return 20 + (bookProgress * 0.8);
       }
       return 20;
-    case 'verifying':
-      return 95;
-    case 'ingesting':
-      return 99;
-    case 'completed':
     case 'complete':
-    case 'available':
-    case 'done':
-      return 100;
     case 'error':
       return 100;
     default:
@@ -83,18 +61,15 @@ const getStatusProgress = (statusName: string, bookProgress?: number): number =>
 
 // Helper to get progress bar color based on status
 const getProgressBarColor = (statusName: string): string => {
-  const isCompleted = ['completed', 'complete', 'available', 'done'].includes(statusName);
-  if (isCompleted) return 'bg-green-600';
+  if (statusName === 'complete') return 'bg-green-600';
   if (statusName === 'error') return 'bg-red-600';
-  if (statusName === 'queued') return 'bg-gray-600';
-  if (statusName === 'resolving') return 'bg-purple-600';
-  if (statusName === 'bypassing') return 'bg-violet-600';
+  if (statusName === 'queued') return 'bg-amber-600';
+  if (statusName === 'resolving') return 'bg-indigo-600';
   if (statusName === 'downloading') return 'bg-sky-600';
-  if (statusName === 'verifying') return 'bg-cyan-600';
-  if (statusName === 'ingesting') return 'bg-teal-600';
   return 'bg-sky-600';
 };
 
+
 export const DownloadsSidebar = ({
   isOpen,
   onClose,
@@ -119,22 +94,21 @@ export const DownloadsSidebar = ({
   }, [isOpen, onClose]);
 
   // Collect all download items from different status sections
-  const allDownloadItems: Array<{ book: Book; status: string; order: number }> = [];
-  
-  // Priority order for display
-  const statusOrder = ['downloading', 'bypassing', 'resolving', 'queued', 'verifying', 'ingesting', 'error', 'completed', 'complete', 'available', 'done', 'cancelled'];
-  
-  statusOrder.forEach((statusName, index) => {
+  const allDownloadItems: Array<{ book: Book; status: string }> = [];
+
+  const statusTypes = ['downloading', 'resolving', 'queued', 'error', 'complete', 'cancelled'];
+
+  statusTypes.forEach((statusName) => {
     const items = (status as any)[statusName];
     if (items && Object.keys(items).length > 0) {
       Object.values(items).forEach((book: any) => {
-        allDownloadItems.push({ book, status: statusName, order: index });
+        allDownloadItems.push({ book, status: statusName });
       });
     }
   });
 
-  // Sort by status priority
-  allDownloadItems.sort((a, b) => a.order - b.order);
+  // Sort by added_time descending (newest first)
+  allDownloadItems.sort((a, b) => (b.book.added_time || 0) - (a.book.added_time || 0));
 
   const renderDownloadItem = (item: { book: Book; status: string }) => {
     const { book, status: statusName } = item;
@@ -144,23 +118,31 @@ export const DownloadsSidebar = ({
       label: statusName.charAt(0).toUpperCase() + statusName.slice(1),
     };
 
-    const isInProgress = ['queued', 'resolving', 'bypassing', 'downloading', 'verifying', 'ingesting'].includes(statusName);
-    const isCompleted = ['completed', 'complete', 'available', 'done'].includes(statusName);
+    const isInProgress = ['queued', 'resolving', 'downloading'].includes(statusName);
+    const isCompleted = statusName === 'complete';
     const hasError = statusName === 'error';
     
     // Get progress information
     const progress = getStatusProgress(statusName, book.progress);
     const progressBarColor = getProgressBarColor(statusName);
     
-    // Format progress text
-    let progressText = statusStyle.label;
+    // Format progress text - use status_message if available, otherwise fall back to label
+    let progressText = book.status_message || statusStyle.label;
     if (statusName === 'downloading' && book.progress && book.size) {
-      const downloadedMB = (book.progress / 100) * parseFloat(book.size.replace(/[^\d.]/g, ''));
-      progressText = `${downloadedMB.toFixed(1)}mb / ${book.size}`;
+      const sizeValue = parseFloat(book.size.replace(/[^\d.]/g, ''));
+      const sizeUnit = book.size.replace(/[\d.\s]/g, ''); // Extract unit as-is from backend
+      const downloadedSize = (book.progress / 100) * sizeValue;
+      const sizeProgress = `${downloadedSize.toFixed(1)}${sizeUnit} / ${book.size}`;
+      // If there's attempt info in the status message, prepend it to the progress
+      if (book.status_message?.startsWith('Attempt')) {
+        progressText = `${book.status_message} - ${sizeProgress}`;
+      } else {
+        progressText = sizeProgress;
+      }
     } else if (isCompleted) {
       progressText = 'Complete';
     } else if (hasError) {
-      progressText = 'Failed';
+      progressText = book.status_message || 'Failed';
     }
 
     return (
@@ -169,6 +151,22 @@ export const DownloadsSidebar = ({
         className="relative rounded-lg border hover:shadow-md transition-shadow overflow-hidden"
         style={{ borderColor: 'var(--border-muted)', background: 'var(--bg-soft)' }}
       >
+        {/* Cancel/Clear Button - top right corner */}
+        <button
+          type="button"
+          onClick={(e) => {
+            e.stopPropagation();
+            onCancel(book.id);
+          }}
+          className="absolute top-1 right-1 z-10 flex items-center justify-center w-6 h-6 rounded-full hover:bg-red-100 dark:hover:bg-red-900/30 text-gray-500 hover:text-red-600 transition-colors"
+          title={isInProgress ? "Cancel download" : "Clear from list"}
+          aria-label={isInProgress ? "Cancel download" : "Clear from list"}
+        >
+          <svg className="w-3.5 h-3.5" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.5">
+            <path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12" />
+          </svg>
+        </button>
+
         {/* Main content area */}
         <div className="flex gap-2">
           {/* Book Thumbnail - left side */}
@@ -187,8 +185,8 @@ export const DownloadsSidebar = ({
 
           {/* Book Info - right side */}
           <div className="flex-1 min-w-0 flex flex-col justify-between px-3 pt-2 pb-3">
-            {/* Title & Author */}
-            <div className="mb-1">
+            {/* Title & Author - with safe area for cancel/clear button */}
+            <div className="mb-1 pr-6">
               <h3 className="font-semibold text-sm truncate" title={book.title}>
                 {isCompleted && book.download_path ? (
                   <a
@@ -206,47 +204,42 @@ export const DownloadsSidebar = ({
               </p>
             </div>
 
-            {/* Status Badge and Details Row */}
-            <div className="space-y-1">
-              <div className="flex items-center justify-between gap-2">
-                <span
-                  className={`inline-flex items-center px-2 py-0.5 rounded-full text-xs font-medium ${statusStyle.bg} ${statusStyle.text}`}
-                >
-                  {statusStyle.label}
-                </span>
-                
-                {/* Cancel Button for in-progress items */}
-                {isInProgress && (
-                  <button
-                    type="button"
-                    onClick={() => onCancel(book.id)}
-                    className="text-xs px-2 py-1 rounded border hover-action transition-colors"
-                    style={{ borderColor: 'var(--border-muted)' }}
-                    title="Cancel download"
-                  >
-                    ✕
-                  </button>
-                )}
+            {/* Details Row */}
+            <div className="space-y-1 pb-8">
+              <div className="flex items-center gap-2">
+                {/* Format and Size */}
+                <div className="text-xs opacity-70">
+                  {book.format && <span className="uppercase">{book.format}</span>}
+                  {book.format && book.size && <span> • </span>}
+                  {book.size && <span>{book.size}</span>}
+                </div>
               </div>
-
-              {/* Format and Size */}
-              <div className="text-xs opacity-70">
-                {book.format && <span className="uppercase">{book.format}</span>}
-                {book.format && book.size && <span> • </span>}
-                {book.size && <span>{formatSize(book.size)}</span>}
-              </div>
-
-              {/* Error Message */}
-              {hasError && (
-                <p className="text-xs text-red-600">Download failed</p>
-              )}
             </div>
           </div>
         </div>
 
         {/* Progress Bar - absolute positioned at bottom - always visible */}
-        <div className="absolute bottom-0 left-0 right-0">
-          <p className="text-xs opacity-70 mt-0.5 text-right p-2">{progressText}</p>
+        <div className="absolute bottom-0 left-0 right-0 pointer-events-none">
+          {/* ml-16 clears the 64px thumbnail, gap-2 adds spacing */}
+          <div className="flex justify-end p-2 ml-16 gap-2">
+            <span
+              className={`relative px-2 py-0.5 rounded-lg text-xs font-medium text-right ${statusStyle.bg} ${statusStyle.text}`}
+            >
+              {/* Wave animation overlay for in-progress states */}
+              {isInProgress && statusStyle.waveColor && (
+                <span
+                  key={statusName}
+                  className="absolute inset-0 rounded-lg"
+                  style={{
+                    background: `linear-gradient(90deg, transparent 0%, ${statusStyle.waveColor} 50%, transparent 100%)`,
+                    backgroundSize: '200% 100%',
+                    animation: 'wave 2s linear infinite',
+                  }}
+                />
+              )}
+              <span className="relative">{progressText}</span>
+            </span>
+          </div>
           <div className="h-1.5 bg-gray-200 dark:bg-gray-700 overflow-hidden relative">
             <div
               className={`h-full ${progressBarColor} transition-all duration-300 relative overflow-hidden`}

src/frontend/src/components/Header.tsx

@@ -23,6 +23,8 @@ interface HeaderProps {
   authRequired?: boolean;
   isAuthenticated?: boolean;
   onLogout?: () => void;
+  onShowToast?: (message: string, type: 'success' | 'error' | 'info', persistent?: boolean) => string;
+  onRemoveToast?: (id: string) => void;
 }
 
 export const Header = ({ 
@@ -41,6 +43,8 @@ export const Header = ({
   authRequired = false,
   isAuthenticated = false,
   onLogout,
+  onShowToast,
+  onRemoveToast,
 }: HeaderProps) => {
   const [theme, setTheme] = useState<string>('auto');
   const [isDropdownOpen, setIsDropdownOpen] = useState(false);
@@ -303,17 +307,62 @@ export const Header = ({
               {/* Debug Buttons */}
               {debug && (
                 <>
-                  <form action="/debug" method="get" className="w-full">
-                    <button
-                      className="w-full text-left px-4 py-2 hover-surface transition-colors flex items-center gap-3 text-orange-600 dark:text-orange-400"
-                      type="submit"
-                    >
-                      <svg className="w-5 h-5" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth="1.5" stroke="currentColor">
-                        <path strokeLinecap="round" strokeLinejoin="round" d="M12 12.75c1.148 0 2.278.08 3.383.237 1.037.146 1.866.966 1.866 2.013 0 3.728-2.35 6.75-5.25 6.75S6.75 18.728 6.75 15c0-1.046.83-1.867 1.866-2.013A24.204 24.204 0 0112 12.75zm0 0c2.883 0 5.647.508 8.207 1.44a23.91 23.91 0 01-1.152 6.06M12 12.75c-2.883 0-5.647.508-8.208 1.44.125 2.104.52 4.136 1.153 6.06M12 12.75a2.25 2.25 0 002.248-2.354M12 12.75a2.25 2.25 0 01-2.248-2.354M12 8.25c.995 0 1.971-.08 2.922-.236.403-.066.74-.358.795-.762a3.778 3.778 0 00-.399-2.25M12 8.25c-.995 0-1.97-.08-2.922-.236-.402-.066-.74-.358-.795-.762a3.734 3.734 0 01.4-2.253M12 8.25a2.25 2.25 0 00-2.248 2.146M12 8.25a2.25 2.25 0 012.248 2.146M8.683 5a6.032 6.032 0 01-1.155-1.002c.07-.63.27-1.222.574-1.747m.581 2.749A3.75 3.75 0 0115.318 5m0 0c.427-.283.815-.62 1.155-.999a4.471 4.471 0 00-.575-1.752M4.921 6a24.048 24.048 0 00-.392 3.314c1.668.546 3.416.914 5.223 1.082M19.08 6c.205 1.08.337 2.187.392 3.314a23.882 23.882 0 01-5.223 1.082" />
-                      </svg>
-                      <span>Debug</span>
-                    </button>
-                  </form>
+                  <button
+                    className="w-full text-left px-4 py-2 hover-surface transition-colors flex items-center gap-3 text-orange-600 dark:text-orange-400"
+                    onClick={async () => {
+                      closeDropdown();
+                      // Show persistent toast while gathering logs
+                      const loadingToastId = onShowToast?.('Gathering debug logs... This may take a minute.', 'info', true);
+                      try {
+                        const response = await fetch('/api/debug', {
+                          method: 'GET',
+                          credentials: 'include',
+                        });
+                        
+                        // Remove the loading toast
+                        if (loadingToastId) onRemoveToast?.(loadingToastId);
+                        
+                        if (!response.ok) {
+                          const errorData = await response.json().catch(() => ({}));
+                          onShowToast?.(`Debug download failed: ${errorData.error || response.statusText}`, 'error');
+                          return;
+                        }
+                        
+                        // Get the filename from Content-Disposition header or use default
+                        const contentDisposition = response.headers.get('Content-Disposition');
+                        let filename = 'debug.zip';
+                        if (contentDisposition) {
+                          const filenameMatch = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/);
+                          if (filenameMatch && filenameMatch[1]) {
+                            filename = filenameMatch[1].replace(/['"]/g, '');
+                          }
+                        }
+                        
+                        // Create blob and trigger download
+                        const blob = await response.blob();
+                        const url = window.URL.createObjectURL(blob);
+                        const a = document.createElement('a');
+                        a.href = url;
+                        a.download = filename;
+                        document.body.appendChild(a);
+                        a.click();
+                        window.URL.revokeObjectURL(url);
+                        a.remove();
+                        
+                        onShowToast?.('Debug logs downloaded successfully', 'success');
+                      } catch (error) {
+                        // Remove the loading toast on error too
+                        if (loadingToastId) onRemoveToast?.(loadingToastId);
+                        console.error('Debug download error:', error);
+                        onShowToast?.('Debug download failed. Check console for details.', 'error');
+                      }
+                    }}
+                  >
+                    <svg className="w-5 h-5" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth="1.5" stroke="currentColor">
+                      <path strokeLinecap="round" strokeLinejoin="round" d="M12 12.75c1.148 0 2.278.08 3.383.237 1.037.146 1.866.966 1.866 2.013 0 3.728-2.35 6.75-5.25 6.75S6.75 18.728 6.75 15c0-1.046.83-1.867 1.866-2.013A24.204 24.204 0 0112 12.75zm0 0c2.883 0 5.647.508 8.207 1.44a23.91 23.91 0 01-1.152 6.06M12 12.75c-2.883 0-5.647.508-8.208 1.44.125 2.104.52 4.136 1.153 6.06M12 12.75a2.25 2.25 0 002.248-2.354M12 12.75a2.25 2.25 0 01-2.248-2.354M12 8.25c.995 0 1.971-.08 2.922-.236.403-.066.74-.358.795-.762a3.778 3.778 0 00-.399-2.25M12 8.25c-.995 0-1.97-.08-2.922-.236-.402-.066-.74-.358-.795-.762a3.734 3.734 0 01.4-2.253M12 8.25a2.25 2.25 0 00-2.248 2.146M12 8.25a2.25 2.25 0 012.248 2.146M8.683 5a6.032 6.032 0 01-1.155-1.002c.07-.63.27-1.222.574-1.747m.581 2.749A3.75 3.75 0 0115.318 5m0 0c.427-.283.815-.62 1.155-.999a4.471 4.471 0 00-.575-1.752M4.921 6a24.048 24.048 0 00-.392 3.314c1.668.546 3.416.914 5.223 1.082M19.08 6c.205 1.08.337 2.187.392 3.314a23.882 23.882 0 01-5.223 1.082" />
+                    </svg>
+                    <span>Debug</span>
+                  </button>
                   <form action="/api/restart" method="get" className="w-full">
                     <button
                       className="w-full text-left px-4 py-2 hover-surface transition-colors flex items-center gap-3 text-orange-600 dark:text-orange-400"

src/frontend/src/hooks/useRealtimeStatus.ts

@@ -26,7 +26,7 @@ interface UseRealtimeStatusReturn {
  */
 export const useRealtimeStatus = ({
   wsUrl,
-  pollInterval = 5000,
+  pollInterval = 2000, // Reduced from 5s for better UX when WebSocket unavailable
   reconnectAttempts = 3,
 }: UseRealtimeStatusOptions): UseRealtimeStatusReturn => {
   const [status, setStatus] = useState<StatusData>({});
@@ -130,7 +130,7 @@ export const useRealtimeStatus = ({
       socketRef.current = socket;
 
       socket.on('connect', () => {
-        console.log('WebSocket connected successfully');
+        console.log('✅ WebSocket connected successfully via', socket.io.engine.transport.name);
         setConnected(true);
         setIsUsingWebSocket(true);
         setError(null);
@@ -139,6 +139,9 @@ export const useRealtimeStatus = ({
         
         // Stop polling when WebSocket connects
         stopPolling();
+        
+        // Request initial status via WebSocket
+        socket.emit('request_status');
       });
 
       socket.on('disconnect', (reason: string) => {
@@ -174,22 +177,34 @@ export const useRealtimeStatus = ({
         attemptReconnect();
       });
 
-      // Listen for status updates
+      // Listen for status updates (full status refresh)
       socket.on('status_update', (data: StatusData) => {
+        console.debug('[WS] status_update received', Object.keys(data));
         setStatus(data);
         setError(null);
       });
 
-      // Listen for real-time progress updates
+      // Listen for real-time progress updates (incremental)
       socket.on('download_progress', (data: { book_id: string; progress: number; status: string }) => {
+        console.debug('[WS] download_progress:', data.book_id, `${data.progress.toFixed(1)}%`);
         setStatus(prev => {
           const newStatus = { ...prev };
+          
+          // Update progress in downloading state
           if (newStatus.downloading?.[data.book_id]) {
-            newStatus.downloading[data.book_id] = {
-              ...newStatus.downloading[data.book_id],
-              progress: data.progress,
+            newStatus.downloading = {
+              ...newStatus.downloading,
+              [data.book_id]: {
+                ...newStatus.downloading[data.book_id],
+                progress: data.progress,
+              },
             };
           }
+          // Also check resolving state in case status update hasn't arrived yet
+          else if (newStatus.resolving?.[data.book_id]) {
+            // Book is resolving - progress will apply when it moves to downloading
+          }
+          
           return newStatus;
         });
       });

src/frontend/src/hooks/useToast.tsx

@@ -4,13 +4,17 @@ import { Toast } from '../types';
 export const useToast = () => {
   const [toasts, setToasts] = useState<Toast[]>([]);
 
-  const showToast = useCallback((message: string, type: 'info' | 'success' | 'error' = 'info') => {
+  const showToast = useCallback((message: string, type: 'info' | 'success' | 'error' = 'info', persistent: boolean = false): string => {
     const id = Date.now().toString();
     setToasts(prev => [...prev, { id, message, type }]);
     
-    setTimeout(() => {
-      setToasts(prev => prev.filter(t => t.id !== id));
-    }, 4000);
+    if (!persistent) {
+      setTimeout(() => {
+        setToasts(prev => prev.filter(t => t.id !== id));
+      }, 4000);
+    }
+    
+    return id;
   }, []);
 
   const removeToast = useCallback((id: string) => {

src/frontend/src/types/index.ts

@@ -13,20 +13,16 @@ export interface Book {
   description?: string;
   download_path?: string;
   progress?: number;
+  status_message?: string;  // Detailed status message (e.g., "Trying Libgen (2/5)")
+  added_time?: number;  // Timestamp when added to queue
 }
 
 // Status response types
 export interface StatusData {
   queued?: Record<string, Book>;
   resolving?: Record<string, Book>;
-  bypassing?: Record<string, Book>;
   downloading?: Record<string, Book>;
-  verifying?: Record<string, Book>;
-  ingesting?: Record<string, Book>;
   complete?: Record<string, Book>;
-  available?: Record<string, Book>;
-  done?: Record<string, Book>;
-  completed?: Record<string, Book>;
   error?: Record<string, Book>;
   cancelled?: Record<string, Book>;
 }
@@ -36,7 +32,7 @@ export interface ActiveDownloadsResponse {
 }
 
 // Button states
-export type ButtonState = 'download' | 'queued' | 'resolving' | 'bypassing' | 'downloading' | 'verifying' | 'ingesting' | 'completed' | 'error';
+export type ButtonState = 'download' | 'queued' | 'resolving' | 'downloading' | 'complete' | 'error';
 
 export interface ButtonStateInfo {
   text: string;

src/frontend/vite.config.ts

@@ -21,6 +21,7 @@ export default defineConfig({
         changeOrigin: true,
         secure: false,
       },
+      // Proxy debug endpoint (uses /api/debug so it's automatically proxied above)
     },
   },
   build: {

testing/E2E_test.py

@@ -6,7 +6,7 @@ import hashlib
 # Thee server is already running, so let's grab some of the env vars:
 # Use absolute import since the script is run from the root directory
 import env as SERVER_ENV
-from backend import _sanitize_filename # Moved import to top level
+from models import BookInfo
 
 # Now let's test the server:
 port = SERVER_ENV.FLASK_PORT
@@ -109,12 +109,17 @@ print(f"Book {book_id} download confirmed as available.")
 
 # Step 5 : Verify the file exists locally (optional but good)
 print(f"Step 5: Verifying downloaded file exists...")
-# Depend if env.USE_TITLE is true or false, the filename will be different
+# Depend if env.USE_BOOK_TITLE is true or false, the filename will be different
 if SERVER_ENV.USE_BOOK_TITLE:
-    # Ensure book_details is available; might need adjustment if Step 2 failed
-    # Assuming book_details was successfully fetched in Step 2
-    title_to_sanitize = book_details.get('title', book_title) # Use fetched title if available
-    expected_filename = _sanitize_filename(title_to_sanitize) + ".epub" # Add extension
+    # Build expected filename using BookInfo
+    book_info = BookInfo(
+        id=book_id,
+        title=book_details.get('title', ''),
+        author=book_details.get('author'),
+        year=book_details.get('year'),
+        format='epub'
+    )
+    expected_filename = book_info.get_filename()
 else:
     expected_filename = f"{book_id}.epub"
 

tor.sh

@@ -55,6 +55,19 @@ AutomapHostsOnResolve 1
 TransPort 9040
 DNSPort 53
 Log notice file /var/log/tor/notices.log
+
+# Circuit management to prevent stale circuits after inactivity
+MaxCircuitDirtiness 600
+NewCircuitPeriod 30
+CircuitBuildTimeout 60
+LearnCircuitBuildTimeout 0
+
+# Keep circuits alive
+KeepalivePeriod 60
+CircuitStreamTimeout 60
+
+# Prevent connection timeouts
+SocksTimeout 120
 EOF
 
 echo "[*] Setting up DNS..."
@@ -92,10 +105,14 @@ iptables -t nat -A OUTPUT -p tcp --syn -j REDIRECT --to-ports 9040
 # For UDP DNS queries
 iptables -t nat -A OUTPUT -p udp --dport 53 ! -d 127.0.0.1 -j DNAT --to-destination 127.0.0.1:53
 
-
 # For TCP DNS queries (some DNS queries may use TCP)
 iptables -t nat -A OUTPUT -p tcp --dport 53 ! -d 127.0.0.1 -j DNAT --to-destination 127.0.0.1:53
 
+# Note: ICMP (ping) is NOT routed through Tor as Tor only supports TCP.
+# ICMP will use default routing. If you need to test connectivity, use:
+# curl -s https://check.torproject.org/api/ip
+# or: curl -s https://icanhazip.com
+
 echo "[✓] Transparent Tor routing enabled."
 
 sleep 5
@@ -148,5 +165,45 @@ else
     echo "[*] Falling back to container's default timezone: $TZ"
 fi
 
+# Start a background health check process to monitor Tor
+echo "[*] Starting Tor health check monitor..."
+(
+    check_count=0
+    while true; do
+        sleep 300  # Check every 5 minutes
+        check_count=$((check_count + 1))
+        echo "[*] Tor health check #$check_count at $(date)"
+        
+        # Check if Tor service is running
+        if ! service tor status > /dev/null 2>&1; then
+            echo "[!] $(date): Tor service not running, restarting..."
+            service tor restart
+            sleep 10
+        fi
+        
+        # Test DNS resolution through Tor
+        if ! timeout 10 nslookup google.com 127.0.0.1 > /dev/null 2>&1; then
+            echo "[!] $(date): DNS resolution failed, reloading Tor..."
+            service tor reload
+            sleep 5
+            # Verify DNS works after reload
+            if timeout 10 nslookup google.com 127.0.0.1 > /dev/null 2>&1; then
+                echo "[✓] $(date): DNS resolution restored"
+            else
+                echo "[✗] $(date): DNS still failing after reload, restarting Tor..."
+                service tor restart
+                sleep 10
+            fi
+        fi
+        
+        # Send SIGHUP to Tor to rotate circuits (helps with stale circuits)
+        echo "[*] $(date): Rotating Tor circuits..."
+        pkill -HUP tor || true
+    done
+) >> $LOG_FILE 2>&1 &
+
+TOR_MONITOR_PID=$!
+echo "[✓] Tor health check monitor started in background (PID: $TOR_MONITOR_PID)"
+
 # Run the entrypoint script
 echo "[*] End of tor script"

websocket_manager.py

@@ -1,7 +1,9 @@
 """WebSocket manager for real-time status updates."""
 
 import logging
-from typing import Optional, Dict, Any
+import threading
+from typing import Optional, Dict, Any, Callable, List
+
 from flask_socketio import SocketIO, emit
 
 logger = logging.getLogger(__name__)
@@ -12,12 +14,98 @@ class WebSocketManager:
     def __init__(self):
         self.socketio: Optional[SocketIO] = None
         self._enabled = False
+        self._connection_count = 0
+        self._connection_lock = threading.Lock()
+        self._on_first_connect_callbacks: List[Callable[[], None]] = []
+        self._on_all_disconnect_callbacks: List[Callable[[], None]] = []
+        self._needs_rewarm = False  # Flag to trigger warmup callbacks on next connect
         
     def init_app(self, app, socketio: SocketIO):
         """Initialize the WebSocket manager with Flask-SocketIO instance."""
         self.socketio = socketio
         self._enabled = True
         logger.info("WebSocket manager initialized")
+    
+    def register_on_first_connect(self, callback: Callable[[], None]):
+        """Register a callback to be called when the first client connects.
+        
+        This is useful for warming up resources (like the Cloudflare bypasser)
+        when a user starts using the web UI.
+        """
+        self._on_first_connect_callbacks.append(callback)
+        logger.debug(f"Registered on_first_connect callback: {callback.__name__}")
+    
+    def register_on_all_disconnect(self, callback: Callable[[], None]):
+        """Register a callback to be called when all clients disconnect.
+
+        This can be used to trigger cleanup or resource release.
+        """
+        self._on_all_disconnect_callbacks.append(callback)
+        logger.debug(f"Registered on_all_disconnect callback: {callback.__name__}")
+
+    def request_warmup_on_next_connect(self):
+        """Request that warmup callbacks be triggered on the next client connect.
+
+        This is used when resources (like the Cloudflare bypasser) shut down due to
+        inactivity while clients are still connected. The next connect event should
+        trigger warmup even though it's not technically the "first" connection.
+        """
+        with self._connection_lock:
+            self._needs_rewarm = True
+            logger.debug("Warmup requested for next client connect")
+    
+    def client_connected(self):
+        """Track a new client connection. Call this from the connect event handler."""
+        with self._connection_lock:
+            was_zero = self._connection_count == 0
+            needs_rewarm = self._needs_rewarm
+            self._connection_count += 1
+            current_count = self._connection_count
+            # Clear rewarm flag if we're going to trigger warmup
+            if was_zero or needs_rewarm:
+                self._needs_rewarm = False
+
+        logger.debug(f"Client connected. Active connections: {current_count}")
+
+        # Trigger warmup callbacks if this is the first connection OR if rewarm was requested
+        # (rewarm is requested when bypasser shuts down due to idle while clients are connected)
+        if was_zero or needs_rewarm:
+            reason = "First client connected" if was_zero else "Rewarm requested after idle shutdown"
+            logger.info(f"{reason}, triggering warmup callbacks...")
+            for callback in self._on_first_connect_callbacks:
+                try:
+                    # Run callbacks in a separate thread to not block the connection
+                    thread = threading.Thread(target=callback, daemon=True)
+                    thread.start()
+                except Exception as e:
+                    logger.error(f"Error in on_first_connect callback {callback.__name__}: {e}")
+    
+    def client_disconnected(self):
+        """Track a client disconnection. Call this from the disconnect event handler."""
+        with self._connection_lock:
+            self._connection_count = max(0, self._connection_count - 1)
+            current_count = self._connection_count
+            is_now_zero = current_count == 0
+        
+        logger.debug(f"Client disconnected. Active connections: {current_count}")
+        
+        # If all clients have disconnected, trigger cleanup callbacks
+        if is_now_zero:
+            logger.info("All clients disconnected, triggering disconnect callbacks...")
+            for callback in self._on_all_disconnect_callbacks:
+                try:
+                    callback()
+                except Exception as e:
+                    logger.error(f"Error in on_all_disconnect callback {callback.__name__}: {e}")
+    
+    def get_connection_count(self) -> int:
+        """Get the current number of active WebSocket connections."""
+        with self._connection_lock:
+            return self._connection_count
+    
+    def has_active_connections(self) -> bool:
+        """Check if there are any active WebSocket connections."""
+        return self.get_connection_count() > 0
         
     def is_enabled(self) -> bool:
         """Check if WebSocket is enabled and ready."""