diff --git a/app.py b/app.py index fe40df35..59a41a71 100644 --- a/app.py +++ b/app.py @@ -1,24 +1,29 @@ """Flask web application for book download service with URL rewrite support.""" +import io import logging -import io, re, os +import os import sqlite3 import time from datetime import datetime, timedelta from functools import wraps -from flask import Flask, request, jsonify, send_file, send_from_directory, session +from typing import Any, Dict, Tuple, Union + +from flask import Flask, jsonify, request, send_file, send_from_directory, session from flask_cors import CORS from flask_socketio import SocketIO, emit from werkzeug.middleware.proxy_fix import ProxyFix from werkzeug.security import check_password_hash from werkzeug.wrappers import Response -import typing -from logger import setup_logger -from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE, SUPPORTED_FORMATS -from env import FLASK_HOST, FLASK_PORT, CWA_DB_PATH, DEBUG, USING_EXTERNAL_BYPASSER, BUILD_VERSION, RELEASE_VERSION, CALIBRE_WEB_URL import backend - +from book_manager import SearchUnavailable +from config import BOOK_LANGUAGE, SUPPORTED_FORMATS, _SUPPORTED_BOOK_LANGUAGE +from env import ( + BUILD_VERSION, CALIBRE_WEB_URL, CWA_DB_PATH, DEBUG, FLASK_HOST, FLASK_PORT, + RELEASE_VERSION, USING_EXTERNAL_BYPASSER, +) +from logger import setup_logger from models import SearchFilters from websocket_manager import ws_manager @@ -58,7 +63,7 @@ logger.info(f"Flask-SocketIO initialized with async_mode='{async_mode}'") # Rate limiting for login attempts # Structure: {username: {'count': int, 'lockout_until': datetime}} -failed_login_attempts: typing.Dict[str, typing.Dict[str, typing.Any]] = {} +failed_login_attempts: Dict[str, Dict[str, Any]] = {} MAX_LOGIN_ATTEMPTS = 10 LOCKOUT_DURATION_MINUTES = 30 @@ -124,15 +129,45 @@ if DEBUG: } }) -# Custom log filter to exclude routine status endpoint polling +# Custom log filter to exclude routine status endpoint polling and WebSocket noise class StatusEndpointFilter(logging.Filter): - """Filter out routine status endpoint requests to reduce log noise.""" + """Filter out routine status endpoint requests and WebSocket upgrade errors to reduce log noise.""" def filter(self, record): - # Exclude GET /api/status requests if hasattr(record, 'getMessage'): message = record.getMessage() + # Exclude GET /api/status requests (polling noise) if 'GET /api/status' in message: return False + # Exclude WebSocket upgrade errors (benign - falls back to polling) + if 'write() before start_response' in message: + return False + # Exclude the Error on request line that precedes WebSocket errors + if 'Error on request:' in message and record.levelno == logging.ERROR: + return False + return True + + +class WebSocketErrorFilter(logging.Filter): + """Filter out WebSocket upgrade errors that occur in Werkzeug dev server. + + These errors are benign - Flask-SocketIO automatically falls back to polling transport. + The error occurs because Werkzeug's built-in server doesn't fully support WebSocket upgrades. + """ + def filter(self, record): + # Filter out the AssertionError traceback for WebSocket upgrades + if record.levelno == logging.ERROR: + message = record.getMessage() if hasattr(record, 'getMessage') else str(record.msg) + # Filter out the full traceback that includes the WebSocket assertion error + if 'write() before start_response' in message: + return False + # Also filter the "Error on request" header that precedes it + if hasattr(record, 'exc_info') and record.exc_info: + exc_type = record.exc_info[0] + if exc_type and exc_type.__name__ == 'AssertionError': + # Check if it's the WebSocket-related assertion + exc_value = record.exc_info[1] + if exc_value and 'write() before start_response' in str(exc_value): + return False return True # Flask logger @@ -142,8 +177,9 @@ app.logger.setLevel(logger.level) werkzeug_logger = logging.getLogger('werkzeug') werkzeug_logger.handlers = logger.handlers werkzeug_logger.setLevel(logger.level) -# Add filter to suppress routine status endpoint polling logs +# Add filters to suppress routine status endpoint polling logs and WebSocket upgrade errors werkzeug_logger.addFilter(StatusEndpointFilter()) +werkzeug_logger.addFilter(WebSocketErrorFilter()) # Set up authentication defaults # The secret key will reset every time we restart, which will @@ -210,32 +246,38 @@ def logo() -> Response: @app.route('/favicon.ico') @app.route('/favico') -def favicon(_ : typing.Any = None) -> Response: +def favicon(_: Any = None) -> Response: """ Serve favicon from built frontend assets. """ return send_from_directory(os.path.join(app.root_path, 'frontend-dist'), 'favicon.ico', mimetype='image/vnd.microsoft.icon') -from typing import Union, Tuple +# Register bypasser warmup callback for when first WebSocket client connects +# and shutdown callback for when all clients disconnect +if not USING_EXTERNAL_BYPASSER: + from cloudflare_bypasser import warmup as bypasser_warmup, shutdown_if_idle as bypasser_shutdown + ws_manager.register_on_first_connect(bypasser_warmup) + ws_manager.register_on_all_disconnect(bypasser_shutdown) + logger.info("Registered Cloudflare bypasser warmup/shutdown on WebSocket connect/disconnect") if DEBUG: import subprocess - import time if USING_EXTERNAL_BYPASSER: - STOP_GUI = lambda: None # No-op for external bypasser + STOP_GUI = lambda: None else: from cloudflare_bypasser import _reset_driver as STOP_GUI - @app.route('/debug', methods=['GET']) + @app.route('/api/debug', methods=['GET']) @login_required def debug() -> Union[Response, Tuple[Response, int]]: """ - This will run the /app/debug.sh script, which will generate a debug zip with all the logs + This will run the /app/genDebug.sh script, which will generate a debug zip with all the logs The file will be named /tmp/cwa-book-downloader-debug.zip And then return it to the user """ try: # Run the debug script + logger.info("Debug endpoint called, stopping GUI and generating debug info...") STOP_GUI() time.sleep(1) result = subprocess.run(['/app/genDebug.sh'], capture_output=True, text=True, check=True) @@ -244,9 +286,10 @@ if DEBUG: logger.info(f"Debug script executed: {result.stdout}") debug_file_path = result.stdout.strip().split('\n')[-1] if not os.path.exists(debug_file_path): - logger.error("Debug zip file not found after running debug script") + logger.error(f"Debug zip file not found at: {debug_file_path}") return jsonify({"error": "Failed to generate debug information"}), 500 + logger.info(f"Sending debug file: {debug_file_path}") # Return the file to the user return send_file( debug_file_path, @@ -307,6 +350,9 @@ def api_search() -> Union[Response, Tuple[Response, int]]: try: books = backend.search_books(query, filters) return jsonify(books) + except SearchUnavailable as e: + logger.warning(f"Search unavailable: {e}") + return jsonify({"error": str(e)}), 503 except Exception as e: logger.error_trace(f"Search error: {e}") return jsonify({"error": str(e)}), 500 @@ -431,15 +477,12 @@ def api_local_download() -> Union[Response, Tuple[Response, int]]: if file_data is None: # Book data not found or not available return jsonify({"error": "File not found"}), 404 - # Santize the file name - file_name = book_info.title - file_name = re.sub(r'[\\/:*?"<>|]', '_', file_name.strip())[:245] - file_extension = book_info.format + file_name = book_info.get_filename() # Prepare the file for sending to the client data = io.BytesIO(file_data) return send_file( data, - download_name=f"{file_name}.{file_extension}", + download_name=file_name, as_attachment=True ) @@ -580,7 +623,7 @@ def api_clear_completed() -> Union[Response, Tuple[Response, int]]: removed_count = backend.clear_completed() # Broadcast status update after clearing - if ws_manager and ws_manager.is_enabled(): + if ws_manager: ws_manager.broadcast_status_update(backend.queue_status()) return jsonify({"status": "cleared", "removed_count": removed_count}) @@ -787,6 +830,10 @@ def catch_all(path: str) -> Response: def handle_connect(): """Handle client connection.""" logger.info("WebSocket client connected") + + # Track the connection (triggers warmup callbacks on first connect) + ws_manager.client_connected() + # Send initial status to the newly connected client try: status = backend.queue_status() @@ -798,6 +845,9 @@ def handle_connect(): def handle_disconnect(): """Handle client disconnection.""" logger.info("WebSocket client disconnected") + + # Track the disconnection + ws_manager.client_disconnected() @socketio.on('request_status') def handle_status_request(): diff --git a/backend.py b/backend.py index 46e265eb..d542ce4b 100644 --- a/backend.py +++ b/backend.py @@ -1,34 +1,41 @@ """Backend logic for the book download application.""" -import threading, time -import shutil -from pathlib import Path -from typing import Dict, List, Optional, Any, Tuple -import subprocess import os -from concurrent.futures import ThreadPoolExecutor, Future -from threading import Event +import random +import shutil +import subprocess +import threading +import time +from concurrent.futures import Future, ThreadPoolExecutor +from pathlib import Path +from threading import Event, Lock +from typing import Any, Dict, List, Optional, Tuple -from logger import setup_logger -from config import CUSTOM_SCRIPT -from env import (INGEST_DIR, DOWNLOAD_PATHS, TMP_DIR, MAIN_LOOP_SLEEP_TIME, USE_BOOK_TITLE, - MAX_CONCURRENT_DOWNLOADS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL) -from models import book_queue, BookInfo, QueueStatus, SearchFilters import book_manager +from book_manager import SearchUnavailable +from config import CUSTOM_SCRIPT +from env import ( + DOWNLOAD_PATHS, DOWNLOAD_PROGRESS_UPDATE_INTERVAL, INGEST_DIR, + MAIN_LOOP_SLEEP_TIME, MAX_CONCURRENT_DOWNLOADS, TMP_DIR, USE_BOOK_TITLE, +) +from logger import setup_logger +from models import BookInfo, QueueStatus, SearchFilters, book_queue logger = setup_logger(__name__) -# Import WebSocket manager (will be initialized by app.py) +# WebSocket manager (initialized by app.py) try: from websocket_manager import ws_manager except ImportError: - logger.warning("WebSocket manager not available") ws_manager = None -def _sanitize_filename(filename: str) -> str: - """Sanitize a filename by replacing spaces with underscores and removing invalid characters.""" - keepcharacters = (' ','.','_') - return "".join(c for c in filename if c.isalnum() or c in keepcharacters).rstrip() +# Progress update throttling - track last broadcast time per book +_progress_last_broadcast: Dict[str, float] = {} +_progress_lock = Lock() + +# Stall detection - track last activity time per download +_last_activity: Dict[str, float] = {} +STALL_TIMEOUT = 300 # 5 minutes without progress/status update = stalled def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]: """Search for books matching the query. @@ -43,6 +50,9 @@ def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]: try: books = book_manager.search_books(query, filters) return [_book_info_to_dict(book) for book in books] + except SearchUnavailable as e: + logger.warning(f"Search unavailable: {e}") + raise except Exception as e: logger.error_trace(f"Error searching books: {e}") return [] @@ -79,7 +89,7 @@ def queue_book(book_id: str, priority: int = 0) -> bool: logger.info(f"Book queued with priority {priority}: {book_info.title}") # Broadcast status update via WebSocket - if ws_manager and ws_manager.is_enabled(): + if ws_manager: ws_manager.broadcast_status_update(queue_status()) return True @@ -162,14 +172,12 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option book_info = book_queue._book_data[book_id] logger.info(f"Starting download: {book_info.title}") - if USE_BOOK_TITLE: - book_name = _sanitize_filename(book_info.title) - else: - book_name = book_id - # If format is not set, use the format of the first download URL - if book_info.format == "": - book_info.format = book_info.download_urls[0].split(".")[-1] - book_name += f".{book_info.format}" + if not book_info.download_urls: + raise ValueError(f"No download URLs available for {book_id}") + + # get_filename() resolves format as side effect + full_name = book_info.get_filename() + book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}" book_path = TMP_DIR / book_name # Check cancellation before download @@ -178,7 +186,11 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option return None progress_callback = lambda progress: update_download_progress(book_id, progress) - status_callback = lambda status: update_download_status(book_id, status) + status_callback = lambda status, message=None: update_download_status(book_id, status, message) + + # Set status to resolving immediately when processing starts + update_download_status(book_id, "resolving") + success_download_url = book_manager.download_book(book_info, book_path, progress_callback, cancel_flag, status_callback) # Stop progress updates @@ -201,29 +213,30 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option book_path.unlink() return None - # Update status to verifying - book_queue.update_status(book_id, QueueStatus.VERIFYING) - if ws_manager and ws_manager.is_enabled(): - ws_manager.broadcast_status_update(queue_status()) - logger.info(f"Verifying download: {book_info.title}") + logger.debug(f"Post-processing download: {book_info.title}") if CUSTOM_SCRIPT: logger.info(f"Running custom script: {CUSTOM_SCRIPT}") subprocess.run([CUSTOM_SCRIPT, book_path]) - - # Update status to ingesting - book_queue.update_status(book_id, QueueStatus.INGESTING) - if ws_manager and ws_manager.is_enabled(): - ws_manager.broadcast_status_update(queue_status()) - if success_download_url and book_info.format == "": - book_info.format = success_download_url.split(".")[-1] - book_name += f".{book_info.format}" + # Regenerate filename with fallback to successful download URL for format + full_name = book_info.get_filename(success_download_url) + book_name = full_name if USE_BOOK_TITLE else f"{book_id}.{book_info.format or 'bin'}" final_dir = _prepare_download_folder(book_info) intermediate_path = final_dir / f"{book_id}.crdownload" final_path = final_dir / book_name + # Handle file already exists - add suffix to avoid overwrite + if final_path.exists(): + base = final_path.stem + ext = final_path.suffix + counter = 1 + while final_path.exists(): + final_path = final_dir / f"{base}_{counter}{ext}" + counter += 1 + logger.info(f"File already exists, saving as: {final_path.name}") + if os.path.exists(book_path): logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}") try: @@ -256,23 +269,60 @@ def _download_book_with_cancellation(book_id: str, cancel_flag: Event) -> Option return None def update_download_progress(book_id: str, progress: float) -> None: - """Update download progress.""" - book_queue.update_progress(book_id, progress) - - # Broadcast progress via WebSocket - if ws_manager and ws_manager.is_enabled(): - ws_manager.broadcast_download_progress(book_id, progress, 'downloading') + """Update download progress with throttled WebSocket broadcasts. -def update_download_status(book_id: str, status: str) -> None: - """Update download status.""" + Progress is always stored in the queue, but WebSocket broadcasts are + throttled to avoid flooding clients with updates. Broadcasts occur: + - At most once per DOWNLOAD_PROGRESS_UPDATE_INTERVAL seconds + - Always at 0% (start) and 100% (complete) + - On significant progress jumps (>10%) + """ + book_queue.update_progress(book_id, progress) + + # Track activity for stall detection + with _progress_lock: + _last_activity[book_id] = time.time() + + # Broadcast progress via WebSocket with throttling + if ws_manager: + current_time = time.time() + should_broadcast = False + + with _progress_lock: + last_broadcast = _progress_last_broadcast.get(book_id, 0) + last_progress = _progress_last_broadcast.get(f"{book_id}_progress", 0) + time_elapsed = current_time - last_broadcast + + # Always broadcast at start (0%) or completion (>=99%) + if progress <= 1 or progress >= 99: + should_broadcast = True + # Broadcast if enough time has passed (convert interval from seconds) + elif time_elapsed >= DOWNLOAD_PROGRESS_UPDATE_INTERVAL: + should_broadcast = True + # Broadcast on significant progress jumps (>10%) + elif progress - last_progress >= 10: + should_broadcast = True + + if should_broadcast: + _progress_last_broadcast[book_id] = current_time + _progress_last_broadcast[f"{book_id}_progress"] = progress + + if should_broadcast: + ws_manager.broadcast_download_progress(book_id, progress, 'downloading') + +def update_download_status(book_id: str, status: str, message: Optional[str] = None) -> None: + """Update download status with optional detailed message. + + Args: + book_id: Book identifier + status: Status string (e.g., 'resolving', 'downloading') + message: Optional detailed status message for UI display + """ # Map string status to QueueStatus enum status_map = { 'queued': QueueStatus.QUEUED, 'resolving': QueueStatus.RESOLVING, - 'bypassing': QueueStatus.BYPASSING, 'downloading': QueueStatus.DOWNLOADING, - 'verifying': QueueStatus.VERIFYING, - 'ingesting': QueueStatus.INGESTING, 'complete': QueueStatus.COMPLETE, 'available': QueueStatus.AVAILABLE, 'error': QueueStatus.ERROR, @@ -283,9 +333,17 @@ def update_download_status(book_id: str, status: str) -> None: queue_status_enum = status_map.get(status.lower()) if queue_status_enum: book_queue.update_status(book_id, queue_status_enum) - + + # Track activity for stall detection + with _progress_lock: + _last_activity[book_id] = time.time() + + # Update status message if provided (empty string clears the message) + if message is not None: + book_queue.update_status_message(book_id, message) + # Broadcast status update via WebSocket - if ws_manager and ws_manager.is_enabled(): + if ws_manager: ws_manager.broadcast_status_update(queue_status()) def cancel_download(book_id: str) -> bool: @@ -340,17 +398,27 @@ def clear_completed() -> int: """Clear all completed downloads from tracking.""" return book_queue.clear_completed() +def _cleanup_progress_tracking(book_id: str) -> None: + """Clean up progress tracking data for a completed/cancelled download.""" + with _progress_lock: + _progress_last_broadcast.pop(book_id, None) + _progress_last_broadcast.pop(f"{book_id}_progress", None) + _last_activity.pop(book_id, None) + def _process_single_download(book_id: str, cancel_flag: Event) -> None: """Process a single download job.""" try: # Status will be updated through callbacks during download process - # (resolving -> bypassing -> downloading -> verifying -> ingesting -> complete) + # (resolving -> downloading -> complete) download_path = _download_book_with_cancellation(book_id, cancel_flag) + # Clean up progress tracking + _cleanup_progress_tracking(book_id) + if cancel_flag.is_set(): book_queue.update_status(book_id, QueueStatus.CANCELLED) # Broadcast cancellation - if ws_manager and ws_manager.is_enabled(): + if ws_manager: ws_manager.broadcast_status_update(queue_status()) return @@ -363,23 +431,26 @@ def _process_single_download(book_id: str, cancel_flag: Event) -> None: book_queue.update_status(book_id, new_status) # Broadcast final status (completed or error) - if ws_manager and ws_manager.is_enabled(): + if ws_manager: ws_manager.broadcast_status_update(queue_status()) - logger.info( - f"Book {book_id} download {'successful' if download_path else 'failed'}" - ) except Exception as e: + # Clean up progress tracking even on error + _cleanup_progress_tracking(book_id) + if not cancel_flag.is_set(): logger.error_trace(f"Error in download processing: {e}") book_queue.update_status(book_id, QueueStatus.ERROR) + # Set error message if not already set by download_book() + if book_id in book_queue._book_data and not book_queue._book_data[book_id].status_message: + book_queue.update_status_message(book_id, f"Download failed: {type(e).__name__}: {str(e)}") else: logger.info(f"Download cancelled: {book_id}") book_queue.update_status(book_id, QueueStatus.CANCELLED) # Broadcast error/cancelled status - if ws_manager and ws_manager.is_enabled(): + if ws_manager: ws_manager.broadcast_status_update(queue_status()) def concurrent_download_loop() -> None: @@ -398,16 +469,32 @@ def concurrent_download_loop() -> None: future.result() # This will raise any exceptions from the worker except Exception as e: logger.error_trace(f"Future exception for {book_id}: {e}") - + + # Check for stalled downloads (no activity in STALL_TIMEOUT seconds) + current_time = time.time() + with _progress_lock: + for future, book_id in list(active_futures.items()): + last_active = _last_activity.get(book_id, current_time) + if current_time - last_active > STALL_TIMEOUT: + logger.warning(f"Download stalled for {book_id}, cancelling") + book_queue.cancel_download(book_id) + book_queue.update_status_message(book_id, f"Download stalled (no activity for {STALL_TIMEOUT}s)") + # Start new downloads if we have capacity while len(active_futures) < MAX_CONCURRENT_DOWNLOADS: next_download = book_queue.get_next() if not next_download: break - + + # Stagger concurrent downloads to avoid rate limiting on shared download servers + # Only delay if other downloads are already active + if active_futures: + stagger_delay = random.uniform(2, 5) + logger.debug(f"Staggering download start by {stagger_delay:.1f}s") + time.sleep(stagger_delay) + book_id, cancel_flag = next_download - logger.info(f"Starting concurrent download: {book_id}") - + # Submit download job to thread pool future = executor.submit(_process_single_download, book_id, cancel_flag) active_futures[future] = book_id diff --git a/book_manager.py b/book_manager.py index 3579036a..deb412c5 100644 --- a/book_manager.py +++ b/book_manager.py @@ -1,19 +1,37 @@ """Book download manager handling search and retrieval operations.""" -import time, json, os, re +import itertools +import json +import re +import time from pathlib import Path -from urllib.parse import quote -from typing import List, Optional, Dict, Union, Callable from threading import Event -from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet +from typing import Callable, Dict, List, Optional +from urllib.parse import quote + +from bs4 import BeautifulSoup, NavigableString, Tag import downloader +import network +from config import BOOK_LANGUAGE, SUPPORTED_FORMATS +from env import AA_DONATOR_KEY, ALLOW_USE_WELIB, DEBUG_SKIP_SOURCES, DOWNLOAD_PATHS, PRIORITIZE_WELIB, USE_CF_BYPASS from logger import setup_logger -from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL -from env import AA_DONATOR_KEY, USE_CF_BYPASS, PRIORITIZE_WELIB, ALLOW_USE_WELIB, DOWNLOAD_PATHS from models import BookInfo, SearchFilters + logger = setup_logger(__name__) +# Round-robin counter for AA slow download source rotation +# Distributes concurrent downloads across different partner mirrors +_aa_slow_rotation = itertools.count() + +if DEBUG_SKIP_SOURCES: + logger.warning("DEBUG_SKIP_SOURCES active: skipping sources %s", DEBUG_SKIP_SOURCES) + + +class SearchUnavailable(Exception): + """Raised when Anna's Archive cannot be reached via any mirror/DNS.""" + pass + def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: @@ -62,8 +80,10 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: ) index += 1 + selector = network.AAMirrorSelector() + url = ( - f"{AA_BASE_URL}" + f"{network.get_aa_base_url()}" f"/search?index=&page=1&display=table" f"&acc=aa_download&acc=external_download" f"&ext={'&ext='.join(formats_to_use)}" @@ -71,13 +91,14 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: f"{filters_query}" ) - html = downloader.html_get_page(url) + html = downloader.html_get_page(url, selector=selector) if not html: - raise Exception("Failed to fetch search results") + # Network/mirror exhaustion path bubbles up so API can notify clients + raise SearchUnavailable("Unable to reach Anna's Archive. Network restricted or mirrors are blocked.") if "No files found." in html: logger.info(f"No books found for query: {query}") - raise Exception("No books found. Please try another query.") + return [] soup = BeautifulSoup(html, "html.parser") tbody: Tag | NavigableString | None = soup.find("table") @@ -143,8 +164,9 @@ def get_book_info(book_id: str) -> BookInfo: Returns: BookInfo: Detailed book information """ - url = f"{AA_BASE_URL}/md5/{book_id}" - html = downloader.html_get_page(url) + url = f"{network.get_aa_base_url()}/md5/{book_id}" + selector = network.AAMirrorSelector() + html = downloader.html_get_page(url, selector=selector) if not html: raise Exception(f"Failed to fetch book info for ID: {book_id}") @@ -174,58 +196,75 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div") divs = list(data.children) - every_url = soup.find_all("a") - slow_urls_no_waitlist = set() - slow_urls_with_waitlist = set() - external_urls_libgen = set() - external_urls_z_lib = set() - external_urls_welib = set() + # Collect download URLs by source type (lists preserve page order, dedup inline) + slow_urls_no_waitlist: list[str] = [] + slow_urls_with_waitlist: list[str] = [] + external_urls_libgen: list[str] = [] + external_urls_z_lib: list[str] = [] - for url in every_url: + def _append_unique(lst: list[str], href: str) -> None: + if href and href not in lst: + lst.append(href) + + for anchor in soup.find_all("a"): try: - if url.text.strip().lower().startswith("slow partner server"): - if ( - url.next is not None - and url.next.next is not None - and "waitlist" in url.next.next.strip().lower() - ): - internal_text = url.next.next.strip().lower() - if "no waitlist" in internal_text: - slow_urls_no_waitlist.add(url["href"]) - else: - slow_urls_with_waitlist.add(url["href"]) - elif ( - url.next is not None - and url.next.next is not None - and "click “GET” at the top" in url.next.next.text.strip() - ): - libgen_url = url["href"] - # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ? - libgen_url = re.sub(r'libgen\.(lc|is|bz|st)', 'libgen.gl', url["href"]) + text = anchor.text.strip().lower() + href = anchor.get("href", "") + next_text = "" + if anchor.next and anchor.next.next: + next_text = getattr(anchor.next.next, 'text', str(anchor.next.next)).strip().lower() - external_urls_libgen.add(libgen_url) - elif url.text.strip().lower().startswith("z-lib"): - if ".onion/" not in url["href"]: - external_urls_z_lib.add(url["href"]) + if text.startswith("slow partner server") and "waitlist" in next_text: + if "no waitlist" in next_text: + _append_unique(slow_urls_no_waitlist, href) + else: + _append_unique(slow_urls_with_waitlist, href) + elif 'libgen.li' in href: + # Normalize libgen domains + libgen_url = re.sub(r'libgen\.(li|lc|is|bz|st)', 'libgen.gl', href) + _append_unique(external_urls_libgen, libgen_url) + elif text.startswith("z-lib") and ".onion/" not in href: + _append_unique(external_urls_z_lib, href) except: pass - external_urls_welib = _get_download_urls_from_welib(book_id) if USE_CF_BYPASS else set() + logger.debug( + "Source inventory for %s -> aa_no_wait=%d, aa_wait=%d, libgen=%d, zlib=%d", + book_id, + len(slow_urls_no_waitlist), + len(slow_urls_with_waitlist), + len(external_urls_libgen), + len(external_urls_z_lib), + ) urls = [] - urls += list(external_urls_welib) if PRIORITIZE_WELIB else [] - urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else [] - urls += list(external_urls_libgen) - urls += list(external_urls_welib) if not PRIORITIZE_WELIB else [] - urls += list(slow_urls_with_waitlist) if USE_CF_BYPASS else [] - urls += list(external_urls_z_lib) + + # Priority: reliable sources first, then external fallbacks + # 1. AA slow (no waitlist) - instant but can be slow + # 2. Libgen - instant, external + # 3. AA slow (waitlist) - has countdown timer but faster once started + # Note: Z-Library disabled - download tokens are session-bound + urls += slow_urls_no_waitlist if USE_CF_BYPASS else [] + urls += external_urls_libgen + urls += slow_urls_with_waitlist if USE_CF_BYPASS else [] for i in range(len(urls)): - urls[i] = downloader.get_absolute_url(AA_BASE_URL, urls[i]) + urls[i] = downloader.get_absolute_url(network.get_aa_base_url(), urls[i]) # Remove empty urls urls = [url for url in urls if url != ""] + # Tag AA slow URLs with detailed source type for skip/retry tracking + base_url = network.get_aa_base_url() + for rel_url in slow_urls_no_waitlist: + abs_url = downloader.get_absolute_url(base_url, rel_url) + if abs_url: + _url_source_types[abs_url] = "aa-slow-nowait" + for rel_url in slow_urls_with_waitlist: + abs_url = downloader.get_absolute_url(base_url, rel_url) + if abs_url: + _url_source_types[abs_url] = "aa-slow-wait" + # Filter out divs that are not text original_divs = divs divs = [div for div in divs if div.text.strip() != ""] @@ -241,7 +280,8 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: if format == "" and f.strip().lower() in SUPPORTED_FORMATS: format = f.strip().lower() if size == "" and any(u in f.strip().lower() for u in ["mb", "kb", "gb"]): - size = f.strip().lower() + # Preserve original case but uppercase the unit (e.g., "5.2 mb" -> "5.2 MB") + size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE) if content == "": for ct in DOWNLOAD_PATHS.keys(): if ct in f.strip().lower(): @@ -253,7 +293,8 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: if format == "" and stripped and " " not in stripped: format = stripped if size == "" and "." in stripped: - size = stripped + # Uppercase any size units + size = re.sub(r'(kb|mb|gb|tb)', lambda m: m.group(1).upper(), f.strip(), flags=re.IGNORECASE) book_title = _find_in_divs(divs, "🔍")[0].strip("🔍").strip() @@ -265,8 +306,8 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: preview=preview, title=book_title, content=content, - publisher=_find_in_divs(divs, "icon-[mdi--company]", isClass=True)[0], - author=_find_in_divs(divs, "icon-[mdi--user-edit]", isClass=True)[0], + publisher=_find_in_divs(divs, "icon-[mdi--company]", is_class=True)[0], + author=_find_in_divs(divs, "icon-[mdi--user-edit]", is_class=True)[0], format=format, size=size, description=description, @@ -289,32 +330,85 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo: return book_info -def _find_in_divs(divs: List[str], text: str, isClass: bool = False) -> List[str]: - divs_found = [] +def _find_in_divs(divs: List, text: str, is_class: bool = False) -> List[str]: + """Find divs containing text or having a specific class.""" + results = [] for div in divs: - if isClass: - if div.find(class_ = text): - divs_found.append(div.text.strip()) - else: - if text in div.text.strip(): - divs_found.append(div.text.strip()) - return divs_found + if is_class: + if div.find(class_=text): + results.append(div.text.strip()) + elif text in div.text.strip(): + results.append(div.text.strip()) + return results -def _get_download_urls_from_welib(book_id: str) -> set[str]: - if ALLOW_USE_WELIB == False: - return set() - """Get download urls from welib.org.""" - url = f"https://welib.org/md5/{book_id}" - logger.info(f"Getting download urls from welib.org for {book_id}. While this uses the bypasser, it will not start downloading them yet.") - html = downloader.html_get_page(url, use_bypasser=True) - if not html: +# Download source definitions: (log_label, friendly_name, url_patterns) +_DOWNLOAD_SOURCES = [ + ("welib", "Welib", ["welib.org"]), + ("aa-fast", "Anna's Archive (Fast)", ["/dyn/api/fast_download"]), + ("aa-slow-wait", "Anna's Archive (Waitlist)", []), # Matched via _url_source_types + ("aa-slow-nowait", "Anna's Archive", []), # Matched via _url_source_types + ("aa-slow", "Anna's Archive", ["/slow_download/", "annas-"]), # Fallback for untagged AA URLs + ("libgen", "Libgen", ["libgen"]), + ("zlib", "Z-Library", ["z-lib", "zlibrary"]), +] + +# Track detailed source types for AA slow URLs (populated during get_book_info) +_url_source_types: dict[str, str] = {} + + +def _get_source_info(link: str) -> tuple[str, str]: + """Get source label and friendly name for a download link. + + Args: + link: Download URL + + Returns: + Tuple of (log_label, friendly_name) + """ + # Check detailed source type mapping first (for AA slow distinction) + if link in _url_source_types: + detailed_label = _url_source_types[link] + for log_label, friendly_name, _ in _DOWNLOAD_SOURCES: + if log_label == detailed_label: + return log_label, friendly_name + + for log_label, friendly_name, patterns in _DOWNLOAD_SOURCES: + if patterns and any(pattern in link for pattern in patterns): + return log_label, friendly_name + return "unknown", "Mirror" + + +def _label_source(link: str) -> str: + """Get lightweight source tag for logging/metrics.""" + return _get_source_info(link)[0] + + +def _friendly_source_name(link: str) -> str: + """Get user-friendly name for a download source.""" + return _get_source_info(link)[1] + +def _get_download_urls_from_welib(book_id: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> list[str]: + """Get download URLs from welib.org (bypasser required).""" + if not ALLOW_USE_WELIB: return [] + url = f"https://welib.org/md5/{book_id}" + logger.info(f"Fetching welib.org download URLs for {book_id}") + try: + html = downloader.html_get_page(url, use_bypasser=True, selector=selector or network.AAMirrorSelector(), cancel_flag=cancel_flag) + except Exception as exc: + logger.error_trace(f"Welib fetch failed for {book_id}: {exc}") + return [] + if not html: + logger.warning(f"Welib page empty for {book_id}") + return [] + soup = BeautifulSoup(html, "html.parser") - download_links = soup.find_all("a", href=True) - download_links = [link["href"] for link in download_links] - download_links = [link for link in download_links if "/slow_download/" in link] - download_links = [downloader.get_absolute_url(url, link) for link in download_links] - return set(download_links) + links = [ + downloader.get_absolute_url(url, a["href"]) + for a in soup.find_all("a", href=True) + if "/slow_download/" in a["href"] + ] + return list(dict.fromkeys(links)) # Dedupe while preserving order def _get_next_value_div(label_div: Tag) -> Optional[Tag]: """Find the next sibling div that holds the value for a metadata label.""" @@ -400,7 +494,14 @@ def _extract_book_metadata(metadata_divs) -> Dict[str, List[str]]: } -def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str], None]] = None) -> Optional[str]: +# After N consecutive failures of the same source type, skip remaining sources of that type +SOURCE_FAILURE_THRESHOLD = 4 + +# Minimum valid file size in bytes (10KB) - anything smaller is likely an error page +MIN_VALID_FILE_SIZE = 10 * 1024 + + +def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None) -> Optional[str]: """Download a book from available sources. Args: @@ -408,88 +509,277 @@ def download_book(book_info: BookInfo, book_path: Path, progress_callback: Optio title: Book title for logging progress_callback: Optional callback for download progress updates cancel_flag: Optional cancellation flag - status_callback: Optional callback for status updates + status_callback: Optional callback for status updates (status, message) Returns: str: Download URL if successful, None otherwise """ + selector = network.AAMirrorSelector() + if len(book_info.download_urls) == 0: book_info = get_book_info(book_info.id) - download_links = book_info.download_urls + download_links = list(book_info.download_urls) # If AA_DONATOR_KEY is set, use the fast download URL. Else try other sources. if AA_DONATOR_KEY != "": download_links.insert( 0, - f"{AA_BASE_URL}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}", + f"{network.get_aa_base_url()}/dyn/api/fast_download.json?md5={book_info.id}&key={AA_DONATOR_KEY}", ) - for link in download_links: - try: - # Update status to resolving before attempting download URL fetch - if status_callback: - status_callback("resolving") + # Preserve order but drop duplicates to avoid retrying the same host + download_links = list(dict.fromkeys(download_links)) - download_url = _get_download_url(link, book_info.title, cancel_flag, status_callback) - if download_url != "": - # Update status to downloading before starting actual download - if status_callback: - status_callback("downloading") + # Round-robin rotation for AA slow download URLs to distribute load across mirrors + # This prevents all concurrent downloads from hitting the same partner server first + # Rotate aa-slow-nowait and aa-slow-wait independently to preserve priority ordering + rotation_value = next(_aa_slow_rotation) - logger.info(f"Downloading `{book_info.title}` from `{download_url}`") + def _rotate_category_in_place(links: list, source_type: str) -> int: + """Rotate URLs of a specific source type within the list, preserving their positions.""" + indices = [i for i, u in enumerate(links) if _url_source_types.get(u) == source_type] + if len(indices) <= 1: + return 0 + rotation = rotation_value % len(indices) + if rotation == 0: + return 0 + # Extract values, rotate, put back + values = [links[i] for i in indices] + rotated = values[rotation:] + values[:rotation] + for idx, val in zip(indices, rotated): + links[idx] = val + return rotation - data = downloader.download_url(download_url, book_info.size or "", progress_callback, cancel_flag) - if not data: - raise Exception("No data received") + nowait_rotation = _rotate_category_in_place(download_links, "aa-slow-nowait") + wait_rotation = _rotate_category_in_place(download_links, "aa-slow-wait") - logger.info(f"Download finished. Writing to {book_path}") - with open(book_path, "wb") as f: - f.write(data.getbuffer()) - logger.info(f"Writing `{book_info.title}` successfully") - return download_url + if nowait_rotation or wait_rotation: + logger.info(f"AA source rotation: nowait={nowait_rotation}, wait={wait_rotation}") - except Exception as e: - logger.error_trace(f"Failed to download from {link}: {e}") + links_queue = download_links + + # Fetch welib URLs upfront when prioritized + welib_fallback_loaded = "welib" in DEBUG_SKIP_SOURCES # Skip welib entirely if in debug skip list + if USE_CF_BYPASS and PRIORITIZE_WELIB and ALLOW_USE_WELIB and not welib_fallback_loaded: + logger.info("Fetching welib.org download URLs (PRIORITIZE_WELIB enabled)") + if status_callback: + status_callback("resolving", "Fetching welib sources...") + welib_links = _get_download_urls_from_welib(book_info.id, selector=selector, cancel_flag=cancel_flag) + if welib_links: + links_queue = welib_links + [l for l in links_queue if l not in welib_links] + welib_fallback_loaded = True + + total_sources = len(links_queue) + + # Handle case where no download sources are available + if total_sources == 0: + logger.warning(f"No download sources available for: {book_info.title}") + if status_callback: + status_callback("error", "No download sources found") + return None + + # Track consecutive failures per source type to skip after threshold + source_failures: dict[str, int] = {} + # Iterate with index so we can append welib links later + idx = 0 + while idx < len(links_queue): + link = links_queue[idx] + source_label = _label_source(link) + friendly_name = _friendly_source_name(link) + + # Debug: skip sources for testing fallback chains + if source_label in DEBUG_SKIP_SOURCES: + logger.info("DEBUG_SKIP_SOURCES: skipping %s (%s)", source_label, link) + idx += 1 continue + # Skip source types that have failed too many times + if source_failures.get(source_label, 0) >= SOURCE_FAILURE_THRESHOLD: + logger.info("Skipping %s - source type '%s' failed %d times", link, source_label, SOURCE_FAILURE_THRESHOLD) + idx += 1 + continue + + try: + current_pos = idx + 1 + # Update total if we added more sources + total_sources = len(links_queue) + + logger.info("Trying download source [%s]: %s (%d/%d)", source_label, link, current_pos, total_sources) + + # Build source context for status messages (e.g., "Welib (1/12)") + source_context = f"{friendly_name} (Server #{current_pos})" + + # Update status with simple message showing which source we're trying + if status_callback: + status_callback("resolving", f"Trying {source_context}") + + download_url = _get_download_url(link, book_info.title, cancel_flag, status_callback, selector, source_context) + if download_url == "": + raise Exception("No download URL resolved") + + logger.info("Resolved download URL [%s]: %s", source_label, download_url) + + # Pass source page as referer (required by some sites) + data = downloader.download_url(download_url, book_info.size or "", progress_callback, cancel_flag, selector, status_callback, referer=link) + if not data: + raise Exception("No data received from download") + + # Validate file size - reject suspiciously small files + file_size = data.tell() + if file_size < MIN_VALID_FILE_SIZE: + logger.warning(f"Downloaded file too small ({file_size} bytes), likely an error page") + raise Exception(f"File too small ({file_size} bytes)") + + logger.debug(f"Download finished ({file_size} bytes). Writing to {book_path}") + data.seek(0) # Reset buffer position before writing + with open(book_path, "wb") as f: + f.write(data.getbuffer()) + return download_url + + except Exception as e: + logger.warning(f"Failed to download from {link} (source={source_label}): {e}") + source_failures[source_label] = source_failures.get(source_label, 0) + 1 + idx += 1 + # If we exhausted primary links and haven't loaded welib yet, fetch them lazily + if ( + idx >= len(links_queue) + and not welib_fallback_loaded + and USE_CF_BYPASS + and ALLOW_USE_WELIB + ): + welib_selector = selector # reuse AA mirror selector for consistency + welib_links = _get_download_urls_from_welib(book_info.id, selector=welib_selector, cancel_flag=cancel_flag) + welib_fallback_loaded = True + if welib_links: + new_links = [wl for wl in welib_links if wl not in links_queue] + if new_links: + logger.info("Adding welib fallback links (%d)", len(new_links)) + links_queue.extend(new_links) + # continue loop to try newly added links + continue + + # All sources exhausted - report final error to UI + if status_callback: + status_callback("error", f"All {len(links_queue)} sources failed") + return None -def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str], None]] = None) -> str: - """Extract actual download URL from various source pages.""" +def _get_download_url(link: str, title: str, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None, selector: Optional[network.AAMirrorSelector] = None, source_context: Optional[str] = None) -> str: + """Extract actual download URL from various source pages. + + Args: + link: URL to extract download link from + title: Book title for logging + cancel_flag: Optional cancellation flag + status_callback: Optional callback for status updates + selector: Optional AA mirror selector + source_context: Optional context string like "Welib (1/12)" for status messages + """ + sel = selector or network.AAMirrorSelector() + # AA fast download API (JSON response) + if link.startswith(f"{network.get_aa_base_url()}/dyn/api/fast_download.json"): + page = downloader.html_get_page(link, selector=sel, cancel_flag=cancel_flag) + return downloader.get_absolute_url(link, json.loads(page).get("download_url", "")) + + html = downloader.html_get_page(link, selector=sel, cancel_flag=cancel_flag) + if not html: + return "" + + soup = BeautifulSoup(html, "html.parser") url = "" - if link.startswith(f"{AA_BASE_URL}/dyn/api/fast_download.json"): - page = downloader.html_get_page(link, status_callback=status_callback) - url = json.loads(page).get("download_url") + # Z-Library + if link.startswith("https://z-lib."): + dl = soup.find("a", href=True, class_="addDownloadedBook") + url = dl["href"] if dl else "" + + # AA slow download / partner servers + elif "/slow_download/" in link: + url = _extract_slow_download_url(soup, link, title, cancel_flag, status_callback, sel, source_context) + + # Libgen (GET button) else: - html = downloader.html_get_page(link, status_callback=status_callback) - - if html == "": - return "" - - soup = BeautifulSoup(html, "html.parser") - - if link.startswith("https://z-lib."): - download_link = soup.find_all("a", href=True, class_="addDownloadedBook") - if download_link: - url = download_link[0]["href"] - elif "/slow_download/" in link: - download_links = soup.find_all("a", href=True, string="📚 Download now") - if not download_links: - countdown = soup.find_all("span", class_="js-partner-countdown") - if countdown: - sleep_time = int(countdown[0].text) - logger.info(f"Waiting {sleep_time}s for {title}") - if cancel_flag is not None and cancel_flag.wait(timeout=sleep_time): - logger.info(f"Cancelled wait for {title}") - return "" - url = _get_download_url(link, title, cancel_flag, status_callback) - else: - url = download_links[0]["href"] - else: - url = soup.find_all("a", string="GET")[0]["href"] + get_btn = soup.find("a", string="GET") + url = get_btn["href"] if get_btn else "" return downloader.get_absolute_url(link, url) + + +def _extract_slow_download_url(soup: BeautifulSoup, link: str, title: str, cancel_flag: Optional[Event], status_callback, selector, source_context: Optional[str] = None) -> str: + """Extract download URL from AA slow download pages.""" + # Try "Download now" button variations + dl_link = soup.find("a", href=True, string="📚 Download now") + if not dl_link: + dl_link = soup.find("a", href=True, string=lambda s: s and "Download now" in s) + if dl_link: + return dl_link["href"] + + # Try finding URL in gray background span (AA's copy URL format) + # The URL appears as plain text in http://... + for span in soup.find_all("span", class_=lambda c: c and "bg-gray-200" in c): + text = span.get_text(strip=True) + if text.startswith("http://") or text.startswith("https://"): + return text + + # Try "copy this URL" pattern (legacy) + copy_text = soup.find(string=lambda s: s and "copy this url" in s.lower()) + if copy_text and copy_text.parent: + parent = copy_text.parent + next_link = parent.find_next("a", href=True) + if next_link and next_link.get("href"): + return next_link["href"] + code_elem = parent.find_next("code") + if code_elem: + return code_elem.get_text(strip=True) + for sibling in parent.find_next_siblings(): + text = sibling.get_text(strip=True) if hasattr(sibling, 'get_text') else str(sibling).strip() + if text.startswith("http"): + return text + + # Check for countdown timer (waitlist) + countdown = soup.find("span", class_="js-partner-countdown") + if countdown: + # Cap countdown at 10 minutes to prevent malformed HTML from blocking indefinitely + MAX_COUNTDOWN_SECONDS = 600 + try: + raw_countdown = int(countdown.text) + except (ValueError, TypeError): + logger.warning(f"Invalid countdown value '{countdown.text}', skipping wait") + raw_countdown = 0 + sleep_time = min(raw_countdown, MAX_COUNTDOWN_SECONDS) + if raw_countdown > MAX_COUNTDOWN_SECONDS: + logger.warning(f"Countdown {raw_countdown}s exceeds max, capping at {MAX_COUNTDOWN_SECONDS}s") + logger.info(f"Waiting {sleep_time}s for {title}") + + # Live countdown with status updates + remaining = sleep_time + while remaining > 0: + # Format countdown message with source context + if source_context: + wait_msg = f"{source_context} - Waiting {remaining}s" + else: + wait_msg = f"Waiting {remaining}s" + + if status_callback: + status_callback("resolving", wait_msg) + + # Wait 1 second (or until cancelled) + if cancel_flag and cancel_flag.wait(timeout=1): + logger.info(f"Cancelled wait for {title}") + return "" + + remaining -= 1 + + # After countdown, update status and re-fetch + if status_callback and source_context: + status_callback("resolving", f"{source_context} - Fetching...") + + return _get_download_url(link, title, cancel_flag, status_callback, selector, source_context) + + # Debug fallback + link_texts = [a.get_text(strip=True)[:50] for a in soup.find_all("a", href=True)[:10]] + logger.warning(f"No download URL found. First 10 links: {link_texts}") + return "" diff --git a/cloudflare_bypasser.py b/cloudflare_bypasser.py index 0ddf460e..525504b2 100644 --- a/cloudflare_bypasser.py +++ b/cloudflare_bypasser.py @@ -1,30 +1,47 @@ -import time import os -import socket -from urllib.parse import urlparse -import threading -import env -from env import LOG_DIR, DEBUG +import random import signal -from datetime import datetime +import socket import subprocess -import requests +import threading +import time +import traceback +from datetime import datetime +from threading import Event from typing import Optional +from urllib.parse import urlparse -# --- SeleniumBase Import --- + +class BypassCancelledException(Exception): + """Raised when a bypass operation is cancelled.""" + pass + +import requests from seleniumbase import Driver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.common.exceptions import TimeoutException +import env import network +from config import PROXIES, RECORDING_DIR, VIRTUAL_SCREEN_SIZE +from env import AA_DONATOR_KEY, BYPASS_WARMUP_ON_CONNECT, DEBUG, DEFAULT_SLEEP, LOG_DIR, MAX_RETRY from logger import setup_logger -from env import MAX_RETRY, DEFAULT_SLEEP -from config import PROXIES, CUSTOM_DNS, DOH_SERVER, VIRTUAL_SCREEN_SIZE, RECORDING_DIR logger = setup_logger(__name__) -network.init() + +# Challenge detection indicators +CLOUDFLARE_INDICATORS = [ + "just a moment", + "verify you are human", + "verifying you are human", + "cloudflare.com/products/turnstile", +] + +DDOS_GUARD_INDICATORS = [ + "ddos-guard", + "ddos guard", + "checking your browser before accessing", + "complete the manual check to continue", + "could not verify your browser automatically", +] DRIVER = None DISPLAY = { @@ -33,85 +50,213 @@ DISPLAY = { } LAST_USED = None LOCKED = threading.Lock() -TENTATIVE_CURRENT_URL = None + +# Flag to track if DNS rotated while a bypass was in progress +# Chrome will be restarted after the current operation completes +_dns_rotation_pending = False +_dns_rotation_lock = threading.Lock() + +# Cookie storage - shared with requests library for Cloudflare bypass +# Structure: {domain: {cookie_name: {value, expiry, ...}}} +_cf_cookies: dict[str, dict] = {} +_cf_cookies_lock = threading.Lock() + +# Protection cookie names we care about (Cloudflare and DDoS-Guard) +CF_COOKIE_NAMES = {'cf_clearance', '__cf_bm', 'cf_chl_2', 'cf_chl_prog'} +DDG_COOKIE_NAMES = {'__ddg1_', '__ddg2_', '__ddg5_', '__ddg8_', '__ddg9_', '__ddg10_', '__ddgid_', '__ddgmark_', 'ddg_last_challenge'} + +# Domains requiring full session cookies (not just protection cookies) +FULL_COOKIE_DOMAINS = {'z-lib.fm', 'z-lib.gs', 'z-lib.id', 'z-library.sk', 'zlibrary-global.se'} + + +def _extract_cookies_from_driver(driver, url: str) -> None: + """Extract cookies from Chrome after successful bypass.""" + try: + parsed = urlparse(url) + domain = parsed.hostname or "" + if not domain: + return + + # Get base domain for storage and full-cookie check + base_domain = '.'.join(domain.split('.')[-2:]) if '.' in domain else domain + extract_all = base_domain in FULL_COOKIE_DOMAINS + + cookies = driver.get_cookies() + cookies_found = {} + + for cookie in cookies: + name = cookie.get('name', '') + + if extract_all: + should_extract = True + else: + is_cf = name in CF_COOKIE_NAMES or name.startswith('cf_') + is_ddg = name in DDG_COOKIE_NAMES or name.startswith('__ddg') + should_extract = is_cf or is_ddg + + if should_extract: + cookies_found[name] = { + 'value': cookie.get('value', ''), + 'domain': cookie.get('domain', domain), + 'path': cookie.get('path', '/'), + 'expiry': cookie.get('expiry'), + 'secure': cookie.get('secure', True), + 'httpOnly': cookie.get('httpOnly', True), + } + + if cookies_found: + with _cf_cookies_lock: + _cf_cookies[base_domain] = cookies_found + cookie_type = "all" if extract_all else "protection" + logger.debug(f"Extracted {len(cookies_found)} {cookie_type} cookies for {base_domain}") + + except Exception as e: + logger.debug(f"Failed to extract cookies: {e}") + + +def get_cf_cookies_for_domain(domain: str) -> dict[str, str]: + """Get stored cookies for a domain. Returns empty dict if none available.""" + if not domain: + return {} + + # Get base domain + base_domain = '.'.join(domain.split('.')[-2:]) if '.' in domain else domain + + with _cf_cookies_lock: + cookies = _cf_cookies.get(base_domain, {}) + if not cookies: + return {} + + # Check if cf_clearance exists and hasn't expired + cf_clearance = cookies.get('cf_clearance', {}) + if cf_clearance: + expiry = cf_clearance.get('expiry') + if expiry and time.time() > expiry: + logger.debug(f"CF cookies expired for {base_domain}") + _cf_cookies.pop(base_domain, None) + return {} + + # Return simple name->value dict for requests + return {name: c['value'] for name, c in cookies.items()} + + +def has_valid_cf_cookies(domain: str) -> bool: + """Check if we have valid Cloudflare cookies for a domain.""" + return bool(get_cf_cookies_for_domain(domain)) + + +def clear_cf_cookies(domain: str = None) -> None: + """Clear stored Cloudflare cookies. If domain is None, clear all.""" + with _cf_cookies_lock: + if domain: + base_domain = '.'.join(domain.split('.')[-2:]) if '.' in domain else domain + _cf_cookies.pop(base_domain, None) + else: + _cf_cookies.clear() + def _reset_pyautogui_display_state(): try: import pyautogui import Xlib.display - pyautogui._pyautogui_x11._display = ( - Xlib.display.Display(os.environ['DISPLAY']) - ) + pyautogui._pyautogui_x11._display = Xlib.display.Display(os.environ['DISPLAY']) except Exception as e: logger.warning(f"Error resetting pyautogui display state: {e}") -def _is_bypassed(sb, escape_emojis : bool = True) -> bool: - """Enhanced bypass detection with more comprehensive checks""" +def _get_page_info(sb) -> tuple[str, str, str]: + """Extract page title, body text, and current URL safely.""" try: - # Get page information with error handling - try: - title = sb.get_title().lower() - except: - title = "" - - try: - body = sb.get_text("body").lower() - except: - body = "" - - try: - current_url = sb.get_current_url() - except: - current_url = "" + title = sb.get_title().lower() + except: + title = "" + try: + body = sb.get_text("body").lower() + except: + body = "" + try: + current_url = sb.get_current_url() + except: + current_url = "" + return title, body, current_url + + +def _check_indicators(title: str, body: str, indicators: list[str]) -> Optional[str]: + """Check if any indicator is present in title or body. Returns the found indicator or None.""" + for indicator in indicators: + if indicator in title or indicator in body: + return indicator + return None + +def _has_cloudflare_patterns(body: str, url: str) -> bool: + """Check for Cloudflare-specific patterns in body or URL.""" + return "cf-" in body or "cloudflare" in url.lower() or "/cdn-cgi/" in url + +def _detect_challenge_type(sb) -> str: + """Detect what type of challenge we're facing. + + Returns: + str: 'cloudflare', 'ddos_guard', or 'none' if no challenge detected + """ + try: + title, body, current_url = _get_page_info(sb) - # Check if page is too long, if so we are probably bypassed - if len(body.strip()) > 100000: - logger.debug(f"Page content too long, we are probably bypassed len: {len(body.strip())}") + # DDOS-Guard indicators + if found := _check_indicators(title, body, DDOS_GUARD_INDICATORS): + logger.debug(f"DDOS-Guard indicator found: '{found}'") + return "ddos_guard" + + # Cloudflare indicators + if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS): + logger.debug(f"Cloudflare indicator found: '{found}'") + return "cloudflare" + + # Check URL patterns + if _has_cloudflare_patterns(body, current_url): + return "cloudflare" + + return "none" + except Exception as e: + logger.warning(f"Error detecting challenge type: {e}") + return "none" + +def _is_bypassed(sb, escape_emojis: bool = True) -> bool: + """Check if the protection has been bypassed.""" + try: + title, body, current_url = _get_page_info(sb) + body_len = len(body.strip()) + + # Long page content = probably bypassed + if body_len > 100000: + logger.debug(f"Page content too long, probably bypassed (len: {body_len})") return True - # Detect if there is an emoji in the page, any utf8 emoji, if so we are probably bypassed + # Multiple emojis = probably real content if escape_emojis: import emoji - emoji_list = emoji.emoji_list(body) - if len(emoji_list) >= 3: - logger.debug(f"Detected emoji in page, we are probably bypassed len: {len(emoji_list)}") + if len(emoji.emoji_list(body)) >= 3: + logger.debug("Detected emojis in page, probably bypassed") return True - # Enhanced verification texts for newer Cloudflare versions - verification_texts = [ - "just a moment", - "verify you are human", - "verifying you are human", - "cloudflare.com/products/turnstile/?utm_source=turnstile" - ] + # Check for protection indicators (means NOT bypassed) + if found := _check_indicators(title, body, CLOUDFLARE_INDICATORS + DDOS_GUARD_INDICATORS): + logger.debug(f"Protection indicator found: '{found}'") + return False - # Check for Cloudflare indicators - for text in verification_texts: - if text in title or text in body: - logger.debug(f"Cloudflare indicator found: '{text}' in page") - return False - - # Additional checks for specific Cloudflare patterns - if "cf-" in body or "cloudflare" in current_url.lower(): + # Cloudflare URL patterns + if _has_cloudflare_patterns(body, current_url): logger.debug("Cloudflare patterns detected in page") return False - # Check if we're still on a challenge page (common Cloudflare pattern) - if "/cdn-cgi/" in current_url: - logger.debug("Still on Cloudflare CDN challenge page") - return False - - # If page is mostly empty, it might still be loading - if len(body.strip()) < 50: + # Page too short = still loading + if body_len < 50: logger.debug("Page content too short, might still be loading") return False - logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {len(body)}") + logger.debug(f"Bypass check passed - Title: '{title[:100]}', Body length: {body_len}") return True except Exception as e: logger.warning(f"Error checking bypass status: {e}") - # If we can't check, assume we're not bypassed return False def _bypass_method_1(sb) -> bool: @@ -172,7 +317,6 @@ def _bypass_method_3(sb) -> bool: try: logger.debug("Attempting bypass method 3: stealth approach") # Wait a random amount to appear more human - import random wait_time = random.uniform(8, 15) time.sleep(wait_time) @@ -201,40 +345,137 @@ def _bypass_method_3(sb) -> bool: logger.debug(f"Method 3 failed: {e}") return False -def _bypass(sb, max_retries: int = MAX_RETRY) -> None: - """Enhanced bypass function with multiple strategies""" - try_count = 0 - methods = [_bypass_method_1, _bypass_method_2, _bypass_method_3] - - while not _is_bypassed(sb): - if try_count >= max_retries: - logger.warning("Exceeded maximum retries. Bypass failed.") - break - - method_index = try_count % len(methods) - method = methods[method_index] +def _bypass_ddos_guard_method_1(sb) -> bool: + """DDOS-Guard bypass: Use SeleniumBase's captcha handling (most reliable)""" + try: + logger.debug("Attempting DDOS-Guard bypass: SeleniumBase uc_gui methods") + time.sleep(random.uniform(2, 4)) - logger.info(f"Bypass attempt {try_count + 1} / {max_retries} using {method.__name__}") + # SeleniumBase's uc_gui_click_captcha often works for DDOS-Guard too + try: + sb.uc_gui_click_captcha() + time.sleep(random.uniform(3, 5)) + if _is_bypassed(sb): + return True + except Exception as e: + logger.debug(f"uc_gui_click_captcha failed: {e}") - try_count += 1 + # Fallback: Try clicking visible checkbox-like elements + checkbox_patterns = [ + "//input[@type='checkbox']", + "//*[contains(@class, 'checkbox')]", + "//*[contains(@class, 'cb-')]", + ] + for pattern in checkbox_patterns: + try: + elements = sb.find_elements(f"xpath:{pattern}") + for elem in elements: + if elem.is_displayed(): + logger.debug(f"Clicking element with pattern: {pattern}") + elem.click() + time.sleep(random.uniform(3, 5)) + if _is_bypassed(sb): + return True + except: + continue + + return False + except Exception as e: + logger.debug(f"DDOS-Guard method 1 failed: {e}") + return False - # Progressive backoff: wait longer between retries - wait_time = min(DEFAULT_SLEEP * (try_count - 1), 15) +def _bypass_ddos_guard_method_2(sb) -> bool: + """DDOS-Guard bypass: Use pyautogui to click estimated checkbox location""" + try: + logger.debug("Attempting DDOS-Guard bypass: pyautogui coordinate click") + time.sleep(random.uniform(2, 4)) + + import pyautogui + window_size = sb.get_window_size() + width = window_size.get("width", 1920) + height = window_size.get("height", 1080) + + # DDOS-Guard checkbox is typically around 35% from left, 55% from top + checkbox_x = int(width * 0.35) + random.randint(-5, 5) + checkbox_y = int(height * 0.55) + random.randint(-5, 5) + + logger.debug(f"Clicking at coordinates: ({checkbox_x}, {checkbox_y})") + pyautogui.moveTo(checkbox_x, checkbox_y, duration=random.uniform(0.3, 0.7)) + time.sleep(random.uniform(0.1, 0.3)) + pyautogui.click() + time.sleep(random.uniform(3, 5)) + + return _is_bypassed(sb) + except ImportError: + logger.debug("pyautogui not available") + return False + except Exception as e: + logger.debug(f"DDOS-Guard method 2 failed: {e}") + return False + +def _bypass(sb, max_retries: int = MAX_RETRY, cancel_flag: Optional[Event] = None) -> bool: + """Bypass function with strategies for Cloudflare and DDOS-Guard protection. + + Returns True if bypass succeeded, False otherwise. + """ + cloudflare_methods = [_bypass_method_1, _bypass_method_2, _bypass_method_3] + ddos_guard_methods = [_bypass_ddos_guard_method_1, _bypass_ddos_guard_method_2] + + for try_count in range(max_retries): + # Check for cancellation before each attempt + if cancel_flag and cancel_flag.is_set(): + logger.info("Bypass cancelled by user") + raise BypassCancelledException("Bypass cancelled") + + if _is_bypassed(sb): + return True + + challenge_type = _detect_challenge_type(sb) + logger.info(f"Detected challenge type: {challenge_type}") + + if challenge_type == "ddos_guard": + methods = ddos_guard_methods + elif challenge_type == "cloudflare": + methods = cloudflare_methods + else: + methods = cloudflare_methods + ddos_guard_methods + + method = methods[try_count % len(methods)] + logger.info(f"Bypass attempt {try_count + 1}/{max_retries} using {method.__name__}") + + # Progressive backoff with cancellation checks + wait_time = min(DEFAULT_SLEEP * try_count, 15) if wait_time > 0: logger.info(f"Waiting {wait_time}s before trying...") - time.sleep(wait_time) + # Check cancellation during wait (check every second) + for _ in range(int(wait_time)): + if cancel_flag and cancel_flag.is_set(): + logger.info("Bypass cancelled during wait") + raise BypassCancelledException("Bypass cancelled") + time.sleep(1) try: if method(sb): logger.info(f"Bypass successful using {method.__name__}") - return + return True + except BypassCancelledException: + raise except Exception as e: logger.warning(f"Exception in {method.__name__}: {e}") logger.info(f"Bypass method {method.__name__} failed.") + logger.warning("Exceeded maximum retries. Bypass failed.") + return False + def _get_chromium_args(): + """Build Chrome arguments dynamically, pre-resolving hostnames via Python's DNS. + Instead of trying to configure Chrome's DNS (which is unreliable), we pre-resolve + AA hostnames using Python's patched socket (which uses DoH/custom DNS) and pass + the resolved IPs directly to Chrome via --host-resolver-rules. This bypasses + Chrome's DNS entirely for those hosts. + """ arguments = [ # Ignore certificate and SSL errors (similar to curl's --insecure) "--ignore-certificate-errors", @@ -258,40 +499,64 @@ def _get_chromium_args(): if proxy_url: arguments.append(f'--proxy-server={proxy_url}') - # --- Add Custom DNS settings --- + # --- Pre-resolve AA hostnames and map them directly in Chrome --- + # This bypasses Chrome's DNS entirely - we resolve via Python's patched socket.getaddrinfo + # (which uses DoH/Cloudflare when system DNS fails) and tell Chrome to use those IPs + host_rules = [] + try: - if len(CUSTOM_DNS) > 0: - if DOH_SERVER: - logger.info(f"Configuring DNS over HTTPS (DoH) with server: {DOH_SERVER}") - - # TODO: This is probably broken and a halucination, - # but it should still default to google DOH so its fine... - arguments.extend(['--enable-features=DnsOverHttps', '--dns-over-https-mode=secure', f'--dns-over-https-servers="{DOH_SERVER}"']) - doh_hostname = urlparse(DOH_SERVER).hostname - if doh_hostname: - try: - arguments.append(f'--host-resolver-rules=MAP {doh_hostname} {socket.gethostbyname(doh_hostname)}') - except socket.gaierror: - logger.warning(f"Could not resolve DoH hostname: {doh_hostname}") - elif CUSTOM_DNS: - arguments.append(f'--dns-server="{",".join(CUSTOM_DNS)}"') - arguments.append(f'--disable-features=DnsOverHttps') + aa_urls = network.get_available_aa_urls() + for url in aa_urls: + hostname = urlparse(url).hostname + if hostname: + try: + # Use socket.getaddrinfo which IS patched by our network module + # (DoH/Cloudflare if system DNS failed) + # getaddrinfo returns: [(family, type, proto, canonname, sockaddr), ...] + # sockaddr for IPv4 is (ip, port) + results = socket.getaddrinfo(hostname, 443, socket.AF_INET) + if results: + ip = results[0][4][0] # First result, sockaddr tuple, IP address + host_rules.append(f"MAP {hostname} {ip}") + logger.debug(f"Chrome: Pre-resolved {hostname} -> {ip}") + else: + logger.warning(f"Chrome: No addresses returned for {hostname}") + except socket.gaierror as e: + logger.warning(f"Chrome: Could not pre-resolve {hostname}: {e}") + + if host_rules: + # Join all rules with comma, e.g. "MAP host1 ip1, MAP host2 ip2" + rules_str = ", ".join(host_rules) + arguments.append(f'--host-resolver-rules={rules_str}') + logger.info(f"Chrome: Using host resolver rules for {len(host_rules)} hosts") + else: + logger.warning("Chrome: No hosts could be pre-resolved, Chrome will use its own DNS") + except Exception as e: - logger.error_trace(f"Error configuring DNS settings: {e}") + logger.error_trace(f"Error pre-resolving hostnames for Chrome: {e}") + return arguments -CHROMIUM_ARGS = _get_chromium_args() +def _get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): + # Check for cancellation before starting + if cancel_flag and cancel_flag.is_set(): + logger.info("Bypass cancelled before starting") + raise BypassCancelledException("Bypass cancelled") -def _get(url, retry : int = MAX_RETRY): try: logger.info(f"SB_GET: {url}") sb = _get_driver() - + # Enhanced page loading with better error handling logger.debug("Opening URL with SeleniumBase...") sb.uc_open_with_reconnect(url, DEFAULT_SLEEP) time.sleep(DEFAULT_SLEEP) - + + # Check for cancellation after page load + if cancel_flag and cancel_flag.is_set(): + logger.info("Bypass cancelled after page load") + raise BypassCancelledException("Bypass cancelled") + # Log current page title and URL for debugging try: current_url = sb.get_current_url() @@ -299,13 +564,13 @@ def _get(url, retry : int = MAX_RETRY): logger.debug(f"Page loaded - URL: {current_url}, Title: {current_title}") except Exception as debug_e: logger.debug(f"Could not get page info: {debug_e}") - - # Attempt bypass + + # Attempt bypass with cancellation support logger.debug("Starting bypass process...") - _bypass(sb) - - if _is_bypassed(sb): + if _bypass(sb, cancel_flag=cancel_flag): logger.info("Bypass successful.") + # Extract cookies for sharing with requests library + _extract_cookies_from_driver(sb, url) return sb.page_source else: logger.warning("Bypass completed but page still shows Cloudflare protection") @@ -315,34 +580,54 @@ def _get(url, retry : int = MAX_RETRY): logger.debug(f"Page content: {page_text}") except: pass - + + except BypassCancelledException: + raise except Exception as e: - # Enhanced error logging with full stack trace - import traceback error_details = f"Exception type: {type(e).__name__}, Message: {str(e)}" stack_trace = traceback.format_exc() - + if retry == 0: logger.error(f"Failed to initialize browser after all retries: {error_details}") logger.debug(f"Full stack trace: {stack_trace}") _reset_driver() raise e - + logger.warning(f"Failed to bypass Cloudflare (retry {MAX_RETRY - retry + 1}/{MAX_RETRY}): {error_details}") logger.debug(f"Stack trace: {stack_trace}") - - # Reset driver on certain errors - if "WebDriverException" in str(type(e)) or "SessionNotCreatedException" in str(type(e)): - logger.info("Resetting driver due to WebDriver error...") - _reset_driver() - - return _get(url, retry - 1) -def get(url, retry : int = MAX_RETRY): - global LOCKED, TENTATIVE_CURRENT_URL, LAST_USED + # Reset driver on certain errors + error_type = type(e).__name__ + if error_type in ("WebDriverException", "SessionNotCreatedException", "TimeoutException", "MaxRetryError"): + logger.info("Restarting bypasser due to browser error...") + _reset_driver() + + # Check for cancellation before retry + if cancel_flag and cancel_flag.is_set(): + logger.info("Bypass cancelled before retry") + raise BypassCancelledException("Bypass cancelled") + + return _get(url, retry - 1, cancel_flag) + +def get(url, retry: int = MAX_RETRY, cancel_flag: Optional[Event] = None): + """Fetch a URL with protection bypass.""" + global LAST_USED with LOCKED: - TENTATIVE_CURRENT_URL = url - ret = _get(url, retry) + # Check for cookies AFTER acquiring lock - another request may have + # completed bypass while we were waiting, making Chrome unnecessary + parsed = urlparse(url) + cookies = get_cf_cookies_for_domain(parsed.hostname or "") + if cookies: + try: + response = requests.get(url, cookies=cookies, proxies=PROXIES, timeout=(5, 10)) + if response.status_code == 200: + logger.debug(f"Cookies available after lock wait - skipped Chrome") + LAST_USED = time.time() + return response.text + except Exception: + pass # Fall through to Chrome bypass + + ret = _get(url, retry, cancel_flag) LAST_USED = time.time() return ret @@ -350,108 +635,218 @@ def _init_driver(): global DRIVER if DRIVER: _reset_driver() - driver = Driver(uc=True, headless=False, size=f"{VIRTUAL_SCREEN_SIZE[0]},{VIRTUAL_SCREEN_SIZE[1]}", chromium_arg=CHROMIUM_ARGS) + # Build Chrome args dynamically to pick up current DNS settings from network module + chromium_args = _get_chromium_args() + logger.debug(f"Initializing Chrome driver with args: {chromium_args}") + driver = Driver(uc=True, headless=False, size=f"{VIRTUAL_SCREEN_SIZE[0]},{VIRTUAL_SCREEN_SIZE[1]}", chromium_arg=chromium_args) + driver.set_page_load_timeout(60) DRIVER = driver time.sleep(DEFAULT_SLEEP) return driver +def _ensure_display_initialized(): + """Initialize virtual display if needed. Must be called with LOCKED held.""" + global DISPLAY + if DISPLAY["xvfb"] is not None: + return + if not (env.DOCKERMODE and env.USE_CF_BYPASS): + return + + from pyvirtualdisplay import Display + display = Display(visible=False, size=VIRTUAL_SCREEN_SIZE) + display.start() + DISPLAY["xvfb"] = display + logger.info("Virtual display started") + time.sleep(DEFAULT_SLEEP) + _reset_pyautogui_display_state() + + def _get_driver(): - global DRIVER, DISPLAY - global LAST_USED + global DRIVER, DISPLAY, LAST_USED logger.info("Getting driver...") LAST_USED = time.time() - if env.DOCKERMODE and env.USE_CF_BYPASS and not DISPLAY["xvfb"]: - from pyvirtualdisplay import Display - display = Display(visible=False, size=VIRTUAL_SCREEN_SIZE) - display.start() - logger.info("Display started") - DISPLAY["xvfb"] = display - time.sleep(DEFAULT_SLEEP) - _reset_pyautogui_display_state() + + _ensure_display_initialized() + + # Start FFmpeg recording on first actual bypass request (not during warmup) + # This ensures we only record active bypass sessions, not idle time + if env.DEBUG and DISPLAY["xvfb"] and not DISPLAY["ffmpeg"]: + display = DISPLAY["xvfb"] + timestamp = datetime.now().strftime("%y%m%d-%H%M%S") + output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4" - if env.DEBUG: - timestamp = datetime.now().strftime("%y%m%d-%H%M%S") - output_file = RECORDING_DIR / f"screen_recording_{timestamp}.mp4" - - ffmpeg_cmd = [ - "ffmpeg", - "-y", - "-f", "x11grab", - "-video_size", f"{VIRTUAL_SCREEN_SIZE[0]}x{VIRTUAL_SCREEN_SIZE[1]}", - "-i", f":{display.display}", - "-c:v", "libx264", - "-preset", "ultrafast", # or "veryfast" (trade speed for slightly better compression) - "-maxrate", "700k", # Slightly higher bitrate for text clarity - "-bufsize", "1400k", # Buffer size (2x maxrate) - "-crf", "36", # Adjust as needed: higher = smaller, lower = better quality (23 is visually lossless) - "-pix_fmt", "yuv420p", # Crucial for compatibility with most players - "-tune", "animation", # Optimize encoding for screen content - "-x264-params", "bframes=0:deblock=-1,-1", # Optimize for text, disable b-frames and deblocking - "-r", "15", # Reduce frame rate (if content allows) - "-an", # Disable audio recording (if not needed) - output_file.as_posix(), - "-nostats", "-loglevel", "0" - ] - logger.info("Starting FFmpeg recording to %s", output_file) - logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}") - DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd) + ffmpeg_cmd = [ + "ffmpeg", + "-y", + "-f", "x11grab", + "-video_size", f"{VIRTUAL_SCREEN_SIZE[0]}x{VIRTUAL_SCREEN_SIZE[1]}", + "-i", f":{display.display}", + "-c:v", "libx264", + "-preset", "ultrafast", # or "veryfast" (trade speed for slightly better compression) + "-maxrate", "700k", # Slightly higher bitrate for text clarity + "-bufsize", "1400k", # Buffer size (2x maxrate) + "-crf", "36", # Adjust as needed: higher = smaller, lower = better quality (23 is visually lossless) + "-pix_fmt", "yuv420p", # Crucial for compatibility with most players + "-tune", "animation", # Optimize encoding for screen content + "-x264-params", "bframes=0:deblock=-1,-1", # Optimize for text, disable b-frames and deblocking + "-r", "15", # Reduce frame rate (if content allows) + "-an", # Disable audio recording (if not needed) + output_file.as_posix(), + "-nostats", "-loglevel", "0" + ] + logger.info("Starting FFmpeg recording to %s", output_file) + logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}") + DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd) + if not DRIVER: return _init_driver() logger.log_resource_usage() return DRIVER def _reset_driver(): + """Reset the browser driver and cleanup all associated processes.""" logger.log_resource_usage() - logger.info("Resetting driver...") + logger.info("Shutting down Cloudflare bypasser...") global DRIVER, DISPLAY + + # Quit driver if DRIVER: try: DRIVER.quit() - DRIVER = None except Exception as e: logger.warning(f"Error quitting driver: {e}") - time.sleep(0.5) + DRIVER = None + + # Stop virtual display if DISPLAY["xvfb"]: try: DISPLAY["xvfb"].stop() - DISPLAY["xvfb"] = None except Exception as e: logger.warning(f"Error stopping display: {e}") - time.sleep(0.5) - try: - os.system("pkill -f Xvfb") - except Exception as e: - logger.debug(f"Error killing Xvfb: {e}") - time.sleep(0.5) + DISPLAY["xvfb"] = None + + # Stop ffmpeg recording if DISPLAY["ffmpeg"]: try: DISPLAY["ffmpeg"].send_signal(signal.SIGINT) - DISPLAY["ffmpeg"] = None except Exception as e: logger.debug(f"Error stopping ffmpeg: {e}") - time.sleep(0.5) - try: - os.system("pkill -f ffmpeg") - except Exception as e: - logger.debug(f"Error killing ffmpeg: {e}") + DISPLAY["ffmpeg"] = None + + # Kill any lingering processes time.sleep(0.5) + for process in ["Xvfb", "ffmpeg", "chrom"]: + try: + os.system(f"pkill -f {process}") + except Exception as e: + logger.debug(f"Error killing {process}: {e}") + + time.sleep(0.5) + logger.info("Cloudflare bypasser shut down (browser and display stopped)") + logger.log_resource_usage() + +def _restart_chrome_only(): + """Restart just Chrome (not the display) to pick up new DNS settings. + + Called when DNS provider rotates in auto mode. The display is kept running + to avoid the slower full restart. Chrome will be re-initialized with fresh + pre-resolved IPs from the new DNS provider. + """ + global DRIVER, LAST_USED + + logger.debug("Restarting Chrome to apply new DNS settings...") + + # Quit existing driver + if DRIVER: + try: + DRIVER.quit() + except Exception as e: + logger.debug(f"Error quitting driver during DNS rotation restart: {e}") + DRIVER = None + + # Kill any lingering Chrome processes (same pattern as _reset_driver) try: os.system("pkill -f chrom") except Exception as e: - logger.debug(f"Error killing chrom: {e}") + logger.debug(f"Error killing chrome processes: {e}") + time.sleep(0.5) - logger.info("Driver reset.") - logger.log_resource_usage() + + # Re-initialize driver with new DNS settings + # _get_chromium_args() will re-resolve hostnames using the new DNS + try: + _init_driver() + LAST_USED = time.time() + logger.debug("Chrome restarted with updated DNS settings") + except Exception as e: + logger.warning(f"Failed to restart Chrome after DNS rotation: {e}") + # Don't raise - the bypasser can try again on next request + + +def _on_dns_rotation(provider_name: str, servers: list, doh_url: str) -> None: + """Callback invoked when network.py rotates DNS provider. + + If Chrome is currently running, schedule a restart in the background. + This is async to avoid blocking the request that triggered DNS rotation. + """ + global DRIVER, _dns_rotation_pending + + if DRIVER is None: + return + + # Always set pending flag - the restart will happen asynchronously + # This avoids blocking the current request (which triggered DNS rotation) + with _dns_rotation_lock: + if _dns_rotation_pending: + return # Already scheduled + _dns_rotation_pending = True + + def _async_restart(): + global _dns_rotation_pending + logger.debug(f"DNS rotated to {provider_name} - restarting Chrome in background") + with LOCKED: + # Clear flag before restart (under lock to be safe) + with _dns_rotation_lock: + _dns_rotation_pending = False + _restart_chrome_only() + + restart_thread = threading.Thread(target=_async_restart, daemon=True) + restart_thread.start() + def _cleanup_driver(): - global LOCKED + """Reset driver after inactivity timeout. + + Uses a longer timeout (4x) when UI clients are connected to avoid + resetting while users are actively browsing. After all clients disconnect, + the standard timeout applies as a grace period before shutdown. + """ global LAST_USED + + # Check for active UI connections + try: + from websocket_manager import ws_manager + has_active_clients = ws_manager.has_active_connections() + except ImportError: + ws_manager = None + has_active_clients = False + + # Use longer timeout when UI is connected (user might be browsing) + timeout_minutes = env.BYPASS_RELEASE_INACTIVE_MIN + if has_active_clients: + timeout_minutes *= 4 # 20 min default when UI open vs 5 min after disconnect + with LOCKED: - if LAST_USED: - if time.time() - LAST_USED >= env.BYPASS_RELEASE_INACTIVE_MIN * 60: - _reset_driver() - LAST_USED = None - logger.info("Driver reset due to inactivity.") + if LAST_USED and time.time() - LAST_USED >= timeout_minutes * 60: + logger.info(f"Cloudflare bypasser idle for {timeout_minutes} min - shutting down to free resources") + _reset_driver() + LAST_USED = None + + # If clients are still connected, request warmup on next connect so the + # bypasser restarts when the user becomes active again + if has_active_clients and ws_manager: + ws_manager.request_warmup_on_next_connect() + logger.debug("Requested warmup on next client connect (clients still connected)") def _cleanup_loop(): while True: @@ -463,27 +858,146 @@ def _init_cleanup_thread(): cleanup_thread.daemon = True cleanup_thread.start() -def wait_for_result(func, timeout : int = 10, condition : any = True): - start_time = time.time() - while time.time() - start_time < timeout: - result = func() - if condition(result): - return result - time.sleep(0.5) - return None +def warmup(): + """Pre-initialize the virtual display and Chrome browser to eliminate cold start time. + + This function can be called when a user connects to the web UI to + warm up the bypasser environment before it's actually needed. + Both the display and Chrome driver are initialized so the first + bypass request is nearly instant. + + Warmup is skipped in the following scenarios: + - BYPASS_WARMUP_ON_CONNECT is false (explicit disable) + - Not running in Docker mode + - USE_CF_BYPASS is disabled + - AA_DONATOR_KEY is set (user has fast downloads, bypass rarely needed) + + Note: Even when warmup is skipped, the bypasser can still start on-demand + when actually needed for a download. + """ + global DRIVER, LAST_USED + + if not BYPASS_WARMUP_ON_CONNECT: + logger.debug("Bypasser warmup disabled via BYPASS_WARMUP_ON_CONNECT") + return + + if not env.DOCKERMODE: + logger.debug("Bypasser warmup skipped - not in Docker mode") + return + + if not env.USE_CF_BYPASS: + logger.debug("Bypasser warmup skipped - CF bypass disabled") + return + + if AA_DONATOR_KEY: + logger.debug("Bypasser warmup skipped - AA donator key set (fast downloads available)") + return + + with LOCKED: + if is_warmed_up(): + logger.debug("Bypasser already fully warmed up") + return + + logger.info("Warming up Cloudflare bypasser (pre-initializing display and browser)...") + + try: + # Initialize virtual display (FFmpeg recording starts later on first actual request) + _ensure_display_initialized() + + # Initialize Chrome driver + if DRIVER is None: + logger.info("Pre-initializing Chrome browser...") + _init_driver() + LAST_USED = time.time() + logger.info("Chrome browser ready") + + logger.info("Bypasser warmup complete - ready for instant bypass") + logger.log_resource_usage() + + except Exception as e: + logger.warning(f"Failed to warm up bypasser: {e}") + +def is_warmed_up() -> bool: + """Check if the bypasser is fully warmed up (display and browser initialized).""" + return DISPLAY["xvfb"] is not None and DRIVER is not None + +def shutdown_if_idle(): + """Start the inactivity countdown when all WebSocket clients disconnect. + + Instead of immediately shutting down, this sets LAST_USED to start the + inactivity timer. The cleanup loop will then shut down the bypasser after + BYPASS_RELEASE_INACTIVE_MIN minutes, giving users a grace period to return + (e.g., if they refresh the page or briefly navigate away). + + If there are active downloads, the timer naturally won't trigger until + they complete since LAST_USED gets updated on each bypass operation. + """ + global LAST_USED + + with LOCKED: + if not is_warmed_up(): + logger.debug("Bypasser already shut down") + return + + # Start the inactivity countdown + LAST_USED = time.time() + logger.info(f"All clients disconnected - bypasser will shut down after {env.BYPASS_RELEASE_INACTIVE_MIN} min of inactivity") + _init_cleanup_thread() +# Register for DNS rotation notifications so Chrome can restart with new DNS settings +# Only register if using the internal Chrome bypasser (not external FlareSolverr) +if env.USE_CF_BYPASS and not env.USING_EXTERNAL_BYPASSER: + network.register_dns_rotation_callback(_on_dns_rotation) -def get_bypassed_page(url: str) -> Optional[str]: + +def get_bypassed_page(url: str, selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None) -> Optional[str]: """Fetch HTML content from a URL using the internal Cloudflare Bypasser. Args: url: Target URL + selector: Optional mirror selector for AA URL rewriting + cancel_flag: Optional threading Event to signal cancellation + Returns: str: HTML content if successful, None otherwise - """ - response_html = get(url) + Raises: + BypassCancelledException: If cancel_flag is set during operation + """ + sel = selector or network.AAMirrorSelector() + attempt_url = sel.rewrite(url) + + # Before using Chrome, check if cookies are available (from a previous bypass) + # This helps concurrent downloads avoid unnecessary Chrome usage + parsed = urlparse(attempt_url) + cookies = get_cf_cookies_for_domain(parsed.hostname or "") + if cookies: + try: + logger.debug(f"Trying request with cached cookies before Chrome: {attempt_url}") + response = requests.get(attempt_url, cookies=cookies, proxies=PROXIES, timeout=(5, 10)) + if response.status_code == 200: + logger.debug(f"Cached cookies worked, skipped Chrome bypass") + return response.text + except Exception: + pass # Fall through to Chrome bypass + + try: + response_html = get(attempt_url, cancel_flag=cancel_flag) + except BypassCancelledException: + raise + except Exception: + # Check for cancellation before retry + if cancel_flag and cancel_flag.is_set(): + raise BypassCancelledException("Bypass cancelled") + # On failure, try mirror/DNS rotation for AA-like URLs + new_base, action = sel.next_mirror_or_rotate_dns() + if action in ("mirror", "dns") and new_base: + attempt_url = sel.rewrite(url) + response_html = get(attempt_url, cancel_flag=cancel_flag) + else: + raise + logger.debug(f"Cloudflare Bypasser response length: {len(response_html)}") if response_html.strip() != "": return response_html diff --git a/cloudflare_bypasser_external.py b/cloudflare_bypasser_external.py index 3b64d6a0..d53a8122 100644 --- a/cloudflare_bypasser_external.py +++ b/cloudflare_bypasser_external.py @@ -1,6 +1,17 @@ from logger import setup_logger -from typing import Optional +from threading import Event +from typing import Optional, TYPE_CHECKING import requests +import time +import random + +if TYPE_CHECKING: + import network + + +class BypassCancelledException(Exception): + """Raised when a bypass operation is cancelled.""" + pass try: from env import EXT_BYPASSER_PATH, EXT_BYPASSER_TIMEOUT, EXT_BYPASSER_URL @@ -9,26 +20,140 @@ except ImportError: logger = setup_logger(__name__) +# Connection timeout (seconds) - how long to wait for external bypasser to accept connection +CONNECT_TIMEOUT = 10 +# Maximum read timeout cap (seconds) - hard limit regardless of EXT_BYPASSER_TIMEOUT +MAX_READ_TIMEOUT = 120 +# Buffer added to bypasser's configured timeout (seconds) - accounts for processing overhead +READ_TIMEOUT_BUFFER = 15 +# Retry settings for bypasser failures +MAX_RETRY = 5 +BACKOFF_BASE = 1.0 +BACKOFF_CAP = 10.0 -def get_bypassed_page(url: str) -> Optional[str]: - """Fetch HTML content from a URL using an External Cloudflare Resolver. +def _fetch_via_bypasser(target_url: str) -> Optional[str]: + """Make a single request to the external bypasser service. + Args: - url: Target URL + target_url: The URL to fetch through the bypasser + Returns: - str: HTML content if successful, None otherwise + HTML content if successful, None otherwise """ if not EXT_BYPASSER_URL or not EXT_BYPASSER_PATH: - logger.error("Wrong External Bypass configuration. Please check your environment configuration.") + logger.error("External bypasser not configured. Check EXT_BYPASSER_URL and EXT_BYPASSER_PATH.") return None - ext_url = f"{EXT_BYPASSER_URL}{EXT_BYPASSER_PATH}" + + bypasser_endpoint = f"{EXT_BYPASSER_URL}{EXT_BYPASSER_PATH}" headers = {"Content-Type": "application/json"} - data = { + payload = { "cmd": "request.get", - "url": url, + "url": target_url, "maxTimeout": EXT_BYPASSER_TIMEOUT } - response = requests.post(ext_url, headers=headers, json=data) - response.raise_for_status() - logger.debug(f"External Bypass response for '{url}': {response.json()['status']} - {response.json()['message']}") - return response.json()['solution']['response'] + + # Calculate read timeout: bypasser timeout (ms → s) + buffer, capped at max + read_timeout = min((EXT_BYPASSER_TIMEOUT / 1000) + READ_TIMEOUT_BUFFER, MAX_READ_TIMEOUT) + + try: + response = requests.post( + bypasser_endpoint, + headers=headers, + json=payload, + timeout=(CONNECT_TIMEOUT, read_timeout) + ) + response.raise_for_status() + result = response.json() + + status = result.get('status', 'unknown') + message = result.get('message', '') + logger.debug(f"External bypasser response for '{target_url}': {status} - {message}") + + # Check for error status (bypasser returns status="error" with solution=null on failure) + if status != 'ok': + logger.warning(f"External bypasser failed for '{target_url}': {status} - {message}") + return None + + solution = result.get('solution') + if not solution: + logger.warning(f"External bypasser returned empty solution for '{target_url}'") + return None + + html = solution.get('response', '') + if not html: + logger.warning(f"External bypasser returned empty response for '{target_url}'") + return None + + return html + + except requests.exceptions.Timeout: + logger.warning(f"External bypasser timed out for '{target_url}' (connect: {CONNECT_TIMEOUT}s, read: {read_timeout:.0f}s)") + return None + except requests.exceptions.RequestException as e: + logger.warning(f"External bypasser request failed for '{target_url}': {e}") + return None + except (KeyError, TypeError, ValueError) as e: + logger.warning(f"External bypasser returned malformed response for '{target_url}': {e}") + return None + + +def get_bypassed_page(url: str, selector: Optional["network.AAMirrorSelector"] = None, cancel_flag: Optional[Event] = None) -> Optional[str]: + """Fetch HTML content from a URL using an external Cloudflare bypasser service. + + Retries with exponential backoff and mirror/DNS rotation on failure. + + Args: + url: Target URL to fetch + selector: Mirror selector for AA URL rewriting and rotation + cancel_flag: Optional threading Event to signal cancellation + + Returns: + HTML content if successful, None otherwise + + Raises: + BypassCancelledException: If cancel_flag is set during operation + """ + import network + sel = selector or network.AAMirrorSelector() + + for attempt in range(1, MAX_RETRY + 1): + # Check for cancellation before each attempt + if cancel_flag and cancel_flag.is_set(): + logger.info("External bypasser cancelled by user") + raise BypassCancelledException("Bypass cancelled") + + attempt_url = sel.rewrite(url) + result = _fetch_via_bypasser(attempt_url) + if result: + return result + + if attempt == MAX_RETRY: + break + + # Check for cancellation before backoff wait + if cancel_flag and cancel_flag.is_set(): + logger.info("External bypasser cancelled during retry") + raise BypassCancelledException("Bypass cancelled") + + # Backoff with jitter before retry, checking cancellation during wait + delay = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** (attempt - 1))) + random.random() + logger.info(f"External bypasser attempt {attempt}/{MAX_RETRY} failed, retrying in {delay:.1f}s") + + # Check cancellation during delay (check every second) + for _ in range(int(delay)): + if cancel_flag and cancel_flag.is_set(): + logger.info("External bypasser cancelled during backoff") + raise BypassCancelledException("Bypass cancelled") + time.sleep(1) + # Sleep remaining fraction + remaining = delay - int(delay) + if remaining > 0: + time.sleep(remaining) + + # Rotate mirror/DNS for next attempt + new_base, action = sel.next_mirror_or_rotate_dns() + if action in ("mirror", "dns") and new_base: + logger.info(f"Rotated {action} for retry") + + return None diff --git a/config.py b/config.py index e08b9ff3..9b2d99f7 100644 --- a/config.py +++ b/config.py @@ -35,22 +35,34 @@ logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}") # Network settings _custom_dns = env._CUSTOM_DNS.lower().strip() _doh_server = "" -if _custom_dns == "google": + +if _custom_dns == "auto" or _custom_dns == "": + # Auto mode - DNS provider rotation handled by network.py + # Starts with system DNS, switches to providers from DNS_PROVIDERS on failure + CUSTOM_DNS = [] + _doh_server = "" + logger.info("CUSTOM_DNS: auto (starts with system DNS, rotates on failure)") +elif _custom_dns == "google": CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860:0000:0000:0000:0000:8888", "2001:4860:4860:0000:0000:0000:0000:8844"] - _doh_server = "https://dns.google/dns-query" + _doh_server = "https://dns.google/resolve" + logger.info(f"CUSTOM_DNS: google {CUSTOM_DNS}") elif _custom_dns == "quad9": CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:00fe:0000:0000:0000:0000:0000:00fe", "2620:00fe:0000:0000:0000:0000:0000:0009"] _doh_server = "https://dns.quad9.net/dns-query" + logger.info(f"CUSTOM_DNS: quad9 {CUSTOM_DNS}") elif _custom_dns == "cloudflare": CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700:0000:0000:0000:0000:1111", "2606:4700:4700:0000:0000:0000:0000:1001"] _doh_server = "https://cloudflare-dns.com/dns-query" + logger.info(f"CUSTOM_DNS: cloudflare {CUSTOM_DNS}") elif _custom_dns == "opendns": CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:0119:0035:0000:0000:0000:0000:0035", "2620:0119:0053:0000:0000:0000:0000:0053"] _doh_server = "https://doh.opendns.com/dns-query" + logger.info(f"CUSTOM_DNS: opendns {CUSTOM_DNS}") else: + # Custom DNS IPs provided by user _custom_dns_ip = _custom_dns.split(",") CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()] -logger.info(f"CUSTOM_DNS: {CUSTOM_DNS}") + logger.info(f"CUSTOM_DNS: custom {CUSTOM_DNS}") DOH_SERVER = _doh_server if env.USE_DOH: DOH_SERVER = _doh_server @@ -58,6 +70,15 @@ else: DOH_SERVER = "" logger.info(f"DOH_SERVER: {DOH_SERVER}") +# Warn about external bypasser DNS limitations +if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS: + logger.warning( + "Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, " + "not this application's custom DNS settings. If you experience DNS-related blocks, " + "configure DNS at the Docker/system level for your FlareSolverr container, " + "or consider using the internal bypasser which integrates with the app's DNS system." + ) + # Proxy settings PROXIES = {} if env.HTTP_PROXY: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3cb759ce..ffffc885 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -9,8 +9,6 @@ services: target: cwa-bd environment: DEBUG: true - USE_DOH: true - CUSTOM_DNS: cloudflare volumes: - /tmp/cwa-book-downloader:/tmp/cwa-book-downloader - /tmp/cwa-book-downloader-log:/var/log/cwa-book-downloader diff --git a/downloader.py b/downloader.py index dfb3ce0a..3e336e96 100644 --- a/downloader.py +++ b/downloader.py @@ -1,138 +1,357 @@ """Network operations manager for the book downloader application.""" -import network -network.init() -import requests +import random import time from io import BytesIO -from typing import Optional -from urllib.parse import urlparse -from tqdm import tqdm -from typing import Callable from threading import Event -from logger import setup_logger +from typing import Callable, Optional +from urllib.parse import urlparse + +import requests +from tqdm import tqdm + +import network from config import PROXIES -from env import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS, USING_EXTERNAL_BYPASSER +from env import DEFAULT_SLEEP, MAX_RETRY, USE_CF_BYPASS, USING_EXTERNAL_BYPASSER +from logger import setup_logger + +# Import bypasser if enabled if USE_CF_BYPASS: if USING_EXTERNAL_BYPASSER: from cloudflare_bypasser_external import get_bypassed_page + # External bypasser doesn't share cookies + get_cf_cookies_for_domain = lambda domain: {} else: - from cloudflare_bypasser import get_bypassed_page + from cloudflare_bypasser import get_bypassed_page, get_cf_cookies_for_domain logger = setup_logger(__name__) +# Network settings +REQUEST_TIMEOUT = (5, 10) # (connect, read) +MAX_DOWNLOAD_RETRIES = 2 +MAX_RESUME_ATTEMPTS = 3 +RETRYABLE_CODES = (429, 500, 502, 503, 504) +CONNECTION_ERRORS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout, + requests.exceptions.SSLError, requests.exceptions.ChunkedEncodingError) +DOWNLOAD_HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', +} -def html_get_page(url: str, retry: int = MAX_RETRY, use_bypasser: bool = False, status_callback: Optional[Callable[[str], None]] = None) -> str: - """Fetch HTML content from a URL with retry mechanism. - - Args: - url: Target URL - retry: Number of retry attempts - use_bypasser: Whether to use Cloudflare bypasser - status_callback: Optional callback for status updates - - Returns: - str: HTML content if successful, None otherwise - """ - response = None + +def parse_size_string(size: str) -> Optional[float]: + """Parse a human-readable size string (e.g., '10.5 MB') into bytes.""" + if not size: + return None try: - logger.debug(f"html_get_page: {url}, retry: {retry}, use_bypasser: {use_bypasser}") - if use_bypasser and USE_CF_BYPASS: - if status_callback: - status_callback("bypassing") - logger.info(f"GET Using Cloudflare Bypasser for: {url}") - return get_bypassed_page(url) - else: - logger.info(f"GET: {url}") - response = requests.get(url, proxies=PROXIES) - response.raise_for_status() - logger.debug(f"Success getting: {url}") - time.sleep(1) - return str(response.text) - - except Exception as e: - if retry == 0: - logger.error_trace(f"Failed to fetch page: {url}, error: {e}") - return "" - - if use_bypasser and USE_CF_BYPASS: - logger.warning(f"Exception while using cloudflare bypass for URL: {url}") - logger.warning(f"Exception: {e}") - logger.warning(f"Response: {response}") - elif response is not None and response.status_code == 404: - logger.warning(f"404 error for URL: {url}") - return "" - elif response is not None and response.status_code == 403: - logger.warning(f"403 detected for URL: {url}. Should retry using cloudflare bypass.") - return html_get_page(url, retry - 1, True, status_callback) - - sleep_time = DEFAULT_SLEEP * (MAX_RETRY - retry + 1) - logger.warning( - f"Retrying GET {url} in {sleep_time} seconds due to error: {e}" - ) - time.sleep(sleep_time) - return html_get_page(url, retry - 1, use_bypasser, status_callback) - -def download_url(link: str, size: str = "", progress_callback: Optional[Callable[[float], None]] = None, cancel_flag: Optional[Event] = None) -> Optional[BytesIO]: - """Download content from URL into a BytesIO buffer. - - Args: - link: URL to download from - - Returns: - BytesIO: Buffer containing downloaded content if successful - """ - try: - logger.info(f"Downloading from: {link}") - response = requests.get(link, stream=True, proxies=PROXIES) - response.raise_for_status() - - total_size : float = 0.0 - try: - # we assume size is in MB - total_size = float(size.strip().replace(" ", "").replace(",", ".").upper()[:-2].strip()) * 1024 * 1024 - except: - total_size = float(response.headers.get('content-length', 0)) - - buffer = BytesIO() - - # Initialize the progress bar with your guess - pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading') - for chunk in response.iter_content(chunk_size=1000): - buffer.write(chunk) - pbar.update(len(chunk)) - if progress_callback is not None: - progress_callback(pbar.n * 100.0 / total_size) - if cancel_flag is not None and cancel_flag.is_set(): - logger.info(f"Download cancelled: {link}") - return None - - pbar.close() - if buffer.tell() * 0.1 < total_size * 0.9: - # Check the content of the buffer if its HTML or binary - if response.headers.get('content-type', '').startswith('text/html'): - logger.warn(f"Failed to download content for {link}. Found HTML content instead.") - return None - return buffer - except requests.exceptions.RequestException as e: - logger.error_trace(f"Failed to download from {link}: {e}") + normalized = size.strip().replace(" ", "").replace(",", ".").upper() + multipliers = {"GB": 1024**3, "MB": 1024**2, "KB": 1024} + for suffix, mult in multipliers.items(): + if normalized.endswith(suffix): + return float(normalized[:-2]) * mult + return float(normalized) + except (ValueError, IndexError): return None -def get_absolute_url(base_url: str, url: str) -> str: - """Get absolute URL from relative URL and base URL. +def _backoff_delay(attempt: int, base: float = 0.25, cap: float = 3.0) -> float: + """Exponential backoff with jitter.""" + return min(cap, base * (2 ** (attempt - 1))) + random.random() * base + + +def _get_status_code(e: Exception) -> Optional[int]: + """Extract HTTP status code from an exception, or None if not applicable.""" + if isinstance(e, requests.exceptions.HTTPError) and e.response is not None: + return e.response.status_code + return None + +def _is_retryable_error(e: Exception) -> bool: + """Check if error is retryable (connection error or retryable HTTP status).""" + if isinstance(e, CONNECTION_ERRORS): + return True + status = _get_status_code(e) + return status in RETRYABLE_CODES if status else False + + +def _try_rotation(original_url: str, current_url: str, selector: network.AAMirrorSelector) -> Optional[str]: + """Try mirror/DNS rotation. Returns new URL or None.""" + if current_url.startswith(network.get_aa_base_url()): + new_base, action = selector.next_mirror_or_rotate_dns() + if action in ("mirror", "dns") and new_base: + new_url = selector.rewrite(original_url) + logger.info(f"[{action}] switching to: {new_url}") + return new_url + elif network.should_rotate_dns_for_url(current_url) and network.rotate_dns_provider(): + logger.info(f"[dns-rotate] retrying: {original_url}") + return original_url + return None + + +def html_get_page( + url: str, + retry: int = MAX_RETRY, + use_bypasser: bool = False, + selector: Optional[network.AAMirrorSelector] = None, + cancel_flag: Optional[Event] = None, +) -> str: + """Fetch HTML content from a URL with retry mechanism.""" + selector = selector or network.AAMirrorSelector() + original_url = url + current_url = selector.rewrite(original_url) + use_bypasser_now = use_bypasser + + for attempt in range(1, retry + 1): + # Check for cancellation before each attempt + if cancel_flag and cancel_flag.is_set(): + logger.info(f"html_get_page cancelled before attempt {attempt}") + return "" + + try: + if use_bypasser_now and USE_CF_BYPASS: + logger.info(f"GET (bypasser): {current_url}") + try: + result = get_bypassed_page(current_url, selector, cancel_flag) + return result or "" + except Exception as e: + logger.warning(f"Bypasser error: {type(e).__name__}: {e}") + return "" + + logger.info(f"GET: {current_url}") + # Try with CF cookies if available (from previous bypass) + cookies = {} + if USE_CF_BYPASS: + parsed = urlparse(current_url) + cookies = get_cf_cookies_for_domain(parsed.hostname or "") + response = requests.get(current_url, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies) + response.raise_for_status() + time.sleep(1) + return response.text + + except Exception as e: + status = _get_status_code(e) + + # 403 = Cloudflare/DDoS-Guard protection + if status == 403: + if USE_CF_BYPASS and not use_bypasser_now: + # Before switching to bypasser, check if cookies have become available + # (another concurrent download may have completed bypass and extracted cookies) + parsed = urlparse(current_url) + fresh_cookies = get_cf_cookies_for_domain(parsed.hostname or "") + if fresh_cookies and not cookies: + # Cookies are now available - retry with cookies before using bypasser + logger.debug(f"403 but cookies now available - retrying with cookies: {current_url}") + continue + logger.info(f"403 detected; switching to bypasser: {current_url}") + use_bypasser_now = True + continue + logger.warning(f"403 error, giving up: {current_url}") + return "" + + # 404 = Not found + if status == 404: + logger.warning(f"404 error: {current_url}") + return "" + + # Try mirror/DNS rotation on retryable errors + if _is_retryable_error(e): + new_url = _try_rotation(original_url, current_url, selector) + if new_url: + current_url = new_url + continue + + # Retry with backoff + if attempt < retry: + logger.warning(f"Retry {attempt}/{retry} for {current_url}: {type(e).__name__}: {e}") + time.sleep(_backoff_delay(attempt)) + else: + logger.error(f"Giving up after {retry} attempts: {current_url}") + + return "" + + +def download_url( + link: str, + size: str = "", + progress_callback: Optional[Callable[[float], None]] = None, + cancel_flag: Optional[Event] = None, + _selector: Optional[network.AAMirrorSelector] = None, + status_callback: Optional[Callable[[str, Optional[str]], None]] = None, + referer: Optional[str] = None, +) -> Optional[BytesIO]: + """Download content from URL with automatic retry and resume support.""" + selector = _selector or network.AAMirrorSelector() + current_url = selector.rewrite(link) + + # Build headers with optional referer + headers = DOWNLOAD_HEADERS.copy() + if referer: + headers['Referer'] = referer + total_size = parse_size_string(size) or 0 + + attempt = 0 + + while attempt < MAX_DOWNLOAD_RETRIES: + if cancel_flag and cancel_flag.is_set(): + return None + + buffer = BytesIO() + bytes_downloaded = 0 + + try: + if attempt > 0 and status_callback: + status_callback("resolving", f"Connecting (Attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})") + + logger.info(f"Downloading: {current_url} (attempt {attempt + 1}/{MAX_DOWNLOAD_RETRIES})") + # Try with CF cookies if available + cookies = {} + if USE_CF_BYPASS: + parsed = urlparse(current_url) + cookies = get_cf_cookies_for_domain(parsed.hostname or "") + if cookies: + logger.debug(f"Using {len(cookies)} cookies for {parsed.hostname}: {list(cookies.keys())}") + response = requests.get(current_url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers) + response.raise_for_status() + + if status_callback: + status_callback("downloading", "") + + total_size = total_size or float(response.headers.get('content-length', 0)) + pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading') + + for chunk in response.iter_content(chunk_size=8192): + if chunk: + buffer.write(chunk) + bytes_downloaded += len(chunk) + pbar.update(len(chunk)) + if progress_callback and total_size > 0: + progress_callback(bytes_downloaded * 100.0 / total_size) + if cancel_flag and cancel_flag.is_set(): + pbar.close() + return None + pbar.close() + + # Validate - check we didn't get HTML instead of file + if total_size > 0 and bytes_downloaded < total_size * 0.9: + if response.headers.get('content-type', '').startswith('text/html'): + logger.warning(f"Received HTML instead of file: {current_url}") + return None + + logger.debug(f"Download completed: {bytes_downloaded} bytes") + return buffer + + except requests.exceptions.RequestException as e: + status = _get_status_code(e) + retryable = _is_retryable_error(e) + + # Non-retryable errors + if status in (403, 404): + logger.warning(f"Download failed ({status}): {current_url}") + return None + + # Rate limited - skip to next source immediately + # (waiting doesn't help with concurrent downloads hitting the same server) + if status == 429: + logger.info(f"Rate limited (429) - trying next source") + if status_callback: + status_callback("resolving", "Server busy, trying next...") + return None + + # Timeout - don't retry, server likely overloaded + if isinstance(e, requests.exceptions.Timeout): + logger.warning(f"Timeout: {current_url} - skipping to next source") + if status_callback: + status_callback("resolving", "Server timed out, trying next...") + return None + + # Try to resume if we got some data + if bytes_downloaded > 0 and retryable: + resumed = _try_resume(current_url, buffer, bytes_downloaded, total_size, progress_callback, cancel_flag, headers) + if resumed: + return resumed + + # Try mirror/DNS rotation if nothing downloaded yet + if bytes_downloaded == 0 and retryable: + new_url = _try_rotation(link, current_url, selector) + if new_url: + current_url = new_url + attempt += 1 + continue + + logger.warning(f"Download error: {type(e).__name__}: {e}") + if attempt < MAX_DOWNLOAD_RETRIES - 1: + time.sleep(_backoff_delay(attempt + 1)) + attempt += 1 + + logger.error(f"Download failed after {MAX_DOWNLOAD_RETRIES} attempts: {link}") + return None + + +def _try_resume( + url: str, + buffer: BytesIO, + start_byte: int, + total_size: float, + progress_callback: Optional[Callable[[float], None]], + cancel_flag: Optional[Event], + base_headers: Optional[dict] = None, +) -> Optional[BytesIO]: + """Try to resume an interrupted download.""" + for attempt in range(MAX_RESUME_ATTEMPTS): + logger.info(f"Resuming from {start_byte} bytes (attempt {attempt + 1}/{MAX_RESUME_ATTEMPTS})") + time.sleep(_backoff_delay(attempt + 1, base=0.5, cap=5.0)) + + try: + # Try with CF cookies if available + cookies = {} + if USE_CF_BYPASS: + parsed = urlparse(url) + cookies = get_cf_cookies_for_domain(parsed.hostname or "") + resume_headers = {**(base_headers or DOWNLOAD_HEADERS), 'Range': f'bytes={start_byte}-'} + response = requests.get( + url, stream=True, proxies=PROXIES, timeout=REQUEST_TIMEOUT, + headers=resume_headers, cookies=cookies + ) + + # Check resume support + if response.status_code == 200: # Server doesn't support resume + logger.info("Server doesn't support resume") + return None + if response.status_code == 416: # Range not satisfiable + logger.warning("Range not satisfiable") + return None + if response.status_code != 206: + response.raise_for_status() + + pbar = tqdm(total=total_size, initial=start_byte, unit='B', unit_scale=True, desc='Resuming') + for chunk in response.iter_content(chunk_size=8192): + if chunk: + buffer.write(chunk) + start_byte += len(chunk) + pbar.update(len(chunk)) + if progress_callback and total_size > 0: + progress_callback(start_byte * 100.0 / total_size) + if cancel_flag and cancel_flag.is_set(): + pbar.close() + return None + pbar.close() + + logger.info(f"Resume completed: {start_byte} bytes") + return buffer + + except requests.exceptions.RequestException as e: + logger.debug(f"Resume attempt {attempt + 1} failed: {e}") - Args: - base_url: Base URL - url: Relative URL - """ - if url.strip() == "": - return "" - if url.strip("#") == "": - return "" - if url.startswith("http"): - return url - parsed_url = urlparse(url) - parsed_base = urlparse(base_url) - if parsed_url.netloc == "" or parsed_url.scheme == "": - parsed_url = parsed_url._replace(netloc=parsed_base.netloc, scheme=parsed_base.scheme) - return parsed_url.geturl() + logger.warning(f"Resume failed after {MAX_RESUME_ATTEMPTS} attempts") + return None + + +def get_absolute_url(base_url: str, url: str) -> str: + """Convert a relative URL to absolute using the base URL.""" + url = url.strip() + if not url or url == "#" or url.startswith("http"): + return url if url.startswith("http") else "" + parsed = urlparse(url) + base = urlparse(base_url) + if not parsed.netloc or not parsed.scheme: + parsed = parsed._replace(netloc=base.netloc, scheme=base.scheme) + return parsed.geturl() diff --git a/env.py b/env.py index a920d226..a07d6178 100644 --- a/env.py +++ b/env.py @@ -50,6 +50,10 @@ _CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip() FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0") FLASK_PORT = int(os.getenv("FLASK_PORT", "8084")) DEBUG = string_to_bool(os.getenv("DEBUG", "false")) +# Debug: skip specific download sources for testing fallback chains +# Comma-separated values: aa-fast, aa-slow-nowait, aa-slow-wait, libgen, zlib, welib +_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower() +DEBUG_SKIP_SOURCES = set(s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip()) PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false")) ALLOW_USE_WELIB = string_to_bool(os.getenv("ALLOW_USE_WELIB", "true")) @@ -65,11 +69,12 @@ else: ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true")) MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5")) MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3")) -DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "5")) +DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "1")) DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false")) -_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "").strip() +_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "auto").strip() USE_DOH = string_to_bool(os.getenv("USE_DOH", "false")) BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5")) +BYPASS_WARMUP_ON_CONNECT = string_to_bool(os.getenv("BYPASS_WARMUP_ON_CONNECT", "true")) # Logging settings LOG_FILE = LOG_DIR / "cwa-book-downloader.log" diff --git a/genDebug.sh b/genDebug.sh index 8b78e5dd..d8108732 100755 --- a/genDebug.sh +++ b/genDebug.sh @@ -124,15 +124,15 @@ fi # Add environment variables (redacting sensitive info) env | grep -v -E "(AA_DONATOR_KEY)" | sort > "$LOG_DIR/environment.txt" -echo "--- HTTPBin ---" > $LOG_DIR/network_info.txt +echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt -ehco "" +echo "" >> $LOG_DIR/network_info.txt echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt -ehco "" +echo "" >> $LOG_DIR/network_info.txt echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt -ehco "" +echo "" >> $LOG_DIR/network_info.txt echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt diff --git a/logger.py b/logger.py index 771deccd..440f30fe 100644 --- a/logger.py +++ b/logger.py @@ -23,16 +23,18 @@ class CustomLogger(logging.Logger): self.warning(msg, *args, exc_info=True, **kwargs) def info_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None: - """Log an info message with full stack trace.""" - self.log_resource_usage() + """Log an info message (stack trace only if exception active).""" kwargs.pop('exc_info', None) - self.info(msg, *args, exc_info=True, **kwargs) - + # Only include exc_info if there's actually an exception + has_exception = sys.exc_info()[0] is not None + self.info(msg, *args, exc_info=has_exception, **kwargs) + def debug_trace(self, msg: Any, *args: Any, **kwargs: Any) -> None: - """Log a debug message with full stack trace.""" - self.log_resource_usage() + """Log a debug message (stack trace only if exception active).""" kwargs.pop('exc_info', None) - self.debug(msg, *args, exc_info=True, **kwargs) + # Only include exc_info if there's actually an exception + has_exception = sys.exc_info()[0] is not None + self.debug(msg, *args, exc_info=has_exception, **kwargs) def log_resource_usage(self): import psutil diff --git a/models.py b/models.py index c327c235..bae92eb6 100644 --- a/models.py +++ b/models.py @@ -7,6 +7,7 @@ from datetime import datetime, timedelta from threading import Lock, Event from pathlib import Path import queue +import re import time from env import INGEST_DIR, STATUS_TIMEOUT @@ -14,10 +15,7 @@ class QueueStatus(str, Enum): """Enum for possible book queue statuses.""" QUEUED = "queued" RESOLVING = "resolving" - BYPASSING = "bypassing" DOWNLOADING = "downloading" - VERIFYING = "verifying" - INGESTING = "ingesting" COMPLETE = "complete" AVAILABLE = "available" ERROR = "error" @@ -56,6 +54,45 @@ class BookInfo: download_path: Optional[str] = None priority: int = 0 progress: Optional[float] = None + status_message: Optional[str] = None # Detailed status message for UI display + added_time: Optional[float] = None # Timestamp when added to queue + + def get_filename(self, fallback_url: Optional[str] = None) -> str: + """Build sanitized filename: 'Author - Title (Year).format' + + Resolves format from self.format, download_urls, or fallback_url. + + Args: + fallback_url: URL to extract format from if not already known + + Returns: + Sanitized filename safe for filesystem use + """ + # Resolve format if needed + if not self.format: + for url in (self.download_urls[0] if self.download_urls else None, fallback_url): + if url: + ext = url.split(".")[-1].lower() + if ext and len(ext) <= 5 and ext.isalnum(): + self.format = ext + break + + # Build filename + parts = [] + if self.author: + parts.append(self.author) + parts.append(" - ") + parts.append(self.title) + if self.year: + parts.append(f" ({self.year})") + + filename = "".join(parts) + filename = re.sub(r'[\\/:*?"<>|]', '_', filename.strip())[:245] + + if self.format: + filename = f"{filename}.{self.format}" + + return filename class BookQueue: """Thread-safe book queue manager with priority support and cancellation.""" @@ -81,36 +118,40 @@ class BookQueue: # Don't add if already exists and not in error/done state if book_id in self._status and self._status[book_id] not in [QueueStatus.ERROR, QueueStatus.DONE, QueueStatus.CANCELLED]: return - + + added_time = time.time() book_data.priority = priority - queue_item = QueueItem(book_id, priority, time.time()) + book_data.added_time = added_time + queue_item = QueueItem(book_id, priority, added_time) self._queue.put(queue_item) self._book_data[book_id] = book_data self._update_status(book_id, QueueStatus.QUEUED) def get_next(self) -> Optional[Tuple[str, Event]]: """Get next book ID from queue with cancellation flag. - + Returns: Tuple of (book_id, cancel_flag) or None if queue is empty """ - try: - queue_item = self._queue.get_nowait() - book_id = queue_item.book_id - - with self._lock: - # Check if book was cancelled while in queue - if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED: - return self.get_next() # Recursively get next non-cancelled item - - # Create cancellation flag for this download - cancel_flag = Event() - self._cancel_flags[book_id] = cancel_flag - self._active_downloads[book_id] = True - - return book_id, cancel_flag - except queue.Empty: - return None + # Use iterative approach to avoid stack overflow if many items are cancelled + while True: + try: + queue_item = self._queue.get_nowait() + book_id = queue_item.book_id + + with self._lock: + # Check if book was cancelled while in queue + if book_id in self._status and self._status[book_id] == QueueStatus.CANCELLED: + continue # Skip cancelled items, try next + + # Create cancellation flag for this download + cancel_flag = Event() + self._cancel_flags[book_id] = cancel_flag + self._active_downloads[book_id] = True + + return book_id, cancel_flag + except queue.Empty: + return None def _update_status(self, book_id: str, status: QueueStatus) -> None: """Internal method to update status and timestamp.""" @@ -138,6 +179,12 @@ class BookQueue: with self._lock: if book_id in self._book_data: self._book_data[book_id].progress = progress + + def update_status_message(self, book_id: str, message: str) -> None: + """Update detailed status message for a book.""" + with self._lock: + if book_id in self._book_data: + self._book_data[book_id].status_message = message def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]: """Get current queue status.""" @@ -180,19 +227,19 @@ class BookQueue: return sorted(queue_items, key=lambda x: (x['priority'], x['added_time'])) def cancel_download(self, book_id: str) -> bool: - """Cancel a download and mark it as cancelled. - + """Cancel a download or clear a completed/errored item. + Args: - book_id: Book identifier to cancel - + book_id: Book identifier to cancel or clear + Returns: - bool: True if cancellation was successful + bool: True if cancellation/clearing was successful """ with self._lock: current_status = self._status.get(book_id) - + # Allow cancellation during any active state - if current_status in [QueueStatus.RESOLVING, QueueStatus.BYPASSING, QueueStatus.DOWNLOADING, QueueStatus.VERIFYING, QueueStatus.INGESTING]: + if current_status in [QueueStatus.RESOLVING, QueueStatus.DOWNLOADING]: # Signal active download to stop if book_id in self._cancel_flags: self._cancel_flags[book_id].set() @@ -202,7 +249,15 @@ class BookQueue: # Remove from queue and mark as cancelled self._update_status(book_id, QueueStatus.CANCELLED) return True - + elif current_status in [QueueStatus.COMPLETE, QueueStatus.DONE, QueueStatus.AVAILABLE, QueueStatus.ERROR, QueueStatus.CANCELLED]: + # Clear completed/errored/cancelled items from tracking + self._status.pop(book_id, None) + self._status_timestamps.pop(book_id, None) + self._book_data.pop(book_id, None) + self._cancel_flags.pop(book_id, None) + self._active_downloads.pop(book_id, None) + return True + return False def set_priority(self, book_id: str, new_priority: int) -> bool: @@ -281,6 +336,27 @@ class BookQueue: """Get list of currently active download book IDs.""" with self._lock: return list(self._active_downloads.keys()) + + def has_pending_work(self) -> bool: + """Check if there are any active downloads or queued items. + + This is useful for determining if the bypasser should stay active + even when the UI is closed. + + Returns: + bool: True if there are active downloads or queued items + """ + with self._lock: + # Check for active downloads + if self._active_downloads: + return True + + # Check for queued items (excluding cancelled ones) + for book_id, status in self._status.items(): + if status == QueueStatus.QUEUED: + return True + + return False def clear_completed(self) -> int: """Remove all completed, errored, or cancelled books from tracking. diff --git a/network.py b/network.py index cfcb97dd..44f01c63 100644 --- a/network.py +++ b/network.py @@ -13,9 +13,147 @@ import ipaddress from logger import setup_logger from config import PROXIES, AA_BASE_URL, CUSTOM_DNS, AA_AVAILABLE_URLS, DOH_SERVER import config +import env +from datetime import datetime, timedelta + +# Try to use gevent locks if available (for gevent worker compatibility) +# Fall back to threading locks for non-gevent environments +try: + from gevent.lock import RLock as _RLock + _using_gevent_locks = True +except ImportError: + from threading import RLock as _RLock + _using_gevent_locks = False logger = setup_logger(__name__) +# In-memory state (no disk persistence) +STATE_TTL_DAYS = 30 +_initialized = False +_dns_initialized = False +_aa_initialized = False +state: dict[str, Any] = {} + +# Locks for greenlet-safe initialization and DNS switching +# Use RLock (reentrant lock) since init() calls init_dns() and init_aa() +_init_lock = _RLock() +_dns_switch_lock = _RLock() + +# DNS rotation callbacks - called when DNS provider switches in auto mode +# Callbacks receive (provider_name: str, servers: List[str], doh_url: str) +_dns_rotation_callbacks: List[Callable[[str, List[str], str], None]] = [] +_dns_callback_lock = _RLock() + + +def register_dns_rotation_callback(callback: Callable[[str, List[str], str], None]) -> None: + """Register a callback to be called when DNS provider rotates. + + The callback receives (provider_name, servers, doh_url) as arguments. + Use this to restart components that cache DNS resolution (e.g., Chrome). + """ + with _dns_callback_lock: + if callback not in _dns_rotation_callbacks: + _dns_rotation_callbacks.append(callback) + logger.debug(f"Registered DNS rotation callback: {callback.__name__}") + + +def unregister_dns_rotation_callback(callback: Callable[[str, List[str], str], None]) -> None: + """Unregister a previously registered DNS rotation callback.""" + with _dns_callback_lock: + if callback in _dns_rotation_callbacks: + _dns_rotation_callbacks.remove(callback) + logger.debug(f"Unregistered DNS rotation callback: {callback.__name__}") + + +def _notify_dns_rotation(provider_name: str, servers: List[str], doh_url: str) -> None: + """Notify all registered callbacks about DNS rotation.""" + with _dns_callback_lock: + callbacks = _dns_rotation_callbacks.copy() + + for callback in callbacks: + try: + logger.debug(f"Calling DNS rotation callback: {callback.__name__}") + callback(provider_name, servers, doh_url) + except Exception as e: + logger.warning(f"DNS rotation callback {callback.__name__} failed: {e}") + +def _agent_debug_log(code: str, source: str, reason: str, meta: Optional[dict] = None) -> None: + """Lightweight debug hook for automated runs; safe no-op on failure.""" + try: + logger.debug(f"[agent] code={code} source={source} reason={reason} meta={meta or {}}") + except Exception as exc: + # Avoid raising inside debug logger + logger.debug(f"[agent] log failed: {exc}") + +def _load_state(): + """Return current in-memory network state (no disk persistence).""" + if state.get('chosen_at'): + chosen = datetime.fromisoformat(state['chosen_at']) + if datetime.now() - chosen > timedelta(days=STATE_TTL_DAYS): + state.clear() + return state + +def _save_state(aa_url=None, dns_provider=None): + """Update in-memory network state (no disk persistence).""" + if aa_url: + state['aa_base_url'] = aa_url + if dns_provider: + state['dns_provider'] = dns_provider + state['chosen_at'] = datetime.now().isoformat() + +# AA URL failover state +_current_aa_url_index = 0 +_aa_urls = AA_AVAILABLE_URLS.copy() + +def _ensure_initialized() -> None: + """Lazy guard so runtime setup happens once and late calls still work.""" + global _initialized + if _initialized: + return + with _init_lock: + # Double-check after acquiring lock + if not _initialized: + init() + +# DNS provider definitions: (name, servers, doh_url) +# Note: Google uses /resolve endpoint for JSON API, others use /dns-query +DNS_PROVIDERS = [ + ("cloudflare", ["1.1.1.1", "1.0.0.1"], "https://cloudflare-dns.com/dns-query"), + ("google", ["8.8.8.8", "8.8.4.4"], "https://dns.google/resolve"), + ("quad9", ["9.9.9.9", "149.112.112.112"], "https://dns.quad9.net/dns-query"), + ("opendns", ["208.67.222.222", "208.67.220.220"], "https://doh.opendns.com/dns-query"), +] + +# Domain patterns that should trigger DNS rotation on failure +DNS_ROTATION_DOMAINS = [ + "annas-archive", +] + + +def should_rotate_dns_for_url(url: str) -> bool: + """Check if a URL matches a known source domain for DNS rotation.""" + url_lower = url.lower() + return any(domain in url_lower for domain in DNS_ROTATION_DOMAINS) + + +# DNS state +_current_dns_index = -1 # -1 = system DNS +_dns_exhausted_logged = False + + +def _is_auto_dns_mode() -> bool: + """Check if DNS is in auto-rotation mode.""" + return env._CUSTOM_DNS.lower().strip() == "auto" and not env.USING_TOR + + +def _current_dns_label() -> str: + """Readable label for the active DNS choice.""" + if _current_dns_index >= 0: + return DNS_PROVIDERS[_current_dns_index][0] + if CUSTOM_DNS: + return f"custom {CUSTOM_DNS}" + return "system" + # Common helper functions for DNS resolution def _decode_host(host: Union[str, bytes, None]) -> str: """Convert host to string, handling bytes and None cases.""" @@ -34,7 +172,6 @@ def _decode_port(port: Union[str, bytes, int, None]) -> int: return int(port) def _is_local_address(host_str: str) -> bool: - """Check if an address is local and should bypass custom DNS.""" """Check if an address is local or private and should bypass custom DNS.""" # Localhost checks if (host_str == 'localhost' or @@ -67,17 +204,34 @@ def _is_ip_address(host_str: str) -> bool: except ValueError: return False +def _aa_hostnames() -> List[str]: + """Return hostname portions for all configured AA URLs.""" + return [ + parsed.hostname for parsed in (urllib.parse.urlparse(url) for url in _aa_urls) + if parsed.hostname + ] + +def _is_aa_hostname(host_str: str) -> bool: + """Check if a hostname matches any configured AA mirror host.""" + return any(host_str.endswith(hostname) for hostname in _aa_hostnames()) + # Store the original getaddrinfo function original_getaddrinfo = socket.getaddrinfo class DoHResolver: - """DNS over HTTPS resolver implementation.""" + """DNS over HTTPS resolver implementation with caching.""" + + # Cache TTL in seconds (5 minutes) + CACHE_TTL = 300 + def __init__(self, provider_url: str, hostname: str, ip: str): """Initialize DoH resolver with specified provider.""" self.base_url = provider_url.lower().strip() self.hostname = hostname # Store the hostname for hostname-based skipping self.ip = ip # Store IP for direct connections self.session = requests.Session() + # DNS cache: {(hostname, record_type): (ip_list, timestamp)} + self._cache: dict[tuple[str, str], tuple[List[str], datetime]] = {} # Different headers based on provider if 'google' in self.base_url: @@ -89,6 +243,24 @@ class DoHResolver: 'Accept': 'application/dns-json', }) + def _get_cached(self, hostname: str, record_type: str) -> Optional[List[str]]: + """Get cached DNS result if still valid.""" + key = (hostname, record_type) + if key in self._cache: + ips, timestamp = self._cache[key] + if datetime.now() - timestamp < timedelta(seconds=self.CACHE_TTL): + logger.debug(f"DoH cache hit for {hostname}: {ips}") + return ips + else: + # Cache expired, remove it + del self._cache[key] + return None + + def _set_cached(self, hostname: str, record_type: str, ips: List[str]) -> None: + """Cache DNS result.""" + if ips: # Only cache non-empty results + self._cache[(hostname, record_type)] = (ips, datetime.now()) + def resolve(self, hostname: str, record_type: str) -> List[str]: """Resolve a hostname using DoH. @@ -113,6 +285,11 @@ class DoHResolver: if hostname == self.hostname: logger.debug(f"Skipping DoH resolution for DoH server itself: {hostname}") return [self.ip] + + # Check cache first + cached = self._get_cached(hostname, record_type) + if cached is not None: + return cached try: params = { @@ -124,7 +301,7 @@ class DoHResolver: self.base_url, params=params, proxies=PROXIES, - timeout=5 + timeout=10 # Increased from 5s to handle slow network conditions ) response.raise_for_status() @@ -136,35 +313,31 @@ class DoHResolver: # Extract IP addresses from the response answers = [answer['data'] for answer in data['Answer'] if answer.get('type') == (28 if record_type == 'AAAA' else 1)] - logger.debug(f"Resolved {hostname} to {len(answers)} addresses using DoH: {answers}") + + # Cache the result + self._set_cached(hostname, record_type, answers) + + # Don't log here - the caller (custom_getaddrinfo) will log the final result return answers except Exception as e: logger.warning(f"DoH resolution failed for {hostname}: {e}") return [] -def create_custom_resolver(): - """Create a custom DNS resolver using the configured DNS servers.""" +def create_custom_resolver(servers: Optional[List[str]] = None): + """Create a custom DNS resolver using the specified or configured DNS servers.""" custom_resolver = dns.resolver.Resolver() - custom_resolver.nameservers = CUSTOM_DNS + custom_resolver.nameservers = servers if servers is not None else CUSTOM_DNS return custom_resolver def resolve_with_custom_dns(resolver, hostname: str, record_type: str) -> List[str]: - """Resolve hostname using custom DNS resolver. - - Args: - resolver: The DNS resolver to use - hostname: The hostname to resolve - record_type: The DNS record type (A or AAAA) - - Returns: - List of resolved IP addresses - """ + """Resolve hostname using custom DNS resolver.""" try: answers = resolver.resolve(hostname, record_type) return [str(answer) for answer in answers] - except Exception as e: - logger.debug(f"{record_type} resolution failed for {hostname}: {e}") + except Exception: + # Don't log here - let the caller handle it to prevent spam + # Don't trigger DNS switch here either - caller handles it return [] def create_custom_getaddrinfo( @@ -193,42 +366,63 @@ def create_custom_getaddrinfo( host_str = _decode_host(host) port_int = _decode_port(port) + def _log_results(source: str, provider_label: str, res: Sequence[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]], is_bypass: bool = False) -> None: + """Emit a unified resolver log with the IPs returned. + + Args: + source: Description of resolver source + provider_label: Label for the DNS provider + res: Resolution results + is_bypass: If True, log at DEBUG level (for local/IP addresses) + """ + # Skip logging entirely for localhost to reduce noise + if host_str in ('localhost', '127.0.0.1', '::1'): + return + try: + ips = [entry[4][0] for entry in res if len(entry) >= 5 and entry[4]] + msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}" + if is_bypass: + logger.debug(msg) + else: + logger.info(msg) + except Exception: + pass # Silently ignore logging failures + # Skip custom resolution for IP addresses, local addresses, or if skip check passes if _is_ip_address(host_str) or _is_local_address(host_str) or (skip_check and skip_check(host_str)): - logger.debug(f"Using system DNS for IP address or local/private address: {host_str}") - return original_getaddrinfo(host, port, family, type, proto, flags) + # Quietly bypass custom resolution for IP/local targets + res = original_getaddrinfo(host, port, family, type, proto, flags) + _log_results("system resolver (bypass)", "system", res, is_bypass=True) + return res results: list[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]] = [] try: - # Try IPv6 first if family allows it - if family == 0 or family == socket.AF_INET6: - logger.debug(f"Resolving IPv6 address for {host_str}") - ipv6_answers = resolve_ipv6(host_str) - for answer in ipv6_answers: - results.append((socket.AF_INET6, cast(SocketKind, type), proto, '', (answer, port_int, 0, 0))) - if ipv6_answers: - logger.debug(f"Found {len(ipv6_answers)} IPv6 addresses for {host_str}") - - # Then try IPv4 + # Try IPv4 (IPv6 disabled to avoid noisy AAAA failures) if family == 0 or family == socket.AF_INET: - logger.debug(f"Resolving IPv4 address for {host_str}") ipv4_answers = resolve_ipv4(host_str) for answer in ipv4_answers: results.append((socket.AF_INET, cast(SocketKind, type), proto, '', (answer, port_int))) - if ipv4_answers: - logger.debug(f"Found {len(ipv4_answers)} IPv4 addresses for {host_str}") if results: - logger.debug(f"Resolved {host_str} to {len(results)} addresses") + _log_results("custom resolver", _current_dns_label(), results) return results except Exception as e: logger.warning(f"Custom DNS resolution failed for {host_str}: {e}, falling back to system DNS") + # Trigger DNS switch on failure (if auto mode) + if _is_auto_dns_mode() and not _is_local_address(host_str) and not _is_ip_address(host_str): + # Only switch if we haven't exhausted all providers + if _current_dns_index < len(DNS_PROVIDERS): + logger.info(f"Requesting DNS provider switch after custom resolver failure for {host_str}") + switch_dns_provider() # Fall back to system DNS if custom resolution fails + logger.info(f"Custom DNS returned no addresses for {host_str}; falling back to system resolver") try: - return original_getaddrinfo(host, port, family, type, proto, flags) + res = original_getaddrinfo(host, port, family, type, proto, flags) + _log_results("system resolver (fallback)", "system", res) + return res except Exception as e: logger.error(f"System DNS resolution also failed for {host_str}: {e}") # Last resort: Try to connect to the hostname directly @@ -240,11 +434,45 @@ def create_custom_getaddrinfo( return custom_getaddrinfo -def init_doh_resolver(doh_server: str = DOH_SERVER): - """Initialize DNS over HTTPS resolver. +def create_system_failover_getaddrinfo(): + """Wrap system getaddrinfo to trigger DNS provider switch on failure.""" + _switch_logged: set[str] = set() + + def system_failover_getaddrinfo( + host: Union[str, bytes, None], + port: Union[str, bytes, int, None], + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0 + ) -> Sequence[Tuple[AddressFamily, SocketKind, int, str, Tuple[Any, ...]]]: + host_str = _decode_host(host) + try: + return original_getaddrinfo(host, port, family, type, proto, flags) + except Exception as e: + if host_str not in _switch_logged: + logger.warning(f"System DNS resolution failed for {host_str}: {e}") + + # Trigger DNS switch only in auto mode for non-local targets + if _is_auto_dns_mode() and not _is_ip_address(host_str) and not _is_local_address(host_str): + if _current_dns_index + 1 < len(DNS_PROVIDERS): + if host_str not in _switch_logged: + logger.info(f"Switching DNS provider after system DNS failure for {host_str}") + _switch_logged.add(host_str) + if switch_dns_provider(): + return socket.getaddrinfo(host, port, family, type, proto, flags) + raise + + return system_failover_getaddrinfo + +def _init_doh_resolver_internal(doh_server: str) -> DoHResolver: + """Internal: Initialize DNS over HTTPS resolver with specified server. Args: doh_server: The DoH server URL + + Returns: + Configured DoHResolver instance """ # Pre-resolve the DoH server hostname to prevent recursion url = urllib.parse.urlparse(doh_server) @@ -292,9 +520,14 @@ def init_doh_resolver(doh_server: str = DOH_SERVER): logger.info("DoH resolver successfully configured and activated") return doh_resolver -def init_custom_resolver(): - """Initialize custom DNS resolver using configured DNS servers.""" - custom_resolver = create_custom_resolver() + +def _init_custom_resolver_internal(servers: List[str]): + """Internal: Initialize custom DNS resolver with specified servers. + + Args: + servers: List of DNS server IPs to use + """ + custom_resolver = create_custom_resolver(servers) # Create resolver functions def resolve_ipv4(hostname: str) -> List[str]: @@ -309,32 +542,286 @@ def init_custom_resolver(): logger.info("Custom DNS resolver successfully configured and activated") return custom_resolver -# Initialize DNS resolvers based on configuration + +def init_doh_resolver(doh_server: str = ""): + """Initialize DNS over HTTPS resolver.""" + server = doh_server or DOH_SERVER + if not server: + return None + return _init_doh_resolver_internal(server) + + +def init_custom_resolver(): + """Initialize custom DNS resolver using configured DNS servers.""" + if not CUSTOM_DNS: + return None + return _init_custom_resolver_internal(CUSTOM_DNS) + +def switch_dns_provider() -> bool: + """Switch to next DNS provider (auto mode only).""" + global CUSTOM_DNS, DOH_SERVER, _current_dns_index, _dns_exhausted_logged + + if not _is_auto_dns_mode(): + return False + + with _dns_switch_lock: + if _current_dns_index + 1 >= len(DNS_PROVIDERS): + if not _dns_exhausted_logged: + logger.warning("All DNS providers exhausted, staying with current") + _dns_exhausted_logged = True + return False + + _current_dns_index += 1 + name, servers, doh = DNS_PROVIDERS[_current_dns_index] + CUSTOM_DNS = servers + DOH_SERVER = doh + config.CUSTOM_DNS = servers + config.DOH_SERVER = doh + + logger.warning(f"Switched DNS provider to: {name} (using DoH)") + _save_state(dns_provider=name) + init_dns_resolvers() + + # Notify listeners (e.g., Chrome bypasser) to restart with new DNS + _notify_dns_rotation(name, servers, doh) + return True + + +def rotate_dns_provider() -> bool: + """Rotate DNS provider (auto mode only), cycling back if exhausted.""" + global _current_dns_index, _dns_exhausted_logged + + if not _is_auto_dns_mode(): + return False + + if _current_dns_index + 1 >= len(DNS_PROVIDERS): + logger.warning("DNS rotation: cycling back to first provider") + _current_dns_index = -1 + _dns_exhausted_logged = False + + return switch_dns_provider() + +def rotate_dns_and_reset_aa() -> bool: + """ + Switch DNS provider (auto mode) and reset AA URL list to the first entry. + Returns True if DNS switched; False if no providers left or not in auto mode. + + Note: This function can be called during initialization, so we must NOT call + _ensure_initialized() here to avoid recursive init loops. + """ + if not rotate_dns_provider(): + return False + # Reset AA URL to first available auto option if using auto AA + global AA_BASE_URL, _current_aa_url_index + if AA_BASE_URL == "auto" or AA_BASE_URL in _aa_urls: + _current_aa_url_index = 0 + AA_BASE_URL = _aa_urls[0] + config.AA_BASE_URL = AA_BASE_URL + logger.info(f"After DNS switch, resetting AA URL to: {AA_BASE_URL}") + _save_state(aa_url=AA_BASE_URL) + return True + def init_dns_resolvers(): """Initialize DNS resolvers based on configuration.""" - if len(CUSTOM_DNS) > 0: + global CUSTOM_DNS, DOH_SERVER + + if _is_auto_dns_mode(): + if _current_dns_index >= 0: + name, servers, doh = DNS_PROVIDERS[_current_dns_index] + CUSTOM_DNS = servers + DOH_SERVER = doh + config.CUSTOM_DNS = servers + config.DOH_SERVER = doh + logger.info(f"Using DNS provider: {name} (DoH enabled)") + else: + CUSTOM_DNS = [] + DOH_SERVER = "" + config.CUSTOM_DNS = [] + config.DOH_SERVER = "" + logger.info("Using system DNS (auto mode - will switch on failure)") + socket.getaddrinfo = cast(Any, create_system_failover_getaddrinfo()) + return + + if CUSTOM_DNS: init_custom_resolver() if DOH_SERVER: - init_doh_resolver() + init_doh_resolver(DOH_SERVER) -# Initialize DNS resolvers -init_dns_resolvers() -# Check available AA_BASE_URLs if set to auto -if AA_BASE_URL == "auto": - logger.info(f"AA_BASE_URL: auto, checking available urls {AA_AVAILABLE_URLS}") - for url in AA_AVAILABLE_URLS: - try: - response = requests.get(url, proxies=PROXIES) - if response.status_code == 200: - AA_BASE_URL = url - break - except Exception as e: - logger.error_trace(f"Error checking {url}: {e}") +def _initialize_dns_state() -> None: + """Restore persisted DNS choice or start fresh.""" + global _current_dns_index + + if _is_auto_dns_mode(): + persisted = state.get('dns_provider') if state else None + if persisted: + for i, (name, _, _) in enumerate(DNS_PROVIDERS): + if name == persisted: + _current_dns_index = i + logger.info(f"Restored DNS provider from state: {name}") + return + _current_dns_index = -1 + +def _initialize_aa_state() -> None: + """Restore or probe AA URL state.""" + global AA_BASE_URL, _current_aa_url_index if AA_BASE_URL == "auto": - AA_BASE_URL = AA_AVAILABLE_URLS[0] -config.AA_BASE_URL = AA_BASE_URL -logger.info(f"AA_BASE_URL: {AA_BASE_URL}") + if state.get('aa_base_url') and state['aa_base_url'] in _aa_urls: + _current_aa_url_index = _aa_urls.index(state['aa_base_url']) + AA_BASE_URL = state['aa_base_url'] + else: + logger.info(f"AA_BASE_URL: auto, checking available urls {_aa_urls}") + for i, url in enumerate(_aa_urls): + try: + response = requests.get(url, proxies=PROXIES, timeout=3) + if response.status_code == 200: + _current_aa_url_index = i + AA_BASE_URL = url + _save_state(aa_url=AA_BASE_URL) + break + except Exception: + pass + if AA_BASE_URL == "auto": + AA_BASE_URL = _aa_urls[0] + _current_aa_url_index = 0 + elif AA_BASE_URL not in _aa_urls: + logger.info(f"AA_BASE_URL set to custom value {AA_BASE_URL}; skipping auto-switch") + else: + _current_aa_url_index = _aa_urls.index(AA_BASE_URL) + + config.AA_BASE_URL = AA_BASE_URL + logger.info(f"AA_BASE_URL: {AA_BASE_URL}") + +def init_dns(force: bool = False) -> None: + """Initialize DNS state and resolvers.""" + global state, _dns_initialized + if _dns_initialized and not force: + return + with _init_lock: + # Double-check after acquiring lock + if _dns_initialized and not force: + return + # Set flag BEFORE doing work to prevent recursive calls during init + _dns_initialized = True + try: + logger.debug(f"Initializing DNS (using {'gevent' if _using_gevent_locks else 'threading'} locks)") + state = _load_state() + _initialize_dns_state() + init_dns_resolvers() + except Exception: + _dns_initialized = False + raise + +def init_aa(force: bool = False) -> None: + """Initialize AA mirror selection.""" + global state, _aa_initialized + if _aa_initialized and not force: + return + with _init_lock: + # Double-check after acquiring lock + if _aa_initialized and not force: + return + # Set flag BEFORE doing work to prevent recursive calls during init + _aa_initialized = True + try: + state = _load_state() + _initialize_aa_state() + except Exception: + _aa_initialized = False + raise + +def init(force: bool = False) -> None: + """ + Initialize network state (DNS resolvers and AA mirror selection). + + Called lazily on first network operation. Safe to call repeatedly; + later calls no-op unless force=True. + """ + global _initialized + if _initialized and not force: + return + with _init_lock: + # Double-check after acquiring lock + if _initialized and not force: + return + # Set flag BEFORE doing work to prevent recursive calls during init + # (e.g., DNS failover handlers calling back into init) + _initialized = True + try: + init_dns(force=force) + init_aa(force=force) + except Exception: + # Reset flag on failure so retry is possible + _initialized = False + raise + +def get_aa_base_url(): + """Get current AA base URL.""" + _ensure_initialized() + return AA_BASE_URL + +def get_available_aa_urls(): + """Get list of configured AA URLs (copy).""" + _ensure_initialized() + return _aa_urls.copy() + +def set_aa_url_index(new_index: int) -> bool: + """Set AA base URL by index in available list; returns True if applied.""" + _ensure_initialized() + global AA_BASE_URL, _current_aa_url_index + if new_index < 0 or new_index >= len(_aa_urls): + return False + _current_aa_url_index = new_index + AA_BASE_URL = _aa_urls[_current_aa_url_index] + config.AA_BASE_URL = AA_BASE_URL + logger.info(f"Set AA URL to: {AA_BASE_URL}") + _save_state(aa_url=AA_BASE_URL) + return True + +class AAMirrorSelector: + """ + Small helper to keep AA mirror switching consistent across call sites. + Tracks attempts per DNS cycle and rewrites URLs safely. + """ + def __init__(self) -> None: + self._ensure_fresh_state(reset_attempts=True) + + def _ensure_fresh_state(self, reset_attempts: bool = False) -> None: + _ensure_initialized() + self.aa_urls = get_available_aa_urls() + self._index = self._safe_index(get_aa_base_url()) + self.current_base = self.aa_urls[self._index] if self.aa_urls else "" + if reset_attempts: + self.attempts_this_dns = 0 + + def _safe_index(self, base: str) -> int: + if base in self.aa_urls: + return self.aa_urls.index(base) + return 0 + + def rewrite(self, url: str) -> str: + """Replace any known AA base in url with current_base.""" + for base in self.aa_urls: + if url.startswith(base): + return url.replace(base, self.current_base, 1) + return url + + def next_mirror_or_rotate_dns(self, allow_dns: bool = True) -> tuple[Optional[str], str]: + """ + Advance to next mirror; if exhausted and allowed, rotate DNS and reset to first. + Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'. + """ + self.attempts_this_dns += 1 + if self.attempts_this_dns >= len(self.aa_urls): + if allow_dns and rotate_dns_and_reset_aa(): + self._ensure_fresh_state(reset_attempts=True) + return self.current_base, "dns" + return None, "exhausted" + + next_index = (self._index + 1) % len(self.aa_urls) + set_aa_url_index(next_index) + self._ensure_fresh_state(reset_attempts=False) + return self.current_base, "mirror" # Configure urllib opener with appropriate headers opener = urllib.request.build_opener() @@ -344,7 +831,3 @@ opener.addheaders = [ 'Chrome/129.0.0.0 Safari/537.3') ] urllib.request.install_opener(opener) - -# Need an empty function to be called by downloader.py -def init(): - pass diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index 6db8d871..21773fac 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -53,7 +53,7 @@ function App() { content: '', formats: DEFAULT_FORMAT_SELECTION, }); - const { toasts, showToast } = useToast(); + const { toasts, showToast, removeToast } = useToast(); const updateAdvancedFilters = useCallback((updates: Partial) => { setAdvancedFilters(prev => ({ ...prev, ...updates })); }, []); @@ -80,18 +80,12 @@ function App() { const ongoing = [ currentStatus.queued, currentStatus.resolving, - currentStatus.bypassing, currentStatus.downloading, - currentStatus.verifying, - currentStatus.ingesting, ].reduce((sum, status) => sum + (status ? Object.keys(status).length : 0), 0); - const completed = [ - currentStatus.completed, - currentStatus.complete, - currentStatus.available, - currentStatus.done, - ].reduce((sum, status) => sum + (status ? Object.keys(status).length : 0), 0); + const completed = currentStatus.complete + ? Object.keys(currentStatus.complete).length + : 0; const errored = currentStatus.error ? Object.keys(currentStatus.error).length : 0; @@ -131,21 +125,24 @@ function App() { // Check for completed items const prevDownloadingIds = new Set(Object.keys(prevDownloading)); + const prevResolvingIds = new Set(Object.keys(prev.resolving || {})); const prevQueuedIds = new Set(Object.keys(prevQueued)); - const currAvailable = curr.available || {}; - const currDone = curr.done || {}; + const currComplete = curr.complete || {}; - Object.keys(currAvailable).forEach(bookId => { + Object.keys(currComplete).forEach(bookId => { if (prevDownloadingIds.has(bookId) || prevQueuedIds.has(bookId)) { - const book = currAvailable[bookId]; + const book = currComplete[bookId]; showToast(`${book.title || 'Book'} completed`, 'success'); } }); - Object.keys(currDone).forEach(bookId => { - if (prevDownloadingIds.has(bookId) || prevQueuedIds.has(bookId)) { - const book = currDone[bookId]; - showToast(`${book.title || 'Book'} completed`, 'success'); + // Check for failed items + const currError = curr.error || {}; + Object.keys(currError).forEach(bookId => { + if (prevDownloadingIds.has(bookId) || prevResolvingIds.has(bookId) || prevQueuedIds.has(bookId)) { + const book = currError[bookId]; + const errorMsg = book.status_message || 'Download failed'; + showToast(`${book.title || 'Book'}: ${errorMsg}`, 'error'); } }); }, [showToast]); @@ -229,6 +226,14 @@ function App() { try { const cfg = await getConfig(); setConfig(cfg); + // Update format selection to match supported formats from config + // This ensures PDF is auto-selected when added to SUPPORTED_FORMATS env var + if (cfg?.supported_formats) { + setAdvancedFilters(prev => ({ + ...prev, + formats: cfg.supported_formats, + })); + } } catch (error) { console.error('Failed to load config:', error); // Use defaults if config fails to load @@ -278,6 +283,11 @@ function App() { } else { console.error('Search failed:', error); setBooks([]); + const message = error instanceof Error ? error.message : 'Search failed'; + const friendly = message.includes("Anna's Archive") || message.includes('Network restricted') + ? message + : "Unable to reach Anna's Archive. Network may be restricted or mirrors blocked."; + showToast(friendly, 'error'); } } finally { setIsSearching(false); @@ -304,6 +314,7 @@ function App() { } catch (error) { console.error('Download failed:', error); showToast('Failed to queue download', 'error'); + throw error; // Re-throw so button components can reset their queuing state } }; @@ -333,6 +344,8 @@ function App() { setSearchInput(''); setShowAdvanced(false); setLastSearchQuery(''); + // Use config's supported formats if available, otherwise fall back to default + const resetFormats = config?.supported_formats || DEFAULT_FORMAT_SELECTION; setAdvancedFilters({ isbn: '', author: '', @@ -340,7 +353,7 @@ function App() { lang: [LANGUAGE_OPTION_DEFAULT], sort: '', content: '', - formats: DEFAULT_FORMAT_SELECTION, + formats: resetFormats, }); }; @@ -366,26 +379,11 @@ function App() { if (currentStatus.error && currentStatus.error[bookId]) { return { text: 'Failed', state: 'error' }; } - // Check completed states - if (currentStatus.completed && currentStatus.completed[bookId]) { - return { text: 'Downloaded', state: 'completed' }; - } + // Check completed if (currentStatus.complete && currentStatus.complete[bookId]) { - return { text: 'Downloaded', state: 'completed' }; - } - if (currentStatus.available && currentStatus.available[bookId]) { - return { text: 'Downloaded', state: 'completed' }; - } - if (currentStatus.done && currentStatus.done[bookId]) { - return { text: 'Downloaded', state: 'completed' }; - } - // Check in-progress states with detailed status - if (currentStatus.ingesting && currentStatus.ingesting[bookId]) { - return { text: 'Ingesting', state: 'ingesting' }; - } - if (currentStatus.verifying && currentStatus.verifying[bookId]) { - return { text: 'Verifying', state: 'verifying' }; + return { text: 'Downloaded', state: 'complete' }; } + // Check in-progress states if (currentStatus.downloading && currentStatus.downloading[bookId]) { const book = currentStatus.downloading[bookId]; return { @@ -394,9 +392,6 @@ function App() { progress: book.progress }; } - if (currentStatus.bypassing && currentStatus.bypassing[bookId]) { - return { text: 'Bypassing Cloudflare...', state: 'bypassing' }; - } if (currentStatus.resolving && currentStatus.resolving[bookId]) { return { text: 'Resolving', state: 'resolving' }; } @@ -440,6 +435,8 @@ function App() { }} onAdvancedToggle={() => setShowAdvanced(!showAdvanced)} isLoading={isSearching} + onShowToast={showToast} + onRemoveToast={removeToast} /> = { - queued: { bg: 'bg-amber-500/10', text: 'text-amber-600', label: 'Queued' }, - resolving: { bg: 'bg-indigo-500/10', text: 'text-indigo-600', label: 'Resolving' }, - bypassing: { bg: 'bg-purple-500/10', text: 'text-purple-600', label: 'Bypassing Cloudflare...' }, - downloading: { bg: 'bg-blue-500/10', text: 'text-blue-600', label: 'Downloading' }, - verifying: { bg: 'bg-cyan-500/10', text: 'text-cyan-600', label: 'Verifying' }, - ingesting: { bg: 'bg-teal-500/10', text: 'text-teal-600', label: 'Ingesting' }, - complete: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Complete' }, - completed: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Completed' }, - available: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Available' }, - done: { bg: 'bg-green-500/10', text: 'text-green-600', label: 'Done' }, - error: { bg: 'bg-red-500/10', text: 'text-red-600', label: 'Error' }, - cancelled: { bg: 'bg-gray-500/10', text: 'text-gray-600', label: 'Cancelled' }, -}; - -// Helper to format file size -const formatSize = (sizeStr?: string): string => { - if (!sizeStr) return ''; - return sizeStr; +const STATUS_STYLES: Record = { + queued: { bg: 'bg-amber-500/20', text: 'text-amber-700 dark:text-amber-300', label: 'Queued', waveColor: 'rgba(217, 119, 6, 0.3)' }, + resolving: { bg: 'bg-indigo-500/20', text: 'text-indigo-700 dark:text-indigo-300', label: 'Resolving', waveColor: 'rgba(79, 70, 229, 0.3)' }, + downloading: { bg: 'bg-sky-500/20', text: 'text-sky-700 dark:text-sky-300', label: 'Downloading', waveColor: 'rgba(2, 132, 199, 0.3)' }, + complete: { bg: 'bg-green-500/20', text: 'text-green-700 dark:text-green-300', label: 'Complete', waveColor: '' }, + error: { bg: 'bg-red-500/20', text: 'text-red-700 dark:text-red-300', label: 'Error', waveColor: '' }, + cancelled: { bg: 'bg-gray-500/20', text: 'text-gray-700 dark:text-gray-300', label: 'Cancelled', waveColor: '' }, }; // Add keyframe animation for wave effect const styleSheet = document.createElement('style'); styleSheet.textContent = ` @keyframes wave { - 0% { background-position: -200% 0; } - 100% { background-position: 200% 0; } + 0% { background-position: 200% 0; } + 100% { background-position: -200% 0; } } `; if (!document.head.querySelector('style[data-wave-animation]')) { @@ -56,24 +44,14 @@ const getStatusProgress = (statusName: string, bookProgress?: number): number => case 'queued': return 5; case 'resolving': - return 10; - case 'bypassing': return 15; case 'downloading': - // Map actual progress (0-100) to 20-90 range + // Map actual progress (0-100) to 20-100 range if (typeof bookProgress === 'number') { - return 20 + (bookProgress * 0.7); + return 20 + (bookProgress * 0.8); } return 20; - case 'verifying': - return 95; - case 'ingesting': - return 99; - case 'completed': case 'complete': - case 'available': - case 'done': - return 100; case 'error': return 100; default: @@ -83,18 +61,15 @@ const getStatusProgress = (statusName: string, bookProgress?: number): number => // Helper to get progress bar color based on status const getProgressBarColor = (statusName: string): string => { - const isCompleted = ['completed', 'complete', 'available', 'done'].includes(statusName); - if (isCompleted) return 'bg-green-600'; + if (statusName === 'complete') return 'bg-green-600'; if (statusName === 'error') return 'bg-red-600'; - if (statusName === 'queued') return 'bg-gray-600'; - if (statusName === 'resolving') return 'bg-purple-600'; - if (statusName === 'bypassing') return 'bg-violet-600'; + if (statusName === 'queued') return 'bg-amber-600'; + if (statusName === 'resolving') return 'bg-indigo-600'; if (statusName === 'downloading') return 'bg-sky-600'; - if (statusName === 'verifying') return 'bg-cyan-600'; - if (statusName === 'ingesting') return 'bg-teal-600'; return 'bg-sky-600'; }; + export const DownloadsSidebar = ({ isOpen, onClose, @@ -119,22 +94,21 @@ export const DownloadsSidebar = ({ }, [isOpen, onClose]); // Collect all download items from different status sections - const allDownloadItems: Array<{ book: Book; status: string; order: number }> = []; - - // Priority order for display - const statusOrder = ['downloading', 'bypassing', 'resolving', 'queued', 'verifying', 'ingesting', 'error', 'completed', 'complete', 'available', 'done', 'cancelled']; - - statusOrder.forEach((statusName, index) => { + const allDownloadItems: Array<{ book: Book; status: string }> = []; + + const statusTypes = ['downloading', 'resolving', 'queued', 'error', 'complete', 'cancelled']; + + statusTypes.forEach((statusName) => { const items = (status as any)[statusName]; if (items && Object.keys(items).length > 0) { Object.values(items).forEach((book: any) => { - allDownloadItems.push({ book, status: statusName, order: index }); + allDownloadItems.push({ book, status: statusName }); }); } }); - // Sort by status priority - allDownloadItems.sort((a, b) => a.order - b.order); + // Sort by added_time descending (newest first) + allDownloadItems.sort((a, b) => (b.book.added_time || 0) - (a.book.added_time || 0)); const renderDownloadItem = (item: { book: Book; status: string }) => { const { book, status: statusName } = item; @@ -144,23 +118,31 @@ export const DownloadsSidebar = ({ label: statusName.charAt(0).toUpperCase() + statusName.slice(1), }; - const isInProgress = ['queued', 'resolving', 'bypassing', 'downloading', 'verifying', 'ingesting'].includes(statusName); - const isCompleted = ['completed', 'complete', 'available', 'done'].includes(statusName); + const isInProgress = ['queued', 'resolving', 'downloading'].includes(statusName); + const isCompleted = statusName === 'complete'; const hasError = statusName === 'error'; // Get progress information const progress = getStatusProgress(statusName, book.progress); const progressBarColor = getProgressBarColor(statusName); - // Format progress text - let progressText = statusStyle.label; + // Format progress text - use status_message if available, otherwise fall back to label + let progressText = book.status_message || statusStyle.label; if (statusName === 'downloading' && book.progress && book.size) { - const downloadedMB = (book.progress / 100) * parseFloat(book.size.replace(/[^\d.]/g, '')); - progressText = `${downloadedMB.toFixed(1)}mb / ${book.size}`; + const sizeValue = parseFloat(book.size.replace(/[^\d.]/g, '')); + const sizeUnit = book.size.replace(/[\d.\s]/g, ''); // Extract unit as-is from backend + const downloadedSize = (book.progress / 100) * sizeValue; + const sizeProgress = `${downloadedSize.toFixed(1)}${sizeUnit} / ${book.size}`; + // If there's attempt info in the status message, prepend it to the progress + if (book.status_message?.startsWith('Attempt')) { + progressText = `${book.status_message} - ${sizeProgress}`; + } else { + progressText = sizeProgress; + } } else if (isCompleted) { progressText = 'Complete'; } else if (hasError) { - progressText = 'Failed'; + progressText = book.status_message || 'Failed'; } return ( @@ -169,6 +151,22 @@ export const DownloadsSidebar = ({ className="relative rounded-lg border hover:shadow-md transition-shadow overflow-hidden" style={{ borderColor: 'var(--border-muted)', background: 'var(--bg-soft)' }} > + {/* Cancel/Clear Button - top right corner */} + + {/* Main content area */}
{/* Book Thumbnail - left side */} @@ -187,8 +185,8 @@ export const DownloadsSidebar = ({ {/* Book Info - right side */}
- {/* Title & Author */} -
+ {/* Title & Author - with safe area for cancel/clear button */} +

{isCompleted && book.download_path ? (

- {/* Status Badge and Details Row */} -
-
- - {statusStyle.label} - - - {/* Cancel Button for in-progress items */} - {isInProgress && ( - - )} + {/* Details Row */} +
+
+ {/* Format and Size */} +
+ {book.format && {book.format}} + {book.format && book.size && • } + {book.size && {book.size}} +
- - {/* Format and Size */} -
- {book.format && {book.format}} - {book.format && book.size && • } - {book.size && {formatSize(book.size)}} -
- - {/* Error Message */} - {hasError && ( -

Download failed

- )}
{/* Progress Bar - absolute positioned at bottom - always visible */} -
-

{progressText}

+
+ {/* ml-16 clears the 64px thumbnail, gap-2 adds spacing */} +
+ + {/* Wave animation overlay for in-progress states */} + {isInProgress && statusStyle.waveColor && ( + + )} + {progressText} + +
void; + onShowToast?: (message: string, type: 'success' | 'error' | 'info', persistent?: boolean) => string; + onRemoveToast?: (id: string) => void; } export const Header = ({ @@ -41,6 +43,8 @@ export const Header = ({ authRequired = false, isAuthenticated = false, onLogout, + onShowToast, + onRemoveToast, }: HeaderProps) => { const [theme, setTheme] = useState('auto'); const [isDropdownOpen, setIsDropdownOpen] = useState(false); @@ -303,17 +307,62 @@ export const Header = ({ {/* Debug Buttons */} {debug && ( <> -
- -
+