mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 21:30:26 +01:00
## Changelog ### 🌐 Network Resilience - **Auto DNS rotation**: New `CUSTOM_DNS=auto` mode (now default) starts with system DNS and automatically rotates through Cloudflare, Google, Quad9, and OpenDNS when failures are detected. DNS results are cached to improve performance. - **Mirror failover**: Anna's Archive requests automatically fail over between mirrors (.org, .se, .li) when one is unreachable - **Round-robin source distribution**: Concurrent downloads are distributed across different AA partner servers to avoid rate limiting ### 📥 Download Reliability - **Much more reliable downloads**: Improved parsing of Anna's Archive pages, smarter source prioritization, and better retry logic with exponential backoff - **Download resume support**: Interrupted downloads can now resume from where they left off (if the server supports Range requests) - **Cookie sharing**: Cloudflare bypass cookies are extracted and shared with subsequent requests, often avoiding the need for re-bypass entirely - **Stall detection**: Downloads with no progress for 5 minutes are automatically cancelled and retried - **Staggered concurrent downloads**: Small delays between starting concurrent downloads to avoid hitting rate limits - **Source failure tracking**: After multiple failures from the same source type (e.g., Libgen), that source is temporarily skipped - **Lazy welib loading**: Welib sources are fetched as a fallback only when primary sources fail (unless `PRIORITIZE_WELIB` is enabled) ### 🛡️ Cloudflare & Protection Bypass - **DDOS-Guard support**: Internal bypasser now detects and handles DDOS-Guard challenges with dedicated bypass strategies - **Cancellation support**: Bypass operations can now be cancelled mid-operation when user cancels a download - **Smart warmup**: Chrome driver is pre-warmed when first client connects (controlled by `BYPASS_WARMUP_ON_CONNECT` env var) and shuts down after periods of inactivity ### 🔌 External Bypasser (FlareSolverr) - **Improved resilience**: Retry with exponential backoff, mirror/DNS rotation on failure, and proper timeout handling - **Cancellation support**: External bypasser operations respect cancellation flags ### 🖥️ Web UI Improvements - **Simplified download status**: Removed intermediate states (bypassing, verifying, ingesting) — now just shows Queued → Resolving → Downloading → Complete - **Status messages**: Downloads show detailed status like "Trying Anna's Archive (Server 3)" or "Server busy, trying next...", or live waitlist countdowns. - **Improved download sidebar**: - Downloads sorted by add time (newest first) - X button moved to top-right corner for better UX - Wave animation on in-progress items - Error messages shown directly on failed items - X button on completed/errored items clears them from the list ### ⚙️ Configuration Changes - **`CUSTOM_DNS=auto`** is now the default (previously empty/system DNS) - **`DOWNLOAD_PROGRESS_UPDATE_INTERVAL`** default changed from 5s to 1s for smoother progress - **`BYPASS_WARMUP_ON_CONNECT`** (default: true) — warm up Chrome when first client connects ### 🐛 Bug Fixes - **Download cancellation actually works**: Fixed issue where cancelling downloads didn't properly stop in-progress operations - **WELIB prioritization**: Fixed `PRIORITIZE_WELIB` not being respected - **File exists handling**: Downloads to same filename now get `_1`, `_2` suffix instead of overwriting - **Empty search results**: "No books found" now returns empty list instead of throwing exception - **Search unavailable error**: Network/mirror failures during search now return proper 503 error to client
123 lines
4.8 KiB
Python
123 lines
4.8 KiB
Python
"""Configuration settings for the book downloader application."""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
import json
|
|
import env
|
|
from logger import setup_logger
|
|
|
|
logger = setup_logger(__name__)
|
|
|
|
for key, value in env.__dict__.items():
|
|
if not key.startswith('_'):
|
|
if key == "AA_DONATOR_KEY" and value.strip() != "":
|
|
value = "REDACTED"
|
|
logger.info(f"{key}: {value}")
|
|
|
|
with open("data/book-languages.json") as file:
|
|
_SUPPORTED_BOOK_LANGUAGE = json.load(file)
|
|
|
|
# Directory settings
|
|
BASE_DIR = Path(__file__).resolve().parent
|
|
logger.info(f"BASE_DIR: {BASE_DIR}")
|
|
if env.ENABLE_LOGGING:
|
|
env.LOG_DIR.mkdir(exist_ok=True)
|
|
|
|
# Create necessary directories
|
|
env.TMP_DIR.mkdir(exist_ok=True)
|
|
env.INGEST_DIR.mkdir(exist_ok=True)
|
|
|
|
CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
|
|
logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
|
|
logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
|
|
logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
|
|
|
|
# Network settings
|
|
_custom_dns = env._CUSTOM_DNS.lower().strip()
|
|
_doh_server = ""
|
|
|
|
if _custom_dns == "auto" or _custom_dns == "":
|
|
# Auto mode - DNS provider rotation handled by network.py
|
|
# Starts with system DNS, switches to providers from DNS_PROVIDERS on failure
|
|
CUSTOM_DNS = []
|
|
_doh_server = ""
|
|
logger.info("CUSTOM_DNS: auto (starts with system DNS, rotates on failure)")
|
|
elif _custom_dns == "google":
|
|
CUSTOM_DNS = ["8.8.8.8", "8.8.4.4", "2001:4860:4860:0000:0000:0000:0000:8888", "2001:4860:4860:0000:0000:0000:0000:8844"]
|
|
_doh_server = "https://dns.google/resolve"
|
|
logger.info(f"CUSTOM_DNS: google {CUSTOM_DNS}")
|
|
elif _custom_dns == "quad9":
|
|
CUSTOM_DNS = ["9.9.9.9", "149.112.112.112", "2620:00fe:0000:0000:0000:0000:0000:00fe", "2620:00fe:0000:0000:0000:0000:0000:0009"]
|
|
_doh_server = "https://dns.quad9.net/dns-query"
|
|
logger.info(f"CUSTOM_DNS: quad9 {CUSTOM_DNS}")
|
|
elif _custom_dns == "cloudflare":
|
|
CUSTOM_DNS = ["1.1.1.1", "1.0.0.1", "2606:4700:4700:0000:0000:0000:0000:1111", "2606:4700:4700:0000:0000:0000:0000:1001"]
|
|
_doh_server = "https://cloudflare-dns.com/dns-query"
|
|
logger.info(f"CUSTOM_DNS: cloudflare {CUSTOM_DNS}")
|
|
elif _custom_dns == "opendns":
|
|
CUSTOM_DNS = ["208.67.222.222", "208.67.220.220", "2620:0119:0035:0000:0000:0000:0000:0035", "2620:0119:0053:0000:0000:0000:0000:0053"]
|
|
_doh_server = "https://doh.opendns.com/dns-query"
|
|
logger.info(f"CUSTOM_DNS: opendns {CUSTOM_DNS}")
|
|
else:
|
|
# Custom DNS IPs provided by user
|
|
_custom_dns_ip = _custom_dns.split(",")
|
|
CUSTOM_DNS = [dns.strip() for dns in _custom_dns_ip if dns.replace(":", "").replace(".", "").strip().isdigit()]
|
|
logger.info(f"CUSTOM_DNS: custom {CUSTOM_DNS}")
|
|
DOH_SERVER = _doh_server
|
|
if env.USE_DOH:
|
|
DOH_SERVER = _doh_server
|
|
else:
|
|
DOH_SERVER = ""
|
|
logger.info(f"DOH_SERVER: {DOH_SERVER}")
|
|
|
|
# Warn about external bypasser DNS limitations
|
|
if env.USING_EXTERNAL_BYPASSER and env.USE_CF_BYPASS:
|
|
logger.warning(
|
|
"Using external bypasser (FlareSolverr). Note: FlareSolverr uses its own DNS resolution, "
|
|
"not this application's custom DNS settings. If you experience DNS-related blocks, "
|
|
"configure DNS at the Docker/system level for your FlareSolverr container, "
|
|
"or consider using the internal bypasser which integrates with the app's DNS system."
|
|
)
|
|
|
|
# Proxy settings
|
|
PROXIES = {}
|
|
if env.HTTP_PROXY:
|
|
PROXIES["http"] = env.HTTP_PROXY
|
|
if env.HTTPS_PROXY:
|
|
PROXIES["https"] = env.HTTPS_PROXY
|
|
logger.info(f"PROXIES: {PROXIES}")
|
|
|
|
# Anna's Archive settings
|
|
AA_BASE_URL = env._AA_BASE_URL
|
|
AA_AVAILABLE_URLS = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
|
|
AA_AVAILABLE_URLS.extend(env._AA_ADDITIONAL_URLS.split(","))
|
|
AA_AVAILABLE_URLS = [url.strip() for url in AA_AVAILABLE_URLS if url.strip()]
|
|
|
|
# File format settings
|
|
SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
|
|
logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
|
|
|
|
# Complex language processing logic kept in config.py
|
|
BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
|
|
BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
|
|
if len(BOOK_LANGUAGE) == 0:
|
|
BOOK_LANGUAGE = ['en']
|
|
|
|
# Custom script settings with validation logic
|
|
CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
|
|
if CUSTOM_SCRIPT:
|
|
if not os.path.exists(CUSTOM_SCRIPT):
|
|
logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
|
|
CUSTOM_SCRIPT = ""
|
|
elif not os.access(CUSTOM_SCRIPT, os.X_OK):
|
|
logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
|
|
CUSTOM_SCRIPT = ""
|
|
|
|
# Debugging settings
|
|
if not env.USING_EXTERNAL_BYPASSER:
|
|
# Virtual display settings for debugging internal cloudflare bypasser
|
|
VIRTUAL_SCREEN_SIZE = (1024, 768)
|
|
RECORDING_DIR = env.LOG_DIR / "recording"
|
|
if env.DEBUG:
|
|
RECORDING_DIR.mkdir(parents=True, exist_ok=True)
|