From 43e554b8aef7a11e0d41189102ed5fd745cddf44 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 23 Jan 2026 17:34:44 +0000 Subject: [PATCH] Fix: HTTP grab behavior, logging enforcement (#521) --- entrypoint.sh | 89 +++++++++++++++++--- scripts/generate_env_docs.py | 2 +- shelfmark/download/http.py | 17 +++- shelfmark/release_sources/direct_download.py | 6 +- tor.sh | 39 +++++++-- 5 files changed, 128 insertions(+), 25 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index a710317d..7a2fa03f 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,10 +1,54 @@ #!/bin/bash -LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark -mkdir -p $LOG_DIR -LOG_FILE=${LOG_DIR}/shelfmark_entrypoint.log -# Cleanup any existing files or folders in the log directory -rm -rf $LOG_DIR/* +is_truthy() { + case "${1,,}" in + true|yes|1|y) return 0 ;; + *) return 1 ;; + esac +} + +ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}" +LOG_PIPE_DIR="" +LOG_PIPE="" +TEE_PID="" + +start_file_logging() { + local logfile="$1" + + LOG_PIPE_DIR="$(mktemp -d)" + LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe" + mkfifo "$LOG_PIPE" + + tee -a "$logfile" < "$LOG_PIPE" & + TEE_PID=$! + + exec 3>&1 4>&2 + exec > "$LOG_PIPE" 2>&1 +} + +stop_file_logging() { + if [ -z "${TEE_PID:-}" ]; then + return 0 + fi + + exec 1>&3 2>&4 + exec 3>&- 4>&- + + rm -f "$LOG_PIPE" + rmdir "$LOG_PIPE_DIR" 2>/dev/null || true + + wait "$TEE_PID" 2>/dev/null || true + TEE_PID="" +} + +if is_truthy "$ENABLE_LOGGING_VALUE"; then + LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark + mkdir -p "$LOG_DIR" + LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log" + + # Cleanup any existing files or folders in the log directory + rm -rf "$LOG_DIR"/* +fi ( if [ "$USING_TOR" = "true" ]; then @@ -12,10 +56,16 @@ rm -rf $LOG_DIR/* fi ) -exec 3>&1 4>&2 -exec > >(tee -a $LOG_FILE) 2>&1 +if is_truthy "$ENABLE_LOGGING_VALUE"; then + start_file_logging "$LOG_FILE" +fi + echo "Starting entrypoint script" -echo "Log file: $LOG_FILE" +if is_truthy "$ENABLE_LOGGING_VALUE"; then + echo "Log file: $LOG_FILE" +else + echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)" +fi set -e # Print build version @@ -134,6 +184,24 @@ change_ownership /app change_ownership /var/log/shelfmark change_ownership /tmp/shelfmark +# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver) +# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs +# to extracted image layers that block non-root writes; ensure the runtime UID owns it. +if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then + set +e + SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null) + set -e + + if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then + change_ownership "$SELENIUMBASE_DRIVERS_DIR" + + # If the driver already exists, ensure it's executable for the runtime user. + if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then + chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..." + fi + fi +fi + # Test write to all folders make_writable ${CONFIG_DIR:-/config} make_writable ${INGEST_DIR:-/books} @@ -263,8 +331,5 @@ UMASK_VALUE=${UMASK:-0022} echo "Setting umask to $UMASK_VALUE" umask $UMASK_VALUE -# Stop logging -exec 1>&3 2>&4 -exec 3>&- 4>&- - +stop_file_logging exec sudo -E -u "$USERNAME" HOME=/app $command diff --git a/scripts/generate_env_docs.py b/scripts/generate_env_docs.py index 3428e9fc..50947126 100755 --- a/scripts/generate_env_docs.py +++ b/scripts/generate_env_docs.py @@ -144,7 +144,7 @@ def _generate_bootstrap_env_docs() -> List[str]: }, { "name": "ENABLE_LOGGING", - "description": "Enable file logging to LOG_ROOT/shelfmark/shelfmark.log.", + "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).", "type": "boolean", "default": "true", }, diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index 69ebf64d..e3beb080 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -176,8 +176,14 @@ def html_get_page( selector: Optional[network.AAMirrorSelector] = None, cancel_flag: Optional[Event] = None, status_callback: Optional[Callable[[str, Optional[str]], None]] = None, + allow_bypasser_fallback: bool = True, ) -> str: - """Fetch HTML content from a URL with retry mechanism.""" + """Fetch HTML content from a URL with retry mechanism. + + Args: + allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation + instead of switching to the bypasser. Use for search operations. + """ retry = retry if retry is not None else app_config.MAX_RETRY selector = selector or network.AAMirrorSelector() original_url = url @@ -216,6 +222,15 @@ def html_get_page( # 403 = Cloudflare/DDoS-Guard protection if status == 403: + # If bypasser fallback is disabled, try mirrors instead + if not allow_bypasser_fallback: + new_url = _try_rotation(original_url, current_url, selector) + if new_url: + current_url = new_url + continue + logger.warning(f"403 error, mirrors exhausted: {current_url}") + return "" + if _is_cf_bypass_enabled() and not use_bypasser_now: # Before switching to bypasser, check if cookies have become available # (another concurrent download may have completed bypass and extracted cookies) diff --git a/shelfmark/release_sources/direct_download.py b/shelfmark/release_sources/direct_download.py index b0e2d5c4..fa15084a 100644 --- a/shelfmark/release_sources/direct_download.py +++ b/shelfmark/release_sources/direct_download.py @@ -195,7 +195,7 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: f"{filters_query}" ) - html = downloader.html_get_page(url, selector=selector) + html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False) if not html: # Network/mirror exhaustion path bubbles up so API can notify clients raise SearchUnavailable("Unable to reach download source. Network restricted or mirrors are blocked.") @@ -245,7 +245,7 @@ def get_book_info(book_id: str, fetch_download_count: bool = True) -> BookInfo: """ url = f"{network.get_aa_base_url()}/md5/{book_id}" selector = network.AAMirrorSelector() - html = downloader.html_get_page(url, selector=selector) + html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False) if not html: raise Exception(f"Failed to fetch book info for ID: {book_id}") @@ -398,7 +398,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str, fetch_download_coun if fetch_download_count: try: summary_url = f"{network.get_aa_base_url()}/dyn/md5/summary/{book_id}" - summary_response = downloader.html_get_page(summary_url, selector=network.AAMirrorSelector()) + summary_response = downloader.html_get_page(summary_url, selector=network.AAMirrorSelector(), allow_bypasser_fallback=False) if summary_response: summary_data = json.loads(summary_response) if "downloads_total" in summary_data: diff --git a/tor.sh b/tor.sh index 37b86a5f..e3c530e7 100644 --- a/tor.sh +++ b/tor.sh @@ -1,12 +1,29 @@ #!/bin/bash -LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark -mkdir -p $LOG_DIR -LOG_FILE=${LOG_DIR}/shelfmark_tor.log -exec 3>&1 4>&2 -exec > >(tee -a $LOG_FILE) 2>&1 +is_truthy() { + case "${1,,}" in + true|yes|1|y) return 0 ;; + *) return 1 ;; + esac +} + +ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}" + +LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark +LOG_FILE="${LOG_DIR}/shelfmark_tor.log" + +if is_truthy "$ENABLE_LOGGING_VALUE"; then + mkdir -p "$LOG_DIR" + + exec 3>&1 4>&2 + exec > >(tee -a "$LOG_FILE") 2>&1 +fi echo "Starting tor script" -echo "Log file: $LOG_FILE" +if is_truthy "$ENABLE_LOGGING_VALUE"; then + echo "Log file: $LOG_FILE" +else + echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)" +fi set +x set -e @@ -264,7 +281,7 @@ fi # Start a background circuit rotation process echo "[*] Starting Tor circuit rotation monitor..." -( +rotation_monitor() { rotation_count=0 # Wait for initial stability @@ -295,7 +312,13 @@ echo "[*] Starting Tor circuit rotation monitor..." sleep 300 done -) >> $LOG_FILE 2>&1 & +} + +if is_truthy "$ENABLE_LOGGING_VALUE"; then + rotation_monitor >> "$LOG_FILE" 2>&1 & +else + rotation_monitor & +fi ROTATION_PID=$! echo "[✓] Tor circuit rotation monitor started in background (PID: $ROTATION_PID)"