Fix: HTTP grab behavior, logging enforcement (#521)

entrypoint.sh

@@ -1,10 +1,54 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/shelfmark_entrypoint.log
 
-# Cleanup any existing files or folders in the log directory
-rm -rf $LOG_DIR/*
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+LOG_PIPE_DIR=""
+LOG_PIPE=""
+TEE_PID=""
+
+start_file_logging() {
+    local logfile="$1"
+
+    LOG_PIPE_DIR="$(mktemp -d)"
+    LOG_PIPE="${LOG_PIPE_DIR}/shelfmark-log.pipe"
+    mkfifo "$LOG_PIPE"
+
+    tee -a "$logfile" < "$LOG_PIPE" &
+    TEE_PID=$!
+
+    exec 3>&1 4>&2
+    exec > "$LOG_PIPE" 2>&1
+}
+
+stop_file_logging() {
+    if [ -z "${TEE_PID:-}" ]; then
+        return 0
+    fi
+
+    exec 1>&3 2>&4
+    exec 3>&- 4>&-
+
+    rm -f "$LOG_PIPE"
+    rmdir "$LOG_PIPE_DIR" 2>/dev/null || true
+
+    wait "$TEE_PID" 2>/dev/null || true
+    TEE_PID=""
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+    mkdir -p "$LOG_DIR"
+    LOG_FILE="${LOG_DIR}/shelfmark_entrypoint.log"
+
+    # Cleanup any existing files or folders in the log directory
+    rm -rf "$LOG_DIR"/*
+fi
 
 (
     if [ "$USING_TOR" = "true" ]; then
@@ -12,10 +56,16 @@ rm -rf $LOG_DIR/*
     fi
 )
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    start_file_logging "$LOG_FILE"
+fi
+
 echo "Starting entrypoint script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 set -e
 
 # Print build version
@@ -134,6 +184,24 @@ change_ownership /app
 change_ownership /var/log/shelfmark
 change_ownership /tmp/shelfmark
 
+# SeleniumBase (internal bypasser) writes a patched chromedriver binary (uc_driver)
+# into its own drivers directory. Some NAS/docker setups can apply restrictive ACLs
+# to extracted image layers that block non-root writes; ensure the runtime UID owns it.
+if [ "${USING_EXTERNAL_BYPASSER}" != "true" ]; then
+    set +e
+    SELENIUMBASE_DRIVERS_DIR=$(python3 -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')" 2>/dev/null)
+    set -e
+
+    if [ -n "$SELENIUMBASE_DRIVERS_DIR" ] && [ -d "$SELENIUMBASE_DRIVERS_DIR" ]; then
+        change_ownership "$SELENIUMBASE_DRIVERS_DIR"
+
+        # If the driver already exists, ensure it's executable for the runtime user.
+        if [ -f "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" ]; then
+            chmod +x "${SELENIUMBASE_DRIVERS_DIR}/uc_driver" || echo "Failed to chmod uc_driver, continuing..."
+        fi
+    fi
+fi
+
 # Test write to all folders
 make_writable ${CONFIG_DIR:-/config}
 make_writable ${INGEST_DIR:-/books}
@@ -263,8 +331,5 @@ UMASK_VALUE=${UMASK:-0022}
 echo "Setting umask to $UMASK_VALUE"
 umask $UMASK_VALUE
 
-# Stop logging
-exec 1>&3 2>&4
-exec 3>&- 4>&-
-
+stop_file_logging
 exec sudo -E -u "$USERNAME" HOME=/app $command

scripts/generate_env_docs.py

@@ -144,7 +144,7 @@ def _generate_bootstrap_env_docs() -> List[str]:
         },
         {
             "name": "ENABLE_LOGGING",
-            "description": "Enable file logging to LOG_ROOT/shelfmark/shelfmark.log.",
+            "description": "Enable file logging under LOG_ROOT/shelfmark/ (including shelfmark.log and startup logs).",
             "type": "boolean",
             "default": "true",
         },

shelfmark/download/http.py

@@ -176,8 +176,14 @@ def html_get_page(
     selector: Optional[network.AAMirrorSelector] = None,
     cancel_flag: Optional[Event] = None,
     status_callback: Optional[Callable[[str, Optional[str]], None]] = None,
+    allow_bypasser_fallback: bool = True,
 ) -> str:
-    """Fetch HTML content from a URL with retry mechanism."""
+    """Fetch HTML content from a URL with retry mechanism.
+
+    Args:
+        allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
+            instead of switching to the bypasser. Use for search operations.
+    """
     retry = retry if retry is not None else app_config.MAX_RETRY
     selector = selector or network.AAMirrorSelector()
     original_url = url
@@ -216,6 +222,15 @@ def html_get_page(
 
             # 403 = Cloudflare/DDoS-Guard protection
             if status == 403:
+                # If bypasser fallback is disabled, try mirrors instead
+                if not allow_bypasser_fallback:
+                    new_url = _try_rotation(original_url, current_url, selector)
+                    if new_url:
+                        current_url = new_url
+                        continue
+                    logger.warning(f"403 error, mirrors exhausted: {current_url}")
+                    return ""
+
                 if _is_cf_bypass_enabled() and not use_bypasser_now:
                     # Before switching to bypasser, check if cookies have become available
                     # (another concurrent download may have completed bypass and extracted cookies)

shelfmark/release_sources/direct_download.py

@@ -195,7 +195,7 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
         f"{filters_query}"
     )
 
-    html = downloader.html_get_page(url, selector=selector)
+    html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
     if not html:
         # Network/mirror exhaustion path bubbles up so API can notify clients
         raise SearchUnavailable("Unable to reach download source. Network restricted or mirrors are blocked.")
@@ -245,7 +245,7 @@ def get_book_info(book_id: str, fetch_download_count: bool = True) -> BookInfo:
     """
     url = f"{network.get_aa_base_url()}/md5/{book_id}"
     selector = network.AAMirrorSelector()
-    html = downloader.html_get_page(url, selector=selector)
+    html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
 
     if not html:
         raise Exception(f"Failed to fetch book info for ID: {book_id}")
@@ -398,7 +398,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str, fetch_download_coun
     if fetch_download_count:
         try:
             summary_url = f"{network.get_aa_base_url()}/dyn/md5/summary/{book_id}"
-            summary_response = downloader.html_get_page(summary_url, selector=network.AAMirrorSelector())
+            summary_response = downloader.html_get_page(summary_url, selector=network.AAMirrorSelector(), allow_bypasser_fallback=False)
             if summary_response:
                 summary_data = json.loads(summary_response)
                 if "downloads_total" in summary_data:

tor.sh

@@ -1,12 +1,29 @@
 #!/bin/bash
-LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
-mkdir -p $LOG_DIR
-LOG_FILE=${LOG_DIR}/shelfmark_tor.log
 
-exec 3>&1 4>&2
-exec > >(tee -a $LOG_FILE) 2>&1
+is_truthy() {
+    case "${1,,}" in
+        true|yes|1|y) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ENABLE_LOGGING_VALUE="${ENABLE_LOGGING:-true}"
+
+LOG_DIR=${LOG_ROOT:-/var/log/}/shelfmark
+LOG_FILE="${LOG_DIR}/shelfmark_tor.log"
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    mkdir -p "$LOG_DIR"
+
+    exec 3>&1 4>&2
+    exec > >(tee -a "$LOG_FILE") 2>&1
+fi
 echo "Starting tor script"
-echo "Log file: $LOG_FILE"
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    echo "Log file: $LOG_FILE"
+else
+    echo "File logging disabled (ENABLE_LOGGING=$ENABLE_LOGGING_VALUE)"
+fi
 
 set +x
 set -e
@@ -264,7 +281,7 @@ fi
 
 # Start a background circuit rotation process
 echo "[*] Starting Tor circuit rotation monitor..."
-(
+rotation_monitor() {
     rotation_count=0
 
     # Wait for initial stability
@@ -295,7 +312,13 @@ echo "[*] Starting Tor circuit rotation monitor..."
 
         sleep 300
     done
-) >> $LOG_FILE 2>&1 &
+}
+
+if is_truthy "$ENABLE_LOGGING_VALUE"; then
+    rotation_monitor >> "$LOG_FILE" 2>&1 &
+else
+    rotation_monitor &
+fi
 
 ROTATION_PID=$!
 echo "[✓] Tor circuit rotation monitor started in background (PID: $ROTATION_PID)"