Multiple bug fixes (#93)

backend.py

@@ -4,6 +4,8 @@ import threading, time
 import shutil
 from pathlib import Path
 from typing import Dict, List, Optional, Any, Tuple
+import subprocess
+import os
 
 from logger import setup_logger
 from config import TMP_DIR, MAIN_LOOP_SLEEP_TIME, INGEST_DIR, CUSTOM_SCRIPT
@@ -119,6 +121,7 @@ def _download_book(book_id: str) -> bool:
             raise Exception("Unkown error downloading book")
 
         if CUSTOM_SCRIPT:
+            logger.info(f"Running custom script: {CUSTOM_SCRIPT}")
             subprocess.run([CUSTOM_SCRIPT, book_path])
 
         final_path = INGEST_DIR /  f"{book_id}.{book_info.format}"

book_manager.py

@@ -41,8 +41,9 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
     if filters.sort:
         filters_query += f"&sort={quote(filters.sort)}"
     
-    for value in filters.content:
-        filters_query += f"&content={quote(value)}"
+    if filters.content:
+        for value in filters.content:
+            filters_query += f"&content={quote(value)}"
 
     index = 1
     for filter_type, filter_values in vars(filters).items():

cloudflare_bypasser.py

@@ -8,16 +8,18 @@ from config import MAX_RETRY, DOCKERMODE, DEFAULT_SLEEP, PROXIES
 
 logger = setup_logger(__name__)
 
+_defaultTab : ChromiumTab | None = None
+
 def _search_recursively_shadow_root_with_iframe(ele : ChromiumElementsList) -> ChromiumElementsList | None:
-        if ele.shadow_root:
-            if ele.shadow_root.child().tag == "iframe":
-                return ele.shadow_root.child()
-        else:
-            for child in ele.children():
-                result = _search_recursively_shadow_root_with_iframe(child)
-                if result:
-                    return result
-        return None
+    if ele.shadow_root:
+        if ele.shadow_root.child().tag == "iframe":
+            return ele.shadow_root.child()
+    else:
+        for child in ele.children():
+            result = _search_recursively_shadow_root_with_iframe(child)
+            if result:
+                return result
+    return None
 
 def _search_recursively_shadow_root_with_cf_input(ele : ChromiumElementsList) -> ChromiumElementsList | None:
     if ele.shadow_root:
@@ -68,8 +70,20 @@ def _is_bypassed(driver: ChromiumTab) -> bool:
     try:
         title = driver.title.lower()
         body = driver.ele("tag:body").text.lower()
-        # TODO check body
-        return "just a moment" not in title
+        
+        # Check both title and body for verification messages
+        verification_texts = [
+            "just a moment",
+            "verify you are human",
+            "verifying you are human",
+            "needs to review the security of your connection before proceeding"
+        ]
+        
+        for text in verification_texts:
+            if text in title.lower() or text in body.lower():
+                return False
+                
+        return True
     except Exception as e:
         logger.debug(f"Error checking page title: {e}")
         return False
@@ -134,9 +148,9 @@ def _genScraper() -> ChromiumPage:
     driver = ChromiumPage(addr_or_opts=options)
     return driver
 
-_defaultTab = None
 
 def _reset_browser() -> None:
+    logger.info("Resetting chromiumbrowser")
     if not DOCKERMODE:
         return
     global _defaultTab
@@ -157,10 +171,12 @@ def _init_browser(retry : int = MAX_RETRY) -> ChromiumTab:
         try:
             driver = _genScraper()
             _defaultTab = driver.get_tabs()[0]
+            return _defaultTab
         except Exception as e:
             if retry > 0:
                 _reset_browser()
             else:
+                logger.error(f"Failed to initialize browser: {e}")
                 raise e
     return _init_browser(retry - 1)
 
@@ -172,5 +188,6 @@ def get(url : str, retry : int = MAX_RETRY) -> ChromiumTab:
     except Exception as e:
         if retry > 0:
             return get(url, retry - 1)
+        logger.error(f"Failed to bypass Cloudflare for {url}: {e}")
         raise e
     return defaultTab

config.py

@@ -37,7 +37,7 @@ if http_proxy:
 if https_proxy:
     PROXIES["https"] = https_proxy
 if not PROXIES:
-    PROXIES = None
+    PROXIES = {}
 
 # Anna's Archive settings
 aa_available_urls = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
@@ -46,16 +46,14 @@ AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip("/")
 if AA_BASE_URL == "auto":
     for url in aa_available_urls:
         try:
-            logger.debug(f"Checking {url}")
+            import requests
             response = requests.get(url)
             if response.status_code == 200:
-                logger.debug(f"Found good url: {url}")
                 AA_BASE_URL = url
                 break
         except Exception as e:
-            logger.debug(f"Error checking {url}: {e}")
+            print(f"Error checking {url}: {e}")
 if AA_BASE_URL == "auto":
-    logger.error("No good url found for Anna's Archive, falling back to default")
     AA_BASE_URL = aa_available_urls[0]
 
 # File format settings
@@ -71,10 +69,8 @@ CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
 # check if the script is valid
 if CUSTOM_SCRIPT:
     if not os.path.exists(CUSTOM_SCRIPT):
-        logger.error(f"Custom script {CUSTOM_SCRIPT} does not exist")
         CUSTOM_SCRIPT = ""
     elif not os.access(CUSTOM_SCRIPT, os.X_OK):
-        logger.error(f"Custom script {CUSTOM_SCRIPT} is not executable")
         CUSTOM_SCRIPT = ""
 
 # API settings
@@ -90,6 +86,6 @@ MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", 5))
 # Docker settings
 DOCKERMODE = os.getenv('DOCKERMODE', 'false').lower().strip() in ['true', '1', 'yes', 'y']
 if DOCKERMODE:
-    from pyvirtualdisplay import Display # type: ignore
+    from pyvirtualdisplay import Display
     display = Display(visible=False, size=(800, 600))
     display.start()