From 909f4b403c4233f8d6802f455b9796c31ca9cbcf Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Sun, 16 Mar 2025 03:33:59 -0400 Subject: [PATCH] Multiple bug fixes (#93) --- backend.py | 3 +++ book_manager.py | 5 +++-- cloudflare_bypasser.py | 41 +++++++++++++++++++++++++++++------------ config.py | 12 ++++-------- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/backend.py b/backend.py index 4ebb57d..e84f4c5 100644 --- a/backend.py +++ b/backend.py @@ -4,6 +4,8 @@ import threading, time import shutil from pathlib import Path from typing import Dict, List, Optional, Any, Tuple +import subprocess +import os from logger import setup_logger from config import TMP_DIR, MAIN_LOOP_SLEEP_TIME, INGEST_DIR, CUSTOM_SCRIPT @@ -119,6 +121,7 @@ def _download_book(book_id: str) -> bool: raise Exception("Unkown error downloading book") if CUSTOM_SCRIPT: + logger.info(f"Running custom script: {CUSTOM_SCRIPT}") subprocess.run([CUSTOM_SCRIPT, book_path]) final_path = INGEST_DIR / f"{book_id}.{book_info.format}" diff --git a/book_manager.py b/book_manager.py index 84bf9d8..832994c 100644 --- a/book_manager.py +++ b/book_manager.py @@ -41,8 +41,9 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]: if filters.sort: filters_query += f"&sort={quote(filters.sort)}" - for value in filters.content: - filters_query += f"&content={quote(value)}" + if filters.content: + for value in filters.content: + filters_query += f"&content={quote(value)}" index = 1 for filter_type, filter_values in vars(filters).items(): diff --git a/cloudflare_bypasser.py b/cloudflare_bypasser.py index f3e0c74..6871fcc 100644 --- a/cloudflare_bypasser.py +++ b/cloudflare_bypasser.py @@ -8,16 +8,18 @@ from config import MAX_RETRY, DOCKERMODE, DEFAULT_SLEEP, PROXIES logger = setup_logger(__name__) +_defaultTab : ChromiumTab | None = None + def _search_recursively_shadow_root_with_iframe(ele : ChromiumElementsList) -> ChromiumElementsList | None: - if ele.shadow_root: - if ele.shadow_root.child().tag == "iframe": - return ele.shadow_root.child() - else: - for child in ele.children(): - result = _search_recursively_shadow_root_with_iframe(child) - if result: - return result - return None + if ele.shadow_root: + if ele.shadow_root.child().tag == "iframe": + return ele.shadow_root.child() + else: + for child in ele.children(): + result = _search_recursively_shadow_root_with_iframe(child) + if result: + return result + return None def _search_recursively_shadow_root_with_cf_input(ele : ChromiumElementsList) -> ChromiumElementsList | None: if ele.shadow_root: @@ -68,8 +70,20 @@ def _is_bypassed(driver: ChromiumTab) -> bool: try: title = driver.title.lower() body = driver.ele("tag:body").text.lower() - # TODO check body - return "just a moment" not in title + + # Check both title and body for verification messages + verification_texts = [ + "just a moment", + "verify you are human", + "verifying you are human", + "needs to review the security of your connection before proceeding" + ] + + for text in verification_texts: + if text in title.lower() or text in body.lower(): + return False + + return True except Exception as e: logger.debug(f"Error checking page title: {e}") return False @@ -134,9 +148,9 @@ def _genScraper() -> ChromiumPage: driver = ChromiumPage(addr_or_opts=options) return driver -_defaultTab = None def _reset_browser() -> None: + logger.info("Resetting chromiumbrowser") if not DOCKERMODE: return global _defaultTab @@ -157,10 +171,12 @@ def _init_browser(retry : int = MAX_RETRY) -> ChromiumTab: try: driver = _genScraper() _defaultTab = driver.get_tabs()[0] + return _defaultTab except Exception as e: if retry > 0: _reset_browser() else: + logger.error(f"Failed to initialize browser: {e}") raise e return _init_browser(retry - 1) @@ -172,5 +188,6 @@ def get(url : str, retry : int = MAX_RETRY) -> ChromiumTab: except Exception as e: if retry > 0: return get(url, retry - 1) + logger.error(f"Failed to bypass Cloudflare for {url}: {e}") raise e return defaultTab diff --git a/config.py b/config.py index 5706f65..a462d69 100644 --- a/config.py +++ b/config.py @@ -37,7 +37,7 @@ if http_proxy: if https_proxy: PROXIES["https"] = https_proxy if not PROXIES: - PROXIES = None + PROXIES = {} # Anna's Archive settings aa_available_urls = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"] @@ -46,16 +46,14 @@ AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip("/") if AA_BASE_URL == "auto": for url in aa_available_urls: try: - logger.debug(f"Checking {url}") + import requests response = requests.get(url) if response.status_code == 200: - logger.debug(f"Found good url: {url}") AA_BASE_URL = url break except Exception as e: - logger.debug(f"Error checking {url}: {e}") + print(f"Error checking {url}: {e}") if AA_BASE_URL == "auto": - logger.error("No good url found for Anna's Archive, falling back to default") AA_BASE_URL = aa_available_urls[0] # File format settings @@ -71,10 +69,8 @@ CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip() # check if the script is valid if CUSTOM_SCRIPT: if not os.path.exists(CUSTOM_SCRIPT): - logger.error(f"Custom script {CUSTOM_SCRIPT} does not exist") CUSTOM_SCRIPT = "" elif not os.access(CUSTOM_SCRIPT, os.X_OK): - logger.error(f"Custom script {CUSTOM_SCRIPT} is not executable") CUSTOM_SCRIPT = "" # API settings @@ -90,6 +86,6 @@ MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", 5)) # Docker settings DOCKERMODE = os.getenv('DOCKERMODE', 'false').lower().strip() in ['true', '1', 'yes', 'y'] if DOCKERMODE: - from pyvirtualdisplay import Display # type: ignore + from pyvirtualdisplay import Display display = Display(visible=False, size=(800, 600)) display.start() \ No newline at end of file