Refactor Cloudflare Bypass & Fix Bugs (#126)

Dockerfile

@@ -18,7 +18,8 @@ ENV DEBIAN_FRONTEND=noninteractive \
     # UID/GID will be handled by entrypoint script, but TZ/Locale are still needed
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
-    LC_ALL=en_US.UTF-8
+    LC_ALL=en_US.UTF-8 \
+    APP_ENV=prod
 
 # Set ARG for build-time expansion (FLASK_PORT), ENV for runtime access
 ENV FLASK_PORT=8084
@@ -63,6 +64,11 @@ RUN pip install --no-cache-dir -r requirements.txt && \
     # Clean root's pip cache
     rm -rf /root/.cache
 
+# Add this line to grant read/execute permissions to others
+RUN chmod -R o+rx /usr/bin/chromium && \
+    chmod -R o+rx /usr/bin/chromedriver && \
+    chmod -R o+w /usr/local/lib/python3.10/site-packages/seleniumbase/drivers/
+
 # Our custom wanabe curl
 
 RUN echo "#!/bin/sh" > /usr/local/bin/pyrequests && \

app.py

@@ -10,7 +10,7 @@ import typing
 
 from logger import setup_logger
 from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE
-from env import FLASK_HOST, FLASK_PORT, FLASK_DEBUG
+from env import FLASK_HOST, FLASK_PORT, FLASK_DEBUG, APP_ENV
 import backend
 
 from models import SearchFilters
@@ -249,11 +249,12 @@ def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
     logger.error_trace(f"500 error: {error}")
     return jsonify({"error": "Internal server error"}), 500
 
+
 if __name__ == '__main__':
     # Register all routes with /request prefix
     register_dual_routes(app)
     
-    logger.info(f"Starting Flask application on {FLASK_HOST}:{FLASK_PORT}")
+    logger.info(f"Starting Flask application on {FLASK_HOST}:{FLASK_PORT} IN {APP_ENV} mode")
     app.run(
         host=FLASK_HOST,
         port=FLASK_PORT,

book_manager.py

@@ -339,7 +339,7 @@ def _get_download_url(link: str, title: str) -> str:
                 if countdown:
                     sleep_time = int(countdown[0].text)
                     logger.info(f"Waiting {sleep_time}s for {title}")
-                    time.sleep(sleep_time + 5)
+                    time.sleep(sleep_time)
                     url = _get_download_url(link, title)
             else:
                 url = download_links[0]['href']

cloudflare_bypasser.py

@@ -21,10 +21,21 @@ logger = setup_logger(__name__)
 network.init()
 
 DRIVER = None
+DISPLAY = None
 LAST_USED = None
 LOCKED = threading.Lock()
 TENTATIVE_CURRENT_URL = None
 
+def _reset_pyautogui_display_state():
+    try:
+        import pyautogui
+        import Xlib.display
+        pyautogui._pyautogui_x11._display = (
+                    Xlib.display.Display(os.environ['DISPLAY'])
+                )
+    except Exception as e:
+        logger.warning(f"Error resetting pyautogui display state: {e}")
+
 def _is_bypassed(sb) -> bool:
     try:
         title = sb.get_title().lower()
@@ -78,7 +89,6 @@ def _bypass(sb, max_retries: int = MAX_RETRY) -> None:
                 sb.uc_gui_click_captcha()
             except Exception as e:
                 time.sleep(DEFAULT_SLEEP)
-                sb.reconnect(DEFAULT_SLEEP)
                 sb.uc_gui_click_captcha()
 
         if _is_bypassed(sb):
@@ -127,8 +137,8 @@ def _get(url, retry : int = MAX_RETRY):
     try:
         logger.info(f"SB_GET: {url}")
         sb = _get_driver()
-        sb.uc_open_with_disconnect(url)
-        time.sleep(1)
+        sb.uc_open_with_reconnect(url, DEFAULT_SLEEP)
+        time.sleep(DEFAULT_SLEEP)
         _bypass(sb)
         return sb.page_source
     except Exception as e:
@@ -157,27 +167,49 @@ def _init_driver():
     return driver
 
 def _get_driver():
-    global DRIVER
+    global DRIVER, DISPLAY
     global LAST_USED
     LAST_USED = time.time()
+    if env.DOCKERMODE and env.USE_CF_BYPASS and not DISPLAY:
+        from pyvirtualdisplay import Display
+        display = Display(visible=False, size=(800, 600))
+        display.start()
+        logger.info("Display started")
+        DISPLAY = display
+        time.sleep(DEFAULT_SLEEP)
+        _reset_pyautogui_display_state()
     if not DRIVER:
         return _init_driver()
     return DRIVER
 
 def _reset_driver():
     logger.info("Resetting driver...")
-    global DRIVER
+    global DRIVER, DISPLAY
     if DRIVER:
-        DRIVER.quit()
+        try:
+            DRIVER.quit()
+            DRIVER = None
+        except Exception as e:
+            logger.warning(f"Error quitting driver: {e}")
+        time.sleep(0.5)
+    if DISPLAY:
+        try:
+            DISPLAY.stop()
+            DISPLAY = None
+        except Exception as e:
+            logger.warning(f"Error stopping display: {e}")
+        time.sleep(0.5)
     try:
-        os.system("pkill -f xvfb")
+        os.system("pkill -f Xvfb")
     except Exception as e:
-        logger.warning(f"Error killing xvfb: {e}")
+        logger.warning(f"Error killing Xvfb: {e}")
+    time.sleep(0.5)
     try:
         os.system("pkill -f chrom")
     except Exception as e:
         logger.warning(f"Error killing chrom: {e}")
-    DRIVER = None
+    time.sleep(0.5)
+
 
 def _cleanup_driver():
     global LOCKED

config.py

@@ -90,9 +90,3 @@ if CUSTOM_SCRIPT:
         logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
         CUSTOM_SCRIPT = ""
 
-# Docker settings
-if env.DOCKERMODE and env.USE_CF_BYPASS:
-    from pyvirtualdisplay import Display
-    display = Display(visible=False, size=(800, 600))
-    display.start()
-    logger.info("Display started")

docker-compose.dev.yml

@@ -10,6 +10,7 @@ services:
     environment:
       LOG_LEVEL: debug
       FLASK_DEBUG: true
-      USE_DOH: true
-      CUSTOM_DNS: cloudflare
+      APP_ENV: dev
+     # USE_DOH: true
+     # CUSTOM_DNS: cloudflare
 

docker-compose.tor.dev.yml

@@ -12,4 +12,5 @@ services:
       - NET_RAW
     environment:
       LOG_LEVEL: debug
-      FLASK_DEBUG: true
+      FLASK_DEBUG: true
+      APP_ENV: dev

docker-compose.tor.yml

@@ -9,6 +9,7 @@ services:
       FLASK_DEBUG: false
       ENABLE_TOR: true
       TZ: America/New_York
+      APP_ENV: prod
     ports:
       - 8084:8084
     restart: unless-stopped

docker-compose.yml

@@ -8,6 +8,9 @@ services:
       USE_BOOK_TITLE: true
       FLASK_DEBUG: false
       TZ: America/New_York
+      APP_ENV: prod
+      UID: 1000
+      GID: 100
     ports:
       - 8084:8084
     restart: unless-stopped

entrypoint.sh

@@ -38,5 +38,14 @@ change_ownership /app
 change_ownership /var/log/cwa-book-downloader
 change_ownership /cwa-book-ingest
 
-# Switch to the user (either newly created or existing) and execute the main command
-exec sudo -E -u "$USERNAME" python3 -m app 
+# Set the command to run based on the environment
+is_prod=$(echo "$APP_ENV" | tr '[:upper:]' '[:lower:]')
+if [ "$is_prod" = "prod" ]; then 
+    command="gunicorn -b 0.0.0.0:${FLASK_PORT:-8084} app:app"
+else
+    command="python3 app.py"
+fi
+
+echo "Running command: '$command' as '$USERNAME' in '$APP_ENV' mode"
+
+exec sudo -E -u "$USERNAME" $command

env.py

@@ -32,6 +32,7 @@ DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
 _CUSTOM_DNS = os.getenv("CUSTOM_DNS", "").strip()
 USE_DOH = string_to_bool(os.getenv("USE_DOH", "false"))
 BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
+APP_ENV = os.getenv("APP_ENV", "prod").lower()
 # Logging settings
 LOG_FILE = LOG_DIR / "cwa-bookd-downloader.log"
 

requirements.txt

@@ -6,3 +6,5 @@ pyvirtualdisplay
 dnspython
 pyautogui
 seleniumbase
+gunicorn
+python-xlib