From c2e7acd4731d9ead03c01f8c966ae47298092403 Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Thu, 20 Mar 2025 00:56:54 -0400 Subject: [PATCH] Use a tmp name while downloading files (#99) Should help with #98 --- app.py | 3 +- backend.py | 12 +++++-- book_manager.py | 3 +- cloudflare_bypasser.py | 3 +- config.py | 80 ++++++++++++++++++------------------------ env.py | 31 ++++++++++++++++ logger.py | 2 +- models.py | 2 +- network.py | 6 ++-- 9 files changed, 88 insertions(+), 54 deletions(-) create mode 100644 env.py diff --git a/app.py b/app.py index af560549..0d96942d 100644 --- a/app.py +++ b/app.py @@ -9,7 +9,8 @@ from flask import url_for as flask_url_for import typing from logger import setup_logger -from config import FLASK_HOST, FLASK_PORT, FLASK_DEBUG, _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE +from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE +from env import FLASK_HOST, FLASK_PORT, FLASK_DEBUG import backend from models import SearchFilters diff --git a/backend.py b/backend.py index 96eb23a2..b68e8ba6 100644 --- a/backend.py +++ b/backend.py @@ -8,7 +8,8 @@ import subprocess import os from logger import setup_logger -from config import TMP_DIR, MAIN_LOOP_SLEEP_TIME, INGEST_DIR, CUSTOM_SCRIPT, USE_BOOK_TITLE +from config import CUSTOM_SCRIPT, CROSS_FILE_SYSTEM +from env import INGEST_DIR, TMP_DIR, MAIN_LOOP_SLEEP_TIME, USE_BOOK_TITLE from models import book_queue, BookInfo, QueueStatus, SearchFilters import book_manager @@ -138,7 +139,14 @@ def _download_book(book_id: str) -> bool: final_path = INGEST_DIR / book_name if os.path.exists(book_path): - shutil.move(book_path, final_path) + if CROSS_FILE_SYSTEM: + logger.info(f"Copying book to ingest directory then renaming: {book_path} -> {final_path}.crdownload -> {final_path}") + tmp_path = final_path.with_name(final_path.name + ".crdownload") + shutil.move(book_path, tmp_path) + os.rename(tmp_path, final_path) + else: + logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}") + shutil.move(book_path, final_path) return True except Exception as e: logger.error(f"Error downloading book: {e}") diff --git a/book_manager.py b/book_manager.py index 832994c0..02f19709 100644 --- a/book_manager.py +++ b/book_manager.py @@ -7,7 +7,8 @@ from typing import List, Optional, Dict, Union from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet from logger import setup_logger -from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_DONATOR_KEY, AA_BASE_URL, USE_CF_BYPASS +from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL +from env import AA_DONATOR_KEY, USE_CF_BYPASS from models import BookInfo, SearchFilters import network diff --git a/cloudflare_bypasser.py b/cloudflare_bypasser.py index 6871fccf..4e755021 100644 --- a/cloudflare_bypasser.py +++ b/cloudflare_bypasser.py @@ -4,7 +4,8 @@ from DrissionPage import ChromiumOptions from DrissionPage._functions.elements import ChromiumElementsList # type: ignore from DrissionPage._pages.chromium_tab import ChromiumTab # type: ignore from logger import setup_logger -from config import MAX_RETRY, DOCKERMODE, DEFAULT_SLEEP, PROXIES +from env import MAX_RETRY, DOCKERMODE, DEFAULT_SLEEP +from config import PROXIES logger = setup_logger(__name__) diff --git a/config.py b/config.py index 09850ca2..8a7a61e0 100644 --- a/config.py +++ b/config.py @@ -3,49 +3,46 @@ import os from pathlib import Path import json +import env +from logger import setup_logger +logger = setup_logger(__name__) with open("data/book-languages.json") as file: _SUPPORTED_BOOK_LANGUAGE = json.load(file) # Directory settings BASE_DIR = Path(__file__).resolve().parent -LOG_DIR = Path("/var/log/cwa-book-downloader") -LOG_DIR.mkdir(exist_ok=True) - -TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader")) - -INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest")) -STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", 3600)) - -USE_BOOK_TITLE = os.getenv("USE_BOOK_TITLE", "false").lower() in ["true", "yes", "1", "y"] +logger.info(f"BASE_DIR: {BASE_DIR}") +env.LOG_DIR.mkdir(exist_ok=True) # Create necessary directories -TMP_DIR.mkdir(exist_ok=True) -LOG_DIR.mkdir(exist_ok=True) -INGEST_DIR.mkdir(exist_ok=True) +env.TMP_DIR.mkdir(exist_ok=True) +env.INGEST_DIR.mkdir(exist_ok=True) + +CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev +logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}") +logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}") +logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}") # Network settings -MAX_RETRY = int(os.getenv("MAX_RETRY", 3)) -DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", 5)) -USE_CF_BYPASS = os.getenv("USE_CF_BYPASS", "true").lower() in ["true", "yes", "1", "y"] # Proxy settings PROXIES = {} -http_proxy = os.getenv("HTTP_PROXY", "").strip() -https_proxy = os.getenv("HTTPS_PROXY", "").strip() -if http_proxy: - PROXIES["http"] = http_proxy -if https_proxy: - PROXIES["https"] = https_proxy -if not PROXIES: - PROXIES = {} +if env.HTTP_PROXY: + PROXIES["http"] = env.HTTP_PROXY +if env.HTTPS_PROXY: + PROXIES["https"] = env.HTTPS_PROXY +logger.info(f"PROXIES: {PROXIES}") # Anna's Archive settings aa_available_urls = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"] -AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip() -AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip("/") +aa_additional_urls = env.AA_ADDITIONAL_URLS.split(",") +aa_available_urls.extend(aa_additional_urls) + +AA_BASE_URL = env._AA_BASE_URL if AA_BASE_URL == "auto": + logger.info(f"AA_BASE_URL: auto, checking available urls {aa_available_urls}") for url in aa_available_urls: try: import requests @@ -54,40 +51,33 @@ if AA_BASE_URL == "auto": AA_BASE_URL = url break except Exception as e: - print(f"Error checking {url}: {e}") -if AA_BASE_URL == "auto": - AA_BASE_URL = aa_available_urls[0] + logger.error(f"Error checking {url}: {e}") + if AA_BASE_URL == "auto": + AA_BASE_URL = aa_available_urls[0] +logger.info(f"AA_BASE_URL: {AA_BASE_URL}") # File format settings -SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").split(",") +SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",") +logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}") -BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower().split(',') +# Complex language processing logic kept in config.py +BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',') BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]] if len(BOOK_LANGUAGE) == 0: BOOK_LANGUAGE = ['en'] -# Custom script settings -CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip() -# check if the script is valid +# Custom script settings with validation logic +CUSTOM_SCRIPT = env._CUSTOM_SCRIPT if CUSTOM_SCRIPT: if not os.path.exists(CUSTOM_SCRIPT): + logger.error(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist") CUSTOM_SCRIPT = "" elif not os.access(CUSTOM_SCRIPT, os.X_OK): + logger.error(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable") CUSTOM_SCRIPT = "" -# API settings -FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0") -FLASK_PORT = int(os.getenv("FLASK_PORT", 5003)) -FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() == "true" - -# Logging settings -ENABLE_LOGGING = os.getenv("ENABLE_LOGGING", "true").lower() in ["true", "yes", "1", "y"] -LOG_FILE = LOG_DIR / "cwa-bookd-downloader.log" -MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", 5)) - # Docker settings -DOCKERMODE = os.getenv('DOCKERMODE', 'false').lower().strip() in ['true', '1', 'yes', 'y'] -if DOCKERMODE: +if env.DOCKERMODE and env.USE_CF_BYPASS: from pyvirtualdisplay import Display display = Display(visible=False, size=(800, 600)) display.start() \ No newline at end of file diff --git a/env.py b/env.py new file mode 100644 index 00000000..73084bac --- /dev/null +++ b/env.py @@ -0,0 +1,31 @@ +import os +from pathlib import Path + +def string_to_bool(s: str) -> bool: + return s.lower() in ["true", "yes", "1", "y"] + +LOG_DIR = Path("/var/log/cwa-book-downloader") +TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader")) +INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest")) +STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600")) +USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false")) +MAX_RETRY = int(os.getenv("MAX_RETRY", "3")) +DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5")) +USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true")) +HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip() +HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip() +AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip() +_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip() +AA_ADDITIONAL_URLS = os.getenv("AA_ADITIINAL_URLS", "") +_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower() +_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower() +_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip() +FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0") +FLASK_PORT = int(os.getenv("FLASK_PORT", "5003")) +FLASK_DEBUG = string_to_bool(os.getenv("FLASK_DEBUG", "False")) +ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true")) +MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5")) +DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false")) + +# Logging settings +LOG_FILE = LOG_DIR / "cwa-bookd-downloader.log" \ No newline at end of file diff --git a/logger.py b/logger.py index 1747d605..95740457 100644 --- a/logger.py +++ b/logger.py @@ -4,7 +4,7 @@ import logging import sys from pathlib import Path from logging.handlers import RotatingFileHandler -from config import FLASK_DEBUG, LOG_FILE, ENABLE_LOGGING +from env import FLASK_DEBUG, LOG_FILE, ENABLE_LOGGING def setup_logger(name: str, log_file: Path = LOG_FILE) -> logging.Logger: """Set up and configure a logger instance. diff --git a/models.py b/models.py index 811a0cf5..742b66c5 100644 --- a/models.py +++ b/models.py @@ -6,7 +6,7 @@ from enum import Enum from datetime import datetime, timedelta from threading import Lock -from config import INGEST_DIR, STATUS_TIMEOUT +from env import INGEST_DIR, STATUS_TIMEOUT class QueueStatus(str, Enum): """Enum for possible book queue statuses.""" diff --git a/network.py b/network.py index bc0d0cef..1e4346e4 100644 --- a/network.py +++ b/network.py @@ -8,9 +8,11 @@ from typing import Optional from urllib.parse import urlparse from tqdm import tqdm -import cloudflare_bypasser from logger import setup_logger -from config import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS, PROXIES +from config import PROXIES +from env import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS +if USE_CF_BYPASS: + import cloudflare_bypasser logger = setup_logger(__name__) """Configure urllib opener with appropriate headers."""