starred/shelfmark
1
0
Fork 0
mirror of https://github.com/calibrain/shelfmark.git synced 2026-10-06 07:24:39 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Use a tmp name while downloading files (#99)

Browse Source 
Should help with #98
This commit is contained in:
CaliBrain
2025-03-20 00:56:54 -04:00
committed by GitHub
parent b93bdd59ac
commit c2e7acd473
9 changed files with 88 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
+2 -1
View File
@@ -9,7 +9,8 @@ from flask import url_for as flask_url_for
import typing
from logger import setup_logger
from config import FLASK_HOST, FLASK_PORT, FLASK_DEBUG, _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE
from config import _SUPPORTED_BOOK_LANGUAGE, BOOK_LANGUAGE
from env import FLASK_HOST, FLASK_PORT, FLASK_DEBUG
import backend
from models import SearchFilters
backend.py
+10 -2
View File
@@ -8,7 +8,8 @@ import subprocess
import os
from logger import setup_logger
from config import TMP_DIR, MAIN_LOOP_SLEEP_TIME, INGEST_DIR, CUSTOM_SCRIPT, USE_BOOK_TITLE
from config import CUSTOM_SCRIPT, CROSS_FILE_SYSTEM
from env import INGEST_DIR, TMP_DIR, MAIN_LOOP_SLEEP_TIME, USE_BOOK_TITLE
from models import book_queue, BookInfo, QueueStatus, SearchFilters
import book_manager
@@ -138,7 +139,14 @@ def _download_book(book_id: str) -> bool:
final_path = INGEST_DIR / book_name
if os.path.exists(book_path):
shutil.move(book_path, final_path)
if CROSS_FILE_SYSTEM:
logger.info(f"Copying book to ingest directory then renaming: {book_path} -> {final_path}.crdownload -> {final_path}")
tmp_path = final_path.with_name(final_path.name + ".crdownload")
shutil.move(book_path, tmp_path)
os.rename(tmp_path, final_path)
else:
logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}")
shutil.move(book_path, final_path)
return True
except Exception as e:
logger.error(f"Error downloading book: {e}")
book_manager.py
+2 -1
View File
@@ -7,7 +7,8 @@ from typing import List, Optional, Dict, Union
from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet
from logger import setup_logger
from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_DONATOR_KEY, AA_BASE_URL, USE_CF_BYPASS
from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL
from env import AA_DONATOR_KEY, USE_CF_BYPASS
from models import BookInfo, SearchFilters
import network
cloudflare_bypasser.py
+2 -1
View File
@@ -4,7 +4,8 @@ from DrissionPage import ChromiumOptions
from DrissionPage._functions.elements import ChromiumElementsList # type: ignore
from DrissionPage._pages.chromium_tab import ChromiumTab # type: ignore
from logger import setup_logger
from config import MAX_RETRY, DOCKERMODE, DEFAULT_SLEEP, PROXIES
from env import MAX_RETRY, DOCKERMODE, DEFAULT_SLEEP
from config import PROXIES
logger = setup_logger(__name__)
config.py
+35 -45
View File
@@ -3,49 +3,46 @@
import os
from pathlib import Path
import json
import env
from logger import setup_logger
logger = setup_logger(__name__)
with open("data/book-languages.json") as file:
_SUPPORTED_BOOK_LANGUAGE = json.load(file)
# Directory settings
BASE_DIR = Path(__file__).resolve().parent
LOG_DIR = Path("/var/log/cwa-book-downloader")
LOG_DIR.mkdir(exist_ok=True)
TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", 3600))
USE_BOOK_TITLE = os.getenv("USE_BOOK_TITLE", "false").lower() in ["true", "yes", "1", "y"]
logger.info(f"BASE_DIR: {BASE_DIR}")
env.LOG_DIR.mkdir(exist_ok=True)
# Create necessary directories
TMP_DIR.mkdir(exist_ok=True)
LOG_DIR.mkdir(exist_ok=True)
INGEST_DIR.mkdir(exist_ok=True)
env.TMP_DIR.mkdir(exist_ok=True)
env.INGEST_DIR.mkdir(exist_ok=True)
CROSS_FILE_SYSTEM = os.stat(env.TMP_DIR).st_dev != os.stat(env.INGEST_DIR).st_dev
logger.info(f"STAT TMP_DIR: {os.stat(env.TMP_DIR)}")
logger.info(f"STAT INGEST_DIR: {os.stat(env.INGEST_DIR)}")
logger.info(f"CROSS_FILE_SYSTEM: {CROSS_FILE_SYSTEM}")
# Network settings
MAX_RETRY = int(os.getenv("MAX_RETRY", 3))
DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", 5))
USE_CF_BYPASS = os.getenv("USE_CF_BYPASS", "true").lower() in ["true", "yes", "1", "y"]
# Proxy settings
PROXIES = {}
http_proxy = os.getenv("HTTP_PROXY", "").strip()
https_proxy = os.getenv("HTTPS_PROXY", "").strip()
if http_proxy:
PROXIES["http"] = http_proxy
if https_proxy:
PROXIES["https"] = https_proxy
if not PROXIES:
PROXIES = {}
if env.HTTP_PROXY:
PROXIES["http"] = env.HTTP_PROXY
if env.HTTPS_PROXY:
PROXIES["https"] = env.HTTPS_PROXY
logger.info(f"PROXIES: {PROXIES}")
# Anna's Archive settings
aa_available_urls = ["https://annas-archive.org", "https://annas-archive.se", "https://annas-archive.li"]
AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip("/")
aa_additional_urls = env.AA_ADDITIONAL_URLS.split(",")
aa_available_urls.extend(aa_additional_urls)
AA_BASE_URL = env._AA_BASE_URL
if AA_BASE_URL == "auto":
logger.info(f"AA_BASE_URL: auto, checking available urls {aa_available_urls}")
for url in aa_available_urls:
try:
import requests
@@ -54,40 +51,33 @@ if AA_BASE_URL == "auto":
AA_BASE_URL = url
break
except Exception as e:
print(f"Error checking {url}: {e}")
if AA_BASE_URL == "auto":
AA_BASE_URL = aa_available_urls[0]
logger.error(f"Error checking {url}: {e}")
if AA_BASE_URL == "auto":
AA_BASE_URL = aa_available_urls[0]
logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
# File format settings
SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").split(",")
SUPPORTED_FORMATS = env._SUPPORTED_FORMATS.split(",")
logger.info(f"SUPPORTED_FORMATS: {SUPPORTED_FORMATS}")
BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower().split(',')
# Complex language processing logic kept in config.py
BOOK_LANGUAGE = env._BOOK_LANGUAGE.split(',')
BOOK_LANGUAGE = [l for l in BOOK_LANGUAGE if l in [lang['code'] for lang in _SUPPORTED_BOOK_LANGUAGE]]
if len(BOOK_LANGUAGE) == 0:
BOOK_LANGUAGE = ['en']
# Custom script settings
CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
# check if the script is valid
# Custom script settings with validation logic
CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
if CUSTOM_SCRIPT:
if not os.path.exists(CUSTOM_SCRIPT):
logger.error(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
CUSTOM_SCRIPT = ""
elif not os.access(CUSTOM_SCRIPT, os.X_OK):
logger.error(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
CUSTOM_SCRIPT = ""
# API settings
FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
FLASK_PORT = int(os.getenv("FLASK_PORT", 5003))
FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() == "true"
# Logging settings
ENABLE_LOGGING = os.getenv("ENABLE_LOGGING", "true").lower() in ["true", "yes", "1", "y"]
LOG_FILE = LOG_DIR / "cwa-bookd-downloader.log"
MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", 5))
# Docker settings
DOCKERMODE = os.getenv('DOCKERMODE', 'false').lower().strip() in ['true', '1', 'yes', 'y']
if DOCKERMODE:
if env.DOCKERMODE and env.USE_CF_BYPASS:
from pyvirtualdisplay import Display
display = Display(visible=False, size=(800, 600))
display.start()
env.py
+31
View File
@@ -0,0 +1,31 @@
import os
from pathlib import Path
def string_to_bool(s: str) -> bool:
return s.lower() in ["true", "yes", "1", "y"]
LOG_DIR = Path("/var/log/cwa-book-downloader")
TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
MAX_RETRY = int(os.getenv("MAX_RETRY", "3"))
DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
AA_ADDITIONAL_URLS = os.getenv("AA_ADITIINAL_URLS", "")
_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
FLASK_PORT = int(os.getenv("FLASK_PORT", "5003"))
FLASK_DEBUG = string_to_bool(os.getenv("FLASK_DEBUG", "False"))
ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
# Logging settings
LOG_FILE = LOG_DIR / "cwa-bookd-downloader.log"
logger.py
+1 -1
View File
@@ -4,7 +4,7 @@ import logging
import sys
from pathlib import Path
from logging.handlers import RotatingFileHandler
from config import FLASK_DEBUG, LOG_FILE, ENABLE_LOGGING
from env import FLASK_DEBUG, LOG_FILE, ENABLE_LOGGING
def setup_logger(name: str, log_file: Path = LOG_FILE) -> logging.Logger:
"""Set up and configure a logger instance.
models.py
+1 -1
View File
@@ -6,7 +6,7 @@ from enum import Enum
from datetime import datetime, timedelta
from threading import Lock
from config import INGEST_DIR, STATUS_TIMEOUT
from env import INGEST_DIR, STATUS_TIMEOUT
class QueueStatus(str, Enum):
"""Enum for possible book queue statuses."""
network.py
+4 -2
View File
@@ -8,9 +8,11 @@ from typing import Optional
from urllib.parse import urlparse
from tqdm import tqdm
import cloudflare_bypasser
from logger import setup_logger
from config import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS, PROXIES
from config import PROXIES
from env import MAX_RETRY, DEFAULT_SLEEP, USE_CF_BYPASS
if USE_CF_BYPASS:
import cloudflare_bypasser
logger = setup_logger(__name__)
"""Configure urllib opener with appropriate headers."""