starred/shelfmark
1
0
Fork 0
mirror of https://github.com/calibrain/shelfmark.git synced 2026-10-04 12:21:18 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Fix errors and improve logging (#102)

Browse Source 
Fix #100
Fix #101
This commit is contained in:
CaliBrain
2025-03-22 17:35:12 -04:00
committed by GitHub
parent c2e7acd473
commit 93a3481750
7 changed files with 52 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
+6 -6
View File
@@ -119,7 +119,7 @@ def api_search() -> Union[Response, Tuple[Response, int]]:
books = backend.search_books(query, filters)
return jsonify(books)
except Exception as e:
logger.error(f"Search error: {e}")
logger.error_trace(f"Search error: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/info', methods=['GET'])
@@ -143,7 +143,7 @@ def api_info() -> Union[Response, Tuple[Response, int]]:
return jsonify(book)
return jsonify({"error": "Book not found"}), 404
except Exception as e:
logger.error(f"Info error: {e}")
logger.error_trace(f"Info error: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/download', methods=['GET'])
@@ -167,7 +167,7 @@ def api_download() -> Union[Response, Tuple[Response, int]]:
return jsonify({"status": "queued"})
return jsonify({"error": "Failed to queue book"}), 500
except Exception as e:
logger.error(f"Download error: {e}")
logger.error_trace(f"Download error: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/status', methods=['GET'])
@@ -182,7 +182,7 @@ def api_status() -> Union[Response, Tuple[Response, int]]:
status = backend.queue_status()
return jsonify(status)
except Exception as e:
logger.error(f"Status error: {e}")
logger.error_trace(f"Status error: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/localdownload', methods=['GET'])
@@ -218,7 +218,7 @@ def api_local_download() -> Union[Response, Tuple[Response, int]]:
)
except Exception as e:
logger.error(f"Local download error: {e}")
logger.error_trace(f"Local download error: {e}")
return jsonify({"error": str(e)}), 500
@app.errorhandler(404)
@@ -246,7 +246,7 @@ def internal_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
Returns:
flask.Response: JSON error message with 500 status.
"""
logger.error(f"500 error: {error}")
logger.error_trace(f"500 error: {error}")
return jsonify({"error": "Internal server error"}), 500
if __name__ == '__main__':
backend.py
+6 -6
View File
@@ -34,7 +34,7 @@ def search_books(query: str, filters: SearchFilters) -> List[Dict[str, Any]]:
books = book_manager.search_books(query, filters)
return [_book_info_to_dict(book) for book in books]
except Exception as e:
logger.error(f"Error searching books: {e}")
logger.error_trace(f"Error searching books: {e}")
return []
def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
@@ -50,7 +50,7 @@ def get_book_info(book_id: str) -> Optional[Dict[str, Any]]:
book = book_manager.get_book_info(book_id)
return _book_info_to_dict(book)
except Exception as e:
logger.error(f"Error getting book info: {e}")
logger.error_trace(f"Error getting book info: {e}")
return None
def queue_book(book_id: str) -> bool:
@@ -68,7 +68,7 @@ def queue_book(book_id: str) -> bool:
logger.info(f"Book queued: {book_info.title}")
return True
except Exception as e:
logger.error(f"Error queueing book: {e}")
logger.error_trace(f"Error queueing book: {e}")
return False
def queue_status() -> Dict[str, Dict[str, Any]]:
@@ -99,7 +99,7 @@ def get_book_data(book_id: str) -> Tuple[Optional[bytes], str] :
with open(path, "rb") as f:
return f.read(), book_info.title
except Exception as e:
logger.error(f"Error getting book data: {e}")
logger.error_trace(f"Error getting book data: {e}")
return None, ""
def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]:
@@ -149,7 +149,7 @@ def _download_book(book_id: str) -> bool:
shutil.move(book_path, final_path)
return True
except Exception as e:
logger.error(f"Error downloading book: {e}")
logger.error_trace(f"Error downloading book: {e}")
return False
def download_loop() -> None:
@@ -176,7 +176,7 @@ def download_loop() -> None:
)
except Exception as e:
logger.error(f"Error in download loop: {e}")
logger.error_trace(f"Error in download loop: {e}")
book_queue.update_status(book_id, QueueStatus.ERROR)
# Start download loop in background thread
book_manager.py
+4 -4
View File
@@ -84,7 +84,7 @@ def search_books(query: str, filters: SearchFilters) -> List[BookInfo]:
if book:
books.append(book)
except Exception as e:
logger.error(f"Failed to parse search result row: {e}")
logger.error_trace(f"Failed to parse search result row: {e}")
books.sort(
key=lambda x: (
@@ -115,7 +115,7 @@ def _parse_search_result_row(row: Tag) -> Optional[BookInfo]:
size=cells[10].find('span').next
)
except Exception as e:
logger.error(f"Error parsing search result row: {e}")
logger.error_trace(f"Error parsing search result row: {e}")
return None
def get_book_info(book_id: str) -> BookInfo:
@@ -307,7 +307,7 @@ def download_book(book_info: BookInfo, book_path: Path) -> bool:
return True
except Exception as e:
logger.error(f"Failed to download from {link}: {e}")
logger.error_trace(f"Failed to download from {link}: {e}")
continue
return False
@@ -321,7 +321,7 @@ def _get_download_url(link: str, title: str) -> str:
page = network.html_get_page(link)
url = json.loads(page).get("download_url")
else:
html = network.html_get_page(link, retry=0)
html = network.html_get_page(link)
if html == "":
return ""
cloudflare_bypasser.py
+2 -2
View File
@@ -177,7 +177,7 @@ def _init_browser(retry : int = MAX_RETRY) -> ChromiumTab:
if retry > 0:
_reset_browser()
else:
logger.error(f"Failed to initialize browser: {e}")
logger.error_trace(f"Failed to initialize browser: {e}")
raise e
return _init_browser(retry - 1)
@@ -189,6 +189,6 @@ def get(url : str, retry : int = MAX_RETRY) -> ChromiumTab:
except Exception as e:
if retry > 0:
return get(url, retry - 1)
logger.error(f"Failed to bypass Cloudflare for {url}: {e}")
logger.error_trace(f"Failed to bypass Cloudflare for {url}: {e}")
raise e
return defaultTab
config.py
+9 -4
View File
@@ -8,6 +8,10 @@ from logger import setup_logger
logger = setup_logger(__name__)
for key, value in env.__dict__.items():
if not key.startswith('_'):
logger.info(f"{key}: {value}")
with open("data/book-languages.json") as file:
_SUPPORTED_BOOK_LANGUAGE = json.load(file)
@@ -51,7 +55,7 @@ if AA_BASE_URL == "auto":
AA_BASE_URL = url
break
except Exception as e:
logger.error(f"Error checking {url}: {e}")
logger.error_trace(f"Error checking {url}: {e}")
if AA_BASE_URL == "auto":
AA_BASE_URL = aa_available_urls[0]
logger.info(f"AA_BASE_URL: {AA_BASE_URL}")
@@ -70,14 +74,15 @@ if len(BOOK_LANGUAGE) == 0:
CUSTOM_SCRIPT = env._CUSTOM_SCRIPT
if CUSTOM_SCRIPT:
if not os.path.exists(CUSTOM_SCRIPT):
logger.error(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} does not exist")
CUSTOM_SCRIPT = ""
elif not os.access(CUSTOM_SCRIPT, os.X_OK):
logger.error(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
logger.warn(f"CUSTOM_SCRIPT {CUSTOM_SCRIPT} is not executable")
CUSTOM_SCRIPT = ""
# Docker settings
if env.DOCKERMODE and env.USE_CF_BYPASS:
from pyvirtualdisplay import Display
display = Display(visible=False, size=(800, 600))
display.start()
display.start()
logger.info("Display started")
logger.py
+10 -2
View File
@@ -19,8 +19,16 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> logging.Logger:
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
# Add helper method for error logging with stack trace
def error_trace(self, msg, *args, **kwargs):
"""Log an error message with full stack trace."""
self.error(msg, *args, exc_info=True, **kwargs)
# Attach the helper method to the logger
logger.error_trace = error_trace.__get__(logger)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
'%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
)
# Console handler for Docker output
@@ -50,6 +58,6 @@ def setup_logger(name: str, log_file: Path = LOG_FILE) -> logging.Logger:
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
except Exception as e:
logger.error(f"Failed to create log file: {e}")
logger.error_trace(f"Failed to create log file: {e}", exc_info=True)
return logger
network.py
+15 -9
View File
@@ -36,8 +36,9 @@ def html_get_page(url: str, retry: int = MAX_RETRY, use_bypasser: bool = False)
Returns:
str: HTML content if successful, None otherwise
"""
response = None
try:
logger.debug(f"html_get_page: {url}, retry: {retry}, use_bypasser: {use_bypasser}")
if use_bypasser and USE_CF_BYPASS:
logger.info(f"GET Using Cloudflare Bypasser for: {url}")
response = cloudflare_bypasser.get(url)
@@ -56,19 +57,19 @@ def html_get_page(url: str, retry: int = MAX_RETRY, use_bypasser: bool = False)
except requests.exceptions.RequestException as e:
if retry == 0:
logger.error(f"Failed to fetch page: {url}, error: {e}")
logger.error_trace(f"Failed to fetch page: {url}, error: {e}")
return ""
if response.status_code == 404:
if response is not None and response.status_code == 404:
logger.warning(f"404 error for URL: {url}")
return ""
if response.status_code == 403:
logger.warning(f"403 error for URL: {url}. Should retry using cloudflare bypass.")
if response is not None and response.status_code == 403:
if use_bypasser:
logger.warning(f"403 error while using cloudflare bypass for URL: {url}")
return ""
use_bypasser = True
logger.warning(f"403 detected for URL: {url}. Should retry using cloudflare bypass.")
return html_get_page(url, retry - 1, True)
sleep_time = DEFAULT_SLEEP * (MAX_RETRY - retry + 1)
logger.warning(
@@ -107,9 +108,14 @@ def download_url(link: str, size: str = "") -> Optional[BytesIO]:
pbar.update(len(chunk))
pbar.close()
if buffer.tell() * 0.1 < total_size * 0.9:
# Check the content of the buffer if its HTML or binary
if response.headers.get('content-type', '').startswith('text/html'):
logger.warn(f"Failed to download content for {link}. Found HTML content instead.")
return None
return buffer
except requests.exceptions.RequestException as e:
logger.error(f"Failed to download from {link}: {e}")
logger.error_trace(f"Failed to download from {link}: {e}")
return None
def get_absolute_url(base_url: str, url: str) -> str: