diff --git a/backend.py b/backend.py index 8a29565..941fa1f 100644 --- a/backend.py +++ b/backend.py @@ -95,11 +95,12 @@ def get_book_data(book_id: str) -> Tuple[Optional[bytes], str] : """ try: book_info = book_queue._book_data[book_id] - path = INGEST_DIR / f"{book_id}.epub" + path = book_info.download_path with open(path, "rb") as f: return f.read(), book_info.title except Exception as e: logger.error_trace(f"Error getting book data: {e}") + book_info.download_path = None return None, "" def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]: @@ -109,14 +110,14 @@ def _book_info_to_dict(book: BookInfo) -> Dict[str, Any]: if value is not None } -def _download_book(book_id: str) -> bool: +def _download_book(book_id: str) -> Optional[str]: """Download and process a book. Args: book_id: Book identifier Returns: - bool: True if download and processing successful + str: Path to the downloaded book if successful, None otherwise """ try: book_info = book_queue._book_data[book_id] @@ -136,26 +137,27 @@ def _download_book(book_id: str) -> bool: logger.info(f"Running custom script: {CUSTOM_SCRIPT}") subprocess.run([CUSTOM_SCRIPT, book_path]) + intermediate_path = INGEST_DIR / book_id # Without extension final_path = INGEST_DIR / book_name if os.path.exists(book_path): if CROSS_FILE_SYSTEM: - logger.info(f"Copying book to ingest directory then renaming: {book_path} -> {final_path}.crdownload -> {final_path}") - tmp_path = final_path.with_name(final_path.name + ".crdownload") + logger.info(f"Copying book to ingest directory then renaming: {book_path} -> {intermediate_path} -> {final_path}") try: - shutil.move(book_path, tmp_path) + shutil.move(book_path, intermediate_path) except Exception as e: logger.debug(f"Error moving book: {e}, will try copying instead") - shutil.copy(book_path, tmp_path) + shutil.copy(book_path, intermediate_path) os.remove(book_path) - os.rename(tmp_path, final_path) else: - logger.info(f"Moving book to ingest directory: {book_path} -> {final_path}") - shutil.move(book_path, final_path) - return True + logger.info(f"Moving book to ingest directory: {book_path} -> {intermediate_path}") + shutil.move(book_path, intermediate_path) + logger.info(f"Renaming book: {intermediate_path} -> {final_path}") + os.rename(intermediate_path, final_path) + return str(final_path) except Exception as e: logger.error_trace(f"Error downloading book: {e}") - return False + return None def download_loop() -> None: """Background thread for processing download queue.""" @@ -169,15 +171,17 @@ def download_loop() -> None: try: book_queue.update_status(book_id, QueueStatus.DOWNLOADING) - success = _download_book(book_id) - + download_path = _download_book(book_id) + if download_path: + book_queue.update_download_path(book_id, download_path) + new_status = ( - QueueStatus.AVAILABLE if success else QueueStatus.ERROR + QueueStatus.AVAILABLE if download_path else QueueStatus.ERROR ) book_queue.update_status(book_id, new_status) logger.info( - f"Book {book_id} download {'successful' if success else 'failed'}" + f"Book {book_id} download {'successful' if download_path else 'failed'}" ) except Exception as e: diff --git a/models.py b/models.py index 742b66c..a30b562 100644 --- a/models.py +++ b/models.py @@ -5,7 +5,7 @@ from typing import Dict, List, Optional from enum import Enum from datetime import datetime, timedelta from threading import Lock - +from pathlib import Path from env import INGEST_DIR, STATUS_TIMEOUT class QueueStatus(str, Enum): @@ -30,6 +30,7 @@ class BookInfo: size: Optional[str] = None info: Optional[Dict[str, List[str]]] = None download_urls: List[str] = field(default_factory=list) + download_path: Optional[str] = None class BookQueue: """Thread-safe book queue manager.""" @@ -62,6 +63,11 @@ class BookQueue: """Update status of a book in the queue.""" with self._lock: self._update_status(book_id, status) + + def update_download_path(self, book_id: str, download_path: str) -> None: + """Update the download path of a book in the queue.""" + with self._lock: + self._book_data[book_id].download_path = download_path def get_status(self) -> Dict[QueueStatus, Dict[str, BookInfo]]: """Get current queue status.""" @@ -82,10 +88,14 @@ class BookQueue: to_remove = [] for book_id, status in self._status.items(): + path = self._book_data[book_id].download_path + if path and not Path(path).exists(): + self._book_data[book_id].download_path = None + path = None + # Check for completed downloads if status == QueueStatus.AVAILABLE: - path = INGEST_DIR / f"{book_id}.epub" - if not path.exists(): + if not path: self._update_status(book_id, QueueStatus.DONE) # Check for stale status entries diff --git a/static/js/main.js b/static/js/main.js index 257e870..9f9144d 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -535,7 +535,7 @@ document.addEventListener('DOMContentLoaded', () => { }); let titleElement; - if (status.toLowerCase().includes('available')) { + if (book.download_path != null) { titleElement = utils.createElement('a', { href: `/request/api/localdownload?id=${book.id}`, target: '_blank', diff --git a/testing/E2E_test.py b/testing/E2E_test.py new file mode 100644 index 0000000..281e228 --- /dev/null +++ b/testing/E2E_test.py @@ -0,0 +1,149 @@ +import requests +import time +import os +import hashlib + +# Thee server is already running, so let's grab some of the env vars: +# Use absolute import since the script is run from the root directory +import env as SERVER_ENV +from backend import _sanitize_filename # Moved import to top level + +# Now let's test the server: +port = SERVER_ENV.FLASK_PORT +server_url = f"http://localhost:{port}" +book_title = "077484a10743e5dd5d151013e8c732f4" # "Moby Dick" +# Directory where downloads should appear +download_dir = SERVER_ENV.INGEST_DIR +# Timeout for waiting for download +download_timeout_seconds = 60 * 5 +# Polling interval +poll_interval_seconds = 5 + +# Helper function to check download status +def check_download_status(book_id): + print(f"Polling status for {book_id}...") + start_time = time.time() + while time.time() - start_time < download_timeout_seconds: + try: + status_response = requests.get(f"{server_url}/api/status") + status_response.raise_for_status() # Raise an exception for bad status codes (4xx or 5xx) + status_data = status_response.json() + except requests.exceptions.RequestException as e: + print(f"Error fetching status: {e}. Retrying...") + time.sleep(poll_interval_seconds) + continue + except ValueError: # Includes JSONDecodeError + print(f"Error decoding status JSON. Response text: {status_response.text}. Retrying...") + time.sleep(poll_interval_seconds) + continue + + # Check success conditions based on download_path + for status_key in ["available", "done"]: + if status_key in status_data and book_id in status_data[status_key]: + book_status_info = status_data[status_key].get(book_id) + # Check if the status info is a dictionary and has a non-empty download_path + if isinstance(book_status_info, dict) and book_status_info.get('download_path'): + print(f"Book {book_id} has download path '{book_status_info['download_path']}' in status '{status_key}'.") + return True, book_status_info + + # Check for error status + if "error" in status_data and book_id in status_data["error"]: + book_error_info = status_data["error"].get(book_id, "Unknown error") + print(f"Book {book_id} failed with error: {book_error_info}") + return False, book_error_info + + #print(f"Polling status for {book_id}... Status: {status_data}") + time.sleep(poll_interval_seconds) + + print(f"Timeout waiting for book {book_id} download path to appear.") + return False, None + +# --- Test Execution --- +print("--- Starting E2E Test ---") + +# Step 1 : Search for a book +print(f"Step 1: Searching for book '{book_title}' (moby dick)...") +search_params = {'query': book_title} +search_response = requests.get(f"{server_url}/api/search", params=search_params) +search_response.raise_for_status() +search_results = search_response.json() + +assert isinstance(search_results, list), f"Expected search results to be a list, got {type(search_results)}" +assert len(search_results) > 0, f"No books found for query: {book_title}" +print(f"Found {len(search_results)} potential matches.") + +# Assume the first result is the one we want +book_to_test = search_results[0] +book_id = book_to_test.get('id') +assert book_id, "First search result is missing an 'id'" +print(f"Selected book ID for testing: {book_id}") + +# Step 2 : Get book details +print(f"Step 2: Getting details for book ID: {book_id}...") +info_params = {'id': book_id} +info_response = requests.get(f"{server_url}/api/info", params=info_params) +info_response.raise_for_status() +book_details = info_response.json() + +assert isinstance(book_details, dict), f"Expected book details to be a dict, got {type(book_details)}" +assert book_details.get('id') == book_id, "Book details ID mismatch" +print(f"Successfully retrieved details for '{book_details.get('title', 'N/A')}'") + +# Step 3 : Queue the book for download +print(f"Step 3: Queuing download for book ID: {book_id}...") +download_params = {'id': book_id} +download_response = requests.get(f"{server_url}/api/download", params=download_params) +download_response.raise_for_status() +download_status = download_response.json() + +assert download_status.get('status') == 'queued', f"Expected status 'queued', got {download_status}" +print(f"Book {book_id} successfully queued for download.") + +# Step 4 : Check the download status until available or timeout +print(f"Step 4: Checking download status for book ID: {book_id} (timeout: {download_timeout_seconds}s)...") +is_available, final_status = check_download_status(book_id) + +assert is_available, f"Book download failed or timed out. Final status check: {final_status}" +print(f"Book {book_id} download confirmed as available.") + +# Step 5 : Verify the file exists locally (optional but good) +print(f"Step 5: Verifying downloaded file exists...") +# Depend if env.USE_TITLE is true or false, the filename will be different +if SERVER_ENV.USE_BOOK_TITLE: + # Ensure book_details is available; might need adjustment if Step 2 failed + # Assuming book_details was successfully fetched in Step 2 + title_to_sanitize = book_details.get('title', book_title) # Use fetched title if available + expected_filename = _sanitize_filename(title_to_sanitize) + ".epub" # Add extension +else: + expected_filename = f"{book_id}.epub" + +expected_filepath = os.path.join(download_dir, expected_filename) + +assert os.path.exists(expected_filepath), f"Expected downloaded file not found at: {expected_filepath}" +print(f"Verified file exists: {expected_filepath}") + +# Step 6 : Download the book +print(f"Step 6: Downloading book {book_id}...") +download_response = requests.get(f"{server_url}/request/api/localdownload?id={book_id}") +download_response.raise_for_status() +# Write book to temp file : +temp_file_path = os.path.join("/tmp", f"{book_id}.epub") +with open(temp_file_path, 'wb') as f: + f.write(download_response.content) + +# Compare the downloaded file to the expected file +# compare shasum of the two files +expected_sha256 = hashlib.sha256(open(expected_filepath, 'rb').read()).hexdigest() +downloaded_sha256 = hashlib.sha256(open(temp_file_path, 'rb').read()).hexdigest() +assert expected_sha256 == downloaded_sha256, f"Downloaded file SHA256 mismatch. Expected: {expected_sha256}, Got: {downloaded_sha256}" +print(f"Downloaded file SHA256 matches expected: {expected_sha256}") + +print("--- E2E Test Completed Successfully ---") + + + + + + + +