diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f4d2180..2ab0c09f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.10 + rev: v0.17.1 hooks: - id: ruff-check args: [--fix] diff --git a/docs/dev/plugin-settings.md b/docs/dev/plugin-settings.md index 55247a1c..6bab9a0e 100644 --- a/docs/dev/plugin-settings.md +++ b/docs/dev/plugin-settings.md @@ -23,12 +23,13 @@ from shelfmark.core.settings_registry import ( ActionButton, ) + @register_settings( - name="my_plugin", # Unique identifier - display_name="My Plugin", # Shown in sidebar - icon="wrench", # Icon name - order=100, # Sort order (lower = higher in list) - group="metadata_providers" # Optional: group in sidebar + name="my_plugin", # Unique identifier + display_name="My Plugin", # Shown in sidebar + icon="wrench", # Icon name + order=100, # Sort order (lower = higher in list) + group="metadata_providers", # Optional: group in sidebar ) def my_plugin_settings(): return [ @@ -46,6 +47,7 @@ def my_plugin_settings(): ), ] + def _test_connection(): # Perform connection test return {"success": True, "message": "Connected successfully!"} @@ -59,16 +61,16 @@ Single-line text input for strings. ```python TextField( - key="MY_SETTING", # Config key - label="Setting Name", # Display label - description="Help text", # Optional description below field - default="", # Default value + key="MY_SETTING", # Config key + label="Setting Name", # Display label + description="Help text", # Optional description below field + default="", # Default value placeholder="Enter value", # Placeholder text - max_length=100, # Optional max characters - required=False, # Is this field required? - requires_restart=False, # Does changing this need a restart? - show_when=None, # Conditional visibility (see below) - disabled_when=None, # Conditional disable (see below) + max_length=100, # Optional max characters + required=False, # Is this field required? + requires_restart=False, # Does changing this need a restart? + show_when=None, # Conditional visibility (see below) + disabled_when=None, # Conditional disable (see below) ) ``` @@ -98,7 +100,7 @@ NumberField( default=30, min_value=5, max_value=300, - step=1, # Increment step + step=1, # Increment step required=False, ) ``` @@ -160,13 +162,14 @@ Button that executes a callback function. Does not store a value. ```python ActionButton( - key="test_connection", # Unique key for the action - label="Test Connection", # Button text + key="test_connection", # Unique key for the action + label="Test Connection", # Button text description="Test the API connection", - style="primary", # "default", "primary", or "danger" - callback=my_callback_fn, # Function to execute + style="primary", # "default", "primary", or "danger" + callback=my_callback_fn, # Function to execute ) + def my_callback_fn(): """Callback must return dict with 'success' and 'message' keys.""" try: @@ -182,11 +185,11 @@ Display-only section heading with optional link. Does not store a value. ```python HeadingField( - key="section_heading", # Unique key - title="Configuration", # Heading text + key="section_heading", # Unique key + title="Configuration", # Heading text description="Configure the plugin settings below", link_url="https://example.com/docs", # Optional link - link_text="View Documentation", # Link text + link_text="View Documentation", # Link text ) ``` @@ -204,8 +207,14 @@ CustomComponentField( label="Request Policy Rules", description="Custom editor for policy defaults and matrix rules.", value_fields=[ - SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"), - SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"), + SelectField( + key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download" + ), + SelectField( + key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", + label="Default Audiobook Mode", + default="download", + ), TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]), ], wrap_in_field_wrapper=True, # use standard FieldWrapper label/description layout @@ -263,7 +272,7 @@ NumberField( disabled_when={ "field": "FEATURE_ENABLED", "value": False, - "reason": "Enable the feature first" + "reason": "Enable the feature first", }, ) ``` @@ -285,6 +294,7 @@ register_group( order=50, ) + # Then register settings to the group @register_settings( name="plugin_a", @@ -426,10 +436,7 @@ class MyProvider(MetadataProvider): def __init__(self, api_key: str = None): self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "") - self.base_url = config.get( - "MY_PROVIDER_BASE_URL", - "https://api.myprovider.com/v1" - ) + self.base_url = config.get("MY_PROVIDER_BASE_URL", "https://api.myprovider.com/v1") def is_available(self) -> bool: return bool(self.api_key) diff --git a/docs/dev/release-sources-plugin-guide.md b/docs/dev/release-sources-plugin-guide.md index 37709201..a4fbaa4f 100644 --- a/docs/dev/release-sources-plugin-guide.md +++ b/docs/dev/release-sources-plugin-guide.md @@ -66,10 +66,11 @@ This separation allows: A source has both a `ReleaseSource` (for searching) and a `DownloadHandler` (for downloading). They share the same `name` identifier: ```python -@register_source("my_source") # Search registration +@register_source("my_source") # Search registration class MySource(ReleaseSource): ... -@register_handler("my_source") # Download registration + +@register_handler("my_source") # Download registration class MyHandler(DownloadHandler): ... ``` @@ -142,10 +143,11 @@ from typing import List from shelfmark.metadata_providers import BookMetadata from shelfmark.release_sources import Release, ReleaseColumnConfig + class ReleaseSource(ABC): """Interface for searching a release source.""" - name: str # Internal identifier: "direct_download", "prowlarr" + name: str # Internal identifier: "direct_download", "prowlarr" display_name: str # User-facing name: "Direct Download", "Prowlarr" @abstractmethod @@ -191,14 +193,16 @@ def search(self, book: BookMetadata) -> List[Release]: if book.isbn_13: api_results = self.api.search_isbn(book.isbn_13) for r in api_results: - results.append(Release( - source="my_source", - source_id=r.id, - title=r.title, - format=r.format, - size=r.size, - # ... more fields - )) + results.append( + Release( + source="my_source", + source_id=r.id, + title=r.title, + format=r.format, + size=r.size, + # ... more fields + ) + ) # Strategy 2: Fallback to title + author search if not results: @@ -239,6 +243,7 @@ from typing import Callable, Optional from threading import Event from shelfmark.core.models import DownloadTask + class DownloadHandler(ABC): """Interface for executing downloads from a source.""" @@ -286,6 +291,7 @@ downloading it; the default returns `None`, which the UI reports as "can't inspe ```python from shelfmark.download.postprocess.packs import PackFile + def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None: """Return the release's files (release-relative paths + sizes), or None.""" torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page @@ -380,20 +386,20 @@ Standardized search result returned by all sources: ```python @dataclass class Release: - source: str # "direct_download", "prowlarr", "irc" - source_id: str # Unique ID within that source - title: str # Display title + source: str # "direct_download", "prowlarr", "irc" + source_id: str # Unique ID within that source + title: str # Display title - format: Optional[str] = None # File format: "epub", "mobi", "pdf" - size: Optional[str] = None # Human-readable: "5.2 MB" - size_bytes: Optional[int] = None # For sorting + format: Optional[str] = None # File format: "epub", "mobi", "pdf" + size: Optional[str] = None # Human-readable: "5.2 MB" + size_bytes: Optional[int] = None # For sorting download_url: Optional[str] = None # Direct download URL if available - info_url: Optional[str] = None # Link to tracker/info page + info_url: Optional[str] = None # Link to tracker/info page - protocol: Optional[str] = None # "http", "torrent", "usenet" - indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer" - seeders: Optional[int] = None # For torrents + protocol: Optional[str] = None # "http", "torrent", "usenet" + indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer" + seeders: Optional[int] = None # For torrents extra: Dict = field(default_factory=dict) # Source-specific metadata ``` @@ -415,7 +421,7 @@ Release( "preview": "https://example.com/cover.jpg", "quality": "high", "publisher": "Publisher Name", - } + }, ) ``` @@ -428,15 +434,15 @@ Source-agnostic task in the download queue: ```python @dataclass class DownloadTask: - task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID) - source: str # Handler name: "direct_download" - title: str # Display title + task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID) + source: str # Handler name: "direct_download" + title: str # Display title # Display info (from Release.extra or Release fields) author: Optional[str] = None format: Optional[str] = None size: Optional[str] = None - preview: Optional[str] = None # Cover image URL + preview: Optional[str] = None # Cover image URL # Runtime state (managed by orchestrator) priority: int = 0 @@ -454,8 +460,8 @@ Book information passed to `search()`: ```python @dataclass class BookMetadata: - provider: str # Metadata provider: "hardcover", "openlibrary" - provider_id: str # ID in provider's system + provider: str # Metadata provider: "hardcover", "openlibrary" + provider_id: str # ID in provider's system title: str provider_display_name: Optional[str] = None @@ -466,7 +472,7 @@ class BookMetadata: description: Optional[str] = None publisher: Optional[str] = None publish_year: Optional[int] = None - language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr" + language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr" genres: List[str] = field(default_factory=list) source_url: Optional[str] = None display_fields: List[DisplayField] = field(default_factory=list) @@ -481,13 +487,13 @@ class BookMetadata: ```python from shelfmark.release_sources import register_source, register_handler + @register_source("my_source") -class MySource(ReleaseSource): - ... +class MySource(ReleaseSource): ... + @register_handler("my_source") -class MyHandler(DownloadHandler): - ... +class MyHandler(DownloadHandler): ... ``` ### Registry Functions @@ -544,12 +550,13 @@ from shelfmark.core.settings_registry import ( is_value_from_env, ) + @register_settings( - name="my_source", # Unique identifier - display_name="My Source", # Shown in settings UI - icon="globe", # Icon name (optional) - order=60, # Sort order in sidebar - group="sources", # Grouping in sidebar + name="my_source", # Unique identifier + display_name="My Source", # Shown in settings UI + icon="globe", # Icon name (optional) + order=60, # Sort order in sidebar + group="sources", # Grouping in sidebar ) def my_source_settings(): """Register settings for My Source plugin.""" @@ -559,7 +566,6 @@ def my_source_settings(): title="My Source Configuration", description="Configure connection to My Source API", ), - PasswordField( key="MY_SOURCE_API_KEY", label="API Key", @@ -567,14 +573,12 @@ def my_source_settings(): required=True, env_supported=True, # Can be set via MY_SOURCE_API_KEY env var ), - TextField( key="MY_SOURCE_BASE_URL", label="Base URL", placeholder="https://api.mysource.com", env_supported=True, ), - NumberField( key="MY_SOURCE_TIMEOUT", label="Request Timeout", @@ -583,14 +587,12 @@ def my_source_settings(): min_value=5, max_value=120, ), - CheckboxField( key="MY_SOURCE_ENABLED", label="Enable My Source", description="Include in release searches", default=True, ), - ActionButton( key="test_connection", label="Test Connection", @@ -599,22 +601,19 @@ def my_source_settings(): ), ] + def test_my_source_connection(): """Test connection callback - returns result dict.""" try: # Get current API key - api_key = get_setting_value( - PasswordField(key="MY_SOURCE_API_KEY", label=""), - "my_source" - ) + api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source") if not api_key: return {"success": False, "message": "API key not configured"} # Test the connection response = requests.get( - "https://api.mysource.com/ping", - headers={"Authorization": f"Bearer {api_key}"} + "https://api.mysource.com/ping", headers={"Authorization": f"Bearer {api_key}"} ) if response.status_code == 200: @@ -648,6 +647,7 @@ from shelfmark.core.settings_registry import ( load_config_file, ) + # In your source/handler class def __init__(self): # Load from config file @@ -655,11 +655,9 @@ def __init__(self): self.api_key = config.get("MY_SOURCE_API_KEY") self.base_url = config.get("MY_SOURCE_BASE_URL", "https://api.mysource.com") + # Or use get_setting_value for ENV var priority -api_key = get_setting_value( - PasswordField(key="MY_SOURCE_API_KEY", label=""), - "my_source" -) +api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source") ``` ### Settings Priority @@ -689,6 +687,7 @@ from shelfmark.release_sources import ( LeadingCellType, ) + @classmethod def get_column_config(cls) -> ReleaseColumnConfig: return ReleaseColumnConfig( @@ -759,7 +758,7 @@ For `BADGE` render type, specify colors: ```python # Use frontend color map (defined in colorMaps.ts) -ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc. +ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc. ColumnColorHint(type="map", value="language") # Maps en→blue, de→yellow, etc. # Use static Tailwind class @@ -963,6 +962,7 @@ logger = setup_logger(__name__) # Settings Registration # --------------------------------------------------------------------------- + @register_settings( name="booknet", display_name="BookNet", @@ -980,7 +980,6 @@ def booknet_settings(): link_url="https://booknet.example.com/api-docs", link_text="API Documentation", ), - PasswordField( key="BOOKNET_API_KEY", label="API Key", @@ -988,14 +987,12 @@ def booknet_settings(): required=True, env_supported=True, ), - TextField( key="BOOKNET_BASE_URL", label="Base URL", placeholder="https://api.booknet.example.com", env_supported=True, ), - NumberField( key="BOOKNET_TIMEOUT", label="Request Timeout", @@ -1004,14 +1001,12 @@ def booknet_settings(): min_value=5, max_value=120, ), - CheckboxField( key="BOOKNET_PREFER_EPUB", label="Prefer EPUB format", description="Sort EPUB results first when available", default=True, ), - ActionButton( key="test_connection", label="Test Connection", @@ -1041,7 +1036,7 @@ def _test_booknet_connection() -> Dict: data = response.json() return { "success": True, - "message": f"Connected! {data.get('books_available', 0):,} books available" + "message": f"Connected! {data.get('books_available', 0):,} books available", } elif response.status_code == 401: return {"success": False, "message": "Invalid API key"} @@ -1058,6 +1053,7 @@ def _test_booknet_connection() -> Dict: # Release Source Implementation # --------------------------------------------------------------------------- + @register_source("booknet") class BookNetSource(ReleaseSource): """Search BookNet for book releases.""" @@ -1090,7 +1086,7 @@ class BookNetSource(ReleaseSource): # Sort by preference if self.prefer_epub: - releases.sort(key=lambda r: (0 if r.format == "epub" else 1)) + releases.sort(key=lambda r: 0 if r.format == "epub" else 1) return releases @@ -1164,7 +1160,9 @@ class BookNetSource(ReleaseSource): render_type=ColumnRenderType.BADGE, align=ColumnAlign.CENTER, width="70px", - color_hint=ColumnColorHint(type="static", value="bg-purple-500/20 text-purple-400"), + color_hint=ColumnColorHint( + type="static", value="bg-purple-500/20 text-purple-400" + ), ), ColumnSchema( key="extra.language", @@ -1202,6 +1200,7 @@ class BookNetSource(ReleaseSource): # Download Handler Implementation # --------------------------------------------------------------------------- + @register_handler("booknet") class BookNetHandler(DownloadHandler): """Handle downloads from BookNet.""" @@ -1264,8 +1263,7 @@ class BookNetHandler(DownloadHandler): # Phase 3: Move to final location safe_title = "".join( - c if c.isalnum() or c in " .-_" else "_" - for c in task.title[:100] + c if c.isalnum() or c in " .-_" else "_" for c in task.title[:100] ).strip() final_filename = f"{safe_title}.{task.format or 'epub'}" @@ -1274,7 +1272,9 @@ class BookNetHandler(DownloadHandler): # Avoid overwriting existing files counter = 1 while final_path.exists(): - final_path = Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}" + final_path = ( + Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}" + ) counter += 1 shutil.move(str(temp_path), str(final_path)) @@ -1363,6 +1363,7 @@ def is_available(self) -> bool: except Exception: return False + def search(self, book: BookMetadata) -> List[Release]: """Return empty list on errors, don't raise.""" try: @@ -1446,10 +1447,10 @@ from shelfmark.core.logger import setup_logger logger = setup_logger(__name__) -logger.debug("Detailed debug info") # Development only -logger.info("Normal operation info") # General progress -logger.warning("Recoverable issues") # Fallback used, retry happening -logger.error("Failures") # Unrecoverable errors +logger.debug("Detailed debug info") # Development only +logger.info("Normal operation info") # General progress +logger.warning("Recoverable issues") # Fallback used, retry happening +logger.error("Failures") # Unrecoverable errors ``` --- diff --git a/shelfmark/core/models.py b/shelfmark/core/models.py index 90a427a3..ff7ae9cc 100644 --- a/shelfmark/core/models.py +++ b/shelfmark/core/models.py @@ -158,6 +158,9 @@ class DownloadTask: status: QueueStatus = QueueStatus.QUEUED status_message: str | None = None download_path: str | None = None + # Every file the output step produced, download_path among them. Kept in memory + # only: history persists download_path alone. + download_paths: list[str] = field(default_factory=list) last_error_message: str | None = None last_error_type: str | None = None staged_path: str | None = None diff --git a/shelfmark/download/clients/base_handler.py b/shelfmark/download/clients/base_handler.py index 5bd24f0b..b8275db2 100644 --- a/shelfmark/download/clients/base_handler.py +++ b/shelfmark/download/clients/base_handler.py @@ -194,7 +194,9 @@ def _probe_completed_path(path: Path) -> tuple[bool, OSError | None]: return True, None -def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tuple[str, int, int], ...] | None: +def _completed_file_signature( + path: Path, content_type: str | None +) -> tuple[tuple[str, int, int], ...] | None: """Return sizes and mtimes for supported files in a completed path.""" from shelfmark.download.postprocess.policy import ( get_supported_audiobook_formats, @@ -202,9 +204,7 @@ def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tup ) formats = ( - get_supported_audiobook_formats() - if is_audiobook(content_type) - else get_supported_formats() + get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats() ) supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats} diff --git a/shelfmark/download/orchestrator.py b/shelfmark/download/orchestrator.py index 13371a30..31b74669 100644 --- a/shelfmark/download/orchestrator.py +++ b/shelfmark/download/orchestrator.py @@ -416,6 +416,28 @@ def get_book_path(task_id: str) -> tuple[str | None, DownloadTask | None]: return path, task +def get_book_paths(task_id: str) -> tuple[list[str], DownloadTask | None]: + """Every file a task's download produced that is still on disk. + + A multi-file release (chapters of an audiobook, a pack) lands as several files; + :func:`get_book_path` only knows the first. The list is trusted only while it + still contains download_path, so a later output that recorded no list cannot + hand back an earlier attempt's files. + """ + path, task = get_book_path(task_id) + if path is None or task is None: + return [], task + + if path not in task.download_paths: + return [path], task + + try: + return [p for p in task.download_paths if Path(p).is_file()], task + except OSError as e: + logger.error_trace(f"Error resolving book paths: {e}") + return [path], task + + def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]: """Get downloaded file data for a specific task. diff --git a/shelfmark/download/outputs/folder.py b/shelfmark/download/outputs/folder.py index 8cc91105..4ac37b85 100644 --- a/shelfmark/download/outputs/folder.py +++ b/shelfmark/download/outputs/folder.py @@ -275,4 +275,5 @@ def process_folder_output( message = f"Complete ({len(final_paths)} files)" status_callback("complete", message) + task.download_paths = [str(path) for path in final_paths] return str(final_paths[0]) diff --git a/shelfmark/download/zip_stream.py b/shelfmark/download/zip_stream.py new file mode 100644 index 00000000..5c6b56c6 --- /dev/null +++ b/shelfmark/download/zip_stream.py @@ -0,0 +1,68 @@ +"""Stream several downloaded files to a client as one ZIP, without staging the archive.""" + +import os +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + +_ZIP_STREAM_CHUNK_SIZE = 1024 * 1024 + + +class _ChunkSink: + """Write-only ZipFile target that hands the written bytes back to the caller. + + Having no tell() or seek() makes ZipFile treat it as an unseekable stream: each + entry's CRC and sizes go in a trailing data descriptor instead of being patched + back into the local header, so nothing has to be held until the entry ends. + """ + + def __init__(self) -> None: + self._chunks: list[bytes] = [] + + def write(self, data: bytes) -> int: + self._chunks.append(bytes(data)) + return len(data) + + def flush(self) -> None: + """Nothing is buffered beyond what drain() hands out.""" + + def close(self) -> None: + """Nothing to release; the caller drains what is left after ZipFile closes.""" + + def drain(self) -> Iterator[bytes]: + chunks, self._chunks = self._chunks, [] + yield from chunks + + +def zip_entry_names(paths: list[str]) -> list[tuple[Path, str]]: + """Pair each file with its name inside the archive. + + Names are relative to the files' common parent, so a book's own folders + (``Title/Part 01.m4b``) survive while the library path above them does not. + """ + files = [Path(path) for path in paths] + root = Path(os.path.commonpath([str(file.parent) for file in files])) + return [(file, file.relative_to(root).as_posix()) for file in files] + + +def stream_zip(paths: list[str]) -> Iterator[bytes]: + """Yield an uncompressed ZIP of ``paths``, reading each file once in chunks. + + Stored, not deflated: audiobooks and ebooks are already compressed, so deflating + them would cost CPU for next to nothing. ZIP64 kicks in per entry from the file + size, which a multi-gigabyte audiobook needs. + """ + sink = _ChunkSink() + with zipfile.ZipFile(sink, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive: + for path, arcname in zip_entry_names(paths): + info = zipfile.ZipInfo.from_file(path, arcname, strict_timestamps=False) + info.compress_type = zipfile.ZIP_STORED + with path.open("rb") as source, archive.open(info, "w") as target: + while chunk := source.read(_ZIP_STREAM_CHUNK_SIZE): + target.write(chunk) + yield from sink.drain() + yield from sink.drain() + yield from sink.drain() diff --git a/shelfmark/main.py b/shelfmark/main.py index 15330302..422cc244 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -6,12 +6,14 @@ import os import re import sqlite3 import time +import unicodedata from contextlib import suppress from datetime import UTC, datetime, timedelta from functools import wraps from importlib import import_module from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, NoReturn, cast +from urllib.parse import quote from flask import Flask, g, jsonify, request, send_file, send_from_directory, session from flask_cors import CORS @@ -55,7 +57,12 @@ from shelfmark.core.cwa_user_sync import upsert_cwa_user from shelfmark.core.download_history_service import DownloadHistoryService from shelfmark.core.external_user_linking import upsert_external_user from shelfmark.core.logger import setup_logger -from shelfmark.core.models import TERMINAL_QUEUE_STATUSES, QueueStatus, SearchFilters +from shelfmark.core.models import ( + TERMINAL_QUEUE_STATUSES, + QueueStatus, + SearchFilters, + build_filename, +) from shelfmark.core.notifications import ( NotificationContext, NotificationEvent, @@ -95,6 +102,7 @@ from shelfmark.core.viewer_redaction import ( ) from shelfmark.download import orchestrator as backend from shelfmark.download import warmup +from shelfmark.download.zip_stream import stream_zip from shelfmark.release_sources import ( BrowseRecord, Release, @@ -103,6 +111,7 @@ from shelfmark.release_sources import ( ) if TYPE_CHECKING: + from shelfmark.core.models import DownloadTask from shelfmark.metadata_providers import BookMetadata, MetadataProvider logger = setup_logger(__name__) @@ -1800,16 +1809,43 @@ def api_status() -> Response | tuple[Response, int]: return jsonify({"error": str(e)}), 500 +def _attachment_filename_options(download_name: str) -> dict[str, str]: + """Content-Disposition filename options, RFC 2231-encoded when not plain ASCII. + + Mirrors what send_file does, for responses built from a generator. + """ + try: + download_name.encode("ascii") + except UnicodeEncodeError: + simple = unicodedata.normalize("NFKD", download_name) + simple = simple.encode("ascii", "ignore").decode("ascii") + quoted = quote(download_name, safe="!#$&+^`|~") + return {"filename": simple, "filename*": f"UTF-8''{quoted}"} + return {"filename": download_name} + + +def _zip_download_response(file_paths: list[str], task: DownloadTask) -> Response: + """Send a multi-file download (audiobook chapters, a pack) as one streamed ZIP.""" + download_name = build_filename(task.title or "download", task.author, task.year, "zip") + response = Response(stream_zip(file_paths), mimetype="application/zip") + response.headers.set( + "Content-Disposition", "attachment", **_attachment_filename_options(download_name) + ) + return response + + @app.route("/api/localdownload", methods=["GET"]) @login_required def api_local_download() -> Response | tuple[Response, int]: - """Download an EPUB file from local storage if available. + """Download a completed book from local storage if available. Query Parameters: id (str): Book identifier (MD5 hash) Returns: - flask.Response: The EPUB file if found, otherwise an error response. + flask.Response: The file if found, a ZIP of all of them when the download + produced several (a live task only; history keeps just the first), otherwise + an error response. """ book_id = request.args.get("id", "") @@ -1820,8 +1856,8 @@ def api_local_download() -> Response | tuple[Response, int]: try: # The path, not the bytes: send_file streams it off disk, where reading it # first cost two resident copies of the whole book. - file_path, book_info = backend.get_book_path(book_id) - if file_path is None: + file_paths, book_info = backend.get_book_paths(book_id) + if not file_paths: # Fallback for dismissed/history entries where queue task may no longer exist. if download_history_service is not None: is_admin, db_user_id, can_access_status = _resolve_status_scope() @@ -1853,8 +1889,11 @@ def api_local_download() -> Response | tuple[Response, int]: ): return jsonify({"error": "File not found"}), 404 + if book_info is not None and len(file_paths) > 1: + return _zip_download_response(file_paths, book_info) + file_name = book_info.get_filename() if book_info is not None else Path(book_id).name - return send_file(file_path, download_name=file_name, as_attachment=True) + return send_file(file_paths[0], download_name=file_name, as_attachment=True) except _OPERATIONAL_ERRORS as e: logger.error_trace(f"Local download error: {e}") diff --git a/shelfmark/metadata_providers/databazeknih.py b/shelfmark/metadata_providers/databazeknih.py index c64045a8..ca877add 100644 --- a/shelfmark/metadata_providers/databazeknih.py +++ b/shelfmark/metadata_providers/databazeknih.py @@ -450,7 +450,7 @@ class DatabazeKnihProvider(MetadataProvider): book_id = _id_from_link(str(data.get("url", ""))) if not book_id: # Pages without a JSON-LD url still carry the id on the rating buttons - tag = soup.find(attrs={"data-typeid": True}) + tag = soup.select_one("[data-typeid]") book_id = str(tag["data-typeid"]) if tag else "" title = _clean_text(str(data.get("name", ""))) authors = [_clean_author(n) for n in _ld_names(data.get("author"))] @@ -539,7 +539,8 @@ class DatabazeKnihProvider(MetadataProvider): name = _clean_text(str(link.get("title") or link.get_text())) or None # The part number follows the link, up to the next series link text = [] - for elem in link.parent.next_siblings: + parent = link.parent + for elem in parent.next_siblings if parent else (): if isinstance(elem, Tag): if elem.find("a", href=re.compile(r"^/serie/")): break diff --git a/shelfmark/release_sources/direct_download/registry.py b/shelfmark/release_sources/direct_download/registry.py index 068a3272..28085181 100644 --- a/shelfmark/release_sources/direct_download/registry.py +++ b/shelfmark/release_sources/direct_download/registry.py @@ -40,7 +40,7 @@ def get_unavailable_reason( if not enabled_providers(providers): return ( "Direct Download is not configured. Add at least one Anna's Archive or OceanofPDF mirror URL in " - "Settings. Z-Library, LibGen, Welib, mirrors only download results found through " + "Settings. Z-Library, LibGen and Welib mirrors only download results found through " "Anna's Archive." ) return None diff --git a/skills/shelfmark/SKILL.md b/skills/shelfmark/SKILL.md index 120bd702..98a5551a 100644 --- a/skills/shelfmark/SKILL.md +++ b/skills/shelfmark/SKILL.md @@ -64,12 +64,12 @@ The script navigates to the main page, enters the search query, and waits for "M ```python # Navigate to main page first to reset SPA state -page.goto('http://localhost:8084/') -page.wait_for_load_state('networkidle') +page.goto("http://localhost:8084/") +page.wait_for_load_state("networkidle") # Enter search query and submit -page.fill('input[type="search"]', f'{title} {author}') -page.press('input[type="search"]', 'Enter') +page.fill('input[type="search"]', f"{title} {author}") +page.press('input[type="search"]', "Enter") # Wait for "Most relevant" to appear (indicates search results are fully rendered) page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000) @@ -87,18 +87,20 @@ def parse_books(page): btns = page.query_selector_all('button[data-action="download"]') books = [] for i, btn in enumerate(btns): - content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html() - - title_match = re.search(r']*>(.*?)', content, re.IGNORECASE | re.DOTALL) - title = title_match.group(1).strip() if title_match else 'Unknown' - - author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL) - author = author_match.group(1).strip() if author_match else 'Unknown' - - dl_match = re.search(r'•\s*([\d,]+)', content) - downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0 - - books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads}) + content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html() + + title_match = re.search(r"]*>(.*?)", content, re.IGNORECASE | re.DOTALL) + title = title_match.group(1).strip() if title_match else "Unknown" + + author_match = re.search( + r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL + ) + author = author_match.group(1).strip() if author_match else "Unknown" + + dl_match = re.search(r"•\s*([\d,]+)", content) + downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0 + + books.append({"index": i, "title": title, "author": author, "downloads": downloads}) return books ``` @@ -106,19 +108,19 @@ def parse_books(page): ```python # Filter books matching the search criteria -matching_books = [b for b in books if matches_search(b['title'], b['author'], title, author)] +matching_books = [b for b in books if matches_search(b["title"], b["author"], title, author)] if matching_books: - best = max(matching_books, key=lambda x: x['downloads']) - + best = max(matching_books, key=lambda x: x["downloads"]) + # CRITICAL: Click on the article h3 to open detail view - h3s = page.query_selector_all('article h3') - if best['index'] < len(h3s): - h3s[best['index']].click() + h3s = page.query_selector_all("article h3") + if best["index"] < len(h3s): + h3s[best["index"]].click() page.wait_for_timeout(300) - + # Then click the download button - btn = page.query_selector_all('button[data-action="download"]')[best['index']] + btn = page.query_selector_all('button[data-action="download"]')[best["index"]] btn.click() page.wait_for_timeout(500) ``` @@ -130,14 +132,14 @@ if matching_books: ```python for i in range(60): page.wait_for_timeout(5000) - activity_text = page.inner_text('aside') - - if 'IN PROGRESS' in activity_text: + activity_text = page.inner_text("aside") + + if "IN PROGRESS" in activity_text: print("Download started!") - elif 'Complete' in activity_text or 'Saved' in activity_text: + elif "Complete" in activity_text or "Saved" in activity_text: print("Download complete!") break - elif 'No activity' in activity_text: + elif "No activity" in activity_text: print("Download not started") break ``` @@ -146,7 +148,7 @@ for i in range(60): ```python # Click "Clear Completed" using JavaScript -page.evaluate(''' +page.evaluate(""" () => { for (const b of document.querySelectorAll('button')) { if (b.textContent.includes('Clear Completed')) { @@ -155,7 +157,7 @@ page.evaluate(''' } } } -''') +""") page.wait_for_timeout(2000) ``` diff --git a/skills/shelfmark/download_books.py b/skills/shelfmark/download_books.py index 1d3deeda..0b8b02d0 100755 --- a/skills/shelfmark/download_books.py +++ b/skills/shelfmark/download_books.py @@ -1,41 +1,62 @@ #!/usr/bin/env python3 """Shelfmark Book Downloader - Downloads books from a local Shelfmark instance.""" +from __future__ import annotations + +import argparse +import contextlib +import json +import re +import sys +import time +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from playwright.sync_api import Error as PlaywrightError from playwright.sync_api import sync_playwright -import re, time, sys, argparse, json -from urllib.parse import quote -SHELFMARK_URL = 'http://localhost:8084/' +if TYPE_CHECKING: + from playwright.sync_api import Page -def parse_books(page): +Book = dict[str, Any] + +SHELFMARK_URL = "http://localhost:8084/" + + +def parse_books(page: Page) -> list[Book]: btns = page.query_selector_all('button[data-action="download"]') books = [] for i, btn in enumerate(btns): - content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html() - - title_match = re.search(r']*>(.*?)', content, re.IGNORECASE | re.DOTALL) - title = title_match.group(1).strip() if title_match else 'Unknown' - - author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL) - author = author_match.group(1).strip() if author_match else 'Unknown' - - dl_match = re.search(r'•\s*([\d,]+)', content) - downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0 - - books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads}) + content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html() + + title_match = re.search(r"]*>(.*?)", content, re.IGNORECASE | re.DOTALL) + title = title_match.group(1).strip() if title_match else "Unknown" + + author_match = re.search( + r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL + ) + author = author_match.group(1).strip() if author_match else "Unknown" + + dl_match = re.search(r"•\s*([\d,]+)", content) + downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0 + + books.append({"index": i, "title": title, "author": author, "downloads": downloads}) return books -def do_search(page, title, author, search_type="author"): + +def do_search( + page: Page, title: str, author: str, search_type: str = "author" +) -> tuple[list[Book], list[Book]] | None: """Search with title+author, fall back to title only""" # Navigate to main page first to reset SPA state page.goto(SHELFMARK_URL) - page.wait_for_load_state('networkidle') - + page.wait_for_load_state("networkidle") + # Enter search query and submit - search_query = f'{title} {author}' if search_type == "author" else title + search_query = f"{title} {author}" if search_type == "author" else title page.fill('input[type="search"]', search_query) - page.press('input[type="search"]', 'Enter') - + page.press('input[type="search"]', "Enter") + # Wait for "Most relevant" to appear (indicates search results are fully rendered) try: page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000) @@ -44,227 +65,252 @@ def do_search(page, title, author, search_type="author"): print(" -> Trying title only...") sys.stdout.flush() return do_search(page, title, author, search_type="title") - else: - print(" >> Timeout waiting for search results") - sys.stdout.flush() - return None - + print(" >> Timeout waiting for search results") + sys.stdout.flush() + return None + books = parse_books(page) - - def clean_title_for_match(book_title, search_title): + + def clean_title_for_match(book_title: str, search_title: str) -> bool: """Clean book title to check if it matches the search title""" # Remove common subtitle patterns - clean = re.sub(r'\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$', '', book_title, flags=re.IGNORECASE) + clean = re.sub( + r"\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$", + "", + book_title, + flags=re.IGNORECASE, + ) clean = clean.strip() # Remove trailing punctuation - clean = clean.rstrip(':,;.') + clean = clean.rstrip(":,;.") return clean.lower().strip() == search_title.lower().strip() - - def matches_search(book_title, book_author, search_title, search_author): + + def matches_search( + book_title: str, book_author: str, search_title: str, search_author: str + ) -> bool: """Check if book matches search criteria more strictly""" title_match = search_title.lower() in book_title.lower() author_match = search_author.lower() in book_author.lower() - + # For title+author search, ensure title starts with search title (not just contains it) if title_match and author_match: - return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower()) - + return clean_title_for_match(book_title, search_title) or book_title.lower().startswith( + search_title.lower() + ) + # Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann") if title_match: author_parts = search_author.lower().split() if len(author_parts) >= 2: reversed_author = f"{author_parts[-1]} {author_parts[0]}" - if reversed_author in book_author.lower() or book_author.lower().startswith(reversed_author): - return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower()) - + if reversed_author in book_author.lower() or book_author.lower().startswith( + reversed_author + ): + return clean_title_for_match( + book_title, search_title + ) or book_title.lower().startswith(search_title.lower()) + return False - + if search_type == "author": - matching = [b for b in books if matches_search(b['title'], b['author'], title, author)] + matching = [b for b in books if matches_search(b["title"], b["author"], title, author)] print(f" With author: {len(books)} total, {len(matching)} matching") for b in books[:3]: print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})") sys.stdout.flush() - + if matching: # Check if any matching book is already in the download queue - try: - sidebar_text = page.inner_text('aside') - if 'IN PROGRESS' in sidebar_text: + with contextlib.suppress(PlaywrightError): + sidebar_text = page.inner_text("aside") + if "IN PROGRESS" in sidebar_text: # Get the title of the book currently downloading - current_download = sidebar_text.split('IN PROGRESS')[1].split('—')[0].strip().split('\n')[0].strip() + current_download = ( + sidebar_text.split("IN PROGRESS")[1] + .split("—")[0] + .strip() + .split("\n")[0] + .strip() + ) # Check if the current download matches our search if title.lower() in current_download.lower(): print(f" >> '{current_download}' already downloading, skipping") sys.stdout.flush() return None - except: - pass - + if matching: return matching, books - + # Fall back to title only print(" -> Trying title only...") sys.stdout.flush() return do_search(page, title, author, search_type="title") - + # Title-only search - prefer exact matches first - matching = [b for b in books if title.lower() in b['title'].lower()] - exact_matches = [b for b in matching if clean_title_for_match(b['title'], title)] - + matching = [b for b in books if title.lower() in b["title"].lower()] + exact_matches = [b for b in matching if clean_title_for_match(b["title"], title)] + if exact_matches: matching = exact_matches - - print(f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)") + + print( + f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)" + ) for b in books[:5]: print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})") sys.stdout.flush() - + if not matching: return None - + return (matching, books) -def download_book(page, title, author): + +def download_book(page: Page, title: str, author: str) -> None: result = do_search(page, title, author) if not result: print("\n >>> NOT FOUND") sys.stdout.flush() return - - matching, all_books = result + + matching, _all_books = result if not matching: print("\n >>> NOT FOUND") sys.stdout.flush() return - + # Find the best book that is not disabled btns = page.query_selector_all('button[data-action="download"]') best = None - for b in sorted(matching, key=lambda x: x['downloads'], reverse=True): - if b['index'] < len(btns) and not btns[b['index']].is_disabled(): + for b in sorted(matching, key=lambda x: x["downloads"], reverse=True): + if b["index"] < len(btns) and not btns[b["index"]].is_disabled(): best = b break - + if not best: print("\n >>> All matching books are already in download queue") sys.stdout.flush() return - - print(f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]") + + print( + f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]" + ) sys.stdout.flush() - + # Click on the article h3 to open detail view (required for download to work) - h3s = page.query_selector_all('article h3') - if best['index'] < len(h3s): - h3s[best['index']].click() + h3s = page.query_selector_all("article h3") + if best["index"] < len(h3s): + h3s[best["index"]].click() page.wait_for_timeout(300) else: print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)") sys.stdout.flush() - + # Click the download button - btn = btns[best['index']] + btn = btns[best["index"]] btn.click() - + # Wait a moment for the click to register page.wait_for_timeout(500) - print(f" >> Click sent") + print(" >> Click sent") sys.stdout.flush() - + # Check sidebar immediately time.sleep(2) try: - txt = page.inner_text('aside') - if 'IN PROGRESS' in txt: - print(f" >> Download started!") + txt = page.inner_text("aside") + if "IN PROGRESS" in txt: + print(" >> Download started!") sys.stdout.flush() else: print(f" >> Sidebar after 2s: {txt[:150]}") sys.stdout.flush() except: - print(f" >> Sidebar not visible after 2s") + print(" >> Sidebar not visible after 2s") sys.stdout.flush() - + # Wait for download to complete - check sidebar every 2 seconds, max 5 minutes last_state = None - for attempt in range(150): + for _attempt in range(150): time.sleep(2) try: - txt = page.inner_text('aside') - if 'IN PROGRESS' in txt: - state = 'downloading' - elif 'Complete' in txt or 'Saved' in txt or 'No activity' in txt: - state = 'done' + txt = page.inner_text("aside") + if "IN PROGRESS" in txt: + state = "downloading" + elif "Complete" in txt or "Saved" in txt or "No activity" in txt: + state = "done" else: - state = 'other' - + state = "other" + if state != last_state: - if state == 'downloading': - print(f" >> Download started!") - elif state == 'done': - if 'No activity' in txt: - print(f" >> Download not started (no activity)") + if state == "downloading": + print(" >> Download started!") + elif state == "done": + if "No activity" in txt: + print(" >> Download not started (no activity)") else: - print(f" >> Download complete!") + print(" >> Download complete!") last_state = state - - if state == 'done': + + if state == "done": break except: if last_state is None: - print(f" >> Sidebar not visible yet") - last_state = 'no_sidebar' + print(" >> Sidebar not visible yet") + last_state = "no_sidebar" sys.stdout.flush() else: - if last_state != 'done': + if last_state != "done": print(" >> Timeout waiting for download") sys.stdout.flush() -def clear_completed_downloads(page): + +def clear_completed_downloads(page: Page) -> None: """Clear completed downloads from previous sessions""" - page.evaluate(''' + page.evaluate(""" () => { for (const b of document.querySelectorAll('button')) { if (b.textContent.includes('Clear Completed')) { b.click(); return; } } } - ''') + """) page.wait_for_timeout(2000) -def check_calibre(title): + +def check_calibre(title: str) -> bool: """Check if a book is already in the calibre database.""" import subprocess + cmd = f'calibredb list --search title:="{title}"' - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + result = subprocess.run(cmd, check=False, shell=True, capture_output=True, text=True) output = result.stdout.strip() # calibredb output has header line "id title authors", count non-header lines - lines = [l.strip() for l in output.split('\n') if l.strip()] + lines = [l.strip() for l in output.split("\n") if l.strip()] # Remove header if present (first line starts with 'id') - if lines and lines[0].startswith('id'): + if lines and lines[0].startswith("id"): lines = lines[1:] return len(lines) > 0 -def load_books_from_json(json_str, source): + +def load_books_from_json(json_str: str, source: str) -> list[tuple[str, str]]: """Load books from a JSON string or file path.""" try: data = json.loads(json_str) except json.JSONDecodeError as e: print(f"Error: Invalid JSON in {source}: {e}") sys.exit(1) - + if not isinstance(data, list): print(f"Error: JSON in {source} must be an array of book objects") sys.exit(1) - + books = [] for item in data: if isinstance(item, dict): - if 'title' not in item or 'author' not in item: + if "title" not in item or "author" not in item: print(f"Error: Each book object must have 'title' and 'author' keys: {item}") sys.exit(1) - books.append((item['title'].strip(), item['author'].strip())) + books.append((item["title"].strip(), item["author"].strip())) elif isinstance(item, list) and len(item) >= 2: books.append((str(item[0]).strip(), str(item[1]).strip())) elif isinstance(item, str): @@ -274,80 +320,91 @@ def load_books_from_json(json_str, source): print(f"Warning: Skipping unrecognized book entry: {item}") return books -def main(): - parser = argparse.ArgumentParser(description='Download books from Shelfmark') - parser.add_argument('books', nargs='*', help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects') - parser.add_argument('--file', '-f', help='Path to JSON file containing an array of books') - parser.add_argument('-c', '--check-calibre', action='store_true', help='Check calibre database before downloading, skip if already present') + +def main() -> None: + parser = argparse.ArgumentParser(description="Download books from Shelfmark") + parser.add_argument( + "books", + nargs="*", + help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects', + ) + parser.add_argument("--file", "-f", help="Path to JSON file containing an array of books") + parser.add_argument( + "-c", + "--check-calibre", + action="store_true", + help="Check calibre database before downloading, skip if already present", + ) args = parser.parse_args() - + # Load books from --file, positional JSON arg, or both - BOOKS = [] - + books_to_download = [] + if args.file: try: - with open(args.file, 'r') as f: - BOOKS = load_books_from_json(f.read(), f"file '{args.file}'") + with Path(args.file).open() as f: + books_to_download = load_books_from_json(f.read(), f"file '{args.file}'") except FileNotFoundError: print(f"Error: File not found: {args.file}") sys.exit(1) - except IOError as e: + except OSError as e: print(f"Error reading file: {e}") sys.exit(1) - + for book_arg in args.books: - BOOKS.extend(load_books_from_json(book_arg, 'command line argument')) - - if not BOOKS: + books_to_download.extend(load_books_from_json(book_arg, "command line argument")) + + if not books_to_download: print("Error: No books specified. Use positional JSON args or --file ") sys.exit(1) - + # Check calibre database if requested if args.check_calibre: print("\nChecking calibre database...") sys.stdout.flush() remaining = [] - for title, author in BOOKS: + for title, author in books_to_download: if check_calibre(title): print(f" >> '{title}' already in calibre, skipping") sys.stdout.flush() else: remaining.append((title, author)) - BOOKS = remaining - if not BOOKS: + books_to_download = remaining + if not books_to_download: print("All books already in calibre, nothing to download.") sys.stdout.flush() return - - print(f"\n{'='*60}") - print(f"Downloading {len(BOOKS)} book(s) from Shelfmark") - print('='*60) + + print(f"\n{'=' * 60}") + print(f"Downloading {len(books_to_download)} book(s) from Shelfmark") + print("=" * 60) sys.stdout.flush() - + with sync_playwright() as p: browser = p.chromium.launch() - page = browser.new_page(viewport={'width': 1280, 'height': 900}) + page = browser.new_page(viewport={"width": 1280, "height": 900}) page.goto(SHELFMARK_URL) - page.wait_for_load_state('networkidle') - + page.wait_for_load_state("networkidle") + # Clear completed downloads at start of batch - print(f"\n{'='*60}") + print(f"\n{'=' * 60}") print("Clearing completed downloads from previous sessions...") - print('='*60) + print("=" * 60) sys.stdout.flush() clear_completed_downloads(page) - - for i, (title, author) in enumerate(BOOKS, 1): - print(f"\n{'='*60}") - print(f"[{i}/{len(BOOKS)}] '{title}' by {author}") - print('='*60) + + for i, (title, author) in enumerate(books_to_download, 1): + print(f"\n{'=' * 60}") + print(f"[{i}/{len(books_to_download)}] '{title}' by {author}") + print("=" * 60) sys.stdout.flush() - + download_book(page, title, author) - + browser.close() print("\nDone!") sys.stdout.flush() -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/src/frontend/src/components/activity/activityTypes.ts b/src/frontend/src/components/activity/activityTypes.ts index 62ab2942..6b2ce93a 100644 --- a/src/frontend/src/components/activity/activityTypes.ts +++ b/src/frontend/src/components/activity/activityTypes.ts @@ -1,6 +1,6 @@ import type { RequestRecord } from '../../types'; -export type ActivityKind = 'download' | 'request'; +type ActivityKind = 'download' | 'request'; export type ActivityVisualStatus = | 'queued' diff --git a/src/frontend/src/components/shared/index.ts b/src/frontend/src/components/shared/index.ts index c6983181..fdaecb4b 100644 --- a/src/frontend/src/components/shared/index.ts +++ b/src/frontend/src/components/shared/index.ts @@ -1,4 +1,4 @@ export { DisplayFieldIcon, DisplayFieldBadge, DisplayFieldBadges } from './DisplayFieldIcon'; export { CircularProgress } from './CircularProgress'; export { ToggleSwitch } from './ToggleSwitch'; -export { LibraryBadge, isInLibrary, isCollectionOnly } from './LibraryBadge'; +export { LibraryBadge, isInLibrary } from './LibraryBadge'; diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index b6802247..e621d86b 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -861,9 +861,9 @@ export const getReleases = async ( // Admin user management API -export type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa'; +type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa'; -export interface AdminUserEditCapabilities { +interface AdminUserEditCapabilities { authSource: AdminAuthSource; canSetPassword: boolean; canEditRole: boolean; diff --git a/src/frontend/src/styles.css b/src/frontend/src/styles.css index 9ef13ca5..ef412214 100644 --- a/src/frontend/src/styles.css +++ b/src/frontend/src/styles.css @@ -172,13 +172,7 @@ } body { - font-family: - 'Inter', - -apple-system, - BlinkMacSystemFont, - 'Segoe UI', - Roboto, - sans-serif; + font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; line-height: 1.6; color: var(--text-color); background: var(--background-color); diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts index 4564c9be..14729a61 100644 --- a/src/frontend/src/types/index.ts +++ b/src/frontend/src/types/index.ts @@ -1,5 +1,5 @@ // Search mode constants and type -export const SEARCH_MODE = { +const SEARCH_MODE = { DIRECT: 'direct', UNIVERSAL: 'universal', } as const; @@ -88,7 +88,7 @@ export interface StatusData { } // Button states -export type ButtonState = +type ButtonState = | 'download' | 'queued' | 'resolving' @@ -156,24 +156,24 @@ export interface TextSearchField extends SearchFieldBase { suggestions_min_query_length?: number; } -export interface NumberSearchField extends SearchFieldBase { +interface NumberSearchField extends SearchFieldBase { type: 'NumberSearchField'; min?: number; max?: number; step?: number; } -export interface SelectSearchField extends SearchFieldBase { +interface SelectSearchField extends SearchFieldBase { type: 'SelectSearchField'; options: SortOption[]; } -export interface CheckboxSearchField extends SearchFieldBase { +interface CheckboxSearchField extends SearchFieldBase { type: 'CheckboxSearchField'; default?: boolean; } -export interface DynamicSelectSearchField extends SearchFieldBase { +interface DynamicSelectSearchField extends SearchFieldBase { type: 'DynamicSelectSearchField'; options_endpoint: string; } @@ -185,7 +185,7 @@ export type MetadataSearchField = | CheckboxSearchField | DynamicSelectSearchField; -export type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field'; +type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field'; export interface QueryTargetOption { key: string; @@ -201,12 +201,12 @@ export type ContentType = 'ebook' | 'audiobook'; export type RequestPolicyMode = 'download' | 'request_release' | 'request_book' | 'blocked'; -export interface RequestPolicyDefaults { +interface RequestPolicyDefaults { ebook: RequestPolicyMode; audiobook: RequestPolicyMode; } -export interface RequestPolicySourceMode { +interface RequestPolicySourceMode { source: string; supported_content_types: string[]; browse_results_are_releases?: boolean; @@ -222,7 +222,7 @@ export interface RequestPolicyResponse { source_modes: RequestPolicySourceMode[]; } -export interface RequestContextPayload { +interface RequestContextPayload { source: string; content_type: ContentType; request_level: 'book' | 'release'; @@ -279,14 +279,14 @@ export interface QueuedDownloadResult { export type RequestSubmissionResult = RequestRecord | QueuedDownloadResult; /** How the library holds a book, per format: owned outright, or inside a collection. */ -export type LibraryHolding = 'owned' | 'collection'; +type LibraryHolding = 'owned' | 'collection'; export interface LibraryOwnership { ebook?: LibraryHolding | null; audiobook?: LibraryHolding | null; } -export type BooksOutputMode = 'folder' | 'booklore' | 'email'; +type BooksOutputMode = 'folder' | 'booklore' | 'email'; export interface AppConfig { calibre_web_url: string; @@ -334,7 +334,7 @@ export interface MetadataProvidersResponse { configured_provider_combined: string | null; } -export interface MetadataCapability { +interface MetadataCapability { key: string; field_key?: string; sort?: string; @@ -398,7 +398,7 @@ export interface ReleaseSource { } // Column schema types for plugin-driven release list UI -export type ColumnRenderType = +type ColumnRenderType = | 'text' | 'badge' | 'tags' @@ -408,7 +408,7 @@ export type ColumnRenderType = | 'indexer_protocol' | 'flag_icon' | 'format_content_type'; -export type ColumnAlign = 'left' | 'center' | 'right'; +type ColumnAlign = 'left' | 'center' | 'right'; export interface ColumnColorHint { type: 'map' | 'static'; // 'map' uses colorMaps.ts, 'static' is a fixed Tailwind class @@ -430,7 +430,7 @@ export interface ColumnSchema { } // Leading cell config - what to show in the left-most position of each row -export type LeadingCellType = 'thumbnail' | 'badge' | 'none'; +type LeadingCellType = 'thumbnail' | 'badge' | 'none'; export interface LeadingCellConfig { type: LeadingCellType; @@ -439,13 +439,13 @@ export interface LeadingCellConfig { uppercase?: boolean; // Force uppercase for badge text } -export interface ExtraSortOption { +interface ExtraSortOption { label: string; // Display label in the sort dropdown sort_key: string; // Field to sort by on the Release object default_direction?: 'asc' | 'desc'; // Which way "best first" runs (defaults to desc) } -export interface SourceActionButton { +interface SourceActionButton { label: string; // Button text (e.g., "Refresh search") action: string; // Action type: "expand" triggers expand_search } @@ -483,7 +483,7 @@ export interface Release { } // Search info returned by release sources -export interface SourceSearchInfo { +interface SourceSearchInfo { search_type: 'isbn' | 'title_author' | 'categories' | 'expanded' | 'manual' | 'query'; total_results?: number | string | null; } diff --git a/src/frontend/src/types/settings.ts b/src/frontend/src/types/settings.ts index 67ad96b6..a5dc5e0b 100644 --- a/src/frontend/src/types/settings.ts +++ b/src/frontend/src/types/settings.ts @@ -1,6 +1,6 @@ // Settings field types matching backend settings_registry.py -export type FieldType = +type FieldType = | 'TextField' | 'PasswordField' | 'NumberField' @@ -28,17 +28,17 @@ export interface ShowWhenCondition { notEmpty?: boolean; // If true, show when field has any non-empty value } -export type ShowWhen = ShowWhenCondition | ShowWhenCondition[]; +type ShowWhen = ShowWhenCondition | ShowWhenCondition[]; // Conditional disable configuration -export interface DisabledWhenCondition { +interface DisabledWhenCondition { field: string; // The field key to check value: string | string[] | boolean; // The value(s) that disable this field reason?: string; // Explanation shown when disabled } // Base field interface - common properties -export interface BaseField { +interface BaseField { key: string; label: string; type: FieldType; @@ -139,20 +139,14 @@ export interface CustomComponentFieldConfig extends BaseField { wrapInFieldWrapper?: boolean; // Whether to render with standard FieldWrapper layout } -export interface TableFieldColumnOption { +interface TableFieldColumnOption { value: string; label: string; description?: string; childOf?: string; } -export type TableFieldColumnType = - | 'text' - | 'password' - | 'select' - | 'multiselect' - | 'checkbox' - | 'path'; +type TableFieldColumnType = 'text' | 'password' | 'select' | 'multiselect' | 'checkbox' | 'path'; export interface TableFieldColumn { key: string; diff --git a/src/frontend/src/utils/colorMaps.ts b/src/frontend/src/utils/colorMaps.ts index f5bf1860..9eedc39e 100644 --- a/src/frontend/src/utils/colorMaps.ts +++ b/src/frontend/src/utils/colorMaps.ts @@ -1,3 +1,5 @@ +import type { ColumnColorHint } from '../types'; + // Color styles with transparent backgrounds and contrasting text interface ColorStyle { bg: string; @@ -133,14 +135,6 @@ function getFlagColor(flag?: string): ColorStyle { return FLAG_COLORS[normalized] || DEFAULT_FLAG_COLOR; } -/** - * Color hint type for dynamic column coloring. - */ -interface ColumnColorHint { - type: 'static' | 'map'; - value: string; -} - /** * Get the color style for a value based on a color hint. * Supports both static color classes and dynamic map lookups. diff --git a/src/frontend/src/utils/contentTypePreference.ts b/src/frontend/src/utils/contentTypePreference.ts index 9409430d..d2d4b8bb 100644 --- a/src/frontend/src/utils/contentTypePreference.ts +++ b/src/frontend/src/utils/contentTypePreference.ts @@ -10,7 +10,7 @@ import type { ContentType } from '../types'; export const CONTENT_TYPE_STORAGE_KEY = 'preferred-content-type.v2'; /** Where the current content type came from, which decides what may still override it. */ -export type ContentTypeSource = +type ContentTypeSource = /** Nothing has chosen a tab, so a configured default may still apply. */ | 'unset' /** This browser stored a choice, or something explicit set one. Defaults do not apply. */ diff --git a/tests/core/test_activity_routes_api.py b/tests/core/test_activity_routes_api.py index f1381d6a..7422a3f0 100644 --- a/tests/core/test_activity_routes_api.py +++ b/tests/core/test_activity_routes_api.py @@ -3,8 +3,10 @@ from __future__ import annotations import importlib +import io import sqlite3 import uuid +import zipfile from pathlib import Path from types import SimpleNamespace from unittest.mock import ANY, patch @@ -355,6 +357,46 @@ class TestActivityRoutes: assert response.status_code == 200 assert response.data == file_bytes + def test_localdownload_zips_a_live_multi_file_download(self, main_module, client, tmp_path): + user = _create_user(main_module, prefix="reader") + _set_session(client, user_id=user["username"], db_user_id=user["id"], is_admin=False) + + task_id = f"live-localdownload-{uuid.uuid4().hex[:8]}" + book_dir = tmp_path / "Chapters" + book_dir.mkdir() + parts = [book_dir / "Part 1.mp3", book_dir / "Part 2.mp3"] + for index, part in enumerate(parts, start=1): + part.write_bytes(f"chapter {index}".encode()) + + task = DownloadTask( + task_id=task_id, + source="direct_download", + title="Zipped Audiobook", + author="Some Author", + user_id=user["id"], + username=user["username"], + download_path=str(parts[0]), + download_paths=[str(part) for part in parts], + ) + assert main_module.backend.book_queue.add(task) is True + + try: + main_module.backend.book_queue.update_status(task_id, QueueStatus.COMPLETE) + with patch.object(main_module, "get_auth_mode", return_value="builtin"): + response = client.get(f"/api/localdownload?id={task_id}") + finally: + main_module.backend.book_queue.cancel_download(task_id) + + assert response.status_code == 200 + assert response.mimetype == "application/zip" + assert ( + 'filename="Some Author - Zipped Audiobook.zip"' + in response.headers["Content-Disposition"] + ) + with zipfile.ZipFile(io.BytesIO(response.data)) as archive: + assert archive.namelist() == ["Part 1.mp3", "Part 2.mp3"] + assert archive.read("Part 2.mp3") == b"chapter 2" + def test_localdownload_returns_not_found_for_other_users_live_queue_task( self, main_module, client, tmp_path ): diff --git a/tests/core/test_processing_integration.py b/tests/core/test_processing_integration.py index 0aeb4156..3fd86294 100644 --- a/tests/core/test_processing_integration.py +++ b/tests/core/test_processing_integration.py @@ -343,6 +343,9 @@ def test_multifile_rename_ignores_template_even_with_original_name(tmp_path): files = sorted(path.name for path in ingest.glob("*.mp3")) assert files == ["Part 1 of 2.mp3", "Part 2 of 2.mp3"] assert all("RENAMED" not in name for name in files) + # Browser download serves all of these, not just the returned first file. + assert result in task.download_paths + assert sorted(Path(path).name for path in task.download_paths) == files def test_torrent_hardlink_enabled_copy_fallback_does_not_extract_archives(tmp_path): diff --git a/tests/direct_download/test_source_availability.py b/tests/direct_download/test_source_availability.py index 3a86ec6c..020591e1 100644 --- a/tests/direct_download/test_source_availability.py +++ b/tests/direct_download/test_source_availability.py @@ -55,7 +55,10 @@ def test_direct_download_explains_aa_requirement_when_only_zlib_is_configured(mo source = DirectDownloadSource() - with pytest.raises(SearchUnavailableError, match="Anna's Archive mirror"): + with pytest.raises( + SearchUnavailableError, + match="Anna's Archive or OceanofPDF mirror URL.*only download results found through", + ): source.search(SimpleNamespace(), SimpleNamespace()) diff --git a/tests/download/test_orchestrator_retry.py b/tests/download/test_orchestrator_retry.py index d938c3bb..9caf9117 100644 --- a/tests/download/test_orchestrator_retry.py +++ b/tests/download/test_orchestrator_retry.py @@ -332,3 +332,51 @@ def test_get_book_path_reports_a_missing_file_rather_than_a_dead_path(monkeypatc assert path is None assert returned_task is task + + +def test_get_book_paths_returns_every_file_still_on_disk(monkeypatch, tmp_path): + import shelfmark.download.orchestrator as orchestrator + + parts = [tmp_path / f"Part {n}.mp3" for n in (1, 2, 3)] + for part in parts[:2]: + part.write_bytes(b"audio") + task = DownloadTask( + task_id="task-book-paths-1", + source="direct_download", + title="Chapters", + download_path=str(parts[0]), + download_paths=[str(part) for part in parts], + ) + + mock_queue = MagicMock() + mock_queue.get_task.return_value = task + monkeypatch.setattr(orchestrator, "book_queue", mock_queue) + + paths, returned_task = orchestrator.get_book_paths(task.task_id) + + assert paths == [str(parts[0]), str(parts[1])] + assert returned_task is task + + +def test_get_book_paths_ignores_a_list_that_no_longer_holds_download_path(monkeypatch, tmp_path): + import shelfmark.download.orchestrator as orchestrator + + stale = tmp_path / "old-attempt.mp3" + current = tmp_path / "current.epub" + stale.write_bytes(b"old") + current.write_bytes(b"new") + task = DownloadTask( + task_id="task-book-paths-2", + source="direct_download", + title="Retried", + download_path=str(current), + download_paths=[str(stale)], + ) + + mock_queue = MagicMock() + mock_queue.get_task.return_value = task + monkeypatch.setattr(orchestrator, "book_queue", mock_queue) + + paths, _ = orchestrator.get_book_paths(task.task_id) + + assert paths == [str(current)] diff --git a/tests/download/test_zip_stream.py b/tests/download/test_zip_stream.py new file mode 100644 index 00000000..7dfdc368 --- /dev/null +++ b/tests/download/test_zip_stream.py @@ -0,0 +1,41 @@ +import io +import zipfile + +from shelfmark.download.zip_stream import stream_zip, zip_entry_names + + +def test_stream_zip_holds_every_file_uncompressed(tmp_path): + book = tmp_path / "Author" / "Title" + book.mkdir(parents=True) + (book / "Part 01.m4b").write_bytes(b"chapter one") + (book / "Part 02.m4b").write_bytes(b"chapter two" * 1000) + + data = b"".join(stream_zip([str(book / "Part 01.m4b"), str(book / "Part 02.m4b")])) + + with zipfile.ZipFile(io.BytesIO(data)) as archive: + assert archive.testzip() is None + assert archive.namelist() == ["Part 01.m4b", "Part 02.m4b"] + assert archive.read("Part 01.m4b") == b"chapter one" + assert archive.read("Part 02.m4b") == b"chapter two" * 1000 + assert {info.compress_type for info in archive.infolist()} == {zipfile.ZIP_STORED} + + +def test_stream_zip_yields_while_reading_rather_than_at_the_end(tmp_path, monkeypatch): + import shelfmark.download.zip_stream as zip_stream + + monkeypatch.setattr(zip_stream, "_ZIP_STREAM_CHUNK_SIZE", 16) + big = tmp_path / "big.mp3" + big.write_bytes(b"x" * 160) + + chunks = list(stream_zip([str(big)])) + + assert len(chunks) > 10 + + +def test_zip_entry_names_keep_folders_below_the_common_parent(tmp_path): + first = tmp_path / "library" / "Book One" / "a.mp3" + second = tmp_path / "library" / "Book Two" / "b.mp3" + + names = [name for _, name in zip_entry_names([str(first), str(second)])] + + assert names == ["Book One/a.mp3", "Book Two/b.mp3"]