diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3f4d2180..2ab0c09f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
- id: check-added-large-files
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.10
+ rev: v0.17.1
hooks:
- id: ruff-check
args: [--fix]
diff --git a/docs/dev/plugin-settings.md b/docs/dev/plugin-settings.md
index 55247a1c..6bab9a0e 100644
--- a/docs/dev/plugin-settings.md
+++ b/docs/dev/plugin-settings.md
@@ -23,12 +23,13 @@ from shelfmark.core.settings_registry import (
ActionButton,
)
+
@register_settings(
- name="my_plugin", # Unique identifier
- display_name="My Plugin", # Shown in sidebar
- icon="wrench", # Icon name
- order=100, # Sort order (lower = higher in list)
- group="metadata_providers" # Optional: group in sidebar
+ name="my_plugin", # Unique identifier
+ display_name="My Plugin", # Shown in sidebar
+ icon="wrench", # Icon name
+ order=100, # Sort order (lower = higher in list)
+ group="metadata_providers", # Optional: group in sidebar
)
def my_plugin_settings():
return [
@@ -46,6 +47,7 @@ def my_plugin_settings():
),
]
+
def _test_connection():
# Perform connection test
return {"success": True, "message": "Connected successfully!"}
@@ -59,16 +61,16 @@ Single-line text input for strings.
```python
TextField(
- key="MY_SETTING", # Config key
- label="Setting Name", # Display label
- description="Help text", # Optional description below field
- default="", # Default value
+ key="MY_SETTING", # Config key
+ label="Setting Name", # Display label
+ description="Help text", # Optional description below field
+ default="", # Default value
placeholder="Enter value", # Placeholder text
- max_length=100, # Optional max characters
- required=False, # Is this field required?
- requires_restart=False, # Does changing this need a restart?
- show_when=None, # Conditional visibility (see below)
- disabled_when=None, # Conditional disable (see below)
+ max_length=100, # Optional max characters
+ required=False, # Is this field required?
+ requires_restart=False, # Does changing this need a restart?
+ show_when=None, # Conditional visibility (see below)
+ disabled_when=None, # Conditional disable (see below)
)
```
@@ -98,7 +100,7 @@ NumberField(
default=30,
min_value=5,
max_value=300,
- step=1, # Increment step
+ step=1, # Increment step
required=False,
)
```
@@ -160,13 +162,14 @@ Button that executes a callback function. Does not store a value.
```python
ActionButton(
- key="test_connection", # Unique key for the action
- label="Test Connection", # Button text
+ key="test_connection", # Unique key for the action
+ label="Test Connection", # Button text
description="Test the API connection",
- style="primary", # "default", "primary", or "danger"
- callback=my_callback_fn, # Function to execute
+ style="primary", # "default", "primary", or "danger"
+ callback=my_callback_fn, # Function to execute
)
+
def my_callback_fn():
"""Callback must return dict with 'success' and 'message' keys."""
try:
@@ -182,11 +185,11 @@ Display-only section heading with optional link. Does not store a value.
```python
HeadingField(
- key="section_heading", # Unique key
- title="Configuration", # Heading text
+ key="section_heading", # Unique key
+ title="Configuration", # Heading text
description="Configure the plugin settings below",
link_url="https://example.com/docs", # Optional link
- link_text="View Documentation", # Link text
+ link_text="View Documentation", # Link text
)
```
@@ -204,8 +207,14 @@ CustomComponentField(
label="Request Policy Rules",
description="Custom editor for policy defaults and matrix rules.",
value_fields=[
- SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"),
- SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"),
+ SelectField(
+ key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"
+ ),
+ SelectField(
+ key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
+ label="Default Audiobook Mode",
+ default="download",
+ ),
TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
],
wrap_in_field_wrapper=True, # use standard FieldWrapper label/description layout
@@ -263,7 +272,7 @@ NumberField(
disabled_when={
"field": "FEATURE_ENABLED",
"value": False,
- "reason": "Enable the feature first"
+ "reason": "Enable the feature first",
},
)
```
@@ -285,6 +294,7 @@ register_group(
order=50,
)
+
# Then register settings to the group
@register_settings(
name="plugin_a",
@@ -426,10 +436,7 @@ class MyProvider(MetadataProvider):
def __init__(self, api_key: str = None):
self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
- self.base_url = config.get(
- "MY_PROVIDER_BASE_URL",
- "https://api.myprovider.com/v1"
- )
+ self.base_url = config.get("MY_PROVIDER_BASE_URL", "https://api.myprovider.com/v1")
def is_available(self) -> bool:
return bool(self.api_key)
diff --git a/docs/dev/release-sources-plugin-guide.md b/docs/dev/release-sources-plugin-guide.md
index 37709201..a4fbaa4f 100644
--- a/docs/dev/release-sources-plugin-guide.md
+++ b/docs/dev/release-sources-plugin-guide.md
@@ -66,10 +66,11 @@ This separation allows:
A source has both a `ReleaseSource` (for searching) and a `DownloadHandler` (for downloading). They share the same `name` identifier:
```python
-@register_source("my_source") # Search registration
+@register_source("my_source") # Search registration
class MySource(ReleaseSource): ...
-@register_handler("my_source") # Download registration
+
+@register_handler("my_source") # Download registration
class MyHandler(DownloadHandler): ...
```
@@ -142,10 +143,11 @@ from typing import List
from shelfmark.metadata_providers import BookMetadata
from shelfmark.release_sources import Release, ReleaseColumnConfig
+
class ReleaseSource(ABC):
"""Interface for searching a release source."""
- name: str # Internal identifier: "direct_download", "prowlarr"
+ name: str # Internal identifier: "direct_download", "prowlarr"
display_name: str # User-facing name: "Direct Download", "Prowlarr"
@abstractmethod
@@ -191,14 +193,16 @@ def search(self, book: BookMetadata) -> List[Release]:
if book.isbn_13:
api_results = self.api.search_isbn(book.isbn_13)
for r in api_results:
- results.append(Release(
- source="my_source",
- source_id=r.id,
- title=r.title,
- format=r.format,
- size=r.size,
- # ... more fields
- ))
+ results.append(
+ Release(
+ source="my_source",
+ source_id=r.id,
+ title=r.title,
+ format=r.format,
+ size=r.size,
+ # ... more fields
+ )
+ )
# Strategy 2: Fallback to title + author search
if not results:
@@ -239,6 +243,7 @@ from typing import Callable, Optional
from threading import Event
from shelfmark.core.models import DownloadTask
+
class DownloadHandler(ABC):
"""Interface for executing downloads from a source."""
@@ -286,6 +291,7 @@ downloading it; the default returns `None`, which the UI reports as "can't inspe
```python
from shelfmark.download.postprocess.packs import PackFile
+
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
"""Return the release's files (release-relative paths + sizes), or None."""
torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page
@@ -380,20 +386,20 @@ Standardized search result returned by all sources:
```python
@dataclass
class Release:
- source: str # "direct_download", "prowlarr", "irc"
- source_id: str # Unique ID within that source
- title: str # Display title
+ source: str # "direct_download", "prowlarr", "irc"
+ source_id: str # Unique ID within that source
+ title: str # Display title
- format: Optional[str] = None # File format: "epub", "mobi", "pdf"
- size: Optional[str] = None # Human-readable: "5.2 MB"
- size_bytes: Optional[int] = None # For sorting
+ format: Optional[str] = None # File format: "epub", "mobi", "pdf"
+ size: Optional[str] = None # Human-readable: "5.2 MB"
+ size_bytes: Optional[int] = None # For sorting
download_url: Optional[str] = None # Direct download URL if available
- info_url: Optional[str] = None # Link to tracker/info page
+ info_url: Optional[str] = None # Link to tracker/info page
- protocol: Optional[str] = None # "http", "torrent", "usenet"
- indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer"
- seeders: Optional[int] = None # For torrents
+ protocol: Optional[str] = None # "http", "torrent", "usenet"
+ indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer"
+ seeders: Optional[int] = None # For torrents
extra: Dict = field(default_factory=dict) # Source-specific metadata
```
@@ -415,7 +421,7 @@ Release(
"preview": "https://example.com/cover.jpg",
"quality": "high",
"publisher": "Publisher Name",
- }
+ },
)
```
@@ -428,15 +434,15 @@ Source-agnostic task in the download queue:
```python
@dataclass
class DownloadTask:
- task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID)
- source: str # Handler name: "direct_download"
- title: str # Display title
+ task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID)
+ source: str # Handler name: "direct_download"
+ title: str # Display title
# Display info (from Release.extra or Release fields)
author: Optional[str] = None
format: Optional[str] = None
size: Optional[str] = None
- preview: Optional[str] = None # Cover image URL
+ preview: Optional[str] = None # Cover image URL
# Runtime state (managed by orchestrator)
priority: int = 0
@@ -454,8 +460,8 @@ Book information passed to `search()`:
```python
@dataclass
class BookMetadata:
- provider: str # Metadata provider: "hardcover", "openlibrary"
- provider_id: str # ID in provider's system
+ provider: str # Metadata provider: "hardcover", "openlibrary"
+ provider_id: str # ID in provider's system
title: str
provider_display_name: Optional[str] = None
@@ -466,7 +472,7 @@ class BookMetadata:
description: Optional[str] = None
publisher: Optional[str] = None
publish_year: Optional[int] = None
- language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr"
+ language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr"
genres: List[str] = field(default_factory=list)
source_url: Optional[str] = None
display_fields: List[DisplayField] = field(default_factory=list)
@@ -481,13 +487,13 @@ class BookMetadata:
```python
from shelfmark.release_sources import register_source, register_handler
+
@register_source("my_source")
-class MySource(ReleaseSource):
- ...
+class MySource(ReleaseSource): ...
+
@register_handler("my_source")
-class MyHandler(DownloadHandler):
- ...
+class MyHandler(DownloadHandler): ...
```
### Registry Functions
@@ -544,12 +550,13 @@ from shelfmark.core.settings_registry import (
is_value_from_env,
)
+
@register_settings(
- name="my_source", # Unique identifier
- display_name="My Source", # Shown in settings UI
- icon="globe", # Icon name (optional)
- order=60, # Sort order in sidebar
- group="sources", # Grouping in sidebar
+ name="my_source", # Unique identifier
+ display_name="My Source", # Shown in settings UI
+ icon="globe", # Icon name (optional)
+ order=60, # Sort order in sidebar
+ group="sources", # Grouping in sidebar
)
def my_source_settings():
"""Register settings for My Source plugin."""
@@ -559,7 +566,6 @@ def my_source_settings():
title="My Source Configuration",
description="Configure connection to My Source API",
),
-
PasswordField(
key="MY_SOURCE_API_KEY",
label="API Key",
@@ -567,14 +573,12 @@ def my_source_settings():
required=True,
env_supported=True, # Can be set via MY_SOURCE_API_KEY env var
),
-
TextField(
key="MY_SOURCE_BASE_URL",
label="Base URL",
placeholder="https://api.mysource.com",
env_supported=True,
),
-
NumberField(
key="MY_SOURCE_TIMEOUT",
label="Request Timeout",
@@ -583,14 +587,12 @@ def my_source_settings():
min_value=5,
max_value=120,
),
-
CheckboxField(
key="MY_SOURCE_ENABLED",
label="Enable My Source",
description="Include in release searches",
default=True,
),
-
ActionButton(
key="test_connection",
label="Test Connection",
@@ -599,22 +601,19 @@ def my_source_settings():
),
]
+
def test_my_source_connection():
"""Test connection callback - returns result dict."""
try:
# Get current API key
- api_key = get_setting_value(
- PasswordField(key="MY_SOURCE_API_KEY", label=""),
- "my_source"
- )
+ api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
if not api_key:
return {"success": False, "message": "API key not configured"}
# Test the connection
response = requests.get(
- "https://api.mysource.com/ping",
- headers={"Authorization": f"Bearer {api_key}"}
+ "https://api.mysource.com/ping", headers={"Authorization": f"Bearer {api_key}"}
)
if response.status_code == 200:
@@ -648,6 +647,7 @@ from shelfmark.core.settings_registry import (
load_config_file,
)
+
# In your source/handler class
def __init__(self):
# Load from config file
@@ -655,11 +655,9 @@ def __init__(self):
self.api_key = config.get("MY_SOURCE_API_KEY")
self.base_url = config.get("MY_SOURCE_BASE_URL", "https://api.mysource.com")
+
# Or use get_setting_value for ENV var priority
-api_key = get_setting_value(
- PasswordField(key="MY_SOURCE_API_KEY", label=""),
- "my_source"
-)
+api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
```
### Settings Priority
@@ -689,6 +687,7 @@ from shelfmark.release_sources import (
LeadingCellType,
)
+
@classmethod
def get_column_config(cls) -> ReleaseColumnConfig:
return ReleaseColumnConfig(
@@ -759,7 +758,7 @@ For `BADGE` render type, specify colors:
```python
# Use frontend color map (defined in colorMaps.ts)
-ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc.
+ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc.
ColumnColorHint(type="map", value="language") # Maps en→blue, de→yellow, etc.
# Use static Tailwind class
@@ -963,6 +962,7 @@ logger = setup_logger(__name__)
# Settings Registration
# ---------------------------------------------------------------------------
+
@register_settings(
name="booknet",
display_name="BookNet",
@@ -980,7 +980,6 @@ def booknet_settings():
link_url="https://booknet.example.com/api-docs",
link_text="API Documentation",
),
-
PasswordField(
key="BOOKNET_API_KEY",
label="API Key",
@@ -988,14 +987,12 @@ def booknet_settings():
required=True,
env_supported=True,
),
-
TextField(
key="BOOKNET_BASE_URL",
label="Base URL",
placeholder="https://api.booknet.example.com",
env_supported=True,
),
-
NumberField(
key="BOOKNET_TIMEOUT",
label="Request Timeout",
@@ -1004,14 +1001,12 @@ def booknet_settings():
min_value=5,
max_value=120,
),
-
CheckboxField(
key="BOOKNET_PREFER_EPUB",
label="Prefer EPUB format",
description="Sort EPUB results first when available",
default=True,
),
-
ActionButton(
key="test_connection",
label="Test Connection",
@@ -1041,7 +1036,7 @@ def _test_booknet_connection() -> Dict:
data = response.json()
return {
"success": True,
- "message": f"Connected! {data.get('books_available', 0):,} books available"
+ "message": f"Connected! {data.get('books_available', 0):,} books available",
}
elif response.status_code == 401:
return {"success": False, "message": "Invalid API key"}
@@ -1058,6 +1053,7 @@ def _test_booknet_connection() -> Dict:
# Release Source Implementation
# ---------------------------------------------------------------------------
+
@register_source("booknet")
class BookNetSource(ReleaseSource):
"""Search BookNet for book releases."""
@@ -1090,7 +1086,7 @@ class BookNetSource(ReleaseSource):
# Sort by preference
if self.prefer_epub:
- releases.sort(key=lambda r: (0 if r.format == "epub" else 1))
+ releases.sort(key=lambda r: 0 if r.format == "epub" else 1)
return releases
@@ -1164,7 +1160,9 @@ class BookNetSource(ReleaseSource):
render_type=ColumnRenderType.BADGE,
align=ColumnAlign.CENTER,
width="70px",
- color_hint=ColumnColorHint(type="static", value="bg-purple-500/20 text-purple-400"),
+ color_hint=ColumnColorHint(
+ type="static", value="bg-purple-500/20 text-purple-400"
+ ),
),
ColumnSchema(
key="extra.language",
@@ -1202,6 +1200,7 @@ class BookNetSource(ReleaseSource):
# Download Handler Implementation
# ---------------------------------------------------------------------------
+
@register_handler("booknet")
class BookNetHandler(DownloadHandler):
"""Handle downloads from BookNet."""
@@ -1264,8 +1263,7 @@ class BookNetHandler(DownloadHandler):
# Phase 3: Move to final location
safe_title = "".join(
- c if c.isalnum() or c in " .-_" else "_"
- for c in task.title[:100]
+ c if c.isalnum() or c in " .-_" else "_" for c in task.title[:100]
).strip()
final_filename = f"{safe_title}.{task.format or 'epub'}"
@@ -1274,7 +1272,9 @@ class BookNetHandler(DownloadHandler):
# Avoid overwriting existing files
counter = 1
while final_path.exists():
- final_path = Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
+ final_path = (
+ Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
+ )
counter += 1
shutil.move(str(temp_path), str(final_path))
@@ -1363,6 +1363,7 @@ def is_available(self) -> bool:
except Exception:
return False
+
def search(self, book: BookMetadata) -> List[Release]:
"""Return empty list on errors, don't raise."""
try:
@@ -1446,10 +1447,10 @@ from shelfmark.core.logger import setup_logger
logger = setup_logger(__name__)
-logger.debug("Detailed debug info") # Development only
-logger.info("Normal operation info") # General progress
-logger.warning("Recoverable issues") # Fallback used, retry happening
-logger.error("Failures") # Unrecoverable errors
+logger.debug("Detailed debug info") # Development only
+logger.info("Normal operation info") # General progress
+logger.warning("Recoverable issues") # Fallback used, retry happening
+logger.error("Failures") # Unrecoverable errors
```
---
diff --git a/shelfmark/core/models.py b/shelfmark/core/models.py
index 90a427a3..ff7ae9cc 100644
--- a/shelfmark/core/models.py
+++ b/shelfmark/core/models.py
@@ -158,6 +158,9 @@ class DownloadTask:
status: QueueStatus = QueueStatus.QUEUED
status_message: str | None = None
download_path: str | None = None
+ # Every file the output step produced, download_path among them. Kept in memory
+ # only: history persists download_path alone.
+ download_paths: list[str] = field(default_factory=list)
last_error_message: str | None = None
last_error_type: str | None = None
staged_path: str | None = None
diff --git a/shelfmark/download/clients/base_handler.py b/shelfmark/download/clients/base_handler.py
index 5bd24f0b..b8275db2 100644
--- a/shelfmark/download/clients/base_handler.py
+++ b/shelfmark/download/clients/base_handler.py
@@ -194,7 +194,9 @@ def _probe_completed_path(path: Path) -> tuple[bool, OSError | None]:
return True, None
-def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tuple[str, int, int], ...] | None:
+def _completed_file_signature(
+ path: Path, content_type: str | None
+) -> tuple[tuple[str, int, int], ...] | None:
"""Return sizes and mtimes for supported files in a completed path."""
from shelfmark.download.postprocess.policy import (
get_supported_audiobook_formats,
@@ -202,9 +204,7 @@ def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tup
)
formats = (
- get_supported_audiobook_formats()
- if is_audiobook(content_type)
- else get_supported_formats()
+ get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats()
)
supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats}
diff --git a/shelfmark/download/orchestrator.py b/shelfmark/download/orchestrator.py
index 13371a30..31b74669 100644
--- a/shelfmark/download/orchestrator.py
+++ b/shelfmark/download/orchestrator.py
@@ -416,6 +416,28 @@ def get_book_path(task_id: str) -> tuple[str | None, DownloadTask | None]:
return path, task
+def get_book_paths(task_id: str) -> tuple[list[str], DownloadTask | None]:
+ """Every file a task's download produced that is still on disk.
+
+ A multi-file release (chapters of an audiobook, a pack) lands as several files;
+ :func:`get_book_path` only knows the first. The list is trusted only while it
+ still contains download_path, so a later output that recorded no list cannot
+ hand back an earlier attempt's files.
+ """
+ path, task = get_book_path(task_id)
+ if path is None or task is None:
+ return [], task
+
+ if path not in task.download_paths:
+ return [path], task
+
+ try:
+ return [p for p in task.download_paths if Path(p).is_file()], task
+ except OSError as e:
+ logger.error_trace(f"Error resolving book paths: {e}")
+ return [path], task
+
+
def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]:
"""Get downloaded file data for a specific task.
diff --git a/shelfmark/download/outputs/folder.py b/shelfmark/download/outputs/folder.py
index 8cc91105..4ac37b85 100644
--- a/shelfmark/download/outputs/folder.py
+++ b/shelfmark/download/outputs/folder.py
@@ -275,4 +275,5 @@ def process_folder_output(
message = f"Complete ({len(final_paths)} files)"
status_callback("complete", message)
+ task.download_paths = [str(path) for path in final_paths]
return str(final_paths[0])
diff --git a/shelfmark/download/zip_stream.py b/shelfmark/download/zip_stream.py
new file mode 100644
index 00000000..5c6b56c6
--- /dev/null
+++ b/shelfmark/download/zip_stream.py
@@ -0,0 +1,68 @@
+"""Stream several downloaded files to a client as one ZIP, without staging the archive."""
+
+import os
+import zipfile
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from collections.abc import Iterator
+
+_ZIP_STREAM_CHUNK_SIZE = 1024 * 1024
+
+
+class _ChunkSink:
+ """Write-only ZipFile target that hands the written bytes back to the caller.
+
+ Having no tell() or seek() makes ZipFile treat it as an unseekable stream: each
+ entry's CRC and sizes go in a trailing data descriptor instead of being patched
+ back into the local header, so nothing has to be held until the entry ends.
+ """
+
+ def __init__(self) -> None:
+ self._chunks: list[bytes] = []
+
+ def write(self, data: bytes) -> int:
+ self._chunks.append(bytes(data))
+ return len(data)
+
+ def flush(self) -> None:
+ """Nothing is buffered beyond what drain() hands out."""
+
+ def close(self) -> None:
+ """Nothing to release; the caller drains what is left after ZipFile closes."""
+
+ def drain(self) -> Iterator[bytes]:
+ chunks, self._chunks = self._chunks, []
+ yield from chunks
+
+
+def zip_entry_names(paths: list[str]) -> list[tuple[Path, str]]:
+ """Pair each file with its name inside the archive.
+
+ Names are relative to the files' common parent, so a book's own folders
+ (``Title/Part 01.m4b``) survive while the library path above them does not.
+ """
+ files = [Path(path) for path in paths]
+ root = Path(os.path.commonpath([str(file.parent) for file in files]))
+ return [(file, file.relative_to(root).as_posix()) for file in files]
+
+
+def stream_zip(paths: list[str]) -> Iterator[bytes]:
+ """Yield an uncompressed ZIP of ``paths``, reading each file once in chunks.
+
+ Stored, not deflated: audiobooks and ebooks are already compressed, so deflating
+ them would cost CPU for next to nothing. ZIP64 kicks in per entry from the file
+ size, which a multi-gigabyte audiobook needs.
+ """
+ sink = _ChunkSink()
+ with zipfile.ZipFile(sink, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
+ for path, arcname in zip_entry_names(paths):
+ info = zipfile.ZipInfo.from_file(path, arcname, strict_timestamps=False)
+ info.compress_type = zipfile.ZIP_STORED
+ with path.open("rb") as source, archive.open(info, "w") as target:
+ while chunk := source.read(_ZIP_STREAM_CHUNK_SIZE):
+ target.write(chunk)
+ yield from sink.drain()
+ yield from sink.drain()
+ yield from sink.drain()
diff --git a/shelfmark/main.py b/shelfmark/main.py
index 15330302..422cc244 100644
--- a/shelfmark/main.py
+++ b/shelfmark/main.py
@@ -6,12 +6,14 @@ import os
import re
import sqlite3
import time
+import unicodedata
from contextlib import suppress
from datetime import UTC, datetime, timedelta
from functools import wraps
from importlib import import_module
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, NoReturn, cast
+from urllib.parse import quote
from flask import Flask, g, jsonify, request, send_file, send_from_directory, session
from flask_cors import CORS
@@ -55,7 +57,12 @@ from shelfmark.core.cwa_user_sync import upsert_cwa_user
from shelfmark.core.download_history_service import DownloadHistoryService
from shelfmark.core.external_user_linking import upsert_external_user
from shelfmark.core.logger import setup_logger
-from shelfmark.core.models import TERMINAL_QUEUE_STATUSES, QueueStatus, SearchFilters
+from shelfmark.core.models import (
+ TERMINAL_QUEUE_STATUSES,
+ QueueStatus,
+ SearchFilters,
+ build_filename,
+)
from shelfmark.core.notifications import (
NotificationContext,
NotificationEvent,
@@ -95,6 +102,7 @@ from shelfmark.core.viewer_redaction import (
)
from shelfmark.download import orchestrator as backend
from shelfmark.download import warmup
+from shelfmark.download.zip_stream import stream_zip
from shelfmark.release_sources import (
BrowseRecord,
Release,
@@ -103,6 +111,7 @@ from shelfmark.release_sources import (
)
if TYPE_CHECKING:
+ from shelfmark.core.models import DownloadTask
from shelfmark.metadata_providers import BookMetadata, MetadataProvider
logger = setup_logger(__name__)
@@ -1800,16 +1809,43 @@ def api_status() -> Response | tuple[Response, int]:
return jsonify({"error": str(e)}), 500
+def _attachment_filename_options(download_name: str) -> dict[str, str]:
+ """Content-Disposition filename options, RFC 2231-encoded when not plain ASCII.
+
+ Mirrors what send_file does, for responses built from a generator.
+ """
+ try:
+ download_name.encode("ascii")
+ except UnicodeEncodeError:
+ simple = unicodedata.normalize("NFKD", download_name)
+ simple = simple.encode("ascii", "ignore").decode("ascii")
+ quoted = quote(download_name, safe="!#$&+^`|~")
+ return {"filename": simple, "filename*": f"UTF-8''{quoted}"}
+ return {"filename": download_name}
+
+
+def _zip_download_response(file_paths: list[str], task: DownloadTask) -> Response:
+ """Send a multi-file download (audiobook chapters, a pack) as one streamed ZIP."""
+ download_name = build_filename(task.title or "download", task.author, task.year, "zip")
+ response = Response(stream_zip(file_paths), mimetype="application/zip")
+ response.headers.set(
+ "Content-Disposition", "attachment", **_attachment_filename_options(download_name)
+ )
+ return response
+
+
@app.route("/api/localdownload", methods=["GET"])
@login_required
def api_local_download() -> Response | tuple[Response, int]:
- """Download an EPUB file from local storage if available.
+ """Download a completed book from local storage if available.
Query Parameters:
id (str): Book identifier (MD5 hash)
Returns:
- flask.Response: The EPUB file if found, otherwise an error response.
+ flask.Response: The file if found, a ZIP of all of them when the download
+ produced several (a live task only; history keeps just the first), otherwise
+ an error response.
"""
book_id = request.args.get("id", "")
@@ -1820,8 +1856,8 @@ def api_local_download() -> Response | tuple[Response, int]:
try:
# The path, not the bytes: send_file streams it off disk, where reading it
# first cost two resident copies of the whole book.
- file_path, book_info = backend.get_book_path(book_id)
- if file_path is None:
+ file_paths, book_info = backend.get_book_paths(book_id)
+ if not file_paths:
# Fallback for dismissed/history entries where queue task may no longer exist.
if download_history_service is not None:
is_admin, db_user_id, can_access_status = _resolve_status_scope()
@@ -1853,8 +1889,11 @@ def api_local_download() -> Response | tuple[Response, int]:
):
return jsonify({"error": "File not found"}), 404
+ if book_info is not None and len(file_paths) > 1:
+ return _zip_download_response(file_paths, book_info)
+
file_name = book_info.get_filename() if book_info is not None else Path(book_id).name
- return send_file(file_path, download_name=file_name, as_attachment=True)
+ return send_file(file_paths[0], download_name=file_name, as_attachment=True)
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Local download error: {e}")
diff --git a/shelfmark/metadata_providers/databazeknih.py b/shelfmark/metadata_providers/databazeknih.py
index c64045a8..ca877add 100644
--- a/shelfmark/metadata_providers/databazeknih.py
+++ b/shelfmark/metadata_providers/databazeknih.py
@@ -450,7 +450,7 @@ class DatabazeKnihProvider(MetadataProvider):
book_id = _id_from_link(str(data.get("url", "")))
if not book_id:
# Pages without a JSON-LD url still carry the id on the rating buttons
- tag = soup.find(attrs={"data-typeid": True})
+ tag = soup.select_one("[data-typeid]")
book_id = str(tag["data-typeid"]) if tag else ""
title = _clean_text(str(data.get("name", "")))
authors = [_clean_author(n) for n in _ld_names(data.get("author"))]
@@ -539,7 +539,8 @@ class DatabazeKnihProvider(MetadataProvider):
name = _clean_text(str(link.get("title") or link.get_text())) or None
# The part number follows the link, up to the next series link
text = []
- for elem in link.parent.next_siblings:
+ parent = link.parent
+ for elem in parent.next_siblings if parent else ():
if isinstance(elem, Tag):
if elem.find("a", href=re.compile(r"^/serie/")):
break
diff --git a/shelfmark/release_sources/direct_download/registry.py b/shelfmark/release_sources/direct_download/registry.py
index 068a3272..28085181 100644
--- a/shelfmark/release_sources/direct_download/registry.py
+++ b/shelfmark/release_sources/direct_download/registry.py
@@ -40,7 +40,7 @@ def get_unavailable_reason(
if not enabled_providers(providers):
return (
"Direct Download is not configured. Add at least one Anna's Archive or OceanofPDF mirror URL in "
- "Settings. Z-Library, LibGen, Welib, mirrors only download results found through "
+ "Settings. Z-Library, LibGen and Welib mirrors only download results found through "
"Anna's Archive."
)
return None
diff --git a/skills/shelfmark/SKILL.md b/skills/shelfmark/SKILL.md
index 120bd702..98a5551a 100644
--- a/skills/shelfmark/SKILL.md
+++ b/skills/shelfmark/SKILL.md
@@ -64,12 +64,12 @@ The script navigates to the main page, enters the search query, and waits for "M
```python
# Navigate to main page first to reset SPA state
-page.goto('http://localhost:8084/')
-page.wait_for_load_state('networkidle')
+page.goto("http://localhost:8084/")
+page.wait_for_load_state("networkidle")
# Enter search query and submit
-page.fill('input[type="search"]', f'{title} {author}')
-page.press('input[type="search"]', 'Enter')
+page.fill('input[type="search"]', f"{title} {author}")
+page.press('input[type="search"]', "Enter")
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
@@ -87,18 +87,20 @@ def parse_books(page):
btns = page.query_selector_all('button[data-action="download"]')
books = []
for i, btn in enumerate(btns):
- content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
-
- title_match = re.search(r'
]*>(.*?)
', content, re.IGNORECASE | re.DOTALL)
- title = title_match.group(1).strip() if title_match else 'Unknown'
-
- author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
- author = author_match.group(1).strip() if author_match else 'Unknown'
-
- dl_match = re.search(r'•\s*([\d,]+)', content)
- downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
-
- books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
+ content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
+
+ title_match = re.search(r"]*>(.*?)
", content, re.IGNORECASE | re.DOTALL)
+ title = title_match.group(1).strip() if title_match else "Unknown"
+
+ author_match = re.search(
+ r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
+ )
+ author = author_match.group(1).strip() if author_match else "Unknown"
+
+ dl_match = re.search(r"•\s*([\d,]+)", content)
+ downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
+
+ books.append({"index": i, "title": title, "author": author, "downloads": downloads})
return books
```
@@ -106,19 +108,19 @@ def parse_books(page):
```python
# Filter books matching the search criteria
-matching_books = [b for b in books if matches_search(b['title'], b['author'], title, author)]
+matching_books = [b for b in books if matches_search(b["title"], b["author"], title, author)]
if matching_books:
- best = max(matching_books, key=lambda x: x['downloads'])
-
+ best = max(matching_books, key=lambda x: x["downloads"])
+
# CRITICAL: Click on the article h3 to open detail view
- h3s = page.query_selector_all('article h3')
- if best['index'] < len(h3s):
- h3s[best['index']].click()
+ h3s = page.query_selector_all("article h3")
+ if best["index"] < len(h3s):
+ h3s[best["index"]].click()
page.wait_for_timeout(300)
-
+
# Then click the download button
- btn = page.query_selector_all('button[data-action="download"]')[best['index']]
+ btn = page.query_selector_all('button[data-action="download"]')[best["index"]]
btn.click()
page.wait_for_timeout(500)
```
@@ -130,14 +132,14 @@ if matching_books:
```python
for i in range(60):
page.wait_for_timeout(5000)
- activity_text = page.inner_text('aside')
-
- if 'IN PROGRESS' in activity_text:
+ activity_text = page.inner_text("aside")
+
+ if "IN PROGRESS" in activity_text:
print("Download started!")
- elif 'Complete' in activity_text or 'Saved' in activity_text:
+ elif "Complete" in activity_text or "Saved" in activity_text:
print("Download complete!")
break
- elif 'No activity' in activity_text:
+ elif "No activity" in activity_text:
print("Download not started")
break
```
@@ -146,7 +148,7 @@ for i in range(60):
```python
# Click "Clear Completed" using JavaScript
-page.evaluate('''
+page.evaluate("""
() => {
for (const b of document.querySelectorAll('button')) {
if (b.textContent.includes('Clear Completed')) {
@@ -155,7 +157,7 @@ page.evaluate('''
}
}
}
-''')
+""")
page.wait_for_timeout(2000)
```
diff --git a/skills/shelfmark/download_books.py b/skills/shelfmark/download_books.py
index 1d3deeda..0b8b02d0 100755
--- a/skills/shelfmark/download_books.py
+++ b/skills/shelfmark/download_books.py
@@ -1,41 +1,62 @@
#!/usr/bin/env python3
"""Shelfmark Book Downloader - Downloads books from a local Shelfmark instance."""
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import re
+import sys
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from playwright.sync_api import Error as PlaywrightError
from playwright.sync_api import sync_playwright
-import re, time, sys, argparse, json
-from urllib.parse import quote
-SHELFMARK_URL = 'http://localhost:8084/'
+if TYPE_CHECKING:
+ from playwright.sync_api import Page
-def parse_books(page):
+Book = dict[str, Any]
+
+SHELFMARK_URL = "http://localhost:8084/"
+
+
+def parse_books(page: Page) -> list[Book]:
btns = page.query_selector_all('button[data-action="download"]')
books = []
for i, btn in enumerate(btns):
- content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
-
- title_match = re.search(r']*>(.*?)
', content, re.IGNORECASE | re.DOTALL)
- title = title_match.group(1).strip() if title_match else 'Unknown'
-
- author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
- author = author_match.group(1).strip() if author_match else 'Unknown'
-
- dl_match = re.search(r'•\s*([\d,]+)', content)
- downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
-
- books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
+ content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
+
+ title_match = re.search(r"]*>(.*?)
", content, re.IGNORECASE | re.DOTALL)
+ title = title_match.group(1).strip() if title_match else "Unknown"
+
+ author_match = re.search(
+ r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
+ )
+ author = author_match.group(1).strip() if author_match else "Unknown"
+
+ dl_match = re.search(r"•\s*([\d,]+)", content)
+ downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
+
+ books.append({"index": i, "title": title, "author": author, "downloads": downloads})
return books
-def do_search(page, title, author, search_type="author"):
+
+def do_search(
+ page: Page, title: str, author: str, search_type: str = "author"
+) -> tuple[list[Book], list[Book]] | None:
"""Search with title+author, fall back to title only"""
# Navigate to main page first to reset SPA state
page.goto(SHELFMARK_URL)
- page.wait_for_load_state('networkidle')
-
+ page.wait_for_load_state("networkidle")
+
# Enter search query and submit
- search_query = f'{title} {author}' if search_type == "author" else title
+ search_query = f"{title} {author}" if search_type == "author" else title
page.fill('input[type="search"]', search_query)
- page.press('input[type="search"]', 'Enter')
-
+ page.press('input[type="search"]', "Enter")
+
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
try:
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
@@ -44,227 +65,252 @@ def do_search(page, title, author, search_type="author"):
print(" -> Trying title only...")
sys.stdout.flush()
return do_search(page, title, author, search_type="title")
- else:
- print(" >> Timeout waiting for search results")
- sys.stdout.flush()
- return None
-
+ print(" >> Timeout waiting for search results")
+ sys.stdout.flush()
+ return None
+
books = parse_books(page)
-
- def clean_title_for_match(book_title, search_title):
+
+ def clean_title_for_match(book_title: str, search_title: str) -> bool:
"""Clean book title to check if it matches the search title"""
# Remove common subtitle patterns
- clean = re.sub(r'\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$', '', book_title, flags=re.IGNORECASE)
+ clean = re.sub(
+ r"\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$",
+ "",
+ book_title,
+ flags=re.IGNORECASE,
+ )
clean = clean.strip()
# Remove trailing punctuation
- clean = clean.rstrip(':,;.')
+ clean = clean.rstrip(":,;.")
return clean.lower().strip() == search_title.lower().strip()
-
- def matches_search(book_title, book_author, search_title, search_author):
+
+ def matches_search(
+ book_title: str, book_author: str, search_title: str, search_author: str
+ ) -> bool:
"""Check if book matches search criteria more strictly"""
title_match = search_title.lower() in book_title.lower()
author_match = search_author.lower() in book_author.lower()
-
+
# For title+author search, ensure title starts with search title (not just contains it)
if title_match and author_match:
- return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
-
+ return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(
+ search_title.lower()
+ )
+
# Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann")
if title_match:
author_parts = search_author.lower().split()
if len(author_parts) >= 2:
reversed_author = f"{author_parts[-1]} {author_parts[0]}"
- if reversed_author in book_author.lower() or book_author.lower().startswith(reversed_author):
- return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
-
+ if reversed_author in book_author.lower() or book_author.lower().startswith(
+ reversed_author
+ ):
+ return clean_title_for_match(
+ book_title, search_title
+ ) or book_title.lower().startswith(search_title.lower())
+
return False
-
+
if search_type == "author":
- matching = [b for b in books if matches_search(b['title'], b['author'], title, author)]
+ matching = [b for b in books if matches_search(b["title"], b["author"], title, author)]
print(f" With author: {len(books)} total, {len(matching)} matching")
for b in books[:3]:
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
sys.stdout.flush()
-
+
if matching:
# Check if any matching book is already in the download queue
- try:
- sidebar_text = page.inner_text('aside')
- if 'IN PROGRESS' in sidebar_text:
+ with contextlib.suppress(PlaywrightError):
+ sidebar_text = page.inner_text("aside")
+ if "IN PROGRESS" in sidebar_text:
# Get the title of the book currently downloading
- current_download = sidebar_text.split('IN PROGRESS')[1].split('—')[0].strip().split('\n')[0].strip()
+ current_download = (
+ sidebar_text.split("IN PROGRESS")[1]
+ .split("—")[0]
+ .strip()
+ .split("\n")[0]
+ .strip()
+ )
# Check if the current download matches our search
if title.lower() in current_download.lower():
print(f" >> '{current_download}' already downloading, skipping")
sys.stdout.flush()
return None
- except:
- pass
-
+
if matching:
return matching, books
-
+
# Fall back to title only
print(" -> Trying title only...")
sys.stdout.flush()
return do_search(page, title, author, search_type="title")
-
+
# Title-only search - prefer exact matches first
- matching = [b for b in books if title.lower() in b['title'].lower()]
- exact_matches = [b for b in matching if clean_title_for_match(b['title'], title)]
-
+ matching = [b for b in books if title.lower() in b["title"].lower()]
+ exact_matches = [b for b in matching if clean_title_for_match(b["title"], title)]
+
if exact_matches:
matching = exact_matches
-
- print(f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)")
+
+ print(
+ f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)"
+ )
for b in books[:5]:
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
sys.stdout.flush()
-
+
if not matching:
return None
-
+
return (matching, books)
-def download_book(page, title, author):
+
+def download_book(page: Page, title: str, author: str) -> None:
result = do_search(page, title, author)
if not result:
print("\n >>> NOT FOUND")
sys.stdout.flush()
return
-
- matching, all_books = result
+
+ matching, _all_books = result
if not matching:
print("\n >>> NOT FOUND")
sys.stdout.flush()
return
-
+
# Find the best book that is not disabled
btns = page.query_selector_all('button[data-action="download"]')
best = None
- for b in sorted(matching, key=lambda x: x['downloads'], reverse=True):
- if b['index'] < len(btns) and not btns[b['index']].is_disabled():
+ for b in sorted(matching, key=lambda x: x["downloads"], reverse=True):
+ if b["index"] < len(btns) and not btns[b["index"]].is_disabled():
best = b
break
-
+
if not best:
print("\n >>> All matching books are already in download queue")
sys.stdout.flush()
return
-
- print(f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]")
+
+ print(
+ f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]"
+ )
sys.stdout.flush()
-
+
# Click on the article h3 to open detail view (required for download to work)
- h3s = page.query_selector_all('article h3')
- if best['index'] < len(h3s):
- h3s[best['index']].click()
+ h3s = page.query_selector_all("article h3")
+ if best["index"] < len(h3s):
+ h3s[best["index"]].click()
page.wait_for_timeout(300)
else:
print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)")
sys.stdout.flush()
-
+
# Click the download button
- btn = btns[best['index']]
+ btn = btns[best["index"]]
btn.click()
-
+
# Wait a moment for the click to register
page.wait_for_timeout(500)
- print(f" >> Click sent")
+ print(" >> Click sent")
sys.stdout.flush()
-
+
# Check sidebar immediately
time.sleep(2)
try:
- txt = page.inner_text('aside')
- if 'IN PROGRESS' in txt:
- print(f" >> Download started!")
+ txt = page.inner_text("aside")
+ if "IN PROGRESS" in txt:
+ print(" >> Download started!")
sys.stdout.flush()
else:
print(f" >> Sidebar after 2s: {txt[:150]}")
sys.stdout.flush()
except:
- print(f" >> Sidebar not visible after 2s")
+ print(" >> Sidebar not visible after 2s")
sys.stdout.flush()
-
+
# Wait for download to complete - check sidebar every 2 seconds, max 5 minutes
last_state = None
- for attempt in range(150):
+ for _attempt in range(150):
time.sleep(2)
try:
- txt = page.inner_text('aside')
- if 'IN PROGRESS' in txt:
- state = 'downloading'
- elif 'Complete' in txt or 'Saved' in txt or 'No activity' in txt:
- state = 'done'
+ txt = page.inner_text("aside")
+ if "IN PROGRESS" in txt:
+ state = "downloading"
+ elif "Complete" in txt or "Saved" in txt or "No activity" in txt:
+ state = "done"
else:
- state = 'other'
-
+ state = "other"
+
if state != last_state:
- if state == 'downloading':
- print(f" >> Download started!")
- elif state == 'done':
- if 'No activity' in txt:
- print(f" >> Download not started (no activity)")
+ if state == "downloading":
+ print(" >> Download started!")
+ elif state == "done":
+ if "No activity" in txt:
+ print(" >> Download not started (no activity)")
else:
- print(f" >> Download complete!")
+ print(" >> Download complete!")
last_state = state
-
- if state == 'done':
+
+ if state == "done":
break
except:
if last_state is None:
- print(f" >> Sidebar not visible yet")
- last_state = 'no_sidebar'
+ print(" >> Sidebar not visible yet")
+ last_state = "no_sidebar"
sys.stdout.flush()
else:
- if last_state != 'done':
+ if last_state != "done":
print(" >> Timeout waiting for download")
sys.stdout.flush()
-def clear_completed_downloads(page):
+
+def clear_completed_downloads(page: Page) -> None:
"""Clear completed downloads from previous sessions"""
- page.evaluate('''
+ page.evaluate("""
() => {
for (const b of document.querySelectorAll('button')) {
if (b.textContent.includes('Clear Completed')) { b.click(); return; }
}
}
- ''')
+ """)
page.wait_for_timeout(2000)
-def check_calibre(title):
+
+def check_calibre(title: str) -> bool:
"""Check if a book is already in the calibre database."""
import subprocess
+
cmd = f'calibredb list --search title:="{title}"'
- result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+ result = subprocess.run(cmd, check=False, shell=True, capture_output=True, text=True)
output = result.stdout.strip()
# calibredb output has header line "id title authors", count non-header lines
- lines = [l.strip() for l in output.split('\n') if l.strip()]
+ lines = [l.strip() for l in output.split("\n") if l.strip()]
# Remove header if present (first line starts with 'id')
- if lines and lines[0].startswith('id'):
+ if lines and lines[0].startswith("id"):
lines = lines[1:]
return len(lines) > 0
-def load_books_from_json(json_str, source):
+
+def load_books_from_json(json_str: str, source: str) -> list[tuple[str, str]]:
"""Load books from a JSON string or file path."""
try:
data = json.loads(json_str)
except json.JSONDecodeError as e:
print(f"Error: Invalid JSON in {source}: {e}")
sys.exit(1)
-
+
if not isinstance(data, list):
print(f"Error: JSON in {source} must be an array of book objects")
sys.exit(1)
-
+
books = []
for item in data:
if isinstance(item, dict):
- if 'title' not in item or 'author' not in item:
+ if "title" not in item or "author" not in item:
print(f"Error: Each book object must have 'title' and 'author' keys: {item}")
sys.exit(1)
- books.append((item['title'].strip(), item['author'].strip()))
+ books.append((item["title"].strip(), item["author"].strip()))
elif isinstance(item, list) and len(item) >= 2:
books.append((str(item[0]).strip(), str(item[1]).strip()))
elif isinstance(item, str):
@@ -274,80 +320,91 @@ def load_books_from_json(json_str, source):
print(f"Warning: Skipping unrecognized book entry: {item}")
return books
-def main():
- parser = argparse.ArgumentParser(description='Download books from Shelfmark')
- parser.add_argument('books', nargs='*', help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects')
- parser.add_argument('--file', '-f', help='Path to JSON file containing an array of books')
- parser.add_argument('-c', '--check-calibre', action='store_true', help='Check calibre database before downloading, skip if already present')
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="Download books from Shelfmark")
+ parser.add_argument(
+ "books",
+ nargs="*",
+ help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects',
+ )
+ parser.add_argument("--file", "-f", help="Path to JSON file containing an array of books")
+ parser.add_argument(
+ "-c",
+ "--check-calibre",
+ action="store_true",
+ help="Check calibre database before downloading, skip if already present",
+ )
args = parser.parse_args()
-
+
# Load books from --file, positional JSON arg, or both
- BOOKS = []
-
+ books_to_download = []
+
if args.file:
try:
- with open(args.file, 'r') as f:
- BOOKS = load_books_from_json(f.read(), f"file '{args.file}'")
+ with Path(args.file).open() as f:
+ books_to_download = load_books_from_json(f.read(), f"file '{args.file}'")
except FileNotFoundError:
print(f"Error: File not found: {args.file}")
sys.exit(1)
- except IOError as e:
+ except OSError as e:
print(f"Error reading file: {e}")
sys.exit(1)
-
+
for book_arg in args.books:
- BOOKS.extend(load_books_from_json(book_arg, 'command line argument'))
-
- if not BOOKS:
+ books_to_download.extend(load_books_from_json(book_arg, "command line argument"))
+
+ if not books_to_download:
print("Error: No books specified. Use positional JSON args or --file ")
sys.exit(1)
-
+
# Check calibre database if requested
if args.check_calibre:
print("\nChecking calibre database...")
sys.stdout.flush()
remaining = []
- for title, author in BOOKS:
+ for title, author in books_to_download:
if check_calibre(title):
print(f" >> '{title}' already in calibre, skipping")
sys.stdout.flush()
else:
remaining.append((title, author))
- BOOKS = remaining
- if not BOOKS:
+ books_to_download = remaining
+ if not books_to_download:
print("All books already in calibre, nothing to download.")
sys.stdout.flush()
return
-
- print(f"\n{'='*60}")
- print(f"Downloading {len(BOOKS)} book(s) from Shelfmark")
- print('='*60)
+
+ print(f"\n{'=' * 60}")
+ print(f"Downloading {len(books_to_download)} book(s) from Shelfmark")
+ print("=" * 60)
sys.stdout.flush()
-
+
with sync_playwright() as p:
browser = p.chromium.launch()
- page = browser.new_page(viewport={'width': 1280, 'height': 900})
+ page = browser.new_page(viewport={"width": 1280, "height": 900})
page.goto(SHELFMARK_URL)
- page.wait_for_load_state('networkidle')
-
+ page.wait_for_load_state("networkidle")
+
# Clear completed downloads at start of batch
- print(f"\n{'='*60}")
+ print(f"\n{'=' * 60}")
print("Clearing completed downloads from previous sessions...")
- print('='*60)
+ print("=" * 60)
sys.stdout.flush()
clear_completed_downloads(page)
-
- for i, (title, author) in enumerate(BOOKS, 1):
- print(f"\n{'='*60}")
- print(f"[{i}/{len(BOOKS)}] '{title}' by {author}")
- print('='*60)
+
+ for i, (title, author) in enumerate(books_to_download, 1):
+ print(f"\n{'=' * 60}")
+ print(f"[{i}/{len(books_to_download)}] '{title}' by {author}")
+ print("=" * 60)
sys.stdout.flush()
-
+
download_book(page, title, author)
-
+
browser.close()
print("\nDone!")
sys.stdout.flush()
-if __name__ == '__main__':
+
+if __name__ == "__main__":
main()
diff --git a/src/frontend/src/components/activity/activityTypes.ts b/src/frontend/src/components/activity/activityTypes.ts
index 62ab2942..6b2ce93a 100644
--- a/src/frontend/src/components/activity/activityTypes.ts
+++ b/src/frontend/src/components/activity/activityTypes.ts
@@ -1,6 +1,6 @@
import type { RequestRecord } from '../../types';
-export type ActivityKind = 'download' | 'request';
+type ActivityKind = 'download' | 'request';
export type ActivityVisualStatus =
| 'queued'
diff --git a/src/frontend/src/components/shared/index.ts b/src/frontend/src/components/shared/index.ts
index c6983181..fdaecb4b 100644
--- a/src/frontend/src/components/shared/index.ts
+++ b/src/frontend/src/components/shared/index.ts
@@ -1,4 +1,4 @@
export { DisplayFieldIcon, DisplayFieldBadge, DisplayFieldBadges } from './DisplayFieldIcon';
export { CircularProgress } from './CircularProgress';
export { ToggleSwitch } from './ToggleSwitch';
-export { LibraryBadge, isInLibrary, isCollectionOnly } from './LibraryBadge';
+export { LibraryBadge, isInLibrary } from './LibraryBadge';
diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts
index b6802247..e621d86b 100644
--- a/src/frontend/src/services/api.ts
+++ b/src/frontend/src/services/api.ts
@@ -861,9 +861,9 @@ export const getReleases = async (
// Admin user management API
-export type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
+type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
-export interface AdminUserEditCapabilities {
+interface AdminUserEditCapabilities {
authSource: AdminAuthSource;
canSetPassword: boolean;
canEditRole: boolean;
diff --git a/src/frontend/src/styles.css b/src/frontend/src/styles.css
index 9ef13ca5..ef412214 100644
--- a/src/frontend/src/styles.css
+++ b/src/frontend/src/styles.css
@@ -172,13 +172,7 @@
}
body {
- font-family:
- 'Inter',
- -apple-system,
- BlinkMacSystemFont,
- 'Segoe UI',
- Roboto,
- sans-serif;
+ font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
line-height: 1.6;
color: var(--text-color);
background: var(--background-color);
diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts
index 4564c9be..14729a61 100644
--- a/src/frontend/src/types/index.ts
+++ b/src/frontend/src/types/index.ts
@@ -1,5 +1,5 @@
// Search mode constants and type
-export const SEARCH_MODE = {
+const SEARCH_MODE = {
DIRECT: 'direct',
UNIVERSAL: 'universal',
} as const;
@@ -88,7 +88,7 @@ export interface StatusData {
}
// Button states
-export type ButtonState =
+type ButtonState =
| 'download'
| 'queued'
| 'resolving'
@@ -156,24 +156,24 @@ export interface TextSearchField extends SearchFieldBase {
suggestions_min_query_length?: number;
}
-export interface NumberSearchField extends SearchFieldBase {
+interface NumberSearchField extends SearchFieldBase {
type: 'NumberSearchField';
min?: number;
max?: number;
step?: number;
}
-export interface SelectSearchField extends SearchFieldBase {
+interface SelectSearchField extends SearchFieldBase {
type: 'SelectSearchField';
options: SortOption[];
}
-export interface CheckboxSearchField extends SearchFieldBase {
+interface CheckboxSearchField extends SearchFieldBase {
type: 'CheckboxSearchField';
default?: boolean;
}
-export interface DynamicSelectSearchField extends SearchFieldBase {
+interface DynamicSelectSearchField extends SearchFieldBase {
type: 'DynamicSelectSearchField';
options_endpoint: string;
}
@@ -185,7 +185,7 @@ export type MetadataSearchField =
| CheckboxSearchField
| DynamicSelectSearchField;
-export type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
+type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
export interface QueryTargetOption {
key: string;
@@ -201,12 +201,12 @@ export type ContentType = 'ebook' | 'audiobook';
export type RequestPolicyMode = 'download' | 'request_release' | 'request_book' | 'blocked';
-export interface RequestPolicyDefaults {
+interface RequestPolicyDefaults {
ebook: RequestPolicyMode;
audiobook: RequestPolicyMode;
}
-export interface RequestPolicySourceMode {
+interface RequestPolicySourceMode {
source: string;
supported_content_types: string[];
browse_results_are_releases?: boolean;
@@ -222,7 +222,7 @@ export interface RequestPolicyResponse {
source_modes: RequestPolicySourceMode[];
}
-export interface RequestContextPayload {
+interface RequestContextPayload {
source: string;
content_type: ContentType;
request_level: 'book' | 'release';
@@ -279,14 +279,14 @@ export interface QueuedDownloadResult {
export type RequestSubmissionResult = RequestRecord | QueuedDownloadResult;
/** How the library holds a book, per format: owned outright, or inside a collection. */
-export type LibraryHolding = 'owned' | 'collection';
+type LibraryHolding = 'owned' | 'collection';
export interface LibraryOwnership {
ebook?: LibraryHolding | null;
audiobook?: LibraryHolding | null;
}
-export type BooksOutputMode = 'folder' | 'booklore' | 'email';
+type BooksOutputMode = 'folder' | 'booklore' | 'email';
export interface AppConfig {
calibre_web_url: string;
@@ -334,7 +334,7 @@ export interface MetadataProvidersResponse {
configured_provider_combined: string | null;
}
-export interface MetadataCapability {
+interface MetadataCapability {
key: string;
field_key?: string;
sort?: string;
@@ -398,7 +398,7 @@ export interface ReleaseSource {
}
// Column schema types for plugin-driven release list UI
-export type ColumnRenderType =
+type ColumnRenderType =
| 'text'
| 'badge'
| 'tags'
@@ -408,7 +408,7 @@ export type ColumnRenderType =
| 'indexer_protocol'
| 'flag_icon'
| 'format_content_type';
-export type ColumnAlign = 'left' | 'center' | 'right';
+type ColumnAlign = 'left' | 'center' | 'right';
export interface ColumnColorHint {
type: 'map' | 'static'; // 'map' uses colorMaps.ts, 'static' is a fixed Tailwind class
@@ -430,7 +430,7 @@ export interface ColumnSchema {
}
// Leading cell config - what to show in the left-most position of each row
-export type LeadingCellType = 'thumbnail' | 'badge' | 'none';
+type LeadingCellType = 'thumbnail' | 'badge' | 'none';
export interface LeadingCellConfig {
type: LeadingCellType;
@@ -439,13 +439,13 @@ export interface LeadingCellConfig {
uppercase?: boolean; // Force uppercase for badge text
}
-export interface ExtraSortOption {
+interface ExtraSortOption {
label: string; // Display label in the sort dropdown
sort_key: string; // Field to sort by on the Release object
default_direction?: 'asc' | 'desc'; // Which way "best first" runs (defaults to desc)
}
-export interface SourceActionButton {
+interface SourceActionButton {
label: string; // Button text (e.g., "Refresh search")
action: string; // Action type: "expand" triggers expand_search
}
@@ -483,7 +483,7 @@ export interface Release {
}
// Search info returned by release sources
-export interface SourceSearchInfo {
+interface SourceSearchInfo {
search_type: 'isbn' | 'title_author' | 'categories' | 'expanded' | 'manual' | 'query';
total_results?: number | string | null;
}
diff --git a/src/frontend/src/types/settings.ts b/src/frontend/src/types/settings.ts
index 67ad96b6..a5dc5e0b 100644
--- a/src/frontend/src/types/settings.ts
+++ b/src/frontend/src/types/settings.ts
@@ -1,6 +1,6 @@
// Settings field types matching backend settings_registry.py
-export type FieldType =
+type FieldType =
| 'TextField'
| 'PasswordField'
| 'NumberField'
@@ -28,17 +28,17 @@ export interface ShowWhenCondition {
notEmpty?: boolean; // If true, show when field has any non-empty value
}
-export type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
+type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
// Conditional disable configuration
-export interface DisabledWhenCondition {
+interface DisabledWhenCondition {
field: string; // The field key to check
value: string | string[] | boolean; // The value(s) that disable this field
reason?: string; // Explanation shown when disabled
}
// Base field interface - common properties
-export interface BaseField {
+interface BaseField {
key: string;
label: string;
type: FieldType;
@@ -139,20 +139,14 @@ export interface CustomComponentFieldConfig extends BaseField {
wrapInFieldWrapper?: boolean; // Whether to render with standard FieldWrapper layout
}
-export interface TableFieldColumnOption {
+interface TableFieldColumnOption {
value: string;
label: string;
description?: string;
childOf?: string;
}
-export type TableFieldColumnType =
- | 'text'
- | 'password'
- | 'select'
- | 'multiselect'
- | 'checkbox'
- | 'path';
+type TableFieldColumnType = 'text' | 'password' | 'select' | 'multiselect' | 'checkbox' | 'path';
export interface TableFieldColumn {
key: string;
diff --git a/src/frontend/src/utils/colorMaps.ts b/src/frontend/src/utils/colorMaps.ts
index f5bf1860..9eedc39e 100644
--- a/src/frontend/src/utils/colorMaps.ts
+++ b/src/frontend/src/utils/colorMaps.ts
@@ -1,3 +1,5 @@
+import type { ColumnColorHint } from '../types';
+
// Color styles with transparent backgrounds and contrasting text
interface ColorStyle {
bg: string;
@@ -133,14 +135,6 @@ function getFlagColor(flag?: string): ColorStyle {
return FLAG_COLORS[normalized] || DEFAULT_FLAG_COLOR;
}
-/**
- * Color hint type for dynamic column coloring.
- */
-interface ColumnColorHint {
- type: 'static' | 'map';
- value: string;
-}
-
/**
* Get the color style for a value based on a color hint.
* Supports both static color classes and dynamic map lookups.
diff --git a/src/frontend/src/utils/contentTypePreference.ts b/src/frontend/src/utils/contentTypePreference.ts
index 9409430d..d2d4b8bb 100644
--- a/src/frontend/src/utils/contentTypePreference.ts
+++ b/src/frontend/src/utils/contentTypePreference.ts
@@ -10,7 +10,7 @@ import type { ContentType } from '../types';
export const CONTENT_TYPE_STORAGE_KEY = 'preferred-content-type.v2';
/** Where the current content type came from, which decides what may still override it. */
-export type ContentTypeSource =
+type ContentTypeSource =
/** Nothing has chosen a tab, so a configured default may still apply. */
| 'unset'
/** This browser stored a choice, or something explicit set one. Defaults do not apply. */
diff --git a/tests/core/test_activity_routes_api.py b/tests/core/test_activity_routes_api.py
index f1381d6a..7422a3f0 100644
--- a/tests/core/test_activity_routes_api.py
+++ b/tests/core/test_activity_routes_api.py
@@ -3,8 +3,10 @@
from __future__ import annotations
import importlib
+import io
import sqlite3
import uuid
+import zipfile
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import ANY, patch
@@ -355,6 +357,46 @@ class TestActivityRoutes:
assert response.status_code == 200
assert response.data == file_bytes
+ def test_localdownload_zips_a_live_multi_file_download(self, main_module, client, tmp_path):
+ user = _create_user(main_module, prefix="reader")
+ _set_session(client, user_id=user["username"], db_user_id=user["id"], is_admin=False)
+
+ task_id = f"live-localdownload-{uuid.uuid4().hex[:8]}"
+ book_dir = tmp_path / "Chapters"
+ book_dir.mkdir()
+ parts = [book_dir / "Part 1.mp3", book_dir / "Part 2.mp3"]
+ for index, part in enumerate(parts, start=1):
+ part.write_bytes(f"chapter {index}".encode())
+
+ task = DownloadTask(
+ task_id=task_id,
+ source="direct_download",
+ title="Zipped Audiobook",
+ author="Some Author",
+ user_id=user["id"],
+ username=user["username"],
+ download_path=str(parts[0]),
+ download_paths=[str(part) for part in parts],
+ )
+ assert main_module.backend.book_queue.add(task) is True
+
+ try:
+ main_module.backend.book_queue.update_status(task_id, QueueStatus.COMPLETE)
+ with patch.object(main_module, "get_auth_mode", return_value="builtin"):
+ response = client.get(f"/api/localdownload?id={task_id}")
+ finally:
+ main_module.backend.book_queue.cancel_download(task_id)
+
+ assert response.status_code == 200
+ assert response.mimetype == "application/zip"
+ assert (
+ 'filename="Some Author - Zipped Audiobook.zip"'
+ in response.headers["Content-Disposition"]
+ )
+ with zipfile.ZipFile(io.BytesIO(response.data)) as archive:
+ assert archive.namelist() == ["Part 1.mp3", "Part 2.mp3"]
+ assert archive.read("Part 2.mp3") == b"chapter 2"
+
def test_localdownload_returns_not_found_for_other_users_live_queue_task(
self, main_module, client, tmp_path
):
diff --git a/tests/core/test_processing_integration.py b/tests/core/test_processing_integration.py
index 0aeb4156..3fd86294 100644
--- a/tests/core/test_processing_integration.py
+++ b/tests/core/test_processing_integration.py
@@ -343,6 +343,9 @@ def test_multifile_rename_ignores_template_even_with_original_name(tmp_path):
files = sorted(path.name for path in ingest.glob("*.mp3"))
assert files == ["Part 1 of 2.mp3", "Part 2 of 2.mp3"]
assert all("RENAMED" not in name for name in files)
+ # Browser download serves all of these, not just the returned first file.
+ assert result in task.download_paths
+ assert sorted(Path(path).name for path in task.download_paths) == files
def test_torrent_hardlink_enabled_copy_fallback_does_not_extract_archives(tmp_path):
diff --git a/tests/direct_download/test_source_availability.py b/tests/direct_download/test_source_availability.py
index 3a86ec6c..020591e1 100644
--- a/tests/direct_download/test_source_availability.py
+++ b/tests/direct_download/test_source_availability.py
@@ -55,7 +55,10 @@ def test_direct_download_explains_aa_requirement_when_only_zlib_is_configured(mo
source = DirectDownloadSource()
- with pytest.raises(SearchUnavailableError, match="Anna's Archive mirror"):
+ with pytest.raises(
+ SearchUnavailableError,
+ match="Anna's Archive or OceanofPDF mirror URL.*only download results found through",
+ ):
source.search(SimpleNamespace(), SimpleNamespace())
diff --git a/tests/download/test_orchestrator_retry.py b/tests/download/test_orchestrator_retry.py
index d938c3bb..9caf9117 100644
--- a/tests/download/test_orchestrator_retry.py
+++ b/tests/download/test_orchestrator_retry.py
@@ -332,3 +332,51 @@ def test_get_book_path_reports_a_missing_file_rather_than_a_dead_path(monkeypatc
assert path is None
assert returned_task is task
+
+
+def test_get_book_paths_returns_every_file_still_on_disk(monkeypatch, tmp_path):
+ import shelfmark.download.orchestrator as orchestrator
+
+ parts = [tmp_path / f"Part {n}.mp3" for n in (1, 2, 3)]
+ for part in parts[:2]:
+ part.write_bytes(b"audio")
+ task = DownloadTask(
+ task_id="task-book-paths-1",
+ source="direct_download",
+ title="Chapters",
+ download_path=str(parts[0]),
+ download_paths=[str(part) for part in parts],
+ )
+
+ mock_queue = MagicMock()
+ mock_queue.get_task.return_value = task
+ monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
+
+ paths, returned_task = orchestrator.get_book_paths(task.task_id)
+
+ assert paths == [str(parts[0]), str(parts[1])]
+ assert returned_task is task
+
+
+def test_get_book_paths_ignores_a_list_that_no_longer_holds_download_path(monkeypatch, tmp_path):
+ import shelfmark.download.orchestrator as orchestrator
+
+ stale = tmp_path / "old-attempt.mp3"
+ current = tmp_path / "current.epub"
+ stale.write_bytes(b"old")
+ current.write_bytes(b"new")
+ task = DownloadTask(
+ task_id="task-book-paths-2",
+ source="direct_download",
+ title="Retried",
+ download_path=str(current),
+ download_paths=[str(stale)],
+ )
+
+ mock_queue = MagicMock()
+ mock_queue.get_task.return_value = task
+ monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
+
+ paths, _ = orchestrator.get_book_paths(task.task_id)
+
+ assert paths == [str(current)]
diff --git a/tests/download/test_zip_stream.py b/tests/download/test_zip_stream.py
new file mode 100644
index 00000000..7dfdc368
--- /dev/null
+++ b/tests/download/test_zip_stream.py
@@ -0,0 +1,41 @@
+import io
+import zipfile
+
+from shelfmark.download.zip_stream import stream_zip, zip_entry_names
+
+
+def test_stream_zip_holds_every_file_uncompressed(tmp_path):
+ book = tmp_path / "Author" / "Title"
+ book.mkdir(parents=True)
+ (book / "Part 01.m4b").write_bytes(b"chapter one")
+ (book / "Part 02.m4b").write_bytes(b"chapter two" * 1000)
+
+ data = b"".join(stream_zip([str(book / "Part 01.m4b"), str(book / "Part 02.m4b")]))
+
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
+ assert archive.testzip() is None
+ assert archive.namelist() == ["Part 01.m4b", "Part 02.m4b"]
+ assert archive.read("Part 01.m4b") == b"chapter one"
+ assert archive.read("Part 02.m4b") == b"chapter two" * 1000
+ assert {info.compress_type for info in archive.infolist()} == {zipfile.ZIP_STORED}
+
+
+def test_stream_zip_yields_while_reading_rather_than_at_the_end(tmp_path, monkeypatch):
+ import shelfmark.download.zip_stream as zip_stream
+
+ monkeypatch.setattr(zip_stream, "_ZIP_STREAM_CHUNK_SIZE", 16)
+ big = tmp_path / "big.mp3"
+ big.write_bytes(b"x" * 160)
+
+ chunks = list(stream_zip([str(big)]))
+
+ assert len(chunks) > 10
+
+
+def test_zip_entry_names_keep_folders_below_the_common_parent(tmp_path):
+ first = tmp_path / "library" / "Book One" / "a.mp3"
+ second = tmp_path / "library" / "Book Two" / "b.mp3"
+
+ names = [name for _, name in zip_entry_names([str(first), str(second)])]
+
+ assert names == ["Book One/a.mp3", "Book Two/b.mp3"]