mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-10 12:28:27 +00:00
fix(download): send every file of a multi-file download to the browser (#1489)
Download to Browser only ever sent the first file of a multi-file release (audiobook chapters, packs): the folder output returned final_paths[0] and /api/localdownload served that one path. The folder output now keeps every final path on the in-memory task (download_paths), and /api/localdownload streams them as one uncompressed ZIP named after the book when there is more than one. The archive is never staged on disk or held in memory, and ZIP64 covers large audiobooks. History still stores only the first path, so a re-download from history after a dismiss or restart serves a single file, as before. Also brings every linter back to green on all files: - pre-commit: bump the ruff hook to v0.17.1 to match uv.lock; ruff-format the Python examples in docs/dev and skills - ruff: fix the remaining issues in skills/shelfmark/download_books.py; format base_handler.py - basedpyright: Optional and attrs errors in databazeknih.py - knip: drop 31 unused exports, reuse ColumnColorHint in colorMaps - oxfmt 0.72: styles.css - direct download: fix the "Welib, mirrors" typo and the test that still expected the pre-OceanofPDF message Fixes #1467
This commit is contained in:
1 parent
3025f120eb
commit
10710abd4e
26 files changed
+612
-292
No files matched your search
@@ -6,7 +6,7 @@ repos:
|
||||
- id: check-added-large-files
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.10
|
||||
rev: v0.17.1
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix]
|
||||
|
||||
+36
-29
@@ -23,12 +23,13 @@ from shelfmark.core.settings_registry import (
|
||||
ActionButton,
|
||||
)
|
||||
|
||||
|
||||
@register_settings(
|
||||
name="my_plugin", # Unique identifier
|
||||
display_name="My Plugin", # Shown in sidebar
|
||||
icon="wrench", # Icon name
|
||||
order=100, # Sort order (lower = higher in list)
|
||||
group="metadata_providers" # Optional: group in sidebar
|
||||
name="my_plugin", # Unique identifier
|
||||
display_name="My Plugin", # Shown in sidebar
|
||||
icon="wrench", # Icon name
|
||||
order=100, # Sort order (lower = higher in list)
|
||||
group="metadata_providers", # Optional: group in sidebar
|
||||
)
|
||||
def my_plugin_settings():
|
||||
return [
|
||||
@@ -46,6 +47,7 @@ def my_plugin_settings():
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _test_connection():
|
||||
# Perform connection test
|
||||
return {"success": True, "message": "Connected successfully!"}
|
||||
@@ -59,16 +61,16 @@ Single-line text input for strings.
|
||||
|
||||
```python
|
||||
TextField(
|
||||
key="MY_SETTING", # Config key
|
||||
label="Setting Name", # Display label
|
||||
description="Help text", # Optional description below field
|
||||
default="", # Default value
|
||||
key="MY_SETTING", # Config key
|
||||
label="Setting Name", # Display label
|
||||
description="Help text", # Optional description below field
|
||||
default="", # Default value
|
||||
placeholder="Enter value", # Placeholder text
|
||||
max_length=100, # Optional max characters
|
||||
required=False, # Is this field required?
|
||||
requires_restart=False, # Does changing this need a restart?
|
||||
show_when=None, # Conditional visibility (see below)
|
||||
disabled_when=None, # Conditional disable (see below)
|
||||
max_length=100, # Optional max characters
|
||||
required=False, # Is this field required?
|
||||
requires_restart=False, # Does changing this need a restart?
|
||||
show_when=None, # Conditional visibility (see below)
|
||||
disabled_when=None, # Conditional disable (see below)
|
||||
)
|
||||
```
|
||||
|
||||
@@ -98,7 +100,7 @@ NumberField(
|
||||
default=30,
|
||||
min_value=5,
|
||||
max_value=300,
|
||||
step=1, # Increment step
|
||||
step=1, # Increment step
|
||||
required=False,
|
||||
)
|
||||
```
|
||||
@@ -160,13 +162,14 @@ Button that executes a callback function. Does not store a value.
|
||||
|
||||
```python
|
||||
ActionButton(
|
||||
key="test_connection", # Unique key for the action
|
||||
label="Test Connection", # Button text
|
||||
key="test_connection", # Unique key for the action
|
||||
label="Test Connection", # Button text
|
||||
description="Test the API connection",
|
||||
style="primary", # "default", "primary", or "danger"
|
||||
callback=my_callback_fn, # Function to execute
|
||||
style="primary", # "default", "primary", or "danger"
|
||||
callback=my_callback_fn, # Function to execute
|
||||
)
|
||||
|
||||
|
||||
def my_callback_fn():
|
||||
"""Callback must return dict with 'success' and 'message' keys."""
|
||||
try:
|
||||
@@ -182,11 +185,11 @@ Display-only section heading with optional link. Does not store a value.
|
||||
|
||||
```python
|
||||
HeadingField(
|
||||
key="section_heading", # Unique key
|
||||
title="Configuration", # Heading text
|
||||
key="section_heading", # Unique key
|
||||
title="Configuration", # Heading text
|
||||
description="Configure the plugin settings below",
|
||||
link_url="https://example.com/docs", # Optional link
|
||||
link_text="View Documentation", # Link text
|
||||
link_text="View Documentation", # Link text
|
||||
)
|
||||
```
|
||||
|
||||
@@ -204,8 +207,14 @@ CustomComponentField(
|
||||
label="Request Policy Rules",
|
||||
description="Custom editor for policy defaults and matrix rules.",
|
||||
value_fields=[
|
||||
SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"),
|
||||
SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"),
|
||||
SelectField(
|
||||
key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"
|
||||
),
|
||||
SelectField(
|
||||
key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
|
||||
label="Default Audiobook Mode",
|
||||
default="download",
|
||||
),
|
||||
TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
|
||||
],
|
||||
wrap_in_field_wrapper=True, # use standard FieldWrapper label/description layout
|
||||
@@ -263,7 +272,7 @@ NumberField(
|
||||
disabled_when={
|
||||
"field": "FEATURE_ENABLED",
|
||||
"value": False,
|
||||
"reason": "Enable the feature first"
|
||||
"reason": "Enable the feature first",
|
||||
},
|
||||
)
|
||||
```
|
||||
@@ -285,6 +294,7 @@ register_group(
|
||||
order=50,
|
||||
)
|
||||
|
||||
|
||||
# Then register settings to the group
|
||||
@register_settings(
|
||||
name="plugin_a",
|
||||
@@ -426,10 +436,7 @@ class MyProvider(MetadataProvider):
|
||||
|
||||
def __init__(self, api_key: str = None):
|
||||
self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
|
||||
self.base_url = config.get(
|
||||
"MY_PROVIDER_BASE_URL",
|
||||
"https://api.myprovider.com/v1"
|
||||
)
|
||||
self.base_url = config.get("MY_PROVIDER_BASE_URL", "https://api.myprovider.com/v1")
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool(self.api_key)
|
||||
|
||||
@@ -66,10 +66,11 @@ This separation allows:
|
||||
A source has both a `ReleaseSource` (for searching) and a `DownloadHandler` (for downloading). They share the same `name` identifier:
|
||||
|
||||
```python
|
||||
@register_source("my_source") # Search registration
|
||||
@register_source("my_source") # Search registration
|
||||
class MySource(ReleaseSource): ...
|
||||
|
||||
@register_handler("my_source") # Download registration
|
||||
|
||||
@register_handler("my_source") # Download registration
|
||||
class MyHandler(DownloadHandler): ...
|
||||
```
|
||||
|
||||
@@ -142,10 +143,11 @@ from typing import List
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
from shelfmark.release_sources import Release, ReleaseColumnConfig
|
||||
|
||||
|
||||
class ReleaseSource(ABC):
|
||||
"""Interface for searching a release source."""
|
||||
|
||||
name: str # Internal identifier: "direct_download", "prowlarr"
|
||||
name: str # Internal identifier: "direct_download", "prowlarr"
|
||||
display_name: str # User-facing name: "Direct Download", "Prowlarr"
|
||||
|
||||
@abstractmethod
|
||||
@@ -191,14 +193,16 @@ def search(self, book: BookMetadata) -> List[Release]:
|
||||
if book.isbn_13:
|
||||
api_results = self.api.search_isbn(book.isbn_13)
|
||||
for r in api_results:
|
||||
results.append(Release(
|
||||
source="my_source",
|
||||
source_id=r.id,
|
||||
title=r.title,
|
||||
format=r.format,
|
||||
size=r.size,
|
||||
# ... more fields
|
||||
))
|
||||
results.append(
|
||||
Release(
|
||||
source="my_source",
|
||||
source_id=r.id,
|
||||
title=r.title,
|
||||
format=r.format,
|
||||
size=r.size,
|
||||
# ... more fields
|
||||
)
|
||||
)
|
||||
|
||||
# Strategy 2: Fallback to title + author search
|
||||
if not results:
|
||||
@@ -239,6 +243,7 @@ from typing import Callable, Optional
|
||||
from threading import Event
|
||||
from shelfmark.core.models import DownloadTask
|
||||
|
||||
|
||||
class DownloadHandler(ABC):
|
||||
"""Interface for executing downloads from a source."""
|
||||
|
||||
@@ -286,6 +291,7 @@ downloading it; the default returns `None`, which the UI reports as "can't inspe
|
||||
```python
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""Return the release's files (release-relative paths + sizes), or None."""
|
||||
torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page
|
||||
@@ -380,20 +386,20 @@ Standardized search result returned by all sources:
|
||||
```python
|
||||
@dataclass
|
||||
class Release:
|
||||
source: str # "direct_download", "prowlarr", "irc"
|
||||
source_id: str # Unique ID within that source
|
||||
title: str # Display title
|
||||
source: str # "direct_download", "prowlarr", "irc"
|
||||
source_id: str # Unique ID within that source
|
||||
title: str # Display title
|
||||
|
||||
format: Optional[str] = None # File format: "epub", "mobi", "pdf"
|
||||
size: Optional[str] = None # Human-readable: "5.2 MB"
|
||||
size_bytes: Optional[int] = None # For sorting
|
||||
format: Optional[str] = None # File format: "epub", "mobi", "pdf"
|
||||
size: Optional[str] = None # Human-readable: "5.2 MB"
|
||||
size_bytes: Optional[int] = None # For sorting
|
||||
|
||||
download_url: Optional[str] = None # Direct download URL if available
|
||||
info_url: Optional[str] = None # Link to tracker/info page
|
||||
info_url: Optional[str] = None # Link to tracker/info page
|
||||
|
||||
protocol: Optional[str] = None # "http", "torrent", "usenet"
|
||||
indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer"
|
||||
seeders: Optional[int] = None # For torrents
|
||||
protocol: Optional[str] = None # "http", "torrent", "usenet"
|
||||
indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer"
|
||||
seeders: Optional[int] = None # For torrents
|
||||
|
||||
extra: Dict = field(default_factory=dict) # Source-specific metadata
|
||||
```
|
||||
@@ -415,7 +421,7 @@ Release(
|
||||
"preview": "https://example.com/cover.jpg",
|
||||
"quality": "high",
|
||||
"publisher": "Publisher Name",
|
||||
}
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
@@ -428,15 +434,15 @@ Source-agnostic task in the download queue:
|
||||
```python
|
||||
@dataclass
|
||||
class DownloadTask:
|
||||
task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID)
|
||||
source: str # Handler name: "direct_download"
|
||||
title: str # Display title
|
||||
task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID)
|
||||
source: str # Handler name: "direct_download"
|
||||
title: str # Display title
|
||||
|
||||
# Display info (from Release.extra or Release fields)
|
||||
author: Optional[str] = None
|
||||
format: Optional[str] = None
|
||||
size: Optional[str] = None
|
||||
preview: Optional[str] = None # Cover image URL
|
||||
preview: Optional[str] = None # Cover image URL
|
||||
|
||||
# Runtime state (managed by orchestrator)
|
||||
priority: int = 0
|
||||
@@ -454,8 +460,8 @@ Book information passed to `search()`:
|
||||
```python
|
||||
@dataclass
|
||||
class BookMetadata:
|
||||
provider: str # Metadata provider: "hardcover", "openlibrary"
|
||||
provider_id: str # ID in provider's system
|
||||
provider: str # Metadata provider: "hardcover", "openlibrary"
|
||||
provider_id: str # ID in provider's system
|
||||
title: str
|
||||
|
||||
provider_display_name: Optional[str] = None
|
||||
@@ -466,7 +472,7 @@ class BookMetadata:
|
||||
description: Optional[str] = None
|
||||
publisher: Optional[str] = None
|
||||
publish_year: Optional[int] = None
|
||||
language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr"
|
||||
language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr"
|
||||
genres: List[str] = field(default_factory=list)
|
||||
source_url: Optional[str] = None
|
||||
display_fields: List[DisplayField] = field(default_factory=list)
|
||||
@@ -481,13 +487,13 @@ class BookMetadata:
|
||||
```python
|
||||
from shelfmark.release_sources import register_source, register_handler
|
||||
|
||||
|
||||
@register_source("my_source")
|
||||
class MySource(ReleaseSource):
|
||||
...
|
||||
class MySource(ReleaseSource): ...
|
||||
|
||||
|
||||
@register_handler("my_source")
|
||||
class MyHandler(DownloadHandler):
|
||||
...
|
||||
class MyHandler(DownloadHandler): ...
|
||||
```
|
||||
|
||||
### Registry Functions
|
||||
@@ -544,12 +550,13 @@ from shelfmark.core.settings_registry import (
|
||||
is_value_from_env,
|
||||
)
|
||||
|
||||
|
||||
@register_settings(
|
||||
name="my_source", # Unique identifier
|
||||
display_name="My Source", # Shown in settings UI
|
||||
icon="globe", # Icon name (optional)
|
||||
order=60, # Sort order in sidebar
|
||||
group="sources", # Grouping in sidebar
|
||||
name="my_source", # Unique identifier
|
||||
display_name="My Source", # Shown in settings UI
|
||||
icon="globe", # Icon name (optional)
|
||||
order=60, # Sort order in sidebar
|
||||
group="sources", # Grouping in sidebar
|
||||
)
|
||||
def my_source_settings():
|
||||
"""Register settings for My Source plugin."""
|
||||
@@ -559,7 +566,6 @@ def my_source_settings():
|
||||
title="My Source Configuration",
|
||||
description="Configure connection to My Source API",
|
||||
),
|
||||
|
||||
PasswordField(
|
||||
key="MY_SOURCE_API_KEY",
|
||||
label="API Key",
|
||||
@@ -567,14 +573,12 @@ def my_source_settings():
|
||||
required=True,
|
||||
env_supported=True, # Can be set via MY_SOURCE_API_KEY env var
|
||||
),
|
||||
|
||||
TextField(
|
||||
key="MY_SOURCE_BASE_URL",
|
||||
label="Base URL",
|
||||
placeholder="https://api.mysource.com",
|
||||
env_supported=True,
|
||||
),
|
||||
|
||||
NumberField(
|
||||
key="MY_SOURCE_TIMEOUT",
|
||||
label="Request Timeout",
|
||||
@@ -583,14 +587,12 @@ def my_source_settings():
|
||||
min_value=5,
|
||||
max_value=120,
|
||||
),
|
||||
|
||||
CheckboxField(
|
||||
key="MY_SOURCE_ENABLED",
|
||||
label="Enable My Source",
|
||||
description="Include in release searches",
|
||||
default=True,
|
||||
),
|
||||
|
||||
ActionButton(
|
||||
key="test_connection",
|
||||
label="Test Connection",
|
||||
@@ -599,22 +601,19 @@ def my_source_settings():
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def test_my_source_connection():
|
||||
"""Test connection callback - returns result dict."""
|
||||
try:
|
||||
# Get current API key
|
||||
api_key = get_setting_value(
|
||||
PasswordField(key="MY_SOURCE_API_KEY", label=""),
|
||||
"my_source"
|
||||
)
|
||||
api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
|
||||
|
||||
if not api_key:
|
||||
return {"success": False, "message": "API key not configured"}
|
||||
|
||||
# Test the connection
|
||||
response = requests.get(
|
||||
"https://api.mysource.com/ping",
|
||||
headers={"Authorization": f"Bearer {api_key}"}
|
||||
"https://api.mysource.com/ping", headers={"Authorization": f"Bearer {api_key}"}
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
@@ -648,6 +647,7 @@ from shelfmark.core.settings_registry import (
|
||||
load_config_file,
|
||||
)
|
||||
|
||||
|
||||
# In your source/handler class
|
||||
def __init__(self):
|
||||
# Load from config file
|
||||
@@ -655,11 +655,9 @@ def __init__(self):
|
||||
self.api_key = config.get("MY_SOURCE_API_KEY")
|
||||
self.base_url = config.get("MY_SOURCE_BASE_URL", "https://api.mysource.com")
|
||||
|
||||
|
||||
# Or use get_setting_value for ENV var priority
|
||||
api_key = get_setting_value(
|
||||
PasswordField(key="MY_SOURCE_API_KEY", label=""),
|
||||
"my_source"
|
||||
)
|
||||
api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
|
||||
```
|
||||
|
||||
### Settings Priority
|
||||
@@ -689,6 +687,7 @@ from shelfmark.release_sources import (
|
||||
LeadingCellType,
|
||||
)
|
||||
|
||||
|
||||
@classmethod
|
||||
def get_column_config(cls) -> ReleaseColumnConfig:
|
||||
return ReleaseColumnConfig(
|
||||
@@ -759,7 +758,7 @@ For `BADGE` render type, specify colors:
|
||||
|
||||
```python
|
||||
# Use frontend color map (defined in colorMaps.ts)
|
||||
ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc.
|
||||
ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc.
|
||||
ColumnColorHint(type="map", value="language") # Maps en→blue, de→yellow, etc.
|
||||
|
||||
# Use static Tailwind class
|
||||
@@ -963,6 +962,7 @@ logger = setup_logger(__name__)
|
||||
# Settings Registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@register_settings(
|
||||
name="booknet",
|
||||
display_name="BookNet",
|
||||
@@ -980,7 +980,6 @@ def booknet_settings():
|
||||
link_url="https://booknet.example.com/api-docs",
|
||||
link_text="API Documentation",
|
||||
),
|
||||
|
||||
PasswordField(
|
||||
key="BOOKNET_API_KEY",
|
||||
label="API Key",
|
||||
@@ -988,14 +987,12 @@ def booknet_settings():
|
||||
required=True,
|
||||
env_supported=True,
|
||||
),
|
||||
|
||||
TextField(
|
||||
key="BOOKNET_BASE_URL",
|
||||
label="Base URL",
|
||||
placeholder="https://api.booknet.example.com",
|
||||
env_supported=True,
|
||||
),
|
||||
|
||||
NumberField(
|
||||
key="BOOKNET_TIMEOUT",
|
||||
label="Request Timeout",
|
||||
@@ -1004,14 +1001,12 @@ def booknet_settings():
|
||||
min_value=5,
|
||||
max_value=120,
|
||||
),
|
||||
|
||||
CheckboxField(
|
||||
key="BOOKNET_PREFER_EPUB",
|
||||
label="Prefer EPUB format",
|
||||
description="Sort EPUB results first when available",
|
||||
default=True,
|
||||
),
|
||||
|
||||
ActionButton(
|
||||
key="test_connection",
|
||||
label="Test Connection",
|
||||
@@ -1041,7 +1036,7 @@ def _test_booknet_connection() -> Dict:
|
||||
data = response.json()
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Connected! {data.get('books_available', 0):,} books available"
|
||||
"message": f"Connected! {data.get('books_available', 0):,} books available",
|
||||
}
|
||||
elif response.status_code == 401:
|
||||
return {"success": False, "message": "Invalid API key"}
|
||||
@@ -1058,6 +1053,7 @@ def _test_booknet_connection() -> Dict:
|
||||
# Release Source Implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@register_source("booknet")
|
||||
class BookNetSource(ReleaseSource):
|
||||
"""Search BookNet for book releases."""
|
||||
@@ -1090,7 +1086,7 @@ class BookNetSource(ReleaseSource):
|
||||
|
||||
# Sort by preference
|
||||
if self.prefer_epub:
|
||||
releases.sort(key=lambda r: (0 if r.format == "epub" else 1))
|
||||
releases.sort(key=lambda r: 0 if r.format == "epub" else 1)
|
||||
|
||||
return releases
|
||||
|
||||
@@ -1164,7 +1160,9 @@ class BookNetSource(ReleaseSource):
|
||||
render_type=ColumnRenderType.BADGE,
|
||||
align=ColumnAlign.CENTER,
|
||||
width="70px",
|
||||
color_hint=ColumnColorHint(type="static", value="bg-purple-500/20 text-purple-400"),
|
||||
color_hint=ColumnColorHint(
|
||||
type="static", value="bg-purple-500/20 text-purple-400"
|
||||
),
|
||||
),
|
||||
ColumnSchema(
|
||||
key="extra.language",
|
||||
@@ -1202,6 +1200,7 @@ class BookNetSource(ReleaseSource):
|
||||
# Download Handler Implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@register_handler("booknet")
|
||||
class BookNetHandler(DownloadHandler):
|
||||
"""Handle downloads from BookNet."""
|
||||
@@ -1264,8 +1263,7 @@ class BookNetHandler(DownloadHandler):
|
||||
|
||||
# Phase 3: Move to final location
|
||||
safe_title = "".join(
|
||||
c if c.isalnum() or c in " .-_" else "_"
|
||||
for c in task.title[:100]
|
||||
c if c.isalnum() or c in " .-_" else "_" for c in task.title[:100]
|
||||
).strip()
|
||||
|
||||
final_filename = f"{safe_title}.{task.format or 'epub'}"
|
||||
@@ -1274,7 +1272,9 @@ class BookNetHandler(DownloadHandler):
|
||||
# Avoid overwriting existing files
|
||||
counter = 1
|
||||
while final_path.exists():
|
||||
final_path = Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
|
||||
final_path = (
|
||||
Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
|
||||
)
|
||||
counter += 1
|
||||
|
||||
shutil.move(str(temp_path), str(final_path))
|
||||
@@ -1363,6 +1363,7 @@ def is_available(self) -> bool:
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def search(self, book: BookMetadata) -> List[Release]:
|
||||
"""Return empty list on errors, don't raise."""
|
||||
try:
|
||||
@@ -1446,10 +1447,10 @@ from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
logger.debug("Detailed debug info") # Development only
|
||||
logger.info("Normal operation info") # General progress
|
||||
logger.warning("Recoverable issues") # Fallback used, retry happening
|
||||
logger.error("Failures") # Unrecoverable errors
|
||||
logger.debug("Detailed debug info") # Development only
|
||||
logger.info("Normal operation info") # General progress
|
||||
logger.warning("Recoverable issues") # Fallback used, retry happening
|
||||
logger.error("Failures") # Unrecoverable errors
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -158,6 +158,9 @@ class DownloadTask:
|
||||
status: QueueStatus = QueueStatus.QUEUED
|
||||
status_message: str | None = None
|
||||
download_path: str | None = None
|
||||
# Every file the output step produced, download_path among them. Kept in memory
|
||||
# only: history persists download_path alone.
|
||||
download_paths: list[str] = field(default_factory=list)
|
||||
last_error_message: str | None = None
|
||||
last_error_type: str | None = None
|
||||
staged_path: str | None = None
|
||||
|
||||
@@ -194,7 +194,9 @@ def _probe_completed_path(path: Path) -> tuple[bool, OSError | None]:
|
||||
return True, None
|
||||
|
||||
|
||||
def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tuple[str, int, int], ...] | None:
|
||||
def _completed_file_signature(
|
||||
path: Path, content_type: str | None
|
||||
) -> tuple[tuple[str, int, int], ...] | None:
|
||||
"""Return sizes and mtimes for supported files in a completed path."""
|
||||
from shelfmark.download.postprocess.policy import (
|
||||
get_supported_audiobook_formats,
|
||||
@@ -202,9 +204,7 @@ def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tup
|
||||
)
|
||||
|
||||
formats = (
|
||||
get_supported_audiobook_formats()
|
||||
if is_audiobook(content_type)
|
||||
else get_supported_formats()
|
||||
get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats()
|
||||
)
|
||||
supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats}
|
||||
|
||||
|
||||
@@ -416,6 +416,28 @@ def get_book_path(task_id: str) -> tuple[str | None, DownloadTask | None]:
|
||||
return path, task
|
||||
|
||||
|
||||
def get_book_paths(task_id: str) -> tuple[list[str], DownloadTask | None]:
|
||||
"""Every file a task's download produced that is still on disk.
|
||||
|
||||
A multi-file release (chapters of an audiobook, a pack) lands as several files;
|
||||
:func:`get_book_path` only knows the first. The list is trusted only while it
|
||||
still contains download_path, so a later output that recorded no list cannot
|
||||
hand back an earlier attempt's files.
|
||||
"""
|
||||
path, task = get_book_path(task_id)
|
||||
if path is None or task is None:
|
||||
return [], task
|
||||
|
||||
if path not in task.download_paths:
|
||||
return [path], task
|
||||
|
||||
try:
|
||||
return [p for p in task.download_paths if Path(p).is_file()], task
|
||||
except OSError as e:
|
||||
logger.error_trace(f"Error resolving book paths: {e}")
|
||||
return [path], task
|
||||
|
||||
|
||||
def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]:
|
||||
"""Get downloaded file data for a specific task.
|
||||
|
||||
|
||||
@@ -275,4 +275,5 @@ def process_folder_output(
|
||||
message = f"Complete ({len(final_paths)} files)"
|
||||
status_callback("complete", message)
|
||||
|
||||
task.download_paths = [str(path) for path in final_paths]
|
||||
return str(final_paths[0])
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Stream several downloaded files to a client as one ZIP, without staging the archive."""
|
||||
|
||||
import os
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
_ZIP_STREAM_CHUNK_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
class _ChunkSink:
|
||||
"""Write-only ZipFile target that hands the written bytes back to the caller.
|
||||
|
||||
Having no tell() or seek() makes ZipFile treat it as an unseekable stream: each
|
||||
entry's CRC and sizes go in a trailing data descriptor instead of being patched
|
||||
back into the local header, so nothing has to be held until the entry ends.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._chunks: list[bytes] = []
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
self._chunks.append(bytes(data))
|
||||
return len(data)
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Nothing is buffered beyond what drain() hands out."""
|
||||
|
||||
def close(self) -> None:
|
||||
"""Nothing to release; the caller drains what is left after ZipFile closes."""
|
||||
|
||||
def drain(self) -> Iterator[bytes]:
|
||||
chunks, self._chunks = self._chunks, []
|
||||
yield from chunks
|
||||
|
||||
|
||||
def zip_entry_names(paths: list[str]) -> list[tuple[Path, str]]:
|
||||
"""Pair each file with its name inside the archive.
|
||||
|
||||
Names are relative to the files' common parent, so a book's own folders
|
||||
(``Title/Part 01.m4b``) survive while the library path above them does not.
|
||||
"""
|
||||
files = [Path(path) for path in paths]
|
||||
root = Path(os.path.commonpath([str(file.parent) for file in files]))
|
||||
return [(file, file.relative_to(root).as_posix()) for file in files]
|
||||
|
||||
|
||||
def stream_zip(paths: list[str]) -> Iterator[bytes]:
|
||||
"""Yield an uncompressed ZIP of ``paths``, reading each file once in chunks.
|
||||
|
||||
Stored, not deflated: audiobooks and ebooks are already compressed, so deflating
|
||||
them would cost CPU for next to nothing. ZIP64 kicks in per entry from the file
|
||||
size, which a multi-gigabyte audiobook needs.
|
||||
"""
|
||||
sink = _ChunkSink()
|
||||
with zipfile.ZipFile(sink, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
|
||||
for path, arcname in zip_entry_names(paths):
|
||||
info = zipfile.ZipInfo.from_file(path, arcname, strict_timestamps=False)
|
||||
info.compress_type = zipfile.ZIP_STORED
|
||||
with path.open("rb") as source, archive.open(info, "w") as target:
|
||||
while chunk := source.read(_ZIP_STREAM_CHUNK_SIZE):
|
||||
target.write(chunk)
|
||||
yield from sink.drain()
|
||||
yield from sink.drain()
|
||||
yield from sink.drain()
|
||||
+45
-6
@@ -6,12 +6,14 @@ import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
import unicodedata
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from functools import wraps
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Callable, NoReturn, cast
|
||||
from urllib.parse import quote
|
||||
|
||||
from flask import Flask, g, jsonify, request, send_file, send_from_directory, session
|
||||
from flask_cors import CORS
|
||||
@@ -55,7 +57,12 @@ from shelfmark.core.cwa_user_sync import upsert_cwa_user
|
||||
from shelfmark.core.download_history_service import DownloadHistoryService
|
||||
from shelfmark.core.external_user_linking import upsert_external_user
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.models import TERMINAL_QUEUE_STATUSES, QueueStatus, SearchFilters
|
||||
from shelfmark.core.models import (
|
||||
TERMINAL_QUEUE_STATUSES,
|
||||
QueueStatus,
|
||||
SearchFilters,
|
||||
build_filename,
|
||||
)
|
||||
from shelfmark.core.notifications import (
|
||||
NotificationContext,
|
||||
NotificationEvent,
|
||||
@@ -95,6 +102,7 @@ from shelfmark.core.viewer_redaction import (
|
||||
)
|
||||
from shelfmark.download import orchestrator as backend
|
||||
from shelfmark.download import warmup
|
||||
from shelfmark.download.zip_stream import stream_zip
|
||||
from shelfmark.release_sources import (
|
||||
BrowseRecord,
|
||||
Release,
|
||||
@@ -103,6 +111,7 @@ from shelfmark.release_sources import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.metadata_providers import BookMetadata, MetadataProvider
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -1800,16 +1809,43 @@ def api_status() -> Response | tuple[Response, int]:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
def _attachment_filename_options(download_name: str) -> dict[str, str]:
|
||||
"""Content-Disposition filename options, RFC 2231-encoded when not plain ASCII.
|
||||
|
||||
Mirrors what send_file does, for responses built from a generator.
|
||||
"""
|
||||
try:
|
||||
download_name.encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
simple = unicodedata.normalize("NFKD", download_name)
|
||||
simple = simple.encode("ascii", "ignore").decode("ascii")
|
||||
quoted = quote(download_name, safe="!#$&+^`|~")
|
||||
return {"filename": simple, "filename*": f"UTF-8''{quoted}"}
|
||||
return {"filename": download_name}
|
||||
|
||||
|
||||
def _zip_download_response(file_paths: list[str], task: DownloadTask) -> Response:
|
||||
"""Send a multi-file download (audiobook chapters, a pack) as one streamed ZIP."""
|
||||
download_name = build_filename(task.title or "download", task.author, task.year, "zip")
|
||||
response = Response(stream_zip(file_paths), mimetype="application/zip")
|
||||
response.headers.set(
|
||||
"Content-Disposition", "attachment", **_attachment_filename_options(download_name)
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@app.route("/api/localdownload", methods=["GET"])
|
||||
@login_required
|
||||
def api_local_download() -> Response | tuple[Response, int]:
|
||||
"""Download an EPUB file from local storage if available.
|
||||
"""Download a completed book from local storage if available.
|
||||
|
||||
Query Parameters:
|
||||
id (str): Book identifier (MD5 hash)
|
||||
|
||||
Returns:
|
||||
flask.Response: The EPUB file if found, otherwise an error response.
|
||||
flask.Response: The file if found, a ZIP of all of them when the download
|
||||
produced several (a live task only; history keeps just the first), otherwise
|
||||
an error response.
|
||||
|
||||
"""
|
||||
book_id = request.args.get("id", "")
|
||||
@@ -1820,8 +1856,8 @@ def api_local_download() -> Response | tuple[Response, int]:
|
||||
try:
|
||||
# The path, not the bytes: send_file streams it off disk, where reading it
|
||||
# first cost two resident copies of the whole book.
|
||||
file_path, book_info = backend.get_book_path(book_id)
|
||||
if file_path is None:
|
||||
file_paths, book_info = backend.get_book_paths(book_id)
|
||||
if not file_paths:
|
||||
# Fallback for dismissed/history entries where queue task may no longer exist.
|
||||
if download_history_service is not None:
|
||||
is_admin, db_user_id, can_access_status = _resolve_status_scope()
|
||||
@@ -1853,8 +1889,11 @@ def api_local_download() -> Response | tuple[Response, int]:
|
||||
):
|
||||
return jsonify({"error": "File not found"}), 404
|
||||
|
||||
if book_info is not None and len(file_paths) > 1:
|
||||
return _zip_download_response(file_paths, book_info)
|
||||
|
||||
file_name = book_info.get_filename() if book_info is not None else Path(book_id).name
|
||||
return send_file(file_path, download_name=file_name, as_attachment=True)
|
||||
return send_file(file_paths[0], download_name=file_name, as_attachment=True)
|
||||
|
||||
except _OPERATIONAL_ERRORS as e:
|
||||
logger.error_trace(f"Local download error: {e}")
|
||||
|
||||
@@ -450,7 +450,7 @@ class DatabazeKnihProvider(MetadataProvider):
|
||||
book_id = _id_from_link(str(data.get("url", "")))
|
||||
if not book_id:
|
||||
# Pages without a JSON-LD url still carry the id on the rating buttons
|
||||
tag = soup.find(attrs={"data-typeid": True})
|
||||
tag = soup.select_one("[data-typeid]")
|
||||
book_id = str(tag["data-typeid"]) if tag else ""
|
||||
title = _clean_text(str(data.get("name", "")))
|
||||
authors = [_clean_author(n) for n in _ld_names(data.get("author"))]
|
||||
@@ -539,7 +539,8 @@ class DatabazeKnihProvider(MetadataProvider):
|
||||
name = _clean_text(str(link.get("title") or link.get_text())) or None
|
||||
# The part number follows the link, up to the next series link
|
||||
text = []
|
||||
for elem in link.parent.next_siblings:
|
||||
parent = link.parent
|
||||
for elem in parent.next_siblings if parent else ():
|
||||
if isinstance(elem, Tag):
|
||||
if elem.find("a", href=re.compile(r"^/serie/")):
|
||||
break
|
||||
|
||||
@@ -40,7 +40,7 @@ def get_unavailable_reason(
|
||||
if not enabled_providers(providers):
|
||||
return (
|
||||
"Direct Download is not configured. Add at least one Anna's Archive or OceanofPDF mirror URL in "
|
||||
"Settings. Z-Library, LibGen, Welib, mirrors only download results found through "
|
||||
"Settings. Z-Library, LibGen and Welib mirrors only download results found through "
|
||||
"Anna's Archive."
|
||||
)
|
||||
return None
|
||||
|
||||
+26
-24
@@ -64,12 +64,12 @@ The script navigates to the main page, enters the search query, and waits for "M
|
||||
|
||||
```python
|
||||
# Navigate to main page first to reset SPA state
|
||||
page.goto('http://localhost:8084/')
|
||||
page.wait_for_load_state('networkidle')
|
||||
page.goto("http://localhost:8084/")
|
||||
page.wait_for_load_state("networkidle")
|
||||
|
||||
# Enter search query and submit
|
||||
page.fill('input[type="search"]', f'{title} {author}')
|
||||
page.press('input[type="search"]', 'Enter')
|
||||
page.fill('input[type="search"]', f"{title} {author}")
|
||||
page.press('input[type="search"]', "Enter")
|
||||
|
||||
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
|
||||
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
|
||||
@@ -87,18 +87,20 @@ def parse_books(page):
|
||||
btns = page.query_selector_all('button[data-action="download"]')
|
||||
books = []
|
||||
for i, btn in enumerate(btns):
|
||||
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
|
||||
content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
|
||||
|
||||
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
|
||||
title = title_match.group(1).strip() if title_match else 'Unknown'
|
||||
title_match = re.search(r"<h3[^>]*>(.*?)</h3>", content, re.IGNORECASE | re.DOTALL)
|
||||
title = title_match.group(1).strip() if title_match else "Unknown"
|
||||
|
||||
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
|
||||
author = author_match.group(1).strip() if author_match else 'Unknown'
|
||||
author_match = re.search(
|
||||
r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
author = author_match.group(1).strip() if author_match else "Unknown"
|
||||
|
||||
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
|
||||
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
|
||||
dl_match = re.search(r"<span>•</span>\s*<span>([\d,]+)</span>", content)
|
||||
downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
|
||||
|
||||
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
|
||||
books.append({"index": i, "title": title, "author": author, "downloads": downloads})
|
||||
return books
|
||||
```
|
||||
|
||||
@@ -106,19 +108,19 @@ def parse_books(page):
|
||||
|
||||
```python
|
||||
# Filter books matching the search criteria
|
||||
matching_books = [b for b in books if matches_search(b['title'], b['author'], title, author)]
|
||||
matching_books = [b for b in books if matches_search(b["title"], b["author"], title, author)]
|
||||
|
||||
if matching_books:
|
||||
best = max(matching_books, key=lambda x: x['downloads'])
|
||||
best = max(matching_books, key=lambda x: x["downloads"])
|
||||
|
||||
# CRITICAL: Click on the article h3 to open detail view
|
||||
h3s = page.query_selector_all('article h3')
|
||||
if best['index'] < len(h3s):
|
||||
h3s[best['index']].click()
|
||||
h3s = page.query_selector_all("article h3")
|
||||
if best["index"] < len(h3s):
|
||||
h3s[best["index"]].click()
|
||||
page.wait_for_timeout(300)
|
||||
|
||||
# Then click the download button
|
||||
btn = page.query_selector_all('button[data-action="download"]')[best['index']]
|
||||
btn = page.query_selector_all('button[data-action="download"]')[best["index"]]
|
||||
btn.click()
|
||||
page.wait_for_timeout(500)
|
||||
```
|
||||
@@ -130,14 +132,14 @@ if matching_books:
|
||||
```python
|
||||
for i in range(60):
|
||||
page.wait_for_timeout(5000)
|
||||
activity_text = page.inner_text('aside')
|
||||
activity_text = page.inner_text("aside")
|
||||
|
||||
if 'IN PROGRESS' in activity_text:
|
||||
if "IN PROGRESS" in activity_text:
|
||||
print("Download started!")
|
||||
elif 'Complete' in activity_text or 'Saved' in activity_text:
|
||||
elif "Complete" in activity_text or "Saved" in activity_text:
|
||||
print("Download complete!")
|
||||
break
|
||||
elif 'No activity' in activity_text:
|
||||
elif "No activity" in activity_text:
|
||||
print("Download not started")
|
||||
break
|
||||
```
|
||||
@@ -146,7 +148,7 @@ for i in range(60):
|
||||
|
||||
```python
|
||||
# Click "Clear Completed" using JavaScript
|
||||
page.evaluate('''
|
||||
page.evaluate("""
|
||||
() => {
|
||||
for (const b of document.querySelectorAll('button')) {
|
||||
if (b.textContent.includes('Clear Completed')) {
|
||||
@@ -155,7 +157,7 @@ page.evaluate('''
|
||||
}
|
||||
}
|
||||
}
|
||||
''')
|
||||
""")
|
||||
|
||||
page.wait_for_timeout(2000)
|
||||
```
|
||||
|
||||
+161
-104
@@ -1,40 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shelfmark Book Downloader - Downloads books from a local Shelfmark instance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from playwright.sync_api import Error as PlaywrightError
|
||||
from playwright.sync_api import sync_playwright
|
||||
import re, time, sys, argparse, json
|
||||
from urllib.parse import quote
|
||||
|
||||
SHELFMARK_URL = 'http://localhost:8084/'
|
||||
if TYPE_CHECKING:
|
||||
from playwright.sync_api import Page
|
||||
|
||||
def parse_books(page):
|
||||
Book = dict[str, Any]
|
||||
|
||||
SHELFMARK_URL = "http://localhost:8084/"
|
||||
|
||||
|
||||
def parse_books(page: Page) -> list[Book]:
|
||||
btns = page.query_selector_all('button[data-action="download"]')
|
||||
books = []
|
||||
for i, btn in enumerate(btns):
|
||||
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
|
||||
content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
|
||||
|
||||
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
|
||||
title = title_match.group(1).strip() if title_match else 'Unknown'
|
||||
title_match = re.search(r"<h3[^>]*>(.*?)</h3>", content, re.IGNORECASE | re.DOTALL)
|
||||
title = title_match.group(1).strip() if title_match else "Unknown"
|
||||
|
||||
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
|
||||
author = author_match.group(1).strip() if author_match else 'Unknown'
|
||||
author_match = re.search(
|
||||
r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
author = author_match.group(1).strip() if author_match else "Unknown"
|
||||
|
||||
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
|
||||
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
|
||||
dl_match = re.search(r"<span>•</span>\s*<span>([\d,]+)</span>", content)
|
||||
downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
|
||||
|
||||
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
|
||||
books.append({"index": i, "title": title, "author": author, "downloads": downloads})
|
||||
return books
|
||||
|
||||
def do_search(page, title, author, search_type="author"):
|
||||
|
||||
def do_search(
|
||||
page: Page, title: str, author: str, search_type: str = "author"
|
||||
) -> tuple[list[Book], list[Book]] | None:
|
||||
"""Search with title+author, fall back to title only"""
|
||||
# Navigate to main page first to reset SPA state
|
||||
page.goto(SHELFMARK_URL)
|
||||
page.wait_for_load_state('networkidle')
|
||||
page.wait_for_load_state("networkidle")
|
||||
|
||||
# Enter search query and submit
|
||||
search_query = f'{title} {author}' if search_type == "author" else title
|
||||
search_query = f"{title} {author}" if search_type == "author" else title
|
||||
page.fill('input[type="search"]', search_query)
|
||||
page.press('input[type="search"]', 'Enter')
|
||||
page.press('input[type="search"]', "Enter")
|
||||
|
||||
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
|
||||
try:
|
||||
@@ -44,43 +65,55 @@ def do_search(page, title, author, search_type="author"):
|
||||
print(" -> Trying title only...")
|
||||
sys.stdout.flush()
|
||||
return do_search(page, title, author, search_type="title")
|
||||
else:
|
||||
print(" >> Timeout waiting for search results")
|
||||
sys.stdout.flush()
|
||||
return None
|
||||
print(" >> Timeout waiting for search results")
|
||||
sys.stdout.flush()
|
||||
return None
|
||||
|
||||
books = parse_books(page)
|
||||
|
||||
def clean_title_for_match(book_title, search_title):
|
||||
def clean_title_for_match(book_title: str, search_title: str) -> bool:
|
||||
"""Clean book title to check if it matches the search title"""
|
||||
# Remove common subtitle patterns
|
||||
clean = re.sub(r'\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$', '', book_title, flags=re.IGNORECASE)
|
||||
clean = re.sub(
|
||||
r"\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$",
|
||||
"",
|
||||
book_title,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
clean = clean.strip()
|
||||
# Remove trailing punctuation
|
||||
clean = clean.rstrip(':,;.')
|
||||
clean = clean.rstrip(":,;.")
|
||||
return clean.lower().strip() == search_title.lower().strip()
|
||||
|
||||
def matches_search(book_title, book_author, search_title, search_author):
|
||||
def matches_search(
|
||||
book_title: str, book_author: str, search_title: str, search_author: str
|
||||
) -> bool:
|
||||
"""Check if book matches search criteria more strictly"""
|
||||
title_match = search_title.lower() in book_title.lower()
|
||||
author_match = search_author.lower() in book_author.lower()
|
||||
|
||||
# For title+author search, ensure title starts with search title (not just contains it)
|
||||
if title_match and author_match:
|
||||
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
|
||||
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(
|
||||
search_title.lower()
|
||||
)
|
||||
|
||||
# Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann")
|
||||
if title_match:
|
||||
author_parts = search_author.lower().split()
|
||||
if len(author_parts) >= 2:
|
||||
reversed_author = f"{author_parts[-1]} {author_parts[0]}"
|
||||
if reversed_author in book_author.lower() or book_author.lower().startswith(reversed_author):
|
||||
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
|
||||
if reversed_author in book_author.lower() or book_author.lower().startswith(
|
||||
reversed_author
|
||||
):
|
||||
return clean_title_for_match(
|
||||
book_title, search_title
|
||||
) or book_title.lower().startswith(search_title.lower())
|
||||
|
||||
return False
|
||||
|
||||
if search_type == "author":
|
||||
matching = [b for b in books if matches_search(b['title'], b['author'], title, author)]
|
||||
matching = [b for b in books if matches_search(b["title"], b["author"], title, author)]
|
||||
print(f" With author: {len(books)} total, {len(matching)} matching")
|
||||
for b in books[:3]:
|
||||
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
|
||||
@@ -88,18 +121,22 @@ def do_search(page, title, author, search_type="author"):
|
||||
|
||||
if matching:
|
||||
# Check if any matching book is already in the download queue
|
||||
try:
|
||||
sidebar_text = page.inner_text('aside')
|
||||
if 'IN PROGRESS' in sidebar_text:
|
||||
with contextlib.suppress(PlaywrightError):
|
||||
sidebar_text = page.inner_text("aside")
|
||||
if "IN PROGRESS" in sidebar_text:
|
||||
# Get the title of the book currently downloading
|
||||
current_download = sidebar_text.split('IN PROGRESS')[1].split('—')[0].strip().split('\n')[0].strip()
|
||||
current_download = (
|
||||
sidebar_text.split("IN PROGRESS")[1]
|
||||
.split("—")[0]
|
||||
.strip()
|
||||
.split("\n")[0]
|
||||
.strip()
|
||||
)
|
||||
# Check if the current download matches our search
|
||||
if title.lower() in current_download.lower():
|
||||
print(f" >> '{current_download}' already downloading, skipping")
|
||||
sys.stdout.flush()
|
||||
return None
|
||||
except:
|
||||
pass
|
||||
|
||||
if matching:
|
||||
return matching, books
|
||||
@@ -110,13 +147,15 @@ def do_search(page, title, author, search_type="author"):
|
||||
return do_search(page, title, author, search_type="title")
|
||||
|
||||
# Title-only search - prefer exact matches first
|
||||
matching = [b for b in books if title.lower() in b['title'].lower()]
|
||||
exact_matches = [b for b in matching if clean_title_for_match(b['title'], title)]
|
||||
matching = [b for b in books if title.lower() in b["title"].lower()]
|
||||
exact_matches = [b for b in matching if clean_title_for_match(b["title"], title)]
|
||||
|
||||
if exact_matches:
|
||||
matching = exact_matches
|
||||
|
||||
print(f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)")
|
||||
print(
|
||||
f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)"
|
||||
)
|
||||
for b in books[:5]:
|
||||
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
|
||||
sys.stdout.flush()
|
||||
@@ -126,14 +165,15 @@ def do_search(page, title, author, search_type="author"):
|
||||
|
||||
return (matching, books)
|
||||
|
||||
def download_book(page, title, author):
|
||||
|
||||
def download_book(page: Page, title: str, author: str) -> None:
|
||||
result = do_search(page, title, author)
|
||||
if not result:
|
||||
print("\n >>> NOT FOUND")
|
||||
sys.stdout.flush()
|
||||
return
|
||||
|
||||
matching, all_books = result
|
||||
matching, _all_books = result
|
||||
if not matching:
|
||||
print("\n >>> NOT FOUND")
|
||||
sys.stdout.flush()
|
||||
@@ -142,8 +182,8 @@ def download_book(page, title, author):
|
||||
# Find the best book that is not disabled
|
||||
btns = page.query_selector_all('button[data-action="download"]')
|
||||
best = None
|
||||
for b in sorted(matching, key=lambda x: x['downloads'], reverse=True):
|
||||
if b['index'] < len(btns) and not btns[b['index']].is_disabled():
|
||||
for b in sorted(matching, key=lambda x: x["downloads"], reverse=True):
|
||||
if b["index"] < len(btns) and not btns[b["index"]].is_disabled():
|
||||
best = b
|
||||
break
|
||||
|
||||
@@ -152,101 +192,107 @@ def download_book(page, title, author):
|
||||
sys.stdout.flush()
|
||||
return
|
||||
|
||||
print(f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]")
|
||||
print(
|
||||
f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]"
|
||||
)
|
||||
sys.stdout.flush()
|
||||
|
||||
# Click on the article h3 to open detail view (required for download to work)
|
||||
h3s = page.query_selector_all('article h3')
|
||||
if best['index'] < len(h3s):
|
||||
h3s[best['index']].click()
|
||||
h3s = page.query_selector_all("article h3")
|
||||
if best["index"] < len(h3s):
|
||||
h3s[best["index"]].click()
|
||||
page.wait_for_timeout(300)
|
||||
else:
|
||||
print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Click the download button
|
||||
btn = btns[best['index']]
|
||||
btn = btns[best["index"]]
|
||||
btn.click()
|
||||
|
||||
# Wait a moment for the click to register
|
||||
page.wait_for_timeout(500)
|
||||
print(f" >> Click sent")
|
||||
print(" >> Click sent")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Check sidebar immediately
|
||||
time.sleep(2)
|
||||
try:
|
||||
txt = page.inner_text('aside')
|
||||
if 'IN PROGRESS' in txt:
|
||||
print(f" >> Download started!")
|
||||
txt = page.inner_text("aside")
|
||||
if "IN PROGRESS" in txt:
|
||||
print(" >> Download started!")
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
print(f" >> Sidebar after 2s: {txt[:150]}")
|
||||
sys.stdout.flush()
|
||||
except:
|
||||
print(f" >> Sidebar not visible after 2s")
|
||||
print(" >> Sidebar not visible after 2s")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Wait for download to complete - check sidebar every 2 seconds, max 5 minutes
|
||||
last_state = None
|
||||
for attempt in range(150):
|
||||
for _attempt in range(150):
|
||||
time.sleep(2)
|
||||
try:
|
||||
txt = page.inner_text('aside')
|
||||
if 'IN PROGRESS' in txt:
|
||||
state = 'downloading'
|
||||
elif 'Complete' in txt or 'Saved' in txt or 'No activity' in txt:
|
||||
state = 'done'
|
||||
txt = page.inner_text("aside")
|
||||
if "IN PROGRESS" in txt:
|
||||
state = "downloading"
|
||||
elif "Complete" in txt or "Saved" in txt or "No activity" in txt:
|
||||
state = "done"
|
||||
else:
|
||||
state = 'other'
|
||||
state = "other"
|
||||
|
||||
if state != last_state:
|
||||
if state == 'downloading':
|
||||
print(f" >> Download started!")
|
||||
elif state == 'done':
|
||||
if 'No activity' in txt:
|
||||
print(f" >> Download not started (no activity)")
|
||||
if state == "downloading":
|
||||
print(" >> Download started!")
|
||||
elif state == "done":
|
||||
if "No activity" in txt:
|
||||
print(" >> Download not started (no activity)")
|
||||
else:
|
||||
print(f" >> Download complete!")
|
||||
print(" >> Download complete!")
|
||||
last_state = state
|
||||
|
||||
if state == 'done':
|
||||
if state == "done":
|
||||
break
|
||||
except:
|
||||
if last_state is None:
|
||||
print(f" >> Sidebar not visible yet")
|
||||
last_state = 'no_sidebar'
|
||||
print(" >> Sidebar not visible yet")
|
||||
last_state = "no_sidebar"
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
if last_state != 'done':
|
||||
if last_state != "done":
|
||||
print(" >> Timeout waiting for download")
|
||||
sys.stdout.flush()
|
||||
|
||||
def clear_completed_downloads(page):
|
||||
|
||||
def clear_completed_downloads(page: Page) -> None:
|
||||
"""Clear completed downloads from previous sessions"""
|
||||
page.evaluate('''
|
||||
page.evaluate("""
|
||||
() => {
|
||||
for (const b of document.querySelectorAll('button')) {
|
||||
if (b.textContent.includes('Clear Completed')) { b.click(); return; }
|
||||
}
|
||||
}
|
||||
''')
|
||||
""")
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
def check_calibre(title):
|
||||
|
||||
def check_calibre(title: str) -> bool:
|
||||
"""Check if a book is already in the calibre database."""
|
||||
import subprocess
|
||||
|
||||
cmd = f'calibredb list --search title:="{title}"'
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
result = subprocess.run(cmd, check=False, shell=True, capture_output=True, text=True)
|
||||
output = result.stdout.strip()
|
||||
# calibredb output has header line "id title authors", count non-header lines
|
||||
lines = [l.strip() for l in output.split('\n') if l.strip()]
|
||||
lines = [l.strip() for l in output.split("\n") if l.strip()]
|
||||
# Remove header if present (first line starts with 'id')
|
||||
if lines and lines[0].startswith('id'):
|
||||
if lines and lines[0].startswith("id"):
|
||||
lines = lines[1:]
|
||||
return len(lines) > 0
|
||||
|
||||
def load_books_from_json(json_str, source):
|
||||
|
||||
def load_books_from_json(json_str: str, source: str) -> list[tuple[str, str]]:
|
||||
"""Load books from a JSON string or file path."""
|
||||
try:
|
||||
data = json.loads(json_str)
|
||||
@@ -261,10 +307,10 @@ def load_books_from_json(json_str, source):
|
||||
books = []
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
if 'title' not in item or 'author' not in item:
|
||||
if "title" not in item or "author" not in item:
|
||||
print(f"Error: Each book object must have 'title' and 'author' keys: {item}")
|
||||
sys.exit(1)
|
||||
books.append((item['title'].strip(), item['author'].strip()))
|
||||
books.append((item["title"].strip(), item["author"].strip()))
|
||||
elif isinstance(item, list) and len(item) >= 2:
|
||||
books.append((str(item[0]).strip(), str(item[1]).strip()))
|
||||
elif isinstance(item, str):
|
||||
@@ -274,31 +320,41 @@ def load_books_from_json(json_str, source):
|
||||
print(f"Warning: Skipping unrecognized book entry: {item}")
|
||||
return books
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Download books from Shelfmark')
|
||||
parser.add_argument('books', nargs='*', help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects')
|
||||
parser.add_argument('--file', '-f', help='Path to JSON file containing an array of books')
|
||||
parser.add_argument('-c', '--check-calibre', action='store_true', help='Check calibre database before downloading, skip if already present')
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Download books from Shelfmark")
|
||||
parser.add_argument(
|
||||
"books",
|
||||
nargs="*",
|
||||
help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects',
|
||||
)
|
||||
parser.add_argument("--file", "-f", help="Path to JSON file containing an array of books")
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--check-calibre",
|
||||
action="store_true",
|
||||
help="Check calibre database before downloading, skip if already present",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load books from --file, positional JSON arg, or both
|
||||
BOOKS = []
|
||||
books_to_download = []
|
||||
|
||||
if args.file:
|
||||
try:
|
||||
with open(args.file, 'r') as f:
|
||||
BOOKS = load_books_from_json(f.read(), f"file '{args.file}'")
|
||||
with Path(args.file).open() as f:
|
||||
books_to_download = load_books_from_json(f.read(), f"file '{args.file}'")
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File not found: {args.file}")
|
||||
sys.exit(1)
|
||||
except IOError as e:
|
||||
except OSError as e:
|
||||
print(f"Error reading file: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
for book_arg in args.books:
|
||||
BOOKS.extend(load_books_from_json(book_arg, 'command line argument'))
|
||||
books_to_download.extend(load_books_from_json(book_arg, "command line argument"))
|
||||
|
||||
if not BOOKS:
|
||||
if not books_to_download:
|
||||
print("Error: No books specified. Use positional JSON args or --file <path>")
|
||||
sys.exit(1)
|
||||
|
||||
@@ -307,40 +363,40 @@ def main():
|
||||
print("\nChecking calibre database...")
|
||||
sys.stdout.flush()
|
||||
remaining = []
|
||||
for title, author in BOOKS:
|
||||
for title, author in books_to_download:
|
||||
if check_calibre(title):
|
||||
print(f" >> '{title}' already in calibre, skipping")
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
remaining.append((title, author))
|
||||
BOOKS = remaining
|
||||
if not BOOKS:
|
||||
books_to_download = remaining
|
||||
if not books_to_download:
|
||||
print("All books already in calibre, nothing to download.")
|
||||
sys.stdout.flush()
|
||||
return
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Downloading {len(BOOKS)} book(s) from Shelfmark")
|
||||
print('='*60)
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Downloading {len(books_to_download)} book(s) from Shelfmark")
|
||||
print("=" * 60)
|
||||
sys.stdout.flush()
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch()
|
||||
page = browser.new_page(viewport={'width': 1280, 'height': 900})
|
||||
page = browser.new_page(viewport={"width": 1280, "height": 900})
|
||||
page.goto(SHELFMARK_URL)
|
||||
page.wait_for_load_state('networkidle')
|
||||
page.wait_for_load_state("networkidle")
|
||||
|
||||
# Clear completed downloads at start of batch
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print("Clearing completed downloads from previous sessions...")
|
||||
print('='*60)
|
||||
print("=" * 60)
|
||||
sys.stdout.flush()
|
||||
clear_completed_downloads(page)
|
||||
|
||||
for i, (title, author) in enumerate(BOOKS, 1):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[{i}/{len(BOOKS)}] '{title}' by {author}")
|
||||
print('='*60)
|
||||
for i, (title, author) in enumerate(books_to_download, 1):
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"[{i}/{len(books_to_download)}] '{title}' by {author}")
|
||||
print("=" * 60)
|
||||
sys.stdout.flush()
|
||||
|
||||
download_book(page, title, author)
|
||||
@@ -349,5 +405,6 @@ def main():
|
||||
print("\nDone!")
|
||||
sys.stdout.flush()
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { RequestRecord } from '../../types';
|
||||
|
||||
export type ActivityKind = 'download' | 'request';
|
||||
type ActivityKind = 'download' | 'request';
|
||||
|
||||
export type ActivityVisualStatus =
|
||||
| 'queued'
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export { DisplayFieldIcon, DisplayFieldBadge, DisplayFieldBadges } from './DisplayFieldIcon';
|
||||
export { CircularProgress } from './CircularProgress';
|
||||
export { ToggleSwitch } from './ToggleSwitch';
|
||||
export { LibraryBadge, isInLibrary, isCollectionOnly } from './LibraryBadge';
|
||||
export { LibraryBadge, isInLibrary } from './LibraryBadge';
|
||||
@@ -861,9 +861,9 @@ export const getReleases = async (
|
||||
|
||||
// Admin user management API
|
||||
|
||||
export type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
|
||||
type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
|
||||
|
||||
export interface AdminUserEditCapabilities {
|
||||
interface AdminUserEditCapabilities {
|
||||
authSource: AdminAuthSource;
|
||||
canSetPassword: boolean;
|
||||
canEditRole: boolean;
|
||||
|
||||
@@ -172,13 +172,7 @@
|
||||
}
|
||||
|
||||
body {
|
||||
font-family:
|
||||
'Inter',
|
||||
-apple-system,
|
||||
BlinkMacSystemFont,
|
||||
'Segoe UI',
|
||||
Roboto,
|
||||
sans-serif;
|
||||
font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
line-height: 1.6;
|
||||
color: var(--text-color);
|
||||
background: var(--background-color);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Search mode constants and type
|
||||
export const SEARCH_MODE = {
|
||||
const SEARCH_MODE = {
|
||||
DIRECT: 'direct',
|
||||
UNIVERSAL: 'universal',
|
||||
} as const;
|
||||
@@ -88,7 +88,7 @@ export interface StatusData {
|
||||
}
|
||||
|
||||
// Button states
|
||||
export type ButtonState =
|
||||
type ButtonState =
|
||||
| 'download'
|
||||
| 'queued'
|
||||
| 'resolving'
|
||||
@@ -156,24 +156,24 @@ export interface TextSearchField extends SearchFieldBase {
|
||||
suggestions_min_query_length?: number;
|
||||
}
|
||||
|
||||
export interface NumberSearchField extends SearchFieldBase {
|
||||
interface NumberSearchField extends SearchFieldBase {
|
||||
type: 'NumberSearchField';
|
||||
min?: number;
|
||||
max?: number;
|
||||
step?: number;
|
||||
}
|
||||
|
||||
export interface SelectSearchField extends SearchFieldBase {
|
||||
interface SelectSearchField extends SearchFieldBase {
|
||||
type: 'SelectSearchField';
|
||||
options: SortOption[];
|
||||
}
|
||||
|
||||
export interface CheckboxSearchField extends SearchFieldBase {
|
||||
interface CheckboxSearchField extends SearchFieldBase {
|
||||
type: 'CheckboxSearchField';
|
||||
default?: boolean;
|
||||
}
|
||||
|
||||
export interface DynamicSelectSearchField extends SearchFieldBase {
|
||||
interface DynamicSelectSearchField extends SearchFieldBase {
|
||||
type: 'DynamicSelectSearchField';
|
||||
options_endpoint: string;
|
||||
}
|
||||
@@ -185,7 +185,7 @@ export type MetadataSearchField =
|
||||
| CheckboxSearchField
|
||||
| DynamicSelectSearchField;
|
||||
|
||||
export type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
|
||||
type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
|
||||
|
||||
export interface QueryTargetOption {
|
||||
key: string;
|
||||
@@ -201,12 +201,12 @@ export type ContentType = 'ebook' | 'audiobook';
|
||||
|
||||
export type RequestPolicyMode = 'download' | 'request_release' | 'request_book' | 'blocked';
|
||||
|
||||
export interface RequestPolicyDefaults {
|
||||
interface RequestPolicyDefaults {
|
||||
ebook: RequestPolicyMode;
|
||||
audiobook: RequestPolicyMode;
|
||||
}
|
||||
|
||||
export interface RequestPolicySourceMode {
|
||||
interface RequestPolicySourceMode {
|
||||
source: string;
|
||||
supported_content_types: string[];
|
||||
browse_results_are_releases?: boolean;
|
||||
@@ -222,7 +222,7 @@ export interface RequestPolicyResponse {
|
||||
source_modes: RequestPolicySourceMode[];
|
||||
}
|
||||
|
||||
export interface RequestContextPayload {
|
||||
interface RequestContextPayload {
|
||||
source: string;
|
||||
content_type: ContentType;
|
||||
request_level: 'book' | 'release';
|
||||
@@ -279,14 +279,14 @@ export interface QueuedDownloadResult {
|
||||
export type RequestSubmissionResult = RequestRecord | QueuedDownloadResult;
|
||||
|
||||
/** How the library holds a book, per format: owned outright, or inside a collection. */
|
||||
export type LibraryHolding = 'owned' | 'collection';
|
||||
type LibraryHolding = 'owned' | 'collection';
|
||||
|
||||
export interface LibraryOwnership {
|
||||
ebook?: LibraryHolding | null;
|
||||
audiobook?: LibraryHolding | null;
|
||||
}
|
||||
|
||||
export type BooksOutputMode = 'folder' | 'booklore' | 'email';
|
||||
type BooksOutputMode = 'folder' | 'booklore' | 'email';
|
||||
|
||||
export interface AppConfig {
|
||||
calibre_web_url: string;
|
||||
@@ -334,7 +334,7 @@ export interface MetadataProvidersResponse {
|
||||
configured_provider_combined: string | null;
|
||||
}
|
||||
|
||||
export interface MetadataCapability {
|
||||
interface MetadataCapability {
|
||||
key: string;
|
||||
field_key?: string;
|
||||
sort?: string;
|
||||
@@ -398,7 +398,7 @@ export interface ReleaseSource {
|
||||
}
|
||||
|
||||
// Column schema types for plugin-driven release list UI
|
||||
export type ColumnRenderType =
|
||||
type ColumnRenderType =
|
||||
| 'text'
|
||||
| 'badge'
|
||||
| 'tags'
|
||||
@@ -408,7 +408,7 @@ export type ColumnRenderType =
|
||||
| 'indexer_protocol'
|
||||
| 'flag_icon'
|
||||
| 'format_content_type';
|
||||
export type ColumnAlign = 'left' | 'center' | 'right';
|
||||
type ColumnAlign = 'left' | 'center' | 'right';
|
||||
|
||||
export interface ColumnColorHint {
|
||||
type: 'map' | 'static'; // 'map' uses colorMaps.ts, 'static' is a fixed Tailwind class
|
||||
@@ -430,7 +430,7 @@ export interface ColumnSchema {
|
||||
}
|
||||
|
||||
// Leading cell config - what to show in the left-most position of each row
|
||||
export type LeadingCellType = 'thumbnail' | 'badge' | 'none';
|
||||
type LeadingCellType = 'thumbnail' | 'badge' | 'none';
|
||||
|
||||
export interface LeadingCellConfig {
|
||||
type: LeadingCellType;
|
||||
@@ -439,13 +439,13 @@ export interface LeadingCellConfig {
|
||||
uppercase?: boolean; // Force uppercase for badge text
|
||||
}
|
||||
|
||||
export interface ExtraSortOption {
|
||||
interface ExtraSortOption {
|
||||
label: string; // Display label in the sort dropdown
|
||||
sort_key: string; // Field to sort by on the Release object
|
||||
default_direction?: 'asc' | 'desc'; // Which way "best first" runs (defaults to desc)
|
||||
}
|
||||
|
||||
export interface SourceActionButton {
|
||||
interface SourceActionButton {
|
||||
label: string; // Button text (e.g., "Refresh search")
|
||||
action: string; // Action type: "expand" triggers expand_search
|
||||
}
|
||||
@@ -483,7 +483,7 @@ export interface Release {
|
||||
}
|
||||
|
||||
// Search info returned by release sources
|
||||
export interface SourceSearchInfo {
|
||||
interface SourceSearchInfo {
|
||||
search_type: 'isbn' | 'title_author' | 'categories' | 'expanded' | 'manual' | 'query';
|
||||
total_results?: number | string | null;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Settings field types matching backend settings_registry.py
|
||||
|
||||
export type FieldType =
|
||||
type FieldType =
|
||||
| 'TextField'
|
||||
| 'PasswordField'
|
||||
| 'NumberField'
|
||||
@@ -28,17 +28,17 @@ export interface ShowWhenCondition {
|
||||
notEmpty?: boolean; // If true, show when field has any non-empty value
|
||||
}
|
||||
|
||||
export type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
|
||||
type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
|
||||
|
||||
// Conditional disable configuration
|
||||
export interface DisabledWhenCondition {
|
||||
interface DisabledWhenCondition {
|
||||
field: string; // The field key to check
|
||||
value: string | string[] | boolean; // The value(s) that disable this field
|
||||
reason?: string; // Explanation shown when disabled
|
||||
}
|
||||
|
||||
// Base field interface - common properties
|
||||
export interface BaseField {
|
||||
interface BaseField {
|
||||
key: string;
|
||||
label: string;
|
||||
type: FieldType;
|
||||
@@ -139,20 +139,14 @@ export interface CustomComponentFieldConfig extends BaseField {
|
||||
wrapInFieldWrapper?: boolean; // Whether to render with standard FieldWrapper layout
|
||||
}
|
||||
|
||||
export interface TableFieldColumnOption {
|
||||
interface TableFieldColumnOption {
|
||||
value: string;
|
||||
label: string;
|
||||
description?: string;
|
||||
childOf?: string;
|
||||
}
|
||||
|
||||
export type TableFieldColumnType =
|
||||
| 'text'
|
||||
| 'password'
|
||||
| 'select'
|
||||
| 'multiselect'
|
||||
| 'checkbox'
|
||||
| 'path';
|
||||
type TableFieldColumnType = 'text' | 'password' | 'select' | 'multiselect' | 'checkbox' | 'path';
|
||||
|
||||
export interface TableFieldColumn {
|
||||
key: string;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { ColumnColorHint } from '../types';
|
||||
|
||||
// Color styles with transparent backgrounds and contrasting text
|
||||
interface ColorStyle {
|
||||
bg: string;
|
||||
@@ -133,14 +135,6 @@ function getFlagColor(flag?: string): ColorStyle {
|
||||
return FLAG_COLORS[normalized] || DEFAULT_FLAG_COLOR;
|
||||
}
|
||||
|
||||
/**
|
||||
* Color hint type for dynamic column coloring.
|
||||
*/
|
||||
interface ColumnColorHint {
|
||||
type: 'static' | 'map';
|
||||
value: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the color style for a value based on a color hint.
|
||||
* Supports both static color classes and dynamic map lookups.
|
||||
|
||||
@@ -10,7 +10,7 @@ import type { ContentType } from '../types';
|
||||
export const CONTENT_TYPE_STORAGE_KEY = 'preferred-content-type.v2';
|
||||
|
||||
/** Where the current content type came from, which decides what may still override it. */
|
||||
export type ContentTypeSource =
|
||||
type ContentTypeSource =
|
||||
/** Nothing has chosen a tab, so a configured default may still apply. */
|
||||
| 'unset'
|
||||
/** This browser stored a choice, or something explicit set one. Defaults do not apply. */
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import io
|
||||
import sqlite3
|
||||
import uuid
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import ANY, patch
|
||||
@@ -355,6 +357,46 @@ class TestActivityRoutes:
|
||||
assert response.status_code == 200
|
||||
assert response.data == file_bytes
|
||||
|
||||
def test_localdownload_zips_a_live_multi_file_download(self, main_module, client, tmp_path):
|
||||
user = _create_user(main_module, prefix="reader")
|
||||
_set_session(client, user_id=user["username"], db_user_id=user["id"], is_admin=False)
|
||||
|
||||
task_id = f"live-localdownload-{uuid.uuid4().hex[:8]}"
|
||||
book_dir = tmp_path / "Chapters"
|
||||
book_dir.mkdir()
|
||||
parts = [book_dir / "Part 1.mp3", book_dir / "Part 2.mp3"]
|
||||
for index, part in enumerate(parts, start=1):
|
||||
part.write_bytes(f"chapter {index}".encode())
|
||||
|
||||
task = DownloadTask(
|
||||
task_id=task_id,
|
||||
source="direct_download",
|
||||
title="Zipped Audiobook",
|
||||
author="Some Author",
|
||||
user_id=user["id"],
|
||||
username=user["username"],
|
||||
download_path=str(parts[0]),
|
||||
download_paths=[str(part) for part in parts],
|
||||
)
|
||||
assert main_module.backend.book_queue.add(task) is True
|
||||
|
||||
try:
|
||||
main_module.backend.book_queue.update_status(task_id, QueueStatus.COMPLETE)
|
||||
with patch.object(main_module, "get_auth_mode", return_value="builtin"):
|
||||
response = client.get(f"/api/localdownload?id={task_id}")
|
||||
finally:
|
||||
main_module.backend.book_queue.cancel_download(task_id)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.mimetype == "application/zip"
|
||||
assert (
|
||||
'filename="Some Author - Zipped Audiobook.zip"'
|
||||
in response.headers["Content-Disposition"]
|
||||
)
|
||||
with zipfile.ZipFile(io.BytesIO(response.data)) as archive:
|
||||
assert archive.namelist() == ["Part 1.mp3", "Part 2.mp3"]
|
||||
assert archive.read("Part 2.mp3") == b"chapter 2"
|
||||
|
||||
def test_localdownload_returns_not_found_for_other_users_live_queue_task(
|
||||
self, main_module, client, tmp_path
|
||||
):
|
||||
|
||||
@@ -343,6 +343,9 @@ def test_multifile_rename_ignores_template_even_with_original_name(tmp_path):
|
||||
files = sorted(path.name for path in ingest.glob("*.mp3"))
|
||||
assert files == ["Part 1 of 2.mp3", "Part 2 of 2.mp3"]
|
||||
assert all("RENAMED" not in name for name in files)
|
||||
# Browser download serves all of these, not just the returned first file.
|
||||
assert result in task.download_paths
|
||||
assert sorted(Path(path).name for path in task.download_paths) == files
|
||||
|
||||
|
||||
def test_torrent_hardlink_enabled_copy_fallback_does_not_extract_archives(tmp_path):
|
||||
|
||||
@@ -55,7 +55,10 @@ def test_direct_download_explains_aa_requirement_when_only_zlib_is_configured(mo
|
||||
|
||||
source = DirectDownloadSource()
|
||||
|
||||
with pytest.raises(SearchUnavailableError, match="Anna's Archive mirror"):
|
||||
with pytest.raises(
|
||||
SearchUnavailableError,
|
||||
match="Anna's Archive or OceanofPDF mirror URL.*only download results found through",
|
||||
):
|
||||
source.search(SimpleNamespace(), SimpleNamespace())
|
||||
|
||||
|
||||
|
||||
@@ -332,3 +332,51 @@ def test_get_book_path_reports_a_missing_file_rather_than_a_dead_path(monkeypatc
|
||||
|
||||
assert path is None
|
||||
assert returned_task is task
|
||||
|
||||
|
||||
def test_get_book_paths_returns_every_file_still_on_disk(monkeypatch, tmp_path):
|
||||
import shelfmark.download.orchestrator as orchestrator
|
||||
|
||||
parts = [tmp_path / f"Part {n}.mp3" for n in (1, 2, 3)]
|
||||
for part in parts[:2]:
|
||||
part.write_bytes(b"audio")
|
||||
task = DownloadTask(
|
||||
task_id="task-book-paths-1",
|
||||
source="direct_download",
|
||||
title="Chapters",
|
||||
download_path=str(parts[0]),
|
||||
download_paths=[str(part) for part in parts],
|
||||
)
|
||||
|
||||
mock_queue = MagicMock()
|
||||
mock_queue.get_task.return_value = task
|
||||
monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
|
||||
|
||||
paths, returned_task = orchestrator.get_book_paths(task.task_id)
|
||||
|
||||
assert paths == [str(parts[0]), str(parts[1])]
|
||||
assert returned_task is task
|
||||
|
||||
|
||||
def test_get_book_paths_ignores_a_list_that_no_longer_holds_download_path(monkeypatch, tmp_path):
|
||||
import shelfmark.download.orchestrator as orchestrator
|
||||
|
||||
stale = tmp_path / "old-attempt.mp3"
|
||||
current = tmp_path / "current.epub"
|
||||
stale.write_bytes(b"old")
|
||||
current.write_bytes(b"new")
|
||||
task = DownloadTask(
|
||||
task_id="task-book-paths-2",
|
||||
source="direct_download",
|
||||
title="Retried",
|
||||
download_path=str(current),
|
||||
download_paths=[str(stale)],
|
||||
)
|
||||
|
||||
mock_queue = MagicMock()
|
||||
mock_queue.get_task.return_value = task
|
||||
monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
|
||||
|
||||
paths, _ = orchestrator.get_book_paths(task.task_id)
|
||||
|
||||
assert paths == [str(current)]
|
||||
@@ -0,0 +1,41 @@
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
from shelfmark.download.zip_stream import stream_zip, zip_entry_names
|
||||
|
||||
|
||||
def test_stream_zip_holds_every_file_uncompressed(tmp_path):
|
||||
book = tmp_path / "Author" / "Title"
|
||||
book.mkdir(parents=True)
|
||||
(book / "Part 01.m4b").write_bytes(b"chapter one")
|
||||
(book / "Part 02.m4b").write_bytes(b"chapter two" * 1000)
|
||||
|
||||
data = b"".join(stream_zip([str(book / "Part 01.m4b"), str(book / "Part 02.m4b")]))
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
assert archive.testzip() is None
|
||||
assert archive.namelist() == ["Part 01.m4b", "Part 02.m4b"]
|
||||
assert archive.read("Part 01.m4b") == b"chapter one"
|
||||
assert archive.read("Part 02.m4b") == b"chapter two" * 1000
|
||||
assert {info.compress_type for info in archive.infolist()} == {zipfile.ZIP_STORED}
|
||||
|
||||
|
||||
def test_stream_zip_yields_while_reading_rather_than_at_the_end(tmp_path, monkeypatch):
|
||||
import shelfmark.download.zip_stream as zip_stream
|
||||
|
||||
monkeypatch.setattr(zip_stream, "_ZIP_STREAM_CHUNK_SIZE", 16)
|
||||
big = tmp_path / "big.mp3"
|
||||
big.write_bytes(b"x" * 160)
|
||||
|
||||
chunks = list(stream_zip([str(big)]))
|
||||
|
||||
assert len(chunks) > 10
|
||||
|
||||
|
||||
def test_zip_entry_names_keep_folders_below_the_common_parent(tmp_path):
|
||||
first = tmp_path / "library" / "Book One" / "a.mp3"
|
||||
second = tmp_path / "library" / "Book Two" / "b.mp3"
|
||||
|
||||
names = [name for _, name in zip_entry_names([str(first), str(second)])]
|
||||
|
||||
assert names == ["Book One/a.mp3", "Book Two/b.mp3"]
|
||||
Reference in new issue
Block a user