mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-10 12:28:27 +00:00
fix(download): send every file of a multi-file download to the browser (#1489)
Download to Browser only ever sent the first file of a multi-file release (audiobook chapters, packs): the folder output returned final_paths[0] and /api/localdownload served that one path. The folder output now keeps every final path on the in-memory task (download_paths), and /api/localdownload streams them as one uncompressed ZIP named after the book when there is more than one. The archive is never staged on disk or held in memory, and ZIP64 covers large audiobooks. History still stores only the first path, so a re-download from history after a dismiss or restart serves a single file, as before. Also brings every linter back to green on all files: - pre-commit: bump the ruff hook to v0.17.1 to match uv.lock; ruff-format the Python examples in docs/dev and skills - ruff: fix the remaining issues in skills/shelfmark/download_books.py; format base_handler.py - basedpyright: Optional and attrs errors in databazeknih.py - knip: drop 31 unused exports, reuse ColumnColorHint in colorMaps - oxfmt 0.72: styles.css - direct download: fix the "Welib, mirrors" typo and the test that still expected the pre-OceanofPDF message Fixes #1467
This commit is contained in:
1 parent
3025f120eb
commit
10710abd4e
26 files changed
+664
-344
No files matched your search
@@ -6,7 +6,7 @@ repos:
|
|||||||
- id: check-added-large-files
|
- id: check-added-large-files
|
||||||
|
|
||||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||||
rev: v0.15.10
|
rev: v0.17.1
|
||||||
hooks:
|
hooks:
|
||||||
- id: ruff-check
|
- id: ruff-check
|
||||||
args: [--fix]
|
args: [--fix]
|
||||||
|
|||||||
+36
-29
@@ -23,12 +23,13 @@ from shelfmark.core.settings_registry import (
|
|||||||
ActionButton,
|
ActionButton,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@register_settings(
|
@register_settings(
|
||||||
name="my_plugin", # Unique identifier
|
name="my_plugin", # Unique identifier
|
||||||
display_name="My Plugin", # Shown in sidebar
|
display_name="My Plugin", # Shown in sidebar
|
||||||
icon="wrench", # Icon name
|
icon="wrench", # Icon name
|
||||||
order=100, # Sort order (lower = higher in list)
|
order=100, # Sort order (lower = higher in list)
|
||||||
group="metadata_providers" # Optional: group in sidebar
|
group="metadata_providers", # Optional: group in sidebar
|
||||||
)
|
)
|
||||||
def my_plugin_settings():
|
def my_plugin_settings():
|
||||||
return [
|
return [
|
||||||
@@ -46,6 +47,7 @@ def my_plugin_settings():
|
|||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _test_connection():
|
def _test_connection():
|
||||||
# Perform connection test
|
# Perform connection test
|
||||||
return {"success": True, "message": "Connected successfully!"}
|
return {"success": True, "message": "Connected successfully!"}
|
||||||
@@ -59,16 +61,16 @@ Single-line text input for strings.
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
TextField(
|
TextField(
|
||||||
key="MY_SETTING", # Config key
|
key="MY_SETTING", # Config key
|
||||||
label="Setting Name", # Display label
|
label="Setting Name", # Display label
|
||||||
description="Help text", # Optional description below field
|
description="Help text", # Optional description below field
|
||||||
default="", # Default value
|
default="", # Default value
|
||||||
placeholder="Enter value", # Placeholder text
|
placeholder="Enter value", # Placeholder text
|
||||||
max_length=100, # Optional max characters
|
max_length=100, # Optional max characters
|
||||||
required=False, # Is this field required?
|
required=False, # Is this field required?
|
||||||
requires_restart=False, # Does changing this need a restart?
|
requires_restart=False, # Does changing this need a restart?
|
||||||
show_when=None, # Conditional visibility (see below)
|
show_when=None, # Conditional visibility (see below)
|
||||||
disabled_when=None, # Conditional disable (see below)
|
disabled_when=None, # Conditional disable (see below)
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -98,7 +100,7 @@ NumberField(
|
|||||||
default=30,
|
default=30,
|
||||||
min_value=5,
|
min_value=5,
|
||||||
max_value=300,
|
max_value=300,
|
||||||
step=1, # Increment step
|
step=1, # Increment step
|
||||||
required=False,
|
required=False,
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
@@ -160,13 +162,14 @@ Button that executes a callback function. Does not store a value.
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
ActionButton(
|
ActionButton(
|
||||||
key="test_connection", # Unique key for the action
|
key="test_connection", # Unique key for the action
|
||||||
label="Test Connection", # Button text
|
label="Test Connection", # Button text
|
||||||
description="Test the API connection",
|
description="Test the API connection",
|
||||||
style="primary", # "default", "primary", or "danger"
|
style="primary", # "default", "primary", or "danger"
|
||||||
callback=my_callback_fn, # Function to execute
|
callback=my_callback_fn, # Function to execute
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def my_callback_fn():
|
def my_callback_fn():
|
||||||
"""Callback must return dict with 'success' and 'message' keys."""
|
"""Callback must return dict with 'success' and 'message' keys."""
|
||||||
try:
|
try:
|
||||||
@@ -182,11 +185,11 @@ Display-only section heading with optional link. Does not store a value.
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
HeadingField(
|
HeadingField(
|
||||||
key="section_heading", # Unique key
|
key="section_heading", # Unique key
|
||||||
title="Configuration", # Heading text
|
title="Configuration", # Heading text
|
||||||
description="Configure the plugin settings below",
|
description="Configure the plugin settings below",
|
||||||
link_url="https://example.com/docs", # Optional link
|
link_url="https://example.com/docs", # Optional link
|
||||||
link_text="View Documentation", # Link text
|
link_text="View Documentation", # Link text
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -204,8 +207,14 @@ CustomComponentField(
|
|||||||
label="Request Policy Rules",
|
label="Request Policy Rules",
|
||||||
description="Custom editor for policy defaults and matrix rules.",
|
description="Custom editor for policy defaults and matrix rules.",
|
||||||
value_fields=[
|
value_fields=[
|
||||||
SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"),
|
SelectField(
|
||||||
SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"),
|
key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"
|
||||||
|
),
|
||||||
|
SelectField(
|
||||||
|
key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
|
||||||
|
label="Default Audiobook Mode",
|
||||||
|
default="download",
|
||||||
|
),
|
||||||
TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
|
TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
|
||||||
],
|
],
|
||||||
wrap_in_field_wrapper=True, # use standard FieldWrapper label/description layout
|
wrap_in_field_wrapper=True, # use standard FieldWrapper label/description layout
|
||||||
@@ -263,7 +272,7 @@ NumberField(
|
|||||||
disabled_when={
|
disabled_when={
|
||||||
"field": "FEATURE_ENABLED",
|
"field": "FEATURE_ENABLED",
|
||||||
"value": False,
|
"value": False,
|
||||||
"reason": "Enable the feature first"
|
"reason": "Enable the feature first",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
@@ -285,6 +294,7 @@ register_group(
|
|||||||
order=50,
|
order=50,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Then register settings to the group
|
# Then register settings to the group
|
||||||
@register_settings(
|
@register_settings(
|
||||||
name="plugin_a",
|
name="plugin_a",
|
||||||
@@ -426,10 +436,7 @@ class MyProvider(MetadataProvider):
|
|||||||
|
|
||||||
def __init__(self, api_key: str = None):
|
def __init__(self, api_key: str = None):
|
||||||
self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
|
self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
|
||||||
self.base_url = config.get(
|
self.base_url = config.get("MY_PROVIDER_BASE_URL", "https://api.myprovider.com/v1")
|
||||||
"MY_PROVIDER_BASE_URL",
|
|
||||||
"https://api.myprovider.com/v1"
|
|
||||||
)
|
|
||||||
|
|
||||||
def is_available(self) -> bool:
|
def is_available(self) -> bool:
|
||||||
return bool(self.api_key)
|
return bool(self.api_key)
|
||||||
|
|||||||
@@ -66,10 +66,11 @@ This separation allows:
|
|||||||
A source has both a `ReleaseSource` (for searching) and a `DownloadHandler` (for downloading). They share the same `name` identifier:
|
A source has both a `ReleaseSource` (for searching) and a `DownloadHandler` (for downloading). They share the same `name` identifier:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@register_source("my_source") # Search registration
|
@register_source("my_source") # Search registration
|
||||||
class MySource(ReleaseSource): ...
|
class MySource(ReleaseSource): ...
|
||||||
|
|
||||||
@register_handler("my_source") # Download registration
|
|
||||||
|
@register_handler("my_source") # Download registration
|
||||||
class MyHandler(DownloadHandler): ...
|
class MyHandler(DownloadHandler): ...
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -142,10 +143,11 @@ from typing import List
|
|||||||
from shelfmark.metadata_providers import BookMetadata
|
from shelfmark.metadata_providers import BookMetadata
|
||||||
from shelfmark.release_sources import Release, ReleaseColumnConfig
|
from shelfmark.release_sources import Release, ReleaseColumnConfig
|
||||||
|
|
||||||
|
|
||||||
class ReleaseSource(ABC):
|
class ReleaseSource(ABC):
|
||||||
"""Interface for searching a release source."""
|
"""Interface for searching a release source."""
|
||||||
|
|
||||||
name: str # Internal identifier: "direct_download", "prowlarr"
|
name: str # Internal identifier: "direct_download", "prowlarr"
|
||||||
display_name: str # User-facing name: "Direct Download", "Prowlarr"
|
display_name: str # User-facing name: "Direct Download", "Prowlarr"
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
@@ -191,14 +193,16 @@ def search(self, book: BookMetadata) -> List[Release]:
|
|||||||
if book.isbn_13:
|
if book.isbn_13:
|
||||||
api_results = self.api.search_isbn(book.isbn_13)
|
api_results = self.api.search_isbn(book.isbn_13)
|
||||||
for r in api_results:
|
for r in api_results:
|
||||||
results.append(Release(
|
results.append(
|
||||||
source="my_source",
|
Release(
|
||||||
source_id=r.id,
|
source="my_source",
|
||||||
title=r.title,
|
source_id=r.id,
|
||||||
format=r.format,
|
title=r.title,
|
||||||
size=r.size,
|
format=r.format,
|
||||||
# ... more fields
|
size=r.size,
|
||||||
))
|
# ... more fields
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# Strategy 2: Fallback to title + author search
|
# Strategy 2: Fallback to title + author search
|
||||||
if not results:
|
if not results:
|
||||||
@@ -239,6 +243,7 @@ from typing import Callable, Optional
|
|||||||
from threading import Event
|
from threading import Event
|
||||||
from shelfmark.core.models import DownloadTask
|
from shelfmark.core.models import DownloadTask
|
||||||
|
|
||||||
|
|
||||||
class DownloadHandler(ABC):
|
class DownloadHandler(ABC):
|
||||||
"""Interface for executing downloads from a source."""
|
"""Interface for executing downloads from a source."""
|
||||||
|
|
||||||
@@ -286,6 +291,7 @@ downloading it; the default returns `None`, which the UI reports as "can't inspe
|
|||||||
```python
|
```python
|
||||||
from shelfmark.download.postprocess.packs import PackFile
|
from shelfmark.download.postprocess.packs import PackFile
|
||||||
|
|
||||||
|
|
||||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||||
"""Return the release's files (release-relative paths + sizes), or None."""
|
"""Return the release's files (release-relative paths + sizes), or None."""
|
||||||
torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page
|
torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page
|
||||||
@@ -380,20 +386,20 @@ Standardized search result returned by all sources:
|
|||||||
```python
|
```python
|
||||||
@dataclass
|
@dataclass
|
||||||
class Release:
|
class Release:
|
||||||
source: str # "direct_download", "prowlarr", "irc"
|
source: str # "direct_download", "prowlarr", "irc"
|
||||||
source_id: str # Unique ID within that source
|
source_id: str # Unique ID within that source
|
||||||
title: str # Display title
|
title: str # Display title
|
||||||
|
|
||||||
format: Optional[str] = None # File format: "epub", "mobi", "pdf"
|
format: Optional[str] = None # File format: "epub", "mobi", "pdf"
|
||||||
size: Optional[str] = None # Human-readable: "5.2 MB"
|
size: Optional[str] = None # Human-readable: "5.2 MB"
|
||||||
size_bytes: Optional[int] = None # For sorting
|
size_bytes: Optional[int] = None # For sorting
|
||||||
|
|
||||||
download_url: Optional[str] = None # Direct download URL if available
|
download_url: Optional[str] = None # Direct download URL if available
|
||||||
info_url: Optional[str] = None # Link to tracker/info page
|
info_url: Optional[str] = None # Link to tracker/info page
|
||||||
|
|
||||||
protocol: Optional[str] = None # "http", "torrent", "usenet"
|
protocol: Optional[str] = None # "http", "torrent", "usenet"
|
||||||
indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer"
|
indexer: Optional[str] = None # Display name: "Direct Download", "My Indexer"
|
||||||
seeders: Optional[int] = None # For torrents
|
seeders: Optional[int] = None # For torrents
|
||||||
|
|
||||||
extra: Dict = field(default_factory=dict) # Source-specific metadata
|
extra: Dict = field(default_factory=dict) # Source-specific metadata
|
||||||
```
|
```
|
||||||
@@ -415,7 +421,7 @@ Release(
|
|||||||
"preview": "https://example.com/cover.jpg",
|
"preview": "https://example.com/cover.jpg",
|
||||||
"quality": "high",
|
"quality": "high",
|
||||||
"publisher": "Publisher Name",
|
"publisher": "Publisher Name",
|
||||||
}
|
},
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -428,15 +434,15 @@ Source-agnostic task in the download queue:
|
|||||||
```python
|
```python
|
||||||
@dataclass
|
@dataclass
|
||||||
class DownloadTask:
|
class DownloadTask:
|
||||||
task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID)
|
task_id: str # Unique ID (e.g., AA MD5, Prowlarr GUID)
|
||||||
source: str # Handler name: "direct_download"
|
source: str # Handler name: "direct_download"
|
||||||
title: str # Display title
|
title: str # Display title
|
||||||
|
|
||||||
# Display info (from Release.extra or Release fields)
|
# Display info (from Release.extra or Release fields)
|
||||||
author: Optional[str] = None
|
author: Optional[str] = None
|
||||||
format: Optional[str] = None
|
format: Optional[str] = None
|
||||||
size: Optional[str] = None
|
size: Optional[str] = None
|
||||||
preview: Optional[str] = None # Cover image URL
|
preview: Optional[str] = None # Cover image URL
|
||||||
|
|
||||||
# Runtime state (managed by orchestrator)
|
# Runtime state (managed by orchestrator)
|
||||||
priority: int = 0
|
priority: int = 0
|
||||||
@@ -454,8 +460,8 @@ Book information passed to `search()`:
|
|||||||
```python
|
```python
|
||||||
@dataclass
|
@dataclass
|
||||||
class BookMetadata:
|
class BookMetadata:
|
||||||
provider: str # Metadata provider: "hardcover", "openlibrary"
|
provider: str # Metadata provider: "hardcover", "openlibrary"
|
||||||
provider_id: str # ID in provider's system
|
provider_id: str # ID in provider's system
|
||||||
title: str
|
title: str
|
||||||
|
|
||||||
provider_display_name: Optional[str] = None
|
provider_display_name: Optional[str] = None
|
||||||
@@ -466,7 +472,7 @@ class BookMetadata:
|
|||||||
description: Optional[str] = None
|
description: Optional[str] = None
|
||||||
publisher: Optional[str] = None
|
publisher: Optional[str] = None
|
||||||
publish_year: Optional[int] = None
|
publish_year: Optional[int] = None
|
||||||
language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr"
|
language: Optional[str] = None # ISO 639-1 code: "en", "de", "fr"
|
||||||
genres: List[str] = field(default_factory=list)
|
genres: List[str] = field(default_factory=list)
|
||||||
source_url: Optional[str] = None
|
source_url: Optional[str] = None
|
||||||
display_fields: List[DisplayField] = field(default_factory=list)
|
display_fields: List[DisplayField] = field(default_factory=list)
|
||||||
@@ -481,13 +487,13 @@ class BookMetadata:
|
|||||||
```python
|
```python
|
||||||
from shelfmark.release_sources import register_source, register_handler
|
from shelfmark.release_sources import register_source, register_handler
|
||||||
|
|
||||||
|
|
||||||
@register_source("my_source")
|
@register_source("my_source")
|
||||||
class MySource(ReleaseSource):
|
class MySource(ReleaseSource): ...
|
||||||
...
|
|
||||||
|
|
||||||
@register_handler("my_source")
|
@register_handler("my_source")
|
||||||
class MyHandler(DownloadHandler):
|
class MyHandler(DownloadHandler): ...
|
||||||
...
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Registry Functions
|
### Registry Functions
|
||||||
@@ -544,12 +550,13 @@ from shelfmark.core.settings_registry import (
|
|||||||
is_value_from_env,
|
is_value_from_env,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@register_settings(
|
@register_settings(
|
||||||
name="my_source", # Unique identifier
|
name="my_source", # Unique identifier
|
||||||
display_name="My Source", # Shown in settings UI
|
display_name="My Source", # Shown in settings UI
|
||||||
icon="globe", # Icon name (optional)
|
icon="globe", # Icon name (optional)
|
||||||
order=60, # Sort order in sidebar
|
order=60, # Sort order in sidebar
|
||||||
group="sources", # Grouping in sidebar
|
group="sources", # Grouping in sidebar
|
||||||
)
|
)
|
||||||
def my_source_settings():
|
def my_source_settings():
|
||||||
"""Register settings for My Source plugin."""
|
"""Register settings for My Source plugin."""
|
||||||
@@ -559,7 +566,6 @@ def my_source_settings():
|
|||||||
title="My Source Configuration",
|
title="My Source Configuration",
|
||||||
description="Configure connection to My Source API",
|
description="Configure connection to My Source API",
|
||||||
),
|
),
|
||||||
|
|
||||||
PasswordField(
|
PasswordField(
|
||||||
key="MY_SOURCE_API_KEY",
|
key="MY_SOURCE_API_KEY",
|
||||||
label="API Key",
|
label="API Key",
|
||||||
@@ -567,14 +573,12 @@ def my_source_settings():
|
|||||||
required=True,
|
required=True,
|
||||||
env_supported=True, # Can be set via MY_SOURCE_API_KEY env var
|
env_supported=True, # Can be set via MY_SOURCE_API_KEY env var
|
||||||
),
|
),
|
||||||
|
|
||||||
TextField(
|
TextField(
|
||||||
key="MY_SOURCE_BASE_URL",
|
key="MY_SOURCE_BASE_URL",
|
||||||
label="Base URL",
|
label="Base URL",
|
||||||
placeholder="https://api.mysource.com",
|
placeholder="https://api.mysource.com",
|
||||||
env_supported=True,
|
env_supported=True,
|
||||||
),
|
),
|
||||||
|
|
||||||
NumberField(
|
NumberField(
|
||||||
key="MY_SOURCE_TIMEOUT",
|
key="MY_SOURCE_TIMEOUT",
|
||||||
label="Request Timeout",
|
label="Request Timeout",
|
||||||
@@ -583,14 +587,12 @@ def my_source_settings():
|
|||||||
min_value=5,
|
min_value=5,
|
||||||
max_value=120,
|
max_value=120,
|
||||||
),
|
),
|
||||||
|
|
||||||
CheckboxField(
|
CheckboxField(
|
||||||
key="MY_SOURCE_ENABLED",
|
key="MY_SOURCE_ENABLED",
|
||||||
label="Enable My Source",
|
label="Enable My Source",
|
||||||
description="Include in release searches",
|
description="Include in release searches",
|
||||||
default=True,
|
default=True,
|
||||||
),
|
),
|
||||||
|
|
||||||
ActionButton(
|
ActionButton(
|
||||||
key="test_connection",
|
key="test_connection",
|
||||||
label="Test Connection",
|
label="Test Connection",
|
||||||
@@ -599,22 +601,19 @@ def my_source_settings():
|
|||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_my_source_connection():
|
def test_my_source_connection():
|
||||||
"""Test connection callback - returns result dict."""
|
"""Test connection callback - returns result dict."""
|
||||||
try:
|
try:
|
||||||
# Get current API key
|
# Get current API key
|
||||||
api_key = get_setting_value(
|
api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
|
||||||
PasswordField(key="MY_SOURCE_API_KEY", label=""),
|
|
||||||
"my_source"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not api_key:
|
if not api_key:
|
||||||
return {"success": False, "message": "API key not configured"}
|
return {"success": False, "message": "API key not configured"}
|
||||||
|
|
||||||
# Test the connection
|
# Test the connection
|
||||||
response = requests.get(
|
response = requests.get(
|
||||||
"https://api.mysource.com/ping",
|
"https://api.mysource.com/ping", headers={"Authorization": f"Bearer {api_key}"}
|
||||||
headers={"Authorization": f"Bearer {api_key}"}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
@@ -648,6 +647,7 @@ from shelfmark.core.settings_registry import (
|
|||||||
load_config_file,
|
load_config_file,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# In your source/handler class
|
# In your source/handler class
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
# Load from config file
|
# Load from config file
|
||||||
@@ -655,11 +655,9 @@ def __init__(self):
|
|||||||
self.api_key = config.get("MY_SOURCE_API_KEY")
|
self.api_key = config.get("MY_SOURCE_API_KEY")
|
||||||
self.base_url = config.get("MY_SOURCE_BASE_URL", "https://api.mysource.com")
|
self.base_url = config.get("MY_SOURCE_BASE_URL", "https://api.mysource.com")
|
||||||
|
|
||||||
|
|
||||||
# Or use get_setting_value for ENV var priority
|
# Or use get_setting_value for ENV var priority
|
||||||
api_key = get_setting_value(
|
api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
|
||||||
PasswordField(key="MY_SOURCE_API_KEY", label=""),
|
|
||||||
"my_source"
|
|
||||||
)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Settings Priority
|
### Settings Priority
|
||||||
@@ -689,6 +687,7 @@ from shelfmark.release_sources import (
|
|||||||
LeadingCellType,
|
LeadingCellType,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_column_config(cls) -> ReleaseColumnConfig:
|
def get_column_config(cls) -> ReleaseColumnConfig:
|
||||||
return ReleaseColumnConfig(
|
return ReleaseColumnConfig(
|
||||||
@@ -759,7 +758,7 @@ For `BADGE` render type, specify colors:
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
# Use frontend color map (defined in colorMaps.ts)
|
# Use frontend color map (defined in colorMaps.ts)
|
||||||
ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc.
|
ColumnColorHint(type="map", value="format") # Maps epub→green, pdf→red, etc.
|
||||||
ColumnColorHint(type="map", value="language") # Maps en→blue, de→yellow, etc.
|
ColumnColorHint(type="map", value="language") # Maps en→blue, de→yellow, etc.
|
||||||
|
|
||||||
# Use static Tailwind class
|
# Use static Tailwind class
|
||||||
@@ -963,6 +962,7 @@ logger = setup_logger(__name__)
|
|||||||
# Settings Registration
|
# Settings Registration
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@register_settings(
|
@register_settings(
|
||||||
name="booknet",
|
name="booknet",
|
||||||
display_name="BookNet",
|
display_name="BookNet",
|
||||||
@@ -980,7 +980,6 @@ def booknet_settings():
|
|||||||
link_url="https://booknet.example.com/api-docs",
|
link_url="https://booknet.example.com/api-docs",
|
||||||
link_text="API Documentation",
|
link_text="API Documentation",
|
||||||
),
|
),
|
||||||
|
|
||||||
PasswordField(
|
PasswordField(
|
||||||
key="BOOKNET_API_KEY",
|
key="BOOKNET_API_KEY",
|
||||||
label="API Key",
|
label="API Key",
|
||||||
@@ -988,14 +987,12 @@ def booknet_settings():
|
|||||||
required=True,
|
required=True,
|
||||||
env_supported=True,
|
env_supported=True,
|
||||||
),
|
),
|
||||||
|
|
||||||
TextField(
|
TextField(
|
||||||
key="BOOKNET_BASE_URL",
|
key="BOOKNET_BASE_URL",
|
||||||
label="Base URL",
|
label="Base URL",
|
||||||
placeholder="https://api.booknet.example.com",
|
placeholder="https://api.booknet.example.com",
|
||||||
env_supported=True,
|
env_supported=True,
|
||||||
),
|
),
|
||||||
|
|
||||||
NumberField(
|
NumberField(
|
||||||
key="BOOKNET_TIMEOUT",
|
key="BOOKNET_TIMEOUT",
|
||||||
label="Request Timeout",
|
label="Request Timeout",
|
||||||
@@ -1004,14 +1001,12 @@ def booknet_settings():
|
|||||||
min_value=5,
|
min_value=5,
|
||||||
max_value=120,
|
max_value=120,
|
||||||
),
|
),
|
||||||
|
|
||||||
CheckboxField(
|
CheckboxField(
|
||||||
key="BOOKNET_PREFER_EPUB",
|
key="BOOKNET_PREFER_EPUB",
|
||||||
label="Prefer EPUB format",
|
label="Prefer EPUB format",
|
||||||
description="Sort EPUB results first when available",
|
description="Sort EPUB results first when available",
|
||||||
default=True,
|
default=True,
|
||||||
),
|
),
|
||||||
|
|
||||||
ActionButton(
|
ActionButton(
|
||||||
key="test_connection",
|
key="test_connection",
|
||||||
label="Test Connection",
|
label="Test Connection",
|
||||||
@@ -1041,7 +1036,7 @@ def _test_booknet_connection() -> Dict:
|
|||||||
data = response.json()
|
data = response.json()
|
||||||
return {
|
return {
|
||||||
"success": True,
|
"success": True,
|
||||||
"message": f"Connected! {data.get('books_available', 0):,} books available"
|
"message": f"Connected! {data.get('books_available', 0):,} books available",
|
||||||
}
|
}
|
||||||
elif response.status_code == 401:
|
elif response.status_code == 401:
|
||||||
return {"success": False, "message": "Invalid API key"}
|
return {"success": False, "message": "Invalid API key"}
|
||||||
@@ -1058,6 +1053,7 @@ def _test_booknet_connection() -> Dict:
|
|||||||
# Release Source Implementation
|
# Release Source Implementation
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@register_source("booknet")
|
@register_source("booknet")
|
||||||
class BookNetSource(ReleaseSource):
|
class BookNetSource(ReleaseSource):
|
||||||
"""Search BookNet for book releases."""
|
"""Search BookNet for book releases."""
|
||||||
@@ -1090,7 +1086,7 @@ class BookNetSource(ReleaseSource):
|
|||||||
|
|
||||||
# Sort by preference
|
# Sort by preference
|
||||||
if self.prefer_epub:
|
if self.prefer_epub:
|
||||||
releases.sort(key=lambda r: (0 if r.format == "epub" else 1))
|
releases.sort(key=lambda r: 0 if r.format == "epub" else 1)
|
||||||
|
|
||||||
return releases
|
return releases
|
||||||
|
|
||||||
@@ -1164,7 +1160,9 @@ class BookNetSource(ReleaseSource):
|
|||||||
render_type=ColumnRenderType.BADGE,
|
render_type=ColumnRenderType.BADGE,
|
||||||
align=ColumnAlign.CENTER,
|
align=ColumnAlign.CENTER,
|
||||||
width="70px",
|
width="70px",
|
||||||
color_hint=ColumnColorHint(type="static", value="bg-purple-500/20 text-purple-400"),
|
color_hint=ColumnColorHint(
|
||||||
|
type="static", value="bg-purple-500/20 text-purple-400"
|
||||||
|
),
|
||||||
),
|
),
|
||||||
ColumnSchema(
|
ColumnSchema(
|
||||||
key="extra.language",
|
key="extra.language",
|
||||||
@@ -1202,6 +1200,7 @@ class BookNetSource(ReleaseSource):
|
|||||||
# Download Handler Implementation
|
# Download Handler Implementation
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@register_handler("booknet")
|
@register_handler("booknet")
|
||||||
class BookNetHandler(DownloadHandler):
|
class BookNetHandler(DownloadHandler):
|
||||||
"""Handle downloads from BookNet."""
|
"""Handle downloads from BookNet."""
|
||||||
@@ -1264,8 +1263,7 @@ class BookNetHandler(DownloadHandler):
|
|||||||
|
|
||||||
# Phase 3: Move to final location
|
# Phase 3: Move to final location
|
||||||
safe_title = "".join(
|
safe_title = "".join(
|
||||||
c if c.isalnum() or c in " .-_" else "_"
|
c if c.isalnum() or c in " .-_" else "_" for c in task.title[:100]
|
||||||
for c in task.title[:100]
|
|
||||||
).strip()
|
).strip()
|
||||||
|
|
||||||
final_filename = f"{safe_title}.{task.format or 'epub'}"
|
final_filename = f"{safe_title}.{task.format or 'epub'}"
|
||||||
@@ -1274,7 +1272,9 @@ class BookNetHandler(DownloadHandler):
|
|||||||
# Avoid overwriting existing files
|
# Avoid overwriting existing files
|
||||||
counter = 1
|
counter = 1
|
||||||
while final_path.exists():
|
while final_path.exists():
|
||||||
final_path = Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
|
final_path = (
|
||||||
|
Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
|
||||||
|
)
|
||||||
counter += 1
|
counter += 1
|
||||||
|
|
||||||
shutil.move(str(temp_path), str(final_path))
|
shutil.move(str(temp_path), str(final_path))
|
||||||
@@ -1363,6 +1363,7 @@ def is_available(self) -> bool:
|
|||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def search(self, book: BookMetadata) -> List[Release]:
|
def search(self, book: BookMetadata) -> List[Release]:
|
||||||
"""Return empty list on errors, don't raise."""
|
"""Return empty list on errors, don't raise."""
|
||||||
try:
|
try:
|
||||||
@@ -1446,10 +1447,10 @@ from shelfmark.core.logger import setup_logger
|
|||||||
|
|
||||||
logger = setup_logger(__name__)
|
logger = setup_logger(__name__)
|
||||||
|
|
||||||
logger.debug("Detailed debug info") # Development only
|
logger.debug("Detailed debug info") # Development only
|
||||||
logger.info("Normal operation info") # General progress
|
logger.info("Normal operation info") # General progress
|
||||||
logger.warning("Recoverable issues") # Fallback used, retry happening
|
logger.warning("Recoverable issues") # Fallback used, retry happening
|
||||||
logger.error("Failures") # Unrecoverable errors
|
logger.error("Failures") # Unrecoverable errors
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -158,6 +158,9 @@ class DownloadTask:
|
|||||||
status: QueueStatus = QueueStatus.QUEUED
|
status: QueueStatus = QueueStatus.QUEUED
|
||||||
status_message: str | None = None
|
status_message: str | None = None
|
||||||
download_path: str | None = None
|
download_path: str | None = None
|
||||||
|
# Every file the output step produced, download_path among them. Kept in memory
|
||||||
|
# only: history persists download_path alone.
|
||||||
|
download_paths: list[str] = field(default_factory=list)
|
||||||
last_error_message: str | None = None
|
last_error_message: str | None = None
|
||||||
last_error_type: str | None = None
|
last_error_type: str | None = None
|
||||||
staged_path: str | None = None
|
staged_path: str | None = None
|
||||||
|
|||||||
@@ -194,7 +194,9 @@ def _probe_completed_path(path: Path) -> tuple[bool, OSError | None]:
|
|||||||
return True, None
|
return True, None
|
||||||
|
|
||||||
|
|
||||||
def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tuple[str, int, int], ...] | None:
|
def _completed_file_signature(
|
||||||
|
path: Path, content_type: str | None
|
||||||
|
) -> tuple[tuple[str, int, int], ...] | None:
|
||||||
"""Return sizes and mtimes for supported files in a completed path."""
|
"""Return sizes and mtimes for supported files in a completed path."""
|
||||||
from shelfmark.download.postprocess.policy import (
|
from shelfmark.download.postprocess.policy import (
|
||||||
get_supported_audiobook_formats,
|
get_supported_audiobook_formats,
|
||||||
@@ -202,9 +204,7 @@ def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tup
|
|||||||
)
|
)
|
||||||
|
|
||||||
formats = (
|
formats = (
|
||||||
get_supported_audiobook_formats()
|
get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats()
|
||||||
if is_audiobook(content_type)
|
|
||||||
else get_supported_formats()
|
|
||||||
)
|
)
|
||||||
supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats}
|
supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats}
|
||||||
|
|
||||||
|
|||||||
@@ -416,6 +416,28 @@ def get_book_path(task_id: str) -> tuple[str | None, DownloadTask | None]:
|
|||||||
return path, task
|
return path, task
|
||||||
|
|
||||||
|
|
||||||
|
def get_book_paths(task_id: str) -> tuple[list[str], DownloadTask | None]:
|
||||||
|
"""Every file a task's download produced that is still on disk.
|
||||||
|
|
||||||
|
A multi-file release (chapters of an audiobook, a pack) lands as several files;
|
||||||
|
:func:`get_book_path` only knows the first. The list is trusted only while it
|
||||||
|
still contains download_path, so a later output that recorded no list cannot
|
||||||
|
hand back an earlier attempt's files.
|
||||||
|
"""
|
||||||
|
path, task = get_book_path(task_id)
|
||||||
|
if path is None or task is None:
|
||||||
|
return [], task
|
||||||
|
|
||||||
|
if path not in task.download_paths:
|
||||||
|
return [path], task
|
||||||
|
|
||||||
|
try:
|
||||||
|
return [p for p in task.download_paths if Path(p).is_file()], task
|
||||||
|
except OSError as e:
|
||||||
|
logger.error_trace(f"Error resolving book paths: {e}")
|
||||||
|
return [path], task
|
||||||
|
|
||||||
|
|
||||||
def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]:
|
def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]:
|
||||||
"""Get downloaded file data for a specific task.
|
"""Get downloaded file data for a specific task.
|
||||||
|
|
||||||
|
|||||||
@@ -275,4 +275,5 @@ def process_folder_output(
|
|||||||
message = f"Complete ({len(final_paths)} files)"
|
message = f"Complete ({len(final_paths)} files)"
|
||||||
status_callback("complete", message)
|
status_callback("complete", message)
|
||||||
|
|
||||||
|
task.download_paths = [str(path) for path in final_paths]
|
||||||
return str(final_paths[0])
|
return str(final_paths[0])
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
"""Stream several downloaded files to a client as one ZIP, without staging the archive."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Iterator
|
||||||
|
|
||||||
|
_ZIP_STREAM_CHUNK_SIZE = 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class _ChunkSink:
|
||||||
|
"""Write-only ZipFile target that hands the written bytes back to the caller.
|
||||||
|
|
||||||
|
Having no tell() or seek() makes ZipFile treat it as an unseekable stream: each
|
||||||
|
entry's CRC and sizes go in a trailing data descriptor instead of being patched
|
||||||
|
back into the local header, so nothing has to be held until the entry ends.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._chunks: list[bytes] = []
|
||||||
|
|
||||||
|
def write(self, data: bytes) -> int:
|
||||||
|
self._chunks.append(bytes(data))
|
||||||
|
return len(data)
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
"""Nothing is buffered beyond what drain() hands out."""
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Nothing to release; the caller drains what is left after ZipFile closes."""
|
||||||
|
|
||||||
|
def drain(self) -> Iterator[bytes]:
|
||||||
|
chunks, self._chunks = self._chunks, []
|
||||||
|
yield from chunks
|
||||||
|
|
||||||
|
|
||||||
|
def zip_entry_names(paths: list[str]) -> list[tuple[Path, str]]:
|
||||||
|
"""Pair each file with its name inside the archive.
|
||||||
|
|
||||||
|
Names are relative to the files' common parent, so a book's own folders
|
||||||
|
(``Title/Part 01.m4b``) survive while the library path above them does not.
|
||||||
|
"""
|
||||||
|
files = [Path(path) for path in paths]
|
||||||
|
root = Path(os.path.commonpath([str(file.parent) for file in files]))
|
||||||
|
return [(file, file.relative_to(root).as_posix()) for file in files]
|
||||||
|
|
||||||
|
|
||||||
|
def stream_zip(paths: list[str]) -> Iterator[bytes]:
|
||||||
|
"""Yield an uncompressed ZIP of ``paths``, reading each file once in chunks.
|
||||||
|
|
||||||
|
Stored, not deflated: audiobooks and ebooks are already compressed, so deflating
|
||||||
|
them would cost CPU for next to nothing. ZIP64 kicks in per entry from the file
|
||||||
|
size, which a multi-gigabyte audiobook needs.
|
||||||
|
"""
|
||||||
|
sink = _ChunkSink()
|
||||||
|
with zipfile.ZipFile(sink, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
|
||||||
|
for path, arcname in zip_entry_names(paths):
|
||||||
|
info = zipfile.ZipInfo.from_file(path, arcname, strict_timestamps=False)
|
||||||
|
info.compress_type = zipfile.ZIP_STORED
|
||||||
|
with path.open("rb") as source, archive.open(info, "w") as target:
|
||||||
|
while chunk := source.read(_ZIP_STREAM_CHUNK_SIZE):
|
||||||
|
target.write(chunk)
|
||||||
|
yield from sink.drain()
|
||||||
|
yield from sink.drain()
|
||||||
|
yield from sink.drain()
|
||||||
+45
-6
@@ -6,12 +6,14 @@ import os
|
|||||||
import re
|
import re
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import time
|
import time
|
||||||
|
import unicodedata
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from importlib import import_module
|
from importlib import import_module
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Any, Callable, NoReturn, cast
|
from typing import TYPE_CHECKING, Any, Callable, NoReturn, cast
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
from flask import Flask, g, jsonify, request, send_file, send_from_directory, session
|
from flask import Flask, g, jsonify, request, send_file, send_from_directory, session
|
||||||
from flask_cors import CORS
|
from flask_cors import CORS
|
||||||
@@ -55,7 +57,12 @@ from shelfmark.core.cwa_user_sync import upsert_cwa_user
|
|||||||
from shelfmark.core.download_history_service import DownloadHistoryService
|
from shelfmark.core.download_history_service import DownloadHistoryService
|
||||||
from shelfmark.core.external_user_linking import upsert_external_user
|
from shelfmark.core.external_user_linking import upsert_external_user
|
||||||
from shelfmark.core.logger import setup_logger
|
from shelfmark.core.logger import setup_logger
|
||||||
from shelfmark.core.models import TERMINAL_QUEUE_STATUSES, QueueStatus, SearchFilters
|
from shelfmark.core.models import (
|
||||||
|
TERMINAL_QUEUE_STATUSES,
|
||||||
|
QueueStatus,
|
||||||
|
SearchFilters,
|
||||||
|
build_filename,
|
||||||
|
)
|
||||||
from shelfmark.core.notifications import (
|
from shelfmark.core.notifications import (
|
||||||
NotificationContext,
|
NotificationContext,
|
||||||
NotificationEvent,
|
NotificationEvent,
|
||||||
@@ -95,6 +102,7 @@ from shelfmark.core.viewer_redaction import (
|
|||||||
)
|
)
|
||||||
from shelfmark.download import orchestrator as backend
|
from shelfmark.download import orchestrator as backend
|
||||||
from shelfmark.download import warmup
|
from shelfmark.download import warmup
|
||||||
|
from shelfmark.download.zip_stream import stream_zip
|
||||||
from shelfmark.release_sources import (
|
from shelfmark.release_sources import (
|
||||||
BrowseRecord,
|
BrowseRecord,
|
||||||
Release,
|
Release,
|
||||||
@@ -103,6 +111,7 @@ from shelfmark.release_sources import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from shelfmark.core.models import DownloadTask
|
||||||
from shelfmark.metadata_providers import BookMetadata, MetadataProvider
|
from shelfmark.metadata_providers import BookMetadata, MetadataProvider
|
||||||
|
|
||||||
logger = setup_logger(__name__)
|
logger = setup_logger(__name__)
|
||||||
@@ -1800,16 +1809,43 @@ def api_status() -> Response | tuple[Response, int]:
|
|||||||
return jsonify({"error": str(e)}), 500
|
return jsonify({"error": str(e)}), 500
|
||||||
|
|
||||||
|
|
||||||
|
def _attachment_filename_options(download_name: str) -> dict[str, str]:
|
||||||
|
"""Content-Disposition filename options, RFC 2231-encoded when not plain ASCII.
|
||||||
|
|
||||||
|
Mirrors what send_file does, for responses built from a generator.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
download_name.encode("ascii")
|
||||||
|
except UnicodeEncodeError:
|
||||||
|
simple = unicodedata.normalize("NFKD", download_name)
|
||||||
|
simple = simple.encode("ascii", "ignore").decode("ascii")
|
||||||
|
quoted = quote(download_name, safe="!#$&+^`|~")
|
||||||
|
return {"filename": simple, "filename*": f"UTF-8''{quoted}"}
|
||||||
|
return {"filename": download_name}
|
||||||
|
|
||||||
|
|
||||||
|
def _zip_download_response(file_paths: list[str], task: DownloadTask) -> Response:
|
||||||
|
"""Send a multi-file download (audiobook chapters, a pack) as one streamed ZIP."""
|
||||||
|
download_name = build_filename(task.title or "download", task.author, task.year, "zip")
|
||||||
|
response = Response(stream_zip(file_paths), mimetype="application/zip")
|
||||||
|
response.headers.set(
|
||||||
|
"Content-Disposition", "attachment", **_attachment_filename_options(download_name)
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
@app.route("/api/localdownload", methods=["GET"])
|
@app.route("/api/localdownload", methods=["GET"])
|
||||||
@login_required
|
@login_required
|
||||||
def api_local_download() -> Response | tuple[Response, int]:
|
def api_local_download() -> Response | tuple[Response, int]:
|
||||||
"""Download an EPUB file from local storage if available.
|
"""Download a completed book from local storage if available.
|
||||||
|
|
||||||
Query Parameters:
|
Query Parameters:
|
||||||
id (str): Book identifier (MD5 hash)
|
id (str): Book identifier (MD5 hash)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
flask.Response: The EPUB file if found, otherwise an error response.
|
flask.Response: The file if found, a ZIP of all of them when the download
|
||||||
|
produced several (a live task only; history keeps just the first), otherwise
|
||||||
|
an error response.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
book_id = request.args.get("id", "")
|
book_id = request.args.get("id", "")
|
||||||
@@ -1820,8 +1856,8 @@ def api_local_download() -> Response | tuple[Response, int]:
|
|||||||
try:
|
try:
|
||||||
# The path, not the bytes: send_file streams it off disk, where reading it
|
# The path, not the bytes: send_file streams it off disk, where reading it
|
||||||
# first cost two resident copies of the whole book.
|
# first cost two resident copies of the whole book.
|
||||||
file_path, book_info = backend.get_book_path(book_id)
|
file_paths, book_info = backend.get_book_paths(book_id)
|
||||||
if file_path is None:
|
if not file_paths:
|
||||||
# Fallback for dismissed/history entries where queue task may no longer exist.
|
# Fallback for dismissed/history entries where queue task may no longer exist.
|
||||||
if download_history_service is not None:
|
if download_history_service is not None:
|
||||||
is_admin, db_user_id, can_access_status = _resolve_status_scope()
|
is_admin, db_user_id, can_access_status = _resolve_status_scope()
|
||||||
@@ -1853,8 +1889,11 @@ def api_local_download() -> Response | tuple[Response, int]:
|
|||||||
):
|
):
|
||||||
return jsonify({"error": "File not found"}), 404
|
return jsonify({"error": "File not found"}), 404
|
||||||
|
|
||||||
|
if book_info is not None and len(file_paths) > 1:
|
||||||
|
return _zip_download_response(file_paths, book_info)
|
||||||
|
|
||||||
file_name = book_info.get_filename() if book_info is not None else Path(book_id).name
|
file_name = book_info.get_filename() if book_info is not None else Path(book_id).name
|
||||||
return send_file(file_path, download_name=file_name, as_attachment=True)
|
return send_file(file_paths[0], download_name=file_name, as_attachment=True)
|
||||||
|
|
||||||
except _OPERATIONAL_ERRORS as e:
|
except _OPERATIONAL_ERRORS as e:
|
||||||
logger.error_trace(f"Local download error: {e}")
|
logger.error_trace(f"Local download error: {e}")
|
||||||
|
|||||||
@@ -450,7 +450,7 @@ class DatabazeKnihProvider(MetadataProvider):
|
|||||||
book_id = _id_from_link(str(data.get("url", "")))
|
book_id = _id_from_link(str(data.get("url", "")))
|
||||||
if not book_id:
|
if not book_id:
|
||||||
# Pages without a JSON-LD url still carry the id on the rating buttons
|
# Pages without a JSON-LD url still carry the id on the rating buttons
|
||||||
tag = soup.find(attrs={"data-typeid": True})
|
tag = soup.select_one("[data-typeid]")
|
||||||
book_id = str(tag["data-typeid"]) if tag else ""
|
book_id = str(tag["data-typeid"]) if tag else ""
|
||||||
title = _clean_text(str(data.get("name", "")))
|
title = _clean_text(str(data.get("name", "")))
|
||||||
authors = [_clean_author(n) for n in _ld_names(data.get("author"))]
|
authors = [_clean_author(n) for n in _ld_names(data.get("author"))]
|
||||||
@@ -539,7 +539,8 @@ class DatabazeKnihProvider(MetadataProvider):
|
|||||||
name = _clean_text(str(link.get("title") or link.get_text())) or None
|
name = _clean_text(str(link.get("title") or link.get_text())) or None
|
||||||
# The part number follows the link, up to the next series link
|
# The part number follows the link, up to the next series link
|
||||||
text = []
|
text = []
|
||||||
for elem in link.parent.next_siblings:
|
parent = link.parent
|
||||||
|
for elem in parent.next_siblings if parent else ():
|
||||||
if isinstance(elem, Tag):
|
if isinstance(elem, Tag):
|
||||||
if elem.find("a", href=re.compile(r"^/serie/")):
|
if elem.find("a", href=re.compile(r"^/serie/")):
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ def get_unavailable_reason(
|
|||||||
if not enabled_providers(providers):
|
if not enabled_providers(providers):
|
||||||
return (
|
return (
|
||||||
"Direct Download is not configured. Add at least one Anna's Archive or OceanofPDF mirror URL in "
|
"Direct Download is not configured. Add at least one Anna's Archive or OceanofPDF mirror URL in "
|
||||||
"Settings. Z-Library, LibGen, Welib, mirrors only download results found through "
|
"Settings. Z-Library, LibGen and Welib mirrors only download results found through "
|
||||||
"Anna's Archive."
|
"Anna's Archive."
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|||||||
+33
-31
@@ -64,12 +64,12 @@ The script navigates to the main page, enters the search query, and waits for "M
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
# Navigate to main page first to reset SPA state
|
# Navigate to main page first to reset SPA state
|
||||||
page.goto('http://localhost:8084/')
|
page.goto("http://localhost:8084/")
|
||||||
page.wait_for_load_state('networkidle')
|
page.wait_for_load_state("networkidle")
|
||||||
|
|
||||||
# Enter search query and submit
|
# Enter search query and submit
|
||||||
page.fill('input[type="search"]', f'{title} {author}')
|
page.fill('input[type="search"]', f"{title} {author}")
|
||||||
page.press('input[type="search"]', 'Enter')
|
page.press('input[type="search"]', "Enter")
|
||||||
|
|
||||||
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
|
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
|
||||||
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
|
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
|
||||||
@@ -87,18 +87,20 @@ def parse_books(page):
|
|||||||
btns = page.query_selector_all('button[data-action="download"]')
|
btns = page.query_selector_all('button[data-action="download"]')
|
||||||
books = []
|
books = []
|
||||||
for i, btn in enumerate(btns):
|
for i, btn in enumerate(btns):
|
||||||
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
|
content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
|
||||||
|
|
||||||
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
|
title_match = re.search(r"<h3[^>]*>(.*?)</h3>", content, re.IGNORECASE | re.DOTALL)
|
||||||
title = title_match.group(1).strip() if title_match else 'Unknown'
|
title = title_match.group(1).strip() if title_match else "Unknown"
|
||||||
|
|
||||||
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
|
author_match = re.search(
|
||||||
author = author_match.group(1).strip() if author_match else 'Unknown'
|
r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
|
||||||
|
)
|
||||||
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
|
author = author_match.group(1).strip() if author_match else "Unknown"
|
||||||
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
|
|
||||||
|
dl_match = re.search(r"<span>•</span>\s*<span>([\d,]+)</span>", content)
|
||||||
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
|
downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
|
||||||
|
|
||||||
|
books.append({"index": i, "title": title, "author": author, "downloads": downloads})
|
||||||
return books
|
return books
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -106,19 +108,19 @@ def parse_books(page):
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
# Filter books matching the search criteria
|
# Filter books matching the search criteria
|
||||||
matching_books = [b for b in books if matches_search(b['title'], b['author'], title, author)]
|
matching_books = [b for b in books if matches_search(b["title"], b["author"], title, author)]
|
||||||
|
|
||||||
if matching_books:
|
if matching_books:
|
||||||
best = max(matching_books, key=lambda x: x['downloads'])
|
best = max(matching_books, key=lambda x: x["downloads"])
|
||||||
|
|
||||||
# CRITICAL: Click on the article h3 to open detail view
|
# CRITICAL: Click on the article h3 to open detail view
|
||||||
h3s = page.query_selector_all('article h3')
|
h3s = page.query_selector_all("article h3")
|
||||||
if best['index'] < len(h3s):
|
if best["index"] < len(h3s):
|
||||||
h3s[best['index']].click()
|
h3s[best["index"]].click()
|
||||||
page.wait_for_timeout(300)
|
page.wait_for_timeout(300)
|
||||||
|
|
||||||
# Then click the download button
|
# Then click the download button
|
||||||
btn = page.query_selector_all('button[data-action="download"]')[best['index']]
|
btn = page.query_selector_all('button[data-action="download"]')[best["index"]]
|
||||||
btn.click()
|
btn.click()
|
||||||
page.wait_for_timeout(500)
|
page.wait_for_timeout(500)
|
||||||
```
|
```
|
||||||
@@ -130,14 +132,14 @@ if matching_books:
|
|||||||
```python
|
```python
|
||||||
for i in range(60):
|
for i in range(60):
|
||||||
page.wait_for_timeout(5000)
|
page.wait_for_timeout(5000)
|
||||||
activity_text = page.inner_text('aside')
|
activity_text = page.inner_text("aside")
|
||||||
|
|
||||||
if 'IN PROGRESS' in activity_text:
|
if "IN PROGRESS" in activity_text:
|
||||||
print("Download started!")
|
print("Download started!")
|
||||||
elif 'Complete' in activity_text or 'Saved' in activity_text:
|
elif "Complete" in activity_text or "Saved" in activity_text:
|
||||||
print("Download complete!")
|
print("Download complete!")
|
||||||
break
|
break
|
||||||
elif 'No activity' in activity_text:
|
elif "No activity" in activity_text:
|
||||||
print("Download not started")
|
print("Download not started")
|
||||||
break
|
break
|
||||||
```
|
```
|
||||||
@@ -146,7 +148,7 @@ for i in range(60):
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
# Click "Clear Completed" using JavaScript
|
# Click "Clear Completed" using JavaScript
|
||||||
page.evaluate('''
|
page.evaluate("""
|
||||||
() => {
|
() => {
|
||||||
for (const b of document.querySelectorAll('button')) {
|
for (const b of document.querySelectorAll('button')) {
|
||||||
if (b.textContent.includes('Clear Completed')) {
|
if (b.textContent.includes('Clear Completed')) {
|
||||||
@@ -155,7 +157,7 @@ page.evaluate('''
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
''')
|
""")
|
||||||
|
|
||||||
page.wait_for_timeout(2000)
|
page.wait_for_timeout(2000)
|
||||||
```
|
```
|
||||||
|
|||||||
+206
-149
@@ -1,41 +1,62 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Shelfmark Book Downloader - Downloads books from a local Shelfmark instance."""
|
"""Shelfmark Book Downloader - Downloads books from a local Shelfmark instance."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import contextlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from playwright.sync_api import Error as PlaywrightError
|
||||||
from playwright.sync_api import sync_playwright
|
from playwright.sync_api import sync_playwright
|
||||||
import re, time, sys, argparse, json
|
|
||||||
from urllib.parse import quote
|
|
||||||
|
|
||||||
SHELFMARK_URL = 'http://localhost:8084/'
|
if TYPE_CHECKING:
|
||||||
|
from playwright.sync_api import Page
|
||||||
|
|
||||||
def parse_books(page):
|
Book = dict[str, Any]
|
||||||
|
|
||||||
|
SHELFMARK_URL = "http://localhost:8084/"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_books(page: Page) -> list[Book]:
|
||||||
btns = page.query_selector_all('button[data-action="download"]')
|
btns = page.query_selector_all('button[data-action="download"]')
|
||||||
books = []
|
books = []
|
||||||
for i, btn in enumerate(btns):
|
for i, btn in enumerate(btns):
|
||||||
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
|
content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
|
||||||
|
|
||||||
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
|
title_match = re.search(r"<h3[^>]*>(.*?)</h3>", content, re.IGNORECASE | re.DOTALL)
|
||||||
title = title_match.group(1).strip() if title_match else 'Unknown'
|
title = title_match.group(1).strip() if title_match else "Unknown"
|
||||||
|
|
||||||
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
|
author_match = re.search(
|
||||||
author = author_match.group(1).strip() if author_match else 'Unknown'
|
r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
|
||||||
|
)
|
||||||
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
|
author = author_match.group(1).strip() if author_match else "Unknown"
|
||||||
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
|
|
||||||
|
dl_match = re.search(r"<span>•</span>\s*<span>([\d,]+)</span>", content)
|
||||||
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
|
downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
|
||||||
|
|
||||||
|
books.append({"index": i, "title": title, "author": author, "downloads": downloads})
|
||||||
return books
|
return books
|
||||||
|
|
||||||
def do_search(page, title, author, search_type="author"):
|
|
||||||
|
def do_search(
|
||||||
|
page: Page, title: str, author: str, search_type: str = "author"
|
||||||
|
) -> tuple[list[Book], list[Book]] | None:
|
||||||
"""Search with title+author, fall back to title only"""
|
"""Search with title+author, fall back to title only"""
|
||||||
# Navigate to main page first to reset SPA state
|
# Navigate to main page first to reset SPA state
|
||||||
page.goto(SHELFMARK_URL)
|
page.goto(SHELFMARK_URL)
|
||||||
page.wait_for_load_state('networkidle')
|
page.wait_for_load_state("networkidle")
|
||||||
|
|
||||||
# Enter search query and submit
|
# Enter search query and submit
|
||||||
search_query = f'{title} {author}' if search_type == "author" else title
|
search_query = f"{title} {author}" if search_type == "author" else title
|
||||||
page.fill('input[type="search"]', search_query)
|
page.fill('input[type="search"]', search_query)
|
||||||
page.press('input[type="search"]', 'Enter')
|
page.press('input[type="search"]', "Enter")
|
||||||
|
|
||||||
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
|
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
|
||||||
try:
|
try:
|
||||||
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
|
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
|
||||||
@@ -44,227 +65,252 @@ def do_search(page, title, author, search_type="author"):
|
|||||||
print(" -> Trying title only...")
|
print(" -> Trying title only...")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
return do_search(page, title, author, search_type="title")
|
return do_search(page, title, author, search_type="title")
|
||||||
else:
|
print(" >> Timeout waiting for search results")
|
||||||
print(" >> Timeout waiting for search results")
|
sys.stdout.flush()
|
||||||
sys.stdout.flush()
|
return None
|
||||||
return None
|
|
||||||
|
|
||||||
books = parse_books(page)
|
books = parse_books(page)
|
||||||
|
|
||||||
def clean_title_for_match(book_title, search_title):
|
def clean_title_for_match(book_title: str, search_title: str) -> bool:
|
||||||
"""Clean book title to check if it matches the search title"""
|
"""Clean book title to check if it matches the search title"""
|
||||||
# Remove common subtitle patterns
|
# Remove common subtitle patterns
|
||||||
clean = re.sub(r'\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$', '', book_title, flags=re.IGNORECASE)
|
clean = re.sub(
|
||||||
|
r"\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$",
|
||||||
|
"",
|
||||||
|
book_title,
|
||||||
|
flags=re.IGNORECASE,
|
||||||
|
)
|
||||||
clean = clean.strip()
|
clean = clean.strip()
|
||||||
# Remove trailing punctuation
|
# Remove trailing punctuation
|
||||||
clean = clean.rstrip(':,;.')
|
clean = clean.rstrip(":,;.")
|
||||||
return clean.lower().strip() == search_title.lower().strip()
|
return clean.lower().strip() == search_title.lower().strip()
|
||||||
|
|
||||||
def matches_search(book_title, book_author, search_title, search_author):
|
def matches_search(
|
||||||
|
book_title: str, book_author: str, search_title: str, search_author: str
|
||||||
|
) -> bool:
|
||||||
"""Check if book matches search criteria more strictly"""
|
"""Check if book matches search criteria more strictly"""
|
||||||
title_match = search_title.lower() in book_title.lower()
|
title_match = search_title.lower() in book_title.lower()
|
||||||
author_match = search_author.lower() in book_author.lower()
|
author_match = search_author.lower() in book_author.lower()
|
||||||
|
|
||||||
# For title+author search, ensure title starts with search title (not just contains it)
|
# For title+author search, ensure title starts with search title (not just contains it)
|
||||||
if title_match and author_match:
|
if title_match and author_match:
|
||||||
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
|
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(
|
||||||
|
search_title.lower()
|
||||||
|
)
|
||||||
|
|
||||||
# Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann")
|
# Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann")
|
||||||
if title_match:
|
if title_match:
|
||||||
author_parts = search_author.lower().split()
|
author_parts = search_author.lower().split()
|
||||||
if len(author_parts) >= 2:
|
if len(author_parts) >= 2:
|
||||||
reversed_author = f"{author_parts[-1]} {author_parts[0]}"
|
reversed_author = f"{author_parts[-1]} {author_parts[0]}"
|
||||||
if reversed_author in book_author.lower() or book_author.lower().startswith(reversed_author):
|
if reversed_author in book_author.lower() or book_author.lower().startswith(
|
||||||
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
|
reversed_author
|
||||||
|
):
|
||||||
|
return clean_title_for_match(
|
||||||
|
book_title, search_title
|
||||||
|
) or book_title.lower().startswith(search_title.lower())
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if search_type == "author":
|
if search_type == "author":
|
||||||
matching = [b for b in books if matches_search(b['title'], b['author'], title, author)]
|
matching = [b for b in books if matches_search(b["title"], b["author"], title, author)]
|
||||||
print(f" With author: {len(books)} total, {len(matching)} matching")
|
print(f" With author: {len(books)} total, {len(matching)} matching")
|
||||||
for b in books[:3]:
|
for b in books[:3]:
|
||||||
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
|
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
if matching:
|
if matching:
|
||||||
# Check if any matching book is already in the download queue
|
# Check if any matching book is already in the download queue
|
||||||
try:
|
with contextlib.suppress(PlaywrightError):
|
||||||
sidebar_text = page.inner_text('aside')
|
sidebar_text = page.inner_text("aside")
|
||||||
if 'IN PROGRESS' in sidebar_text:
|
if "IN PROGRESS" in sidebar_text:
|
||||||
# Get the title of the book currently downloading
|
# Get the title of the book currently downloading
|
||||||
current_download = sidebar_text.split('IN PROGRESS')[1].split('—')[0].strip().split('\n')[0].strip()
|
current_download = (
|
||||||
|
sidebar_text.split("IN PROGRESS")[1]
|
||||||
|
.split("—")[0]
|
||||||
|
.strip()
|
||||||
|
.split("\n")[0]
|
||||||
|
.strip()
|
||||||
|
)
|
||||||
# Check if the current download matches our search
|
# Check if the current download matches our search
|
||||||
if title.lower() in current_download.lower():
|
if title.lower() in current_download.lower():
|
||||||
print(f" >> '{current_download}' already downloading, skipping")
|
print(f" >> '{current_download}' already downloading, skipping")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
return None
|
return None
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if matching:
|
if matching:
|
||||||
return matching, books
|
return matching, books
|
||||||
|
|
||||||
# Fall back to title only
|
# Fall back to title only
|
||||||
print(" -> Trying title only...")
|
print(" -> Trying title only...")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
return do_search(page, title, author, search_type="title")
|
return do_search(page, title, author, search_type="title")
|
||||||
|
|
||||||
# Title-only search - prefer exact matches first
|
# Title-only search - prefer exact matches first
|
||||||
matching = [b for b in books if title.lower() in b['title'].lower()]
|
matching = [b for b in books if title.lower() in b["title"].lower()]
|
||||||
exact_matches = [b for b in matching if clean_title_for_match(b['title'], title)]
|
exact_matches = [b for b in matching if clean_title_for_match(b["title"], title)]
|
||||||
|
|
||||||
if exact_matches:
|
if exact_matches:
|
||||||
matching = exact_matches
|
matching = exact_matches
|
||||||
|
|
||||||
print(f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)")
|
print(
|
||||||
|
f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)"
|
||||||
|
)
|
||||||
for b in books[:5]:
|
for b in books[:5]:
|
||||||
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
|
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
if not matching:
|
if not matching:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return (matching, books)
|
return (matching, books)
|
||||||
|
|
||||||
def download_book(page, title, author):
|
|
||||||
|
def download_book(page: Page, title: str, author: str) -> None:
|
||||||
result = do_search(page, title, author)
|
result = do_search(page, title, author)
|
||||||
if not result:
|
if not result:
|
||||||
print("\n >>> NOT FOUND")
|
print("\n >>> NOT FOUND")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
return
|
return
|
||||||
|
|
||||||
matching, all_books = result
|
matching, _all_books = result
|
||||||
if not matching:
|
if not matching:
|
||||||
print("\n >>> NOT FOUND")
|
print("\n >>> NOT FOUND")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
return
|
return
|
||||||
|
|
||||||
# Find the best book that is not disabled
|
# Find the best book that is not disabled
|
||||||
btns = page.query_selector_all('button[data-action="download"]')
|
btns = page.query_selector_all('button[data-action="download"]')
|
||||||
best = None
|
best = None
|
||||||
for b in sorted(matching, key=lambda x: x['downloads'], reverse=True):
|
for b in sorted(matching, key=lambda x: x["downloads"], reverse=True):
|
||||||
if b['index'] < len(btns) and not btns[b['index']].is_disabled():
|
if b["index"] < len(btns) and not btns[b["index"]].is_disabled():
|
||||||
best = b
|
best = b
|
||||||
break
|
break
|
||||||
|
|
||||||
if not best:
|
if not best:
|
||||||
print("\n >>> All matching books are already in download queue")
|
print("\n >>> All matching books are already in download queue")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
return
|
return
|
||||||
|
|
||||||
print(f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]")
|
print(
|
||||||
|
f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]"
|
||||||
|
)
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
# Click on the article h3 to open detail view (required for download to work)
|
# Click on the article h3 to open detail view (required for download to work)
|
||||||
h3s = page.query_selector_all('article h3')
|
h3s = page.query_selector_all("article h3")
|
||||||
if best['index'] < len(h3s):
|
if best["index"] < len(h3s):
|
||||||
h3s[best['index']].click()
|
h3s[best["index"]].click()
|
||||||
page.wait_for_timeout(300)
|
page.wait_for_timeout(300)
|
||||||
else:
|
else:
|
||||||
print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)")
|
print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
# Click the download button
|
# Click the download button
|
||||||
btn = btns[best['index']]
|
btn = btns[best["index"]]
|
||||||
btn.click()
|
btn.click()
|
||||||
|
|
||||||
# Wait a moment for the click to register
|
# Wait a moment for the click to register
|
||||||
page.wait_for_timeout(500)
|
page.wait_for_timeout(500)
|
||||||
print(f" >> Click sent")
|
print(" >> Click sent")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
# Check sidebar immediately
|
# Check sidebar immediately
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
try:
|
try:
|
||||||
txt = page.inner_text('aside')
|
txt = page.inner_text("aside")
|
||||||
if 'IN PROGRESS' in txt:
|
if "IN PROGRESS" in txt:
|
||||||
print(f" >> Download started!")
|
print(" >> Download started!")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
else:
|
else:
|
||||||
print(f" >> Sidebar after 2s: {txt[:150]}")
|
print(f" >> Sidebar after 2s: {txt[:150]}")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
except:
|
except:
|
||||||
print(f" >> Sidebar not visible after 2s")
|
print(" >> Sidebar not visible after 2s")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
# Wait for download to complete - check sidebar every 2 seconds, max 5 minutes
|
# Wait for download to complete - check sidebar every 2 seconds, max 5 minutes
|
||||||
last_state = None
|
last_state = None
|
||||||
for attempt in range(150):
|
for _attempt in range(150):
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
try:
|
try:
|
||||||
txt = page.inner_text('aside')
|
txt = page.inner_text("aside")
|
||||||
if 'IN PROGRESS' in txt:
|
if "IN PROGRESS" in txt:
|
||||||
state = 'downloading'
|
state = "downloading"
|
||||||
elif 'Complete' in txt or 'Saved' in txt or 'No activity' in txt:
|
elif "Complete" in txt or "Saved" in txt or "No activity" in txt:
|
||||||
state = 'done'
|
state = "done"
|
||||||
else:
|
else:
|
||||||
state = 'other'
|
state = "other"
|
||||||
|
|
||||||
if state != last_state:
|
if state != last_state:
|
||||||
if state == 'downloading':
|
if state == "downloading":
|
||||||
print(f" >> Download started!")
|
print(" >> Download started!")
|
||||||
elif state == 'done':
|
elif state == "done":
|
||||||
if 'No activity' in txt:
|
if "No activity" in txt:
|
||||||
print(f" >> Download not started (no activity)")
|
print(" >> Download not started (no activity)")
|
||||||
else:
|
else:
|
||||||
print(f" >> Download complete!")
|
print(" >> Download complete!")
|
||||||
last_state = state
|
last_state = state
|
||||||
|
|
||||||
if state == 'done':
|
if state == "done":
|
||||||
break
|
break
|
||||||
except:
|
except:
|
||||||
if last_state is None:
|
if last_state is None:
|
||||||
print(f" >> Sidebar not visible yet")
|
print(" >> Sidebar not visible yet")
|
||||||
last_state = 'no_sidebar'
|
last_state = "no_sidebar"
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
else:
|
else:
|
||||||
if last_state != 'done':
|
if last_state != "done":
|
||||||
print(" >> Timeout waiting for download")
|
print(" >> Timeout waiting for download")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
def clear_completed_downloads(page):
|
|
||||||
|
def clear_completed_downloads(page: Page) -> None:
|
||||||
"""Clear completed downloads from previous sessions"""
|
"""Clear completed downloads from previous sessions"""
|
||||||
page.evaluate('''
|
page.evaluate("""
|
||||||
() => {
|
() => {
|
||||||
for (const b of document.querySelectorAll('button')) {
|
for (const b of document.querySelectorAll('button')) {
|
||||||
if (b.textContent.includes('Clear Completed')) { b.click(); return; }
|
if (b.textContent.includes('Clear Completed')) { b.click(); return; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
''')
|
""")
|
||||||
page.wait_for_timeout(2000)
|
page.wait_for_timeout(2000)
|
||||||
|
|
||||||
def check_calibre(title):
|
|
||||||
|
def check_calibre(title: str) -> bool:
|
||||||
"""Check if a book is already in the calibre database."""
|
"""Check if a book is already in the calibre database."""
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
cmd = f'calibredb list --search title:="{title}"'
|
cmd = f'calibredb list --search title:="{title}"'
|
||||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
result = subprocess.run(cmd, check=False, shell=True, capture_output=True, text=True)
|
||||||
output = result.stdout.strip()
|
output = result.stdout.strip()
|
||||||
# calibredb output has header line "id title authors", count non-header lines
|
# calibredb output has header line "id title authors", count non-header lines
|
||||||
lines = [l.strip() for l in output.split('\n') if l.strip()]
|
lines = [l.strip() for l in output.split("\n") if l.strip()]
|
||||||
# Remove header if present (first line starts with 'id')
|
# Remove header if present (first line starts with 'id')
|
||||||
if lines and lines[0].startswith('id'):
|
if lines and lines[0].startswith("id"):
|
||||||
lines = lines[1:]
|
lines = lines[1:]
|
||||||
return len(lines) > 0
|
return len(lines) > 0
|
||||||
|
|
||||||
def load_books_from_json(json_str, source):
|
|
||||||
|
def load_books_from_json(json_str: str, source: str) -> list[tuple[str, str]]:
|
||||||
"""Load books from a JSON string or file path."""
|
"""Load books from a JSON string or file path."""
|
||||||
try:
|
try:
|
||||||
data = json.loads(json_str)
|
data = json.loads(json_str)
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError as e:
|
||||||
print(f"Error: Invalid JSON in {source}: {e}")
|
print(f"Error: Invalid JSON in {source}: {e}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
if not isinstance(data, list):
|
if not isinstance(data, list):
|
||||||
print(f"Error: JSON in {source} must be an array of book objects")
|
print(f"Error: JSON in {source} must be an array of book objects")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
books = []
|
books = []
|
||||||
for item in data:
|
for item in data:
|
||||||
if isinstance(item, dict):
|
if isinstance(item, dict):
|
||||||
if 'title' not in item or 'author' not in item:
|
if "title" not in item or "author" not in item:
|
||||||
print(f"Error: Each book object must have 'title' and 'author' keys: {item}")
|
print(f"Error: Each book object must have 'title' and 'author' keys: {item}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
books.append((item['title'].strip(), item['author'].strip()))
|
books.append((item["title"].strip(), item["author"].strip()))
|
||||||
elif isinstance(item, list) and len(item) >= 2:
|
elif isinstance(item, list) and len(item) >= 2:
|
||||||
books.append((str(item[0]).strip(), str(item[1]).strip()))
|
books.append((str(item[0]).strip(), str(item[1]).strip()))
|
||||||
elif isinstance(item, str):
|
elif isinstance(item, str):
|
||||||
@@ -274,80 +320,91 @@ def load_books_from_json(json_str, source):
|
|||||||
print(f"Warning: Skipping unrecognized book entry: {item}")
|
print(f"Warning: Skipping unrecognized book entry: {item}")
|
||||||
return books
|
return books
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description='Download books from Shelfmark')
|
def main() -> None:
|
||||||
parser.add_argument('books', nargs='*', help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects')
|
parser = argparse.ArgumentParser(description="Download books from Shelfmark")
|
||||||
parser.add_argument('--file', '-f', help='Path to JSON file containing an array of books')
|
parser.add_argument(
|
||||||
parser.add_argument('-c', '--check-calibre', action='store_true', help='Check calibre database before downloading, skip if already present')
|
"books",
|
||||||
|
nargs="*",
|
||||||
|
help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects',
|
||||||
|
)
|
||||||
|
parser.add_argument("--file", "-f", help="Path to JSON file containing an array of books")
|
||||||
|
parser.add_argument(
|
||||||
|
"-c",
|
||||||
|
"--check-calibre",
|
||||||
|
action="store_true",
|
||||||
|
help="Check calibre database before downloading, skip if already present",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Load books from --file, positional JSON arg, or both
|
# Load books from --file, positional JSON arg, or both
|
||||||
BOOKS = []
|
books_to_download = []
|
||||||
|
|
||||||
if args.file:
|
if args.file:
|
||||||
try:
|
try:
|
||||||
with open(args.file, 'r') as f:
|
with Path(args.file).open() as f:
|
||||||
BOOKS = load_books_from_json(f.read(), f"file '{args.file}'")
|
books_to_download = load_books_from_json(f.read(), f"file '{args.file}'")
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
print(f"Error: File not found: {args.file}")
|
print(f"Error: File not found: {args.file}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
except IOError as e:
|
except OSError as e:
|
||||||
print(f"Error reading file: {e}")
|
print(f"Error reading file: {e}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
for book_arg in args.books:
|
for book_arg in args.books:
|
||||||
BOOKS.extend(load_books_from_json(book_arg, 'command line argument'))
|
books_to_download.extend(load_books_from_json(book_arg, "command line argument"))
|
||||||
|
|
||||||
if not BOOKS:
|
if not books_to_download:
|
||||||
print("Error: No books specified. Use positional JSON args or --file <path>")
|
print("Error: No books specified. Use positional JSON args or --file <path>")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# Check calibre database if requested
|
# Check calibre database if requested
|
||||||
if args.check_calibre:
|
if args.check_calibre:
|
||||||
print("\nChecking calibre database...")
|
print("\nChecking calibre database...")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
remaining = []
|
remaining = []
|
||||||
for title, author in BOOKS:
|
for title, author in books_to_download:
|
||||||
if check_calibre(title):
|
if check_calibre(title):
|
||||||
print(f" >> '{title}' already in calibre, skipping")
|
print(f" >> '{title}' already in calibre, skipping")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
else:
|
else:
|
||||||
remaining.append((title, author))
|
remaining.append((title, author))
|
||||||
BOOKS = remaining
|
books_to_download = remaining
|
||||||
if not BOOKS:
|
if not books_to_download:
|
||||||
print("All books already in calibre, nothing to download.")
|
print("All books already in calibre, nothing to download.")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
return
|
return
|
||||||
|
|
||||||
print(f"\n{'='*60}")
|
print(f"\n{'=' * 60}")
|
||||||
print(f"Downloading {len(BOOKS)} book(s) from Shelfmark")
|
print(f"Downloading {len(books_to_download)} book(s) from Shelfmark")
|
||||||
print('='*60)
|
print("=" * 60)
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
with sync_playwright() as p:
|
with sync_playwright() as p:
|
||||||
browser = p.chromium.launch()
|
browser = p.chromium.launch()
|
||||||
page = browser.new_page(viewport={'width': 1280, 'height': 900})
|
page = browser.new_page(viewport={"width": 1280, "height": 900})
|
||||||
page.goto(SHELFMARK_URL)
|
page.goto(SHELFMARK_URL)
|
||||||
page.wait_for_load_state('networkidle')
|
page.wait_for_load_state("networkidle")
|
||||||
|
|
||||||
# Clear completed downloads at start of batch
|
# Clear completed downloads at start of batch
|
||||||
print(f"\n{'='*60}")
|
print(f"\n{'=' * 60}")
|
||||||
print("Clearing completed downloads from previous sessions...")
|
print("Clearing completed downloads from previous sessions...")
|
||||||
print('='*60)
|
print("=" * 60)
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
clear_completed_downloads(page)
|
clear_completed_downloads(page)
|
||||||
|
|
||||||
for i, (title, author) in enumerate(BOOKS, 1):
|
for i, (title, author) in enumerate(books_to_download, 1):
|
||||||
print(f"\n{'='*60}")
|
print(f"\n{'=' * 60}")
|
||||||
print(f"[{i}/{len(BOOKS)}] '{title}' by {author}")
|
print(f"[{i}/{len(books_to_download)}] '{title}' by {author}")
|
||||||
print('='*60)
|
print("=" * 60)
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
download_book(page, title, author)
|
download_book(page, title, author)
|
||||||
|
|
||||||
browser.close()
|
browser.close()
|
||||||
print("\nDone!")
|
print("\nDone!")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { RequestRecord } from '../../types';
|
import type { RequestRecord } from '../../types';
|
||||||
|
|
||||||
export type ActivityKind = 'download' | 'request';
|
type ActivityKind = 'download' | 'request';
|
||||||
|
|
||||||
export type ActivityVisualStatus =
|
export type ActivityVisualStatus =
|
||||||
| 'queued'
|
| 'queued'
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
export { DisplayFieldIcon, DisplayFieldBadge, DisplayFieldBadges } from './DisplayFieldIcon';
|
export { DisplayFieldIcon, DisplayFieldBadge, DisplayFieldBadges } from './DisplayFieldIcon';
|
||||||
export { CircularProgress } from './CircularProgress';
|
export { CircularProgress } from './CircularProgress';
|
||||||
export { ToggleSwitch } from './ToggleSwitch';
|
export { ToggleSwitch } from './ToggleSwitch';
|
||||||
export { LibraryBadge, isInLibrary, isCollectionOnly } from './LibraryBadge';
|
export { LibraryBadge, isInLibrary } from './LibraryBadge';
|
||||||
@@ -861,9 +861,9 @@ export const getReleases = async (
|
|||||||
|
|
||||||
// Admin user management API
|
// Admin user management API
|
||||||
|
|
||||||
export type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
|
type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
|
||||||
|
|
||||||
export interface AdminUserEditCapabilities {
|
interface AdminUserEditCapabilities {
|
||||||
authSource: AdminAuthSource;
|
authSource: AdminAuthSource;
|
||||||
canSetPassword: boolean;
|
canSetPassword: boolean;
|
||||||
canEditRole: boolean;
|
canEditRole: boolean;
|
||||||
|
|||||||
@@ -172,13 +172,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
body {
|
body {
|
||||||
font-family:
|
font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||||
'Inter',
|
|
||||||
-apple-system,
|
|
||||||
BlinkMacSystemFont,
|
|
||||||
'Segoe UI',
|
|
||||||
Roboto,
|
|
||||||
sans-serif;
|
|
||||||
line-height: 1.6;
|
line-height: 1.6;
|
||||||
color: var(--text-color);
|
color: var(--text-color);
|
||||||
background: var(--background-color);
|
background: var(--background-color);
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// Search mode constants and type
|
// Search mode constants and type
|
||||||
export const SEARCH_MODE = {
|
const SEARCH_MODE = {
|
||||||
DIRECT: 'direct',
|
DIRECT: 'direct',
|
||||||
UNIVERSAL: 'universal',
|
UNIVERSAL: 'universal',
|
||||||
} as const;
|
} as const;
|
||||||
@@ -88,7 +88,7 @@ export interface StatusData {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Button states
|
// Button states
|
||||||
export type ButtonState =
|
type ButtonState =
|
||||||
| 'download'
|
| 'download'
|
||||||
| 'queued'
|
| 'queued'
|
||||||
| 'resolving'
|
| 'resolving'
|
||||||
@@ -156,24 +156,24 @@ export interface TextSearchField extends SearchFieldBase {
|
|||||||
suggestions_min_query_length?: number;
|
suggestions_min_query_length?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface NumberSearchField extends SearchFieldBase {
|
interface NumberSearchField extends SearchFieldBase {
|
||||||
type: 'NumberSearchField';
|
type: 'NumberSearchField';
|
||||||
min?: number;
|
min?: number;
|
||||||
max?: number;
|
max?: number;
|
||||||
step?: number;
|
step?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SelectSearchField extends SearchFieldBase {
|
interface SelectSearchField extends SearchFieldBase {
|
||||||
type: 'SelectSearchField';
|
type: 'SelectSearchField';
|
||||||
options: SortOption[];
|
options: SortOption[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CheckboxSearchField extends SearchFieldBase {
|
interface CheckboxSearchField extends SearchFieldBase {
|
||||||
type: 'CheckboxSearchField';
|
type: 'CheckboxSearchField';
|
||||||
default?: boolean;
|
default?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DynamicSelectSearchField extends SearchFieldBase {
|
interface DynamicSelectSearchField extends SearchFieldBase {
|
||||||
type: 'DynamicSelectSearchField';
|
type: 'DynamicSelectSearchField';
|
||||||
options_endpoint: string;
|
options_endpoint: string;
|
||||||
}
|
}
|
||||||
@@ -185,7 +185,7 @@ export type MetadataSearchField =
|
|||||||
| CheckboxSearchField
|
| CheckboxSearchField
|
||||||
| DynamicSelectSearchField;
|
| DynamicSelectSearchField;
|
||||||
|
|
||||||
export type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
|
type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
|
||||||
|
|
||||||
export interface QueryTargetOption {
|
export interface QueryTargetOption {
|
||||||
key: string;
|
key: string;
|
||||||
@@ -201,12 +201,12 @@ export type ContentType = 'ebook' | 'audiobook';
|
|||||||
|
|
||||||
export type RequestPolicyMode = 'download' | 'request_release' | 'request_book' | 'blocked';
|
export type RequestPolicyMode = 'download' | 'request_release' | 'request_book' | 'blocked';
|
||||||
|
|
||||||
export interface RequestPolicyDefaults {
|
interface RequestPolicyDefaults {
|
||||||
ebook: RequestPolicyMode;
|
ebook: RequestPolicyMode;
|
||||||
audiobook: RequestPolicyMode;
|
audiobook: RequestPolicyMode;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RequestPolicySourceMode {
|
interface RequestPolicySourceMode {
|
||||||
source: string;
|
source: string;
|
||||||
supported_content_types: string[];
|
supported_content_types: string[];
|
||||||
browse_results_are_releases?: boolean;
|
browse_results_are_releases?: boolean;
|
||||||
@@ -222,7 +222,7 @@ export interface RequestPolicyResponse {
|
|||||||
source_modes: RequestPolicySourceMode[];
|
source_modes: RequestPolicySourceMode[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RequestContextPayload {
|
interface RequestContextPayload {
|
||||||
source: string;
|
source: string;
|
||||||
content_type: ContentType;
|
content_type: ContentType;
|
||||||
request_level: 'book' | 'release';
|
request_level: 'book' | 'release';
|
||||||
@@ -279,14 +279,14 @@ export interface QueuedDownloadResult {
|
|||||||
export type RequestSubmissionResult = RequestRecord | QueuedDownloadResult;
|
export type RequestSubmissionResult = RequestRecord | QueuedDownloadResult;
|
||||||
|
|
||||||
/** How the library holds a book, per format: owned outright, or inside a collection. */
|
/** How the library holds a book, per format: owned outright, or inside a collection. */
|
||||||
export type LibraryHolding = 'owned' | 'collection';
|
type LibraryHolding = 'owned' | 'collection';
|
||||||
|
|
||||||
export interface LibraryOwnership {
|
export interface LibraryOwnership {
|
||||||
ebook?: LibraryHolding | null;
|
ebook?: LibraryHolding | null;
|
||||||
audiobook?: LibraryHolding | null;
|
audiobook?: LibraryHolding | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type BooksOutputMode = 'folder' | 'booklore' | 'email';
|
type BooksOutputMode = 'folder' | 'booklore' | 'email';
|
||||||
|
|
||||||
export interface AppConfig {
|
export interface AppConfig {
|
||||||
calibre_web_url: string;
|
calibre_web_url: string;
|
||||||
@@ -334,7 +334,7 @@ export interface MetadataProvidersResponse {
|
|||||||
configured_provider_combined: string | null;
|
configured_provider_combined: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MetadataCapability {
|
interface MetadataCapability {
|
||||||
key: string;
|
key: string;
|
||||||
field_key?: string;
|
field_key?: string;
|
||||||
sort?: string;
|
sort?: string;
|
||||||
@@ -398,7 +398,7 @@ export interface ReleaseSource {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Column schema types for plugin-driven release list UI
|
// Column schema types for plugin-driven release list UI
|
||||||
export type ColumnRenderType =
|
type ColumnRenderType =
|
||||||
| 'text'
|
| 'text'
|
||||||
| 'badge'
|
| 'badge'
|
||||||
| 'tags'
|
| 'tags'
|
||||||
@@ -408,7 +408,7 @@ export type ColumnRenderType =
|
|||||||
| 'indexer_protocol'
|
| 'indexer_protocol'
|
||||||
| 'flag_icon'
|
| 'flag_icon'
|
||||||
| 'format_content_type';
|
| 'format_content_type';
|
||||||
export type ColumnAlign = 'left' | 'center' | 'right';
|
type ColumnAlign = 'left' | 'center' | 'right';
|
||||||
|
|
||||||
export interface ColumnColorHint {
|
export interface ColumnColorHint {
|
||||||
type: 'map' | 'static'; // 'map' uses colorMaps.ts, 'static' is a fixed Tailwind class
|
type: 'map' | 'static'; // 'map' uses colorMaps.ts, 'static' is a fixed Tailwind class
|
||||||
@@ -430,7 +430,7 @@ export interface ColumnSchema {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Leading cell config - what to show in the left-most position of each row
|
// Leading cell config - what to show in the left-most position of each row
|
||||||
export type LeadingCellType = 'thumbnail' | 'badge' | 'none';
|
type LeadingCellType = 'thumbnail' | 'badge' | 'none';
|
||||||
|
|
||||||
export interface LeadingCellConfig {
|
export interface LeadingCellConfig {
|
||||||
type: LeadingCellType;
|
type: LeadingCellType;
|
||||||
@@ -439,13 +439,13 @@ export interface LeadingCellConfig {
|
|||||||
uppercase?: boolean; // Force uppercase for badge text
|
uppercase?: boolean; // Force uppercase for badge text
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ExtraSortOption {
|
interface ExtraSortOption {
|
||||||
label: string; // Display label in the sort dropdown
|
label: string; // Display label in the sort dropdown
|
||||||
sort_key: string; // Field to sort by on the Release object
|
sort_key: string; // Field to sort by on the Release object
|
||||||
default_direction?: 'asc' | 'desc'; // Which way "best first" runs (defaults to desc)
|
default_direction?: 'asc' | 'desc'; // Which way "best first" runs (defaults to desc)
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SourceActionButton {
|
interface SourceActionButton {
|
||||||
label: string; // Button text (e.g., "Refresh search")
|
label: string; // Button text (e.g., "Refresh search")
|
||||||
action: string; // Action type: "expand" triggers expand_search
|
action: string; // Action type: "expand" triggers expand_search
|
||||||
}
|
}
|
||||||
@@ -483,7 +483,7 @@ export interface Release {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Search info returned by release sources
|
// Search info returned by release sources
|
||||||
export interface SourceSearchInfo {
|
interface SourceSearchInfo {
|
||||||
search_type: 'isbn' | 'title_author' | 'categories' | 'expanded' | 'manual' | 'query';
|
search_type: 'isbn' | 'title_author' | 'categories' | 'expanded' | 'manual' | 'query';
|
||||||
total_results?: number | string | null;
|
total_results?: number | string | null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// Settings field types matching backend settings_registry.py
|
// Settings field types matching backend settings_registry.py
|
||||||
|
|
||||||
export type FieldType =
|
type FieldType =
|
||||||
| 'TextField'
|
| 'TextField'
|
||||||
| 'PasswordField'
|
| 'PasswordField'
|
||||||
| 'NumberField'
|
| 'NumberField'
|
||||||
@@ -28,17 +28,17 @@ export interface ShowWhenCondition {
|
|||||||
notEmpty?: boolean; // If true, show when field has any non-empty value
|
notEmpty?: boolean; // If true, show when field has any non-empty value
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
|
type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
|
||||||
|
|
||||||
// Conditional disable configuration
|
// Conditional disable configuration
|
||||||
export interface DisabledWhenCondition {
|
interface DisabledWhenCondition {
|
||||||
field: string; // The field key to check
|
field: string; // The field key to check
|
||||||
value: string | string[] | boolean; // The value(s) that disable this field
|
value: string | string[] | boolean; // The value(s) that disable this field
|
||||||
reason?: string; // Explanation shown when disabled
|
reason?: string; // Explanation shown when disabled
|
||||||
}
|
}
|
||||||
|
|
||||||
// Base field interface - common properties
|
// Base field interface - common properties
|
||||||
export interface BaseField {
|
interface BaseField {
|
||||||
key: string;
|
key: string;
|
||||||
label: string;
|
label: string;
|
||||||
type: FieldType;
|
type: FieldType;
|
||||||
@@ -139,20 +139,14 @@ export interface CustomComponentFieldConfig extends BaseField {
|
|||||||
wrapInFieldWrapper?: boolean; // Whether to render with standard FieldWrapper layout
|
wrapInFieldWrapper?: boolean; // Whether to render with standard FieldWrapper layout
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TableFieldColumnOption {
|
interface TableFieldColumnOption {
|
||||||
value: string;
|
value: string;
|
||||||
label: string;
|
label: string;
|
||||||
description?: string;
|
description?: string;
|
||||||
childOf?: string;
|
childOf?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TableFieldColumnType =
|
type TableFieldColumnType = 'text' | 'password' | 'select' | 'multiselect' | 'checkbox' | 'path';
|
||||||
| 'text'
|
|
||||||
| 'password'
|
|
||||||
| 'select'
|
|
||||||
| 'multiselect'
|
|
||||||
| 'checkbox'
|
|
||||||
| 'path';
|
|
||||||
|
|
||||||
export interface TableFieldColumn {
|
export interface TableFieldColumn {
|
||||||
key: string;
|
key: string;
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import type { ColumnColorHint } from '../types';
|
||||||
|
|
||||||
// Color styles with transparent backgrounds and contrasting text
|
// Color styles with transparent backgrounds and contrasting text
|
||||||
interface ColorStyle {
|
interface ColorStyle {
|
||||||
bg: string;
|
bg: string;
|
||||||
@@ -133,14 +135,6 @@ function getFlagColor(flag?: string): ColorStyle {
|
|||||||
return FLAG_COLORS[normalized] || DEFAULT_FLAG_COLOR;
|
return FLAG_COLORS[normalized] || DEFAULT_FLAG_COLOR;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Color hint type for dynamic column coloring.
|
|
||||||
*/
|
|
||||||
interface ColumnColorHint {
|
|
||||||
type: 'static' | 'map';
|
|
||||||
value: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the color style for a value based on a color hint.
|
* Get the color style for a value based on a color hint.
|
||||||
* Supports both static color classes and dynamic map lookups.
|
* Supports both static color classes and dynamic map lookups.
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import type { ContentType } from '../types';
|
|||||||
export const CONTENT_TYPE_STORAGE_KEY = 'preferred-content-type.v2';
|
export const CONTENT_TYPE_STORAGE_KEY = 'preferred-content-type.v2';
|
||||||
|
|
||||||
/** Where the current content type came from, which decides what may still override it. */
|
/** Where the current content type came from, which decides what may still override it. */
|
||||||
export type ContentTypeSource =
|
type ContentTypeSource =
|
||||||
/** Nothing has chosen a tab, so a configured default may still apply. */
|
/** Nothing has chosen a tab, so a configured default may still apply. */
|
||||||
| 'unset'
|
| 'unset'
|
||||||
/** This browser stored a choice, or something explicit set one. Defaults do not apply. */
|
/** This browser stored a choice, or something explicit set one. Defaults do not apply. */
|
||||||
|
|||||||
@@ -3,8 +3,10 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
|
import io
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import uuid
|
import uuid
|
||||||
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import ANY, patch
|
from unittest.mock import ANY, patch
|
||||||
@@ -355,6 +357,46 @@ class TestActivityRoutes:
|
|||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert response.data == file_bytes
|
assert response.data == file_bytes
|
||||||
|
|
||||||
|
def test_localdownload_zips_a_live_multi_file_download(self, main_module, client, tmp_path):
|
||||||
|
user = _create_user(main_module, prefix="reader")
|
||||||
|
_set_session(client, user_id=user["username"], db_user_id=user["id"], is_admin=False)
|
||||||
|
|
||||||
|
task_id = f"live-localdownload-{uuid.uuid4().hex[:8]}"
|
||||||
|
book_dir = tmp_path / "Chapters"
|
||||||
|
book_dir.mkdir()
|
||||||
|
parts = [book_dir / "Part 1.mp3", book_dir / "Part 2.mp3"]
|
||||||
|
for index, part in enumerate(parts, start=1):
|
||||||
|
part.write_bytes(f"chapter {index}".encode())
|
||||||
|
|
||||||
|
task = DownloadTask(
|
||||||
|
task_id=task_id,
|
||||||
|
source="direct_download",
|
||||||
|
title="Zipped Audiobook",
|
||||||
|
author="Some Author",
|
||||||
|
user_id=user["id"],
|
||||||
|
username=user["username"],
|
||||||
|
download_path=str(parts[0]),
|
||||||
|
download_paths=[str(part) for part in parts],
|
||||||
|
)
|
||||||
|
assert main_module.backend.book_queue.add(task) is True
|
||||||
|
|
||||||
|
try:
|
||||||
|
main_module.backend.book_queue.update_status(task_id, QueueStatus.COMPLETE)
|
||||||
|
with patch.object(main_module, "get_auth_mode", return_value="builtin"):
|
||||||
|
response = client.get(f"/api/localdownload?id={task_id}")
|
||||||
|
finally:
|
||||||
|
main_module.backend.book_queue.cancel_download(task_id)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.mimetype == "application/zip"
|
||||||
|
assert (
|
||||||
|
'filename="Some Author - Zipped Audiobook.zip"'
|
||||||
|
in response.headers["Content-Disposition"]
|
||||||
|
)
|
||||||
|
with zipfile.ZipFile(io.BytesIO(response.data)) as archive:
|
||||||
|
assert archive.namelist() == ["Part 1.mp3", "Part 2.mp3"]
|
||||||
|
assert archive.read("Part 2.mp3") == b"chapter 2"
|
||||||
|
|
||||||
def test_localdownload_returns_not_found_for_other_users_live_queue_task(
|
def test_localdownload_returns_not_found_for_other_users_live_queue_task(
|
||||||
self, main_module, client, tmp_path
|
self, main_module, client, tmp_path
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -343,6 +343,9 @@ def test_multifile_rename_ignores_template_even_with_original_name(tmp_path):
|
|||||||
files = sorted(path.name for path in ingest.glob("*.mp3"))
|
files = sorted(path.name for path in ingest.glob("*.mp3"))
|
||||||
assert files == ["Part 1 of 2.mp3", "Part 2 of 2.mp3"]
|
assert files == ["Part 1 of 2.mp3", "Part 2 of 2.mp3"]
|
||||||
assert all("RENAMED" not in name for name in files)
|
assert all("RENAMED" not in name for name in files)
|
||||||
|
# Browser download serves all of these, not just the returned first file.
|
||||||
|
assert result in task.download_paths
|
||||||
|
assert sorted(Path(path).name for path in task.download_paths) == files
|
||||||
|
|
||||||
|
|
||||||
def test_torrent_hardlink_enabled_copy_fallback_does_not_extract_archives(tmp_path):
|
def test_torrent_hardlink_enabled_copy_fallback_does_not_extract_archives(tmp_path):
|
||||||
|
|||||||
@@ -55,7 +55,10 @@ def test_direct_download_explains_aa_requirement_when_only_zlib_is_configured(mo
|
|||||||
|
|
||||||
source = DirectDownloadSource()
|
source = DirectDownloadSource()
|
||||||
|
|
||||||
with pytest.raises(SearchUnavailableError, match="Anna's Archive mirror"):
|
with pytest.raises(
|
||||||
|
SearchUnavailableError,
|
||||||
|
match="Anna's Archive or OceanofPDF mirror URL.*only download results found through",
|
||||||
|
):
|
||||||
source.search(SimpleNamespace(), SimpleNamespace())
|
source.search(SimpleNamespace(), SimpleNamespace())
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -332,3 +332,51 @@ def test_get_book_path_reports_a_missing_file_rather_than_a_dead_path(monkeypatc
|
|||||||
|
|
||||||
assert path is None
|
assert path is None
|
||||||
assert returned_task is task
|
assert returned_task is task
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_book_paths_returns_every_file_still_on_disk(monkeypatch, tmp_path):
|
||||||
|
import shelfmark.download.orchestrator as orchestrator
|
||||||
|
|
||||||
|
parts = [tmp_path / f"Part {n}.mp3" for n in (1, 2, 3)]
|
||||||
|
for part in parts[:2]:
|
||||||
|
part.write_bytes(b"audio")
|
||||||
|
task = DownloadTask(
|
||||||
|
task_id="task-book-paths-1",
|
||||||
|
source="direct_download",
|
||||||
|
title="Chapters",
|
||||||
|
download_path=str(parts[0]),
|
||||||
|
download_paths=[str(part) for part in parts],
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_queue = MagicMock()
|
||||||
|
mock_queue.get_task.return_value = task
|
||||||
|
monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
|
||||||
|
|
||||||
|
paths, returned_task = orchestrator.get_book_paths(task.task_id)
|
||||||
|
|
||||||
|
assert paths == [str(parts[0]), str(parts[1])]
|
||||||
|
assert returned_task is task
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_book_paths_ignores_a_list_that_no_longer_holds_download_path(monkeypatch, tmp_path):
|
||||||
|
import shelfmark.download.orchestrator as orchestrator
|
||||||
|
|
||||||
|
stale = tmp_path / "old-attempt.mp3"
|
||||||
|
current = tmp_path / "current.epub"
|
||||||
|
stale.write_bytes(b"old")
|
||||||
|
current.write_bytes(b"new")
|
||||||
|
task = DownloadTask(
|
||||||
|
task_id="task-book-paths-2",
|
||||||
|
source="direct_download",
|
||||||
|
title="Retried",
|
||||||
|
download_path=str(current),
|
||||||
|
download_paths=[str(stale)],
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_queue = MagicMock()
|
||||||
|
mock_queue.get_task.return_value = task
|
||||||
|
monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
|
||||||
|
|
||||||
|
paths, _ = orchestrator.get_book_paths(task.task_id)
|
||||||
|
|
||||||
|
assert paths == [str(current)]
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import io
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
from shelfmark.download.zip_stream import stream_zip, zip_entry_names
|
||||||
|
|
||||||
|
|
||||||
|
def test_stream_zip_holds_every_file_uncompressed(tmp_path):
|
||||||
|
book = tmp_path / "Author" / "Title"
|
||||||
|
book.mkdir(parents=True)
|
||||||
|
(book / "Part 01.m4b").write_bytes(b"chapter one")
|
||||||
|
(book / "Part 02.m4b").write_bytes(b"chapter two" * 1000)
|
||||||
|
|
||||||
|
data = b"".join(stream_zip([str(book / "Part 01.m4b"), str(book / "Part 02.m4b")]))
|
||||||
|
|
||||||
|
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||||
|
assert archive.testzip() is None
|
||||||
|
assert archive.namelist() == ["Part 01.m4b", "Part 02.m4b"]
|
||||||
|
assert archive.read("Part 01.m4b") == b"chapter one"
|
||||||
|
assert archive.read("Part 02.m4b") == b"chapter two" * 1000
|
||||||
|
assert {info.compress_type for info in archive.infolist()} == {zipfile.ZIP_STORED}
|
||||||
|
|
||||||
|
|
||||||
|
def test_stream_zip_yields_while_reading_rather_than_at_the_end(tmp_path, monkeypatch):
|
||||||
|
import shelfmark.download.zip_stream as zip_stream
|
||||||
|
|
||||||
|
monkeypatch.setattr(zip_stream, "_ZIP_STREAM_CHUNK_SIZE", 16)
|
||||||
|
big = tmp_path / "big.mp3"
|
||||||
|
big.write_bytes(b"x" * 160)
|
||||||
|
|
||||||
|
chunks = list(stream_zip([str(big)]))
|
||||||
|
|
||||||
|
assert len(chunks) > 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_zip_entry_names_keep_folders_below_the_common_parent(tmp_path):
|
||||||
|
first = tmp_path / "library" / "Book One" / "a.mp3"
|
||||||
|
second = tmp_path / "library" / "Book Two" / "b.mp3"
|
||||||
|
|
||||||
|
names = [name for _, name in zip_entry_names([str(first), str(second)])]
|
||||||
|
|
||||||
|
assert names == ["Book One/a.mp3", "Book Two/b.mp3"]
|
||||||
Reference in new issue
Block a user