fix(download): send every file of a multi-file download to the browser (#1489)

Download to Browser only ever sent the first file of a multi-file
release
(audiobook chapters, packs): the folder output returned final_paths[0]
and
/api/localdownload served that one path.

The folder output now keeps every final path on the in-memory task
(download_paths), and /api/localdownload streams them as one
uncompressed
ZIP named after the book when there is more than one. The archive is
never
staged on disk or held in memory, and ZIP64 covers large audiobooks.
History still stores only the first path, so a re-download from history
after a dismiss or restart serves a single file, as before.

Also brings every linter back to green on all files:
- pre-commit: bump the ruff hook to v0.17.1 to match uv.lock;
ruff-format
  the Python examples in docs/dev and skills
- ruff: fix the remaining issues in skills/shelfmark/download_books.py;
  format base_handler.py
- basedpyright: Optional and attrs errors in databazeknih.py
- knip: drop 31 unused exports, reuse ColumnColorHint in colorMaps
- oxfmt 0.72: styles.css
- direct download: fix the "Welib, mirrors" typo and the test that still
  expected the pre-OceanofPDF message

Fixes #1467
This commit is contained in:
CaliBrain authored and GitHub committed 2026-10-20 02:33:53 -04:00
1 parent 3025f120eb
commit 10710abd4e
26 files changed
+549 -229

No files matched your search

+1 -1
View File
@@ -6,7 +6,7 @@ repos:
- id: check-added-large-files
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.10
rev: v0.17.1
hooks:
- id: ruff-check
args: [--fix]
+15 -8
View File
@@ -23,12 +23,13 @@ from shelfmark.core.settings_registry import (
ActionButton,
)
@register_settings(
name="my_plugin", # Unique identifier
display_name="My Plugin", # Shown in sidebar
icon="wrench", # Icon name
order=100, # Sort order (lower = higher in list)
group="metadata_providers" # Optional: group in sidebar
group="metadata_providers", # Optional: group in sidebar
)
def my_plugin_settings():
return [
@@ -46,6 +47,7 @@ def my_plugin_settings():
),
]
def _test_connection():
# Perform connection test
return {"success": True, "message": "Connected successfully!"}
@@ -167,6 +169,7 @@ ActionButton(
callback=my_callback_fn, # Function to execute
)
def my_callback_fn():
"""Callback must return dict with 'success' and 'message' keys."""
try:
@@ -204,8 +207,14 @@ CustomComponentField(
label="Request Policy Rules",
description="Custom editor for policy defaults and matrix rules.",
value_fields=[
SelectField(key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"),
SelectField(key="REQUEST_POLICY_DEFAULT_AUDIOBOOK", label="Default Audiobook Mode", default="download"),
SelectField(
key="REQUEST_POLICY_DEFAULT_EBOOK", label="Default Ebook Mode", default="download"
),
SelectField(
key="REQUEST_POLICY_DEFAULT_AUDIOBOOK",
label="Default Audiobook Mode",
default="download",
),
TableField(key="REQUEST_POLICY_RULES", label="Rules", columns=_rule_columns, default=[]),
],
wrap_in_field_wrapper=True, # use standard FieldWrapper label/description layout
@@ -263,7 +272,7 @@ NumberField(
disabled_when={
"field": "FEATURE_ENABLED",
"value": False,
"reason": "Enable the feature first"
"reason": "Enable the feature first",
},
)
```
@@ -285,6 +294,7 @@ register_group(
order=50,
)
# Then register settings to the group
@register_settings(
name="plugin_a",
@@ -426,10 +436,7 @@ class MyProvider(MetadataProvider):
def __init__(self, api_key: str = None):
self.api_key = api_key or config.get("MY_PROVIDER_API_KEY", "")
self.base_url = config.get(
"MY_PROVIDER_BASE_URL",
"https://api.myprovider.com/v1"
)
self.base_url = config.get("MY_PROVIDER_BASE_URL", "https://api.myprovider.com/v1")
def is_available(self) -> bool:
return bool(self.api_key)
+34 -33
View File
@@ -69,6 +69,7 @@ A source has both a `ReleaseSource` (for searching) and a `DownloadHandler` (for
@register_source("my_source") # Search registration
class MySource(ReleaseSource): ...
@register_handler("my_source") # Download registration
class MyHandler(DownloadHandler): ...
```
@@ -142,6 +143,7 @@ from typing import List
from shelfmark.metadata_providers import BookMetadata
from shelfmark.release_sources import Release, ReleaseColumnConfig
class ReleaseSource(ABC):
"""Interface for searching a release source."""
@@ -191,14 +193,16 @@ def search(self, book: BookMetadata) -> List[Release]:
if book.isbn_13:
api_results = self.api.search_isbn(book.isbn_13)
for r in api_results:
results.append(Release(
results.append(
Release(
source="my_source",
source_id=r.id,
title=r.title,
format=r.format,
size=r.size,
# ... more fields
))
)
)
# Strategy 2: Fallback to title + author search
if not results:
@@ -239,6 +243,7 @@ from typing import Callable, Optional
from threading import Event
from shelfmark.core.models import DownloadTask
class DownloadHandler(ABC):
"""Interface for executing downloads from a source."""
@@ -286,6 +291,7 @@ downloading it; the default returns `None`, which the UI reports as "can't inspe
```python
from shelfmark.download.postprocess.packs import PackFile
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
"""Return the release's files (release-relative paths + sizes), or None."""
torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page
@@ -415,7 +421,7 @@ Release(
"preview": "https://example.com/cover.jpg",
"quality": "high",
"publisher": "Publisher Name",
}
},
)
```
@@ -481,13 +487,13 @@ class BookMetadata:
```python
from shelfmark.release_sources import register_source, register_handler
@register_source("my_source")
class MySource(ReleaseSource):
...
class MySource(ReleaseSource): ...
@register_handler("my_source")
class MyHandler(DownloadHandler):
...
class MyHandler(DownloadHandler): ...
```
### Registry Functions
@@ -544,6 +550,7 @@ from shelfmark.core.settings_registry import (
is_value_from_env,
)
@register_settings(
name="my_source", # Unique identifier
display_name="My Source", # Shown in settings UI
@@ -559,7 +566,6 @@ def my_source_settings():
title="My Source Configuration",
description="Configure connection to My Source API",
),
PasswordField(
key="MY_SOURCE_API_KEY",
label="API Key",
@@ -567,14 +573,12 @@ def my_source_settings():
required=True,
env_supported=True, # Can be set via MY_SOURCE_API_KEY env var
),
TextField(
key="MY_SOURCE_BASE_URL",
label="Base URL",
placeholder="https://api.mysource.com",
env_supported=True,
),
NumberField(
key="MY_SOURCE_TIMEOUT",
label="Request Timeout",
@@ -583,14 +587,12 @@ def my_source_settings():
min_value=5,
max_value=120,
),
CheckboxField(
key="MY_SOURCE_ENABLED",
label="Enable My Source",
description="Include in release searches",
default=True,
),
ActionButton(
key="test_connection",
label="Test Connection",
@@ -599,22 +601,19 @@ def my_source_settings():
),
]
def test_my_source_connection():
"""Test connection callback - returns result dict."""
try:
# Get current API key
api_key = get_setting_value(
PasswordField(key="MY_SOURCE_API_KEY", label=""),
"my_source"
)
api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
if not api_key:
return {"success": False, "message": "API key not configured"}
# Test the connection
response = requests.get(
"https://api.mysource.com/ping",
headers={"Authorization": f"Bearer {api_key}"}
"https://api.mysource.com/ping", headers={"Authorization": f"Bearer {api_key}"}
)
if response.status_code == 200:
@@ -648,6 +647,7 @@ from shelfmark.core.settings_registry import (
load_config_file,
)
# In your source/handler class
def __init__(self):
# Load from config file
@@ -655,11 +655,9 @@ def __init__(self):
self.api_key = config.get("MY_SOURCE_API_KEY")
self.base_url = config.get("MY_SOURCE_BASE_URL", "https://api.mysource.com")
# Or use get_setting_value for ENV var priority
api_key = get_setting_value(
PasswordField(key="MY_SOURCE_API_KEY", label=""),
"my_source"
)
api_key = get_setting_value(PasswordField(key="MY_SOURCE_API_KEY", label=""), "my_source")
```
### Settings Priority
@@ -689,6 +687,7 @@ from shelfmark.release_sources import (
LeadingCellType,
)
@classmethod
def get_column_config(cls) -> ReleaseColumnConfig:
return ReleaseColumnConfig(
@@ -963,6 +962,7 @@ logger = setup_logger(__name__)
# Settings Registration
# ---------------------------------------------------------------------------
@register_settings(
name="booknet",
display_name="BookNet",
@@ -980,7 +980,6 @@ def booknet_settings():
link_url="https://booknet.example.com/api-docs",
link_text="API Documentation",
),
PasswordField(
key="BOOKNET_API_KEY",
label="API Key",
@@ -988,14 +987,12 @@ def booknet_settings():
required=True,
env_supported=True,
),
TextField(
key="BOOKNET_BASE_URL",
label="Base URL",
placeholder="https://api.booknet.example.com",
env_supported=True,
),
NumberField(
key="BOOKNET_TIMEOUT",
label="Request Timeout",
@@ -1004,14 +1001,12 @@ def booknet_settings():
min_value=5,
max_value=120,
),
CheckboxField(
key="BOOKNET_PREFER_EPUB",
label="Prefer EPUB format",
description="Sort EPUB results first when available",
default=True,
),
ActionButton(
key="test_connection",
label="Test Connection",
@@ -1041,7 +1036,7 @@ def _test_booknet_connection() -> Dict:
data = response.json()
return {
"success": True,
"message": f"Connected! {data.get('books_available', 0):,} books available"
"message": f"Connected! {data.get('books_available', 0):,} books available",
}
elif response.status_code == 401:
return {"success": False, "message": "Invalid API key"}
@@ -1058,6 +1053,7 @@ def _test_booknet_connection() -> Dict:
# Release Source Implementation
# ---------------------------------------------------------------------------
@register_source("booknet")
class BookNetSource(ReleaseSource):
"""Search BookNet for book releases."""
@@ -1090,7 +1086,7 @@ class BookNetSource(ReleaseSource):
# Sort by preference
if self.prefer_epub:
releases.sort(key=lambda r: (0 if r.format == "epub" else 1))
releases.sort(key=lambda r: 0 if r.format == "epub" else 1)
return releases
@@ -1164,7 +1160,9 @@ class BookNetSource(ReleaseSource):
render_type=ColumnRenderType.BADGE,
align=ColumnAlign.CENTER,
width="70px",
color_hint=ColumnColorHint(type="static", value="bg-purple-500/20 text-purple-400"),
color_hint=ColumnColorHint(
type="static", value="bg-purple-500/20 text-purple-400"
),
),
ColumnSchema(
key="extra.language",
@@ -1202,6 +1200,7 @@ class BookNetSource(ReleaseSource):
# Download Handler Implementation
# ---------------------------------------------------------------------------
@register_handler("booknet")
class BookNetHandler(DownloadHandler):
"""Handle downloads from BookNet."""
@@ -1264,8 +1263,7 @@ class BookNetHandler(DownloadHandler):
# Phase 3: Move to final location
safe_title = "".join(
c if c.isalnum() or c in " .-_" else "_"
for c in task.title[:100]
c if c.isalnum() or c in " .-_" else "_" for c in task.title[:100]
).strip()
final_filename = f"{safe_title}.{task.format or 'epub'}"
@@ -1274,7 +1272,9 @@ class BookNetHandler(DownloadHandler):
# Avoid overwriting existing files
counter = 1
while final_path.exists():
final_path = Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
final_path = (
Path(INGEST_DIR) / f"{safe_title} ({counter}).{task.format or 'epub'}"
)
counter += 1
shutil.move(str(temp_path), str(final_path))
@@ -1363,6 +1363,7 @@ def is_available(self) -> bool:
except Exception:
return False
def search(self, book: BookMetadata) -> List[Release]:
"""Return empty list on errors, don't raise."""
try:
+3
View File
@@ -158,6 +158,9 @@ class DownloadTask:
status: QueueStatus = QueueStatus.QUEUED
status_message: str | None = None
download_path: str | None = None
# Every file the output step produced, download_path among them. Kept in memory
# only: history persists download_path alone.
download_paths: list[str] = field(default_factory=list)
last_error_message: str | None = None
last_error_type: str | None = None
staged_path: str | None = None
+4 -4
View File
@@ -194,7 +194,9 @@ def _probe_completed_path(path: Path) -> tuple[bool, OSError | None]:
return True, None
def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tuple[str, int, int], ...] | None:
def _completed_file_signature(
path: Path, content_type: str | None
) -> tuple[tuple[str, int, int], ...] | None:
"""Return sizes and mtimes for supported files in a completed path."""
from shelfmark.download.postprocess.policy import (
get_supported_audiobook_formats,
@@ -202,9 +204,7 @@ def _completed_file_signature(path: Path, content_type: str | None) -> tuple[tup
)
formats = (
get_supported_audiobook_formats()
if is_audiobook(content_type)
else get_supported_formats()
get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats()
)
supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats}
+22
View File
@@ -416,6 +416,28 @@ def get_book_path(task_id: str) -> tuple[str | None, DownloadTask | None]:
return path, task
def get_book_paths(task_id: str) -> tuple[list[str], DownloadTask | None]:
"""Every file a task's download produced that is still on disk.
A multi-file release (chapters of an audiobook, a pack) lands as several files;
:func:`get_book_path` only knows the first. The list is trusted only while it
still contains download_path, so a later output that recorded no list cannot
hand back an earlier attempt's files.
"""
path, task = get_book_path(task_id)
if path is None or task is None:
return [], task
if path not in task.download_paths:
return [path], task
try:
return [p for p in task.download_paths if Path(p).is_file()], task
except OSError as e:
logger.error_trace(f"Error resolving book paths: {e}")
return [path], task
def get_book_data(task_id: str) -> tuple[bytes | None, DownloadTask | None]:
"""Get downloaded file data for a specific task.
+1
View File
@@ -275,4 +275,5 @@ def process_folder_output(
message = f"Complete ({len(final_paths)} files)"
status_callback("complete", message)
task.download_paths = [str(path) for path in final_paths]
return str(final_paths[0])
+68
View File
@@ -0,0 +1,68 @@
"""Stream several downloaded files to a client as one ZIP, without staging the archive."""
import os
import zipfile
from pathlib import Path
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Iterator
_ZIP_STREAM_CHUNK_SIZE = 1024 * 1024
class _ChunkSink:
"""Write-only ZipFile target that hands the written bytes back to the caller.
Having no tell() or seek() makes ZipFile treat it as an unseekable stream: each
entry's CRC and sizes go in a trailing data descriptor instead of being patched
back into the local header, so nothing has to be held until the entry ends.
"""
def __init__(self) -> None:
self._chunks: list[bytes] = []
def write(self, data: bytes) -> int:
self._chunks.append(bytes(data))
return len(data)
def flush(self) -> None:
"""Nothing is buffered beyond what drain() hands out."""
def close(self) -> None:
"""Nothing to release; the caller drains what is left after ZipFile closes."""
def drain(self) -> Iterator[bytes]:
chunks, self._chunks = self._chunks, []
yield from chunks
def zip_entry_names(paths: list[str]) -> list[tuple[Path, str]]:
"""Pair each file with its name inside the archive.
Names are relative to the files' common parent, so a book's own folders
(``Title/Part 01.m4b``) survive while the library path above them does not.
"""
files = [Path(path) for path in paths]
root = Path(os.path.commonpath([str(file.parent) for file in files]))
return [(file, file.relative_to(root).as_posix()) for file in files]
def stream_zip(paths: list[str]) -> Iterator[bytes]:
"""Yield an uncompressed ZIP of ``paths``, reading each file once in chunks.
Stored, not deflated: audiobooks and ebooks are already compressed, so deflating
them would cost CPU for next to nothing. ZIP64 kicks in per entry from the file
size, which a multi-gigabyte audiobook needs.
"""
sink = _ChunkSink()
with zipfile.ZipFile(sink, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
for path, arcname in zip_entry_names(paths):
info = zipfile.ZipInfo.from_file(path, arcname, strict_timestamps=False)
info.compress_type = zipfile.ZIP_STORED
with path.open("rb") as source, archive.open(info, "w") as target:
while chunk := source.read(_ZIP_STREAM_CHUNK_SIZE):
target.write(chunk)
yield from sink.drain()
yield from sink.drain()
yield from sink.drain()
+45 -6
View File
@@ -6,12 +6,14 @@ import os
import re
import sqlite3
import time
import unicodedata
from contextlib import suppress
from datetime import UTC, datetime, timedelta
from functools import wraps
from importlib import import_module
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, NoReturn, cast
from urllib.parse import quote
from flask import Flask, g, jsonify, request, send_file, send_from_directory, session
from flask_cors import CORS
@@ -55,7 +57,12 @@ from shelfmark.core.cwa_user_sync import upsert_cwa_user
from shelfmark.core.download_history_service import DownloadHistoryService
from shelfmark.core.external_user_linking import upsert_external_user
from shelfmark.core.logger import setup_logger
from shelfmark.core.models import TERMINAL_QUEUE_STATUSES, QueueStatus, SearchFilters
from shelfmark.core.models import (
TERMINAL_QUEUE_STATUSES,
QueueStatus,
SearchFilters,
build_filename,
)
from shelfmark.core.notifications import (
NotificationContext,
NotificationEvent,
@@ -95,6 +102,7 @@ from shelfmark.core.viewer_redaction import (
)
from shelfmark.download import orchestrator as backend
from shelfmark.download import warmup
from shelfmark.download.zip_stream import stream_zip
from shelfmark.release_sources import (
BrowseRecord,
Release,
@@ -103,6 +111,7 @@ from shelfmark.release_sources import (
)
if TYPE_CHECKING:
from shelfmark.core.models import DownloadTask
from shelfmark.metadata_providers import BookMetadata, MetadataProvider
logger = setup_logger(__name__)
@@ -1800,16 +1809,43 @@ def api_status() -> Response | tuple[Response, int]:
return jsonify({"error": str(e)}), 500
def _attachment_filename_options(download_name: str) -> dict[str, str]:
"""Content-Disposition filename options, RFC 2231-encoded when not plain ASCII.
Mirrors what send_file does, for responses built from a generator.
"""
try:
download_name.encode("ascii")
except UnicodeEncodeError:
simple = unicodedata.normalize("NFKD", download_name)
simple = simple.encode("ascii", "ignore").decode("ascii")
quoted = quote(download_name, safe="!#$&+^`|~")
return {"filename": simple, "filename*": f"UTF-8''{quoted}"}
return {"filename": download_name}
def _zip_download_response(file_paths: list[str], task: DownloadTask) -> Response:
"""Send a multi-file download (audiobook chapters, a pack) as one streamed ZIP."""
download_name = build_filename(task.title or "download", task.author, task.year, "zip")
response = Response(stream_zip(file_paths), mimetype="application/zip")
response.headers.set(
"Content-Disposition", "attachment", **_attachment_filename_options(download_name)
)
return response
@app.route("/api/localdownload", methods=["GET"])
@login_required
def api_local_download() -> Response | tuple[Response, int]:
"""Download an EPUB file from local storage if available.
"""Download a completed book from local storage if available.
Query Parameters:
id (str): Book identifier (MD5 hash)
Returns:
flask.Response: The EPUB file if found, otherwise an error response.
flask.Response: The file if found, a ZIP of all of them when the download
produced several (a live task only; history keeps just the first), otherwise
an error response.
"""
book_id = request.args.get("id", "")
@@ -1820,8 +1856,8 @@ def api_local_download() -> Response | tuple[Response, int]:
try:
# The path, not the bytes: send_file streams it off disk, where reading it
# first cost two resident copies of the whole book.
file_path, book_info = backend.get_book_path(book_id)
if file_path is None:
file_paths, book_info = backend.get_book_paths(book_id)
if not file_paths:
# Fallback for dismissed/history entries where queue task may no longer exist.
if download_history_service is not None:
is_admin, db_user_id, can_access_status = _resolve_status_scope()
@@ -1853,8 +1889,11 @@ def api_local_download() -> Response | tuple[Response, int]:
):
return jsonify({"error": "File not found"}), 404
if book_info is not None and len(file_paths) > 1:
return _zip_download_response(file_paths, book_info)
file_name = book_info.get_filename() if book_info is not None else Path(book_id).name
return send_file(file_path, download_name=file_name, as_attachment=True)
return send_file(file_paths[0], download_name=file_name, as_attachment=True)
except _OPERATIONAL_ERRORS as e:
logger.error_trace(f"Local download error: {e}")
+3 -2
View File
@@ -450,7 +450,7 @@ class DatabazeKnihProvider(MetadataProvider):
book_id = _id_from_link(str(data.get("url", "")))
if not book_id:
# Pages without a JSON-LD url still carry the id on the rating buttons
tag = soup.find(attrs={"data-typeid": True})
tag = soup.select_one("[data-typeid]")
book_id = str(tag["data-typeid"]) if tag else ""
title = _clean_text(str(data.get("name", "")))
authors = [_clean_author(n) for n in _ld_names(data.get("author"))]
@@ -539,7 +539,8 @@ class DatabazeKnihProvider(MetadataProvider):
name = _clean_text(str(link.get("title") or link.get_text())) or None
# The part number follows the link, up to the next series link
text = []
for elem in link.parent.next_siblings:
parent = link.parent
for elem in parent.next_siblings if parent else ():
if isinstance(elem, Tag):
if elem.find("a", href=re.compile(r"^/serie/")):
break
@@ -40,7 +40,7 @@ def get_unavailable_reason(
if not enabled_providers(providers):
return (
"Direct Download is not configured. Add at least one Anna's Archive or OceanofPDF mirror URL in "
"Settings. Z-Library, LibGen, Welib, mirrors only download results found through "
"Settings. Z-Library, LibGen and Welib mirrors only download results found through "
"Anna's Archive."
)
return None
+26 -24
View File
@@ -64,12 +64,12 @@ The script navigates to the main page, enters the search query, and waits for "M
```python
# Navigate to main page first to reset SPA state
page.goto('http://localhost:8084/')
page.wait_for_load_state('networkidle')
page.goto("http://localhost:8084/")
page.wait_for_load_state("networkidle")
# Enter search query and submit
page.fill('input[type="search"]', f'{title} {author}')
page.press('input[type="search"]', 'Enter')
page.fill('input[type="search"]', f"{title} {author}")
page.press('input[type="search"]', "Enter")
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
@@ -87,18 +87,20 @@ def parse_books(page):
btns = page.query_selector_all('button[data-action="download"]')
books = []
for i, btn in enumerate(btns):
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
title = title_match.group(1).strip() if title_match else 'Unknown'
title_match = re.search(r"<h3[^>]*>(.*?)</h3>", content, re.IGNORECASE | re.DOTALL)
title = title_match.group(1).strip() if title_match else "Unknown"
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
author = author_match.group(1).strip() if author_match else 'Unknown'
author_match = re.search(
r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
)
author = author_match.group(1).strip() if author_match else "Unknown"
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
dl_match = re.search(r"<span>•</span>\s*<span>([\d,]+)</span>", content)
downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
books.append({"index": i, "title": title, "author": author, "downloads": downloads})
return books
```
@@ -106,19 +108,19 @@ def parse_books(page):
```python
# Filter books matching the search criteria
matching_books = [b for b in books if matches_search(b['title'], b['author'], title, author)]
matching_books = [b for b in books if matches_search(b["title"], b["author"], title, author)]
if matching_books:
best = max(matching_books, key=lambda x: x['downloads'])
best = max(matching_books, key=lambda x: x["downloads"])
# CRITICAL: Click on the article h3 to open detail view
h3s = page.query_selector_all('article h3')
if best['index'] < len(h3s):
h3s[best['index']].click()
h3s = page.query_selector_all("article h3")
if best["index"] < len(h3s):
h3s[best["index"]].click()
page.wait_for_timeout(300)
# Then click the download button
btn = page.query_selector_all('button[data-action="download"]')[best['index']]
btn = page.query_selector_all('button[data-action="download"]')[best["index"]]
btn.click()
page.wait_for_timeout(500)
```
@@ -130,14 +132,14 @@ if matching_books:
```python
for i in range(60):
page.wait_for_timeout(5000)
activity_text = page.inner_text('aside')
activity_text = page.inner_text("aside")
if 'IN PROGRESS' in activity_text:
if "IN PROGRESS" in activity_text:
print("Download started!")
elif 'Complete' in activity_text or 'Saved' in activity_text:
elif "Complete" in activity_text or "Saved" in activity_text:
print("Download complete!")
break
elif 'No activity' in activity_text:
elif "No activity" in activity_text:
print("Download not started")
break
```
@@ -146,7 +148,7 @@ for i in range(60):
```python
# Click "Clear Completed" using JavaScript
page.evaluate('''
page.evaluate("""
() => {
for (const b of document.querySelectorAll('button')) {
if (b.textContent.includes('Clear Completed')) {
@@ -155,7 +157,7 @@ page.evaluate('''
}
}
}
''')
""")
page.wait_for_timeout(2000)
```
+155 -98
View File
@@ -1,40 +1,61 @@
#!/usr/bin/env python3
"""Shelfmark Book Downloader - Downloads books from a local Shelfmark instance."""
from __future__ import annotations
import argparse
import contextlib
import json
import re
import sys
import time
from pathlib import Path
from typing import TYPE_CHECKING, Any
from playwright.sync_api import Error as PlaywrightError
from playwright.sync_api import sync_playwright
import re, time, sys, argparse, json
from urllib.parse import quote
SHELFMARK_URL = 'http://localhost:8084/'
if TYPE_CHECKING:
from playwright.sync_api import Page
def parse_books(page):
Book = dict[str, Any]
SHELFMARK_URL = "http://localhost:8084/"
def parse_books(page: Page) -> list[Book]:
btns = page.query_selector_all('button[data-action="download"]')
books = []
for i, btn in enumerate(btns):
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
content = btn.evaluate_handle("el => el.parentElement.parentElement").inner_html()
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
title = title_match.group(1).strip() if title_match else 'Unknown'
title_match = re.search(r"<h3[^>]*>(.*?)</h3>", content, re.IGNORECASE | re.DOTALL)
title = title_match.group(1).strip() if title_match else "Unknown"
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
author = author_match.group(1).strip() if author_match else 'Unknown'
author_match = re.search(
r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL
)
author = author_match.group(1).strip() if author_match else "Unknown"
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
dl_match = re.search(r"<span>•</span>\s*<span>([\d,]+)</span>", content)
downloads = int(dl_match.group(1).replace(",", "")) if dl_match else 0
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
books.append({"index": i, "title": title, "author": author, "downloads": downloads})
return books
def do_search(page, title, author, search_type="author"):
def do_search(
page: Page, title: str, author: str, search_type: str = "author"
) -> tuple[list[Book], list[Book]] | None:
"""Search with title+author, fall back to title only"""
# Navigate to main page first to reset SPA state
page.goto(SHELFMARK_URL)
page.wait_for_load_state('networkidle')
page.wait_for_load_state("networkidle")
# Enter search query and submit
search_query = f'{title} {author}' if search_type == "author" else title
search_query = f"{title} {author}" if search_type == "author" else title
page.fill('input[type="search"]', search_query)
page.press('input[type="search"]', 'Enter')
page.press('input[type="search"]', "Enter")
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
try:
@@ -44,43 +65,55 @@ def do_search(page, title, author, search_type="author"):
print(" -> Trying title only...")
sys.stdout.flush()
return do_search(page, title, author, search_type="title")
else:
print(" >> Timeout waiting for search results")
sys.stdout.flush()
return None
books = parse_books(page)
def clean_title_for_match(book_title, search_title):
def clean_title_for_match(book_title: str, search_title: str) -> bool:
"""Clean book title to check if it matches the search title"""
# Remove common subtitle patterns
clean = re.sub(r'\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$', '', book_title, flags=re.IGNORECASE)
clean = re.sub(
r"\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$",
"",
book_title,
flags=re.IGNORECASE,
)
clean = clean.strip()
# Remove trailing punctuation
clean = clean.rstrip(':,;.')
clean = clean.rstrip(":,;.")
return clean.lower().strip() == search_title.lower().strip()
def matches_search(book_title, book_author, search_title, search_author):
def matches_search(
book_title: str, book_author: str, search_title: str, search_author: str
) -> bool:
"""Check if book matches search criteria more strictly"""
title_match = search_title.lower() in book_title.lower()
author_match = search_author.lower() in book_author.lower()
# For title+author search, ensure title starts with search title (not just contains it)
if title_match and author_match:
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(
search_title.lower()
)
# Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann")
if title_match:
author_parts = search_author.lower().split()
if len(author_parts) >= 2:
reversed_author = f"{author_parts[-1]} {author_parts[0]}"
if reversed_author in book_author.lower() or book_author.lower().startswith(reversed_author):
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
if reversed_author in book_author.lower() or book_author.lower().startswith(
reversed_author
):
return clean_title_for_match(
book_title, search_title
) or book_title.lower().startswith(search_title.lower())
return False
if search_type == "author":
matching = [b for b in books if matches_search(b['title'], b['author'], title, author)]
matching = [b for b in books if matches_search(b["title"], b["author"], title, author)]
print(f" With author: {len(books)} total, {len(matching)} matching")
for b in books[:3]:
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
@@ -88,18 +121,22 @@ def do_search(page, title, author, search_type="author"):
if matching:
# Check if any matching book is already in the download queue
try:
sidebar_text = page.inner_text('aside')
if 'IN PROGRESS' in sidebar_text:
with contextlib.suppress(PlaywrightError):
sidebar_text = page.inner_text("aside")
if "IN PROGRESS" in sidebar_text:
# Get the title of the book currently downloading
current_download = sidebar_text.split('IN PROGRESS')[1].split('—')[0].strip().split('\n')[0].strip()
current_download = (
sidebar_text.split("IN PROGRESS")[1]
.split("—")[0]
.strip()
.split("\n")[0]
.strip()
)
# Check if the current download matches our search
if title.lower() in current_download.lower():
print(f" >> '{current_download}' already downloading, skipping")
sys.stdout.flush()
return None
except:
pass
if matching:
return matching, books
@@ -110,13 +147,15 @@ def do_search(page, title, author, search_type="author"):
return do_search(page, title, author, search_type="title")
# Title-only search - prefer exact matches first
matching = [b for b in books if title.lower() in b['title'].lower()]
exact_matches = [b for b in matching if clean_title_for_match(b['title'], title)]
matching = [b for b in books if title.lower() in b["title"].lower()]
exact_matches = [b for b in matching if clean_title_for_match(b["title"], title)]
if exact_matches:
matching = exact_matches
print(f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)")
print(
f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)"
)
for b in books[:5]:
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
sys.stdout.flush()
@@ -126,14 +165,15 @@ def do_search(page, title, author, search_type="author"):
return (matching, books)
def download_book(page, title, author):
def download_book(page: Page, title: str, author: str) -> None:
result = do_search(page, title, author)
if not result:
print("\n >>> NOT FOUND")
sys.stdout.flush()
return
matching, all_books = result
matching, _all_books = result
if not matching:
print("\n >>> NOT FOUND")
sys.stdout.flush()
@@ -142,8 +182,8 @@ def download_book(page, title, author):
# Find the best book that is not disabled
btns = page.query_selector_all('button[data-action="download"]')
best = None
for b in sorted(matching, key=lambda x: x['downloads'], reverse=True):
if b['index'] < len(btns) and not btns[b['index']].is_disabled():
for b in sorted(matching, key=lambda x: x["downloads"], reverse=True):
if b["index"] < len(btns) and not btns[b["index"]].is_disabled():
best = b
break
@@ -152,101 +192,107 @@ def download_book(page, title, author):
sys.stdout.flush()
return
print(f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]")
print(
f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]"
)
sys.stdout.flush()
# Click on the article h3 to open detail view (required for download to work)
h3s = page.query_selector_all('article h3')
if best['index'] < len(h3s):
h3s[best['index']].click()
h3s = page.query_selector_all("article h3")
if best["index"] < len(h3s):
h3s[best["index"]].click()
page.wait_for_timeout(300)
else:
print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)")
sys.stdout.flush()
# Click the download button
btn = btns[best['index']]
btn = btns[best["index"]]
btn.click()
# Wait a moment for the click to register
page.wait_for_timeout(500)
print(f" >> Click sent")
print(" >> Click sent")
sys.stdout.flush()
# Check sidebar immediately
time.sleep(2)
try:
txt = page.inner_text('aside')
if 'IN PROGRESS' in txt:
print(f" >> Download started!")
txt = page.inner_text("aside")
if "IN PROGRESS" in txt:
print(" >> Download started!")
sys.stdout.flush()
else:
print(f" >> Sidebar after 2s: {txt[:150]}")
sys.stdout.flush()
except:
print(f" >> Sidebar not visible after 2s")
print(" >> Sidebar not visible after 2s")
sys.stdout.flush()
# Wait for download to complete - check sidebar every 2 seconds, max 5 minutes
last_state = None
for attempt in range(150):
for _attempt in range(150):
time.sleep(2)
try:
txt = page.inner_text('aside')
if 'IN PROGRESS' in txt:
state = 'downloading'
elif 'Complete' in txt or 'Saved' in txt or 'No activity' in txt:
state = 'done'
txt = page.inner_text("aside")
if "IN PROGRESS" in txt:
state = "downloading"
elif "Complete" in txt or "Saved" in txt or "No activity" in txt:
state = "done"
else:
state = 'other'
state = "other"
if state != last_state:
if state == 'downloading':
print(f" >> Download started!")
elif state == 'done':
if 'No activity' in txt:
print(f" >> Download not started (no activity)")
if state == "downloading":
print(" >> Download started!")
elif state == "done":
if "No activity" in txt:
print(" >> Download not started (no activity)")
else:
print(f" >> Download complete!")
print(" >> Download complete!")
last_state = state
if state == 'done':
if state == "done":
break
except:
if last_state is None:
print(f" >> Sidebar not visible yet")
last_state = 'no_sidebar'
print(" >> Sidebar not visible yet")
last_state = "no_sidebar"
sys.stdout.flush()
else:
if last_state != 'done':
if last_state != "done":
print(" >> Timeout waiting for download")
sys.stdout.flush()
def clear_completed_downloads(page):
def clear_completed_downloads(page: Page) -> None:
"""Clear completed downloads from previous sessions"""
page.evaluate('''
page.evaluate("""
() => {
for (const b of document.querySelectorAll('button')) {
if (b.textContent.includes('Clear Completed')) { b.click(); return; }
}
}
''')
""")
page.wait_for_timeout(2000)
def check_calibre(title):
def check_calibre(title: str) -> bool:
"""Check if a book is already in the calibre database."""
import subprocess
cmd = f'calibredb list --search title:="{title}"'
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
result = subprocess.run(cmd, check=False, shell=True, capture_output=True, text=True)
output = result.stdout.strip()
# calibredb output has header line "id title authors", count non-header lines
lines = [l.strip() for l in output.split('\n') if l.strip()]
lines = [l.strip() for l in output.split("\n") if l.strip()]
# Remove header if present (first line starts with 'id')
if lines and lines[0].startswith('id'):
if lines and lines[0].startswith("id"):
lines = lines[1:]
return len(lines) > 0
def load_books_from_json(json_str, source):
def load_books_from_json(json_str: str, source: str) -> list[tuple[str, str]]:
"""Load books from a JSON string or file path."""
try:
data = json.loads(json_str)
@@ -261,10 +307,10 @@ def load_books_from_json(json_str, source):
books = []
for item in data:
if isinstance(item, dict):
if 'title' not in item or 'author' not in item:
if "title" not in item or "author" not in item:
print(f"Error: Each book object must have 'title' and 'author' keys: {item}")
sys.exit(1)
books.append((item['title'].strip(), item['author'].strip()))
books.append((item["title"].strip(), item["author"].strip()))
elif isinstance(item, list) and len(item) >= 2:
books.append((str(item[0]).strip(), str(item[1]).strip()))
elif isinstance(item, str):
@@ -274,31 +320,41 @@ def load_books_from_json(json_str, source):
print(f"Warning: Skipping unrecognized book entry: {item}")
return books
def main():
parser = argparse.ArgumentParser(description='Download books from Shelfmark')
parser.add_argument('books', nargs='*', help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects')
parser.add_argument('--file', '-f', help='Path to JSON file containing an array of books')
parser.add_argument('-c', '--check-calibre', action='store_true', help='Check calibre database before downloading, skip if already present')
def main() -> None:
parser = argparse.ArgumentParser(description="Download books from Shelfmark")
parser.add_argument(
"books",
nargs="*",
help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects',
)
parser.add_argument("--file", "-f", help="Path to JSON file containing an array of books")
parser.add_argument(
"-c",
"--check-calibre",
action="store_true",
help="Check calibre database before downloading, skip if already present",
)
args = parser.parse_args()
# Load books from --file, positional JSON arg, or both
BOOKS = []
books_to_download = []
if args.file:
try:
with open(args.file, 'r') as f:
BOOKS = load_books_from_json(f.read(), f"file '{args.file}'")
with Path(args.file).open() as f:
books_to_download = load_books_from_json(f.read(), f"file '{args.file}'")
except FileNotFoundError:
print(f"Error: File not found: {args.file}")
sys.exit(1)
except IOError as e:
except OSError as e:
print(f"Error reading file: {e}")
sys.exit(1)
for book_arg in args.books:
BOOKS.extend(load_books_from_json(book_arg, 'command line argument'))
books_to_download.extend(load_books_from_json(book_arg, "command line argument"))
if not BOOKS:
if not books_to_download:
print("Error: No books specified. Use positional JSON args or --file <path>")
sys.exit(1)
@@ -307,40 +363,40 @@ def main():
print("\nChecking calibre database...")
sys.stdout.flush()
remaining = []
for title, author in BOOKS:
for title, author in books_to_download:
if check_calibre(title):
print(f" >> '{title}' already in calibre, skipping")
sys.stdout.flush()
else:
remaining.append((title, author))
BOOKS = remaining
if not BOOKS:
books_to_download = remaining
if not books_to_download:
print("All books already in calibre, nothing to download.")
sys.stdout.flush()
return
print(f"\n{'=' * 60}")
print(f"Downloading {len(BOOKS)} book(s) from Shelfmark")
print('='*60)
print(f"Downloading {len(books_to_download)} book(s) from Shelfmark")
print("=" * 60)
sys.stdout.flush()
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page(viewport={'width': 1280, 'height': 900})
page = browser.new_page(viewport={"width": 1280, "height": 900})
page.goto(SHELFMARK_URL)
page.wait_for_load_state('networkidle')
page.wait_for_load_state("networkidle")
# Clear completed downloads at start of batch
print(f"\n{'=' * 60}")
print("Clearing completed downloads from previous sessions...")
print('='*60)
print("=" * 60)
sys.stdout.flush()
clear_completed_downloads(page)
for i, (title, author) in enumerate(BOOKS, 1):
for i, (title, author) in enumerate(books_to_download, 1):
print(f"\n{'=' * 60}")
print(f"[{i}/{len(BOOKS)}] '{title}' by {author}")
print('='*60)
print(f"[{i}/{len(books_to_download)}] '{title}' by {author}")
print("=" * 60)
sys.stdout.flush()
download_book(page, title, author)
@@ -349,5 +405,6 @@ def main():
print("\nDone!")
sys.stdout.flush()
if __name__ == '__main__':
if __name__ == "__main__":
main()
@@ -1,6 +1,6 @@
import type { RequestRecord } from '../../types';
export type ActivityKind = 'download' | 'request';
type ActivityKind = 'download' | 'request';
export type ActivityVisualStatus =
| 'queued'
+1 -1
View File
@@ -1,4 +1,4 @@
export { DisplayFieldIcon, DisplayFieldBadge, DisplayFieldBadges } from './DisplayFieldIcon';
export { CircularProgress } from './CircularProgress';
export { ToggleSwitch } from './ToggleSwitch';
export { LibraryBadge, isInLibrary, isCollectionOnly } from './LibraryBadge';
export { LibraryBadge, isInLibrary } from './LibraryBadge';
+2 -2
View File
@@ -861,9 +861,9 @@ export const getReleases = async (
// Admin user management API
export type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
type AdminAuthSource = 'builtin' | 'oidc' | 'proxy' | 'cwa';
export interface AdminUserEditCapabilities {
interface AdminUserEditCapabilities {
authSource: AdminAuthSource;
canSetPassword: boolean;
canEditRole: boolean;
+1 -7
View File
@@ -172,13 +172,7 @@
}
body {
font-family:
'Inter',
-apple-system,
BlinkMacSystemFont,
'Segoe UI',
Roboto,
sans-serif;
font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
line-height: 1.6;
color: var(--text-color);
background: var(--background-color);
+19 -19
View File
@@ -1,5 +1,5 @@
// Search mode constants and type
export const SEARCH_MODE = {
const SEARCH_MODE = {
DIRECT: 'direct',
UNIVERSAL: 'universal',
} as const;
@@ -88,7 +88,7 @@ export interface StatusData {
}
// Button states
export type ButtonState =
type ButtonState =
| 'download'
| 'queued'
| 'resolving'
@@ -156,24 +156,24 @@ export interface TextSearchField extends SearchFieldBase {
suggestions_min_query_length?: number;
}
export interface NumberSearchField extends SearchFieldBase {
interface NumberSearchField extends SearchFieldBase {
type: 'NumberSearchField';
min?: number;
max?: number;
step?: number;
}
export interface SelectSearchField extends SearchFieldBase {
interface SelectSearchField extends SearchFieldBase {
type: 'SelectSearchField';
options: SortOption[];
}
export interface CheckboxSearchField extends SearchFieldBase {
interface CheckboxSearchField extends SearchFieldBase {
type: 'CheckboxSearchField';
default?: boolean;
}
export interface DynamicSelectSearchField extends SearchFieldBase {
interface DynamicSelectSearchField extends SearchFieldBase {
type: 'DynamicSelectSearchField';
options_endpoint: string;
}
@@ -185,7 +185,7 @@ export type MetadataSearchField =
| CheckboxSearchField
| DynamicSelectSearchField;
export type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
type QueryTargetSource = 'general' | 'manual' | 'direct-field' | 'provider-field';
export interface QueryTargetOption {
key: string;
@@ -201,12 +201,12 @@ export type ContentType = 'ebook' | 'audiobook';
export type RequestPolicyMode = 'download' | 'request_release' | 'request_book' | 'blocked';
export interface RequestPolicyDefaults {
interface RequestPolicyDefaults {
ebook: RequestPolicyMode;
audiobook: RequestPolicyMode;
}
export interface RequestPolicySourceMode {
interface RequestPolicySourceMode {
source: string;
supported_content_types: string[];
browse_results_are_releases?: boolean;
@@ -222,7 +222,7 @@ export interface RequestPolicyResponse {
source_modes: RequestPolicySourceMode[];
}
export interface RequestContextPayload {
interface RequestContextPayload {
source: string;
content_type: ContentType;
request_level: 'book' | 'release';
@@ -279,14 +279,14 @@ export interface QueuedDownloadResult {
export type RequestSubmissionResult = RequestRecord | QueuedDownloadResult;
/** How the library holds a book, per format: owned outright, or inside a collection. */
export type LibraryHolding = 'owned' | 'collection';
type LibraryHolding = 'owned' | 'collection';
export interface LibraryOwnership {
ebook?: LibraryHolding | null;
audiobook?: LibraryHolding | null;
}
export type BooksOutputMode = 'folder' | 'booklore' | 'email';
type BooksOutputMode = 'folder' | 'booklore' | 'email';
export interface AppConfig {
calibre_web_url: string;
@@ -334,7 +334,7 @@ export interface MetadataProvidersResponse {
configured_provider_combined: string | null;
}
export interface MetadataCapability {
interface MetadataCapability {
key: string;
field_key?: string;
sort?: string;
@@ -398,7 +398,7 @@ export interface ReleaseSource {
}
// Column schema types for plugin-driven release list UI
export type ColumnRenderType =
type ColumnRenderType =
| 'text'
| 'badge'
| 'tags'
@@ -408,7 +408,7 @@ export type ColumnRenderType =
| 'indexer_protocol'
| 'flag_icon'
| 'format_content_type';
export type ColumnAlign = 'left' | 'center' | 'right';
type ColumnAlign = 'left' | 'center' | 'right';
export interface ColumnColorHint {
type: 'map' | 'static'; // 'map' uses colorMaps.ts, 'static' is a fixed Tailwind class
@@ -430,7 +430,7 @@ export interface ColumnSchema {
}
// Leading cell config - what to show in the left-most position of each row
export type LeadingCellType = 'thumbnail' | 'badge' | 'none';
type LeadingCellType = 'thumbnail' | 'badge' | 'none';
export interface LeadingCellConfig {
type: LeadingCellType;
@@ -439,13 +439,13 @@ export interface LeadingCellConfig {
uppercase?: boolean; // Force uppercase for badge text
}
export interface ExtraSortOption {
interface ExtraSortOption {
label: string; // Display label in the sort dropdown
sort_key: string; // Field to sort by on the Release object
default_direction?: 'asc' | 'desc'; // Which way "best first" runs (defaults to desc)
}
export interface SourceActionButton {
interface SourceActionButton {
label: string; // Button text (e.g., "Refresh search")
action: string; // Action type: "expand" triggers expand_search
}
@@ -483,7 +483,7 @@ export interface Release {
}
// Search info returned by release sources
export interface SourceSearchInfo {
interface SourceSearchInfo {
search_type: 'isbn' | 'title_author' | 'categories' | 'expanded' | 'manual' | 'query';
total_results?: number | string | null;
}
+6 -12
View File
@@ -1,6 +1,6 @@
// Settings field types matching backend settings_registry.py
export type FieldType =
type FieldType =
| 'TextField'
| 'PasswordField'
| 'NumberField'
@@ -28,17 +28,17 @@ export interface ShowWhenCondition {
notEmpty?: boolean; // If true, show when field has any non-empty value
}
export type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
type ShowWhen = ShowWhenCondition | ShowWhenCondition[];
// Conditional disable configuration
export interface DisabledWhenCondition {
interface DisabledWhenCondition {
field: string; // The field key to check
value: string | string[] | boolean; // The value(s) that disable this field
reason?: string; // Explanation shown when disabled
}
// Base field interface - common properties
export interface BaseField {
interface BaseField {
key: string;
label: string;
type: FieldType;
@@ -139,20 +139,14 @@ export interface CustomComponentFieldConfig extends BaseField {
wrapInFieldWrapper?: boolean; // Whether to render with standard FieldWrapper layout
}
export interface TableFieldColumnOption {
interface TableFieldColumnOption {
value: string;
label: string;
description?: string;
childOf?: string;
}
export type TableFieldColumnType =
| 'text'
| 'password'
| 'select'
| 'multiselect'
| 'checkbox'
| 'path';
type TableFieldColumnType = 'text' | 'password' | 'select' | 'multiselect' | 'checkbox' | 'path';
export interface TableFieldColumn {
key: string;
+2 -8
View File
@@ -1,3 +1,5 @@
import type { ColumnColorHint } from '../types';
// Color styles with transparent backgrounds and contrasting text
interface ColorStyle {
bg: string;
@@ -133,14 +135,6 @@ function getFlagColor(flag?: string): ColorStyle {
return FLAG_COLORS[normalized] || DEFAULT_FLAG_COLOR;
}
/**
* Color hint type for dynamic column coloring.
*/
interface ColumnColorHint {
type: 'static' | 'map';
value: string;
}
/**
* Get the color style for a value based on a color hint.
* Supports both static color classes and dynamic map lookups.
@@ -10,7 +10,7 @@ import type { ContentType } from '../types';
export const CONTENT_TYPE_STORAGE_KEY = 'preferred-content-type.v2';
/** Where the current content type came from, which decides what may still override it. */
export type ContentTypeSource =
type ContentTypeSource =
/** Nothing has chosen a tab, so a configured default may still apply. */
| 'unset'
/** This browser stored a choice, or something explicit set one. Defaults do not apply. */
+42
View File
@@ -3,8 +3,10 @@
from __future__ import annotations
import importlib
import io
import sqlite3
import uuid
import zipfile
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import ANY, patch
@@ -355,6 +357,46 @@ class TestActivityRoutes:
assert response.status_code == 200
assert response.data == file_bytes
def test_localdownload_zips_a_live_multi_file_download(self, main_module, client, tmp_path):
user = _create_user(main_module, prefix="reader")
_set_session(client, user_id=user["username"], db_user_id=user["id"], is_admin=False)
task_id = f"live-localdownload-{uuid.uuid4().hex[:8]}"
book_dir = tmp_path / "Chapters"
book_dir.mkdir()
parts = [book_dir / "Part 1.mp3", book_dir / "Part 2.mp3"]
for index, part in enumerate(parts, start=1):
part.write_bytes(f"chapter {index}".encode())
task = DownloadTask(
task_id=task_id,
source="direct_download",
title="Zipped Audiobook",
author="Some Author",
user_id=user["id"],
username=user["username"],
download_path=str(parts[0]),
download_paths=[str(part) for part in parts],
)
assert main_module.backend.book_queue.add(task) is True
try:
main_module.backend.book_queue.update_status(task_id, QueueStatus.COMPLETE)
with patch.object(main_module, "get_auth_mode", return_value="builtin"):
response = client.get(f"/api/localdownload?id={task_id}")
finally:
main_module.backend.book_queue.cancel_download(task_id)
assert response.status_code == 200
assert response.mimetype == "application/zip"
assert (
'filename="Some Author - Zipped Audiobook.zip"'
in response.headers["Content-Disposition"]
)
with zipfile.ZipFile(io.BytesIO(response.data)) as archive:
assert archive.namelist() == ["Part 1.mp3", "Part 2.mp3"]
assert archive.read("Part 2.mp3") == b"chapter 2"
def test_localdownload_returns_not_found_for_other_users_live_queue_task(
self, main_module, client, tmp_path
):
@@ -343,6 +343,9 @@ def test_multifile_rename_ignores_template_even_with_original_name(tmp_path):
files = sorted(path.name for path in ingest.glob("*.mp3"))
assert files == ["Part 1 of 2.mp3", "Part 2 of 2.mp3"]
assert all("RENAMED" not in name for name in files)
# Browser download serves all of these, not just the returned first file.
assert result in task.download_paths
assert sorted(Path(path).name for path in task.download_paths) == files
def test_torrent_hardlink_enabled_copy_fallback_does_not_extract_archives(tmp_path):
@@ -55,7 +55,10 @@ def test_direct_download_explains_aa_requirement_when_only_zlib_is_configured(mo
source = DirectDownloadSource()
with pytest.raises(SearchUnavailableError, match="Anna's Archive mirror"):
with pytest.raises(
SearchUnavailableError,
match="Anna's Archive or OceanofPDF mirror URL.*only download results found through",
):
source.search(SimpleNamespace(), SimpleNamespace())
+48
View File
@@ -332,3 +332,51 @@ def test_get_book_path_reports_a_missing_file_rather_than_a_dead_path(monkeypatc
assert path is None
assert returned_task is task
def test_get_book_paths_returns_every_file_still_on_disk(monkeypatch, tmp_path):
import shelfmark.download.orchestrator as orchestrator
parts = [tmp_path / f"Part {n}.mp3" for n in (1, 2, 3)]
for part in parts[:2]:
part.write_bytes(b"audio")
task = DownloadTask(
task_id="task-book-paths-1",
source="direct_download",
title="Chapters",
download_path=str(parts[0]),
download_paths=[str(part) for part in parts],
)
mock_queue = MagicMock()
mock_queue.get_task.return_value = task
monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
paths, returned_task = orchestrator.get_book_paths(task.task_id)
assert paths == [str(parts[0]), str(parts[1])]
assert returned_task is task
def test_get_book_paths_ignores_a_list_that_no_longer_holds_download_path(monkeypatch, tmp_path):
import shelfmark.download.orchestrator as orchestrator
stale = tmp_path / "old-attempt.mp3"
current = tmp_path / "current.epub"
stale.write_bytes(b"old")
current.write_bytes(b"new")
task = DownloadTask(
task_id="task-book-paths-2",
source="direct_download",
title="Retried",
download_path=str(current),
download_paths=[str(stale)],
)
mock_queue = MagicMock()
mock_queue.get_task.return_value = task
monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
paths, _ = orchestrator.get_book_paths(task.task_id)
assert paths == [str(current)]
+41
View File
@@ -0,0 +1,41 @@
import io
import zipfile
from shelfmark.download.zip_stream import stream_zip, zip_entry_names
def test_stream_zip_holds_every_file_uncompressed(tmp_path):
book = tmp_path / "Author" / "Title"
book.mkdir(parents=True)
(book / "Part 01.m4b").write_bytes(b"chapter one")
(book / "Part 02.m4b").write_bytes(b"chapter two" * 1000)
data = b"".join(stream_zip([str(book / "Part 01.m4b"), str(book / "Part 02.m4b")]))
with zipfile.ZipFile(io.BytesIO(data)) as archive:
assert archive.testzip() is None
assert archive.namelist() == ["Part 01.m4b", "Part 02.m4b"]
assert archive.read("Part 01.m4b") == b"chapter one"
assert archive.read("Part 02.m4b") == b"chapter two" * 1000
assert {info.compress_type for info in archive.infolist()} == {zipfile.ZIP_STORED}
def test_stream_zip_yields_while_reading_rather_than_at_the_end(tmp_path, monkeypatch):
import shelfmark.download.zip_stream as zip_stream
monkeypatch.setattr(zip_stream, "_ZIP_STREAM_CHUNK_SIZE", 16)
big = tmp_path / "big.mp3"
big.write_bytes(b"x" * 160)
chunks = list(stream_zip([str(big)]))
assert len(chunks) > 10
def test_zip_entry_names_keep_folders_below_the_common_parent(tmp_path):
first = tmp_path / "library" / "Book One" / "a.mp3"
second = tmp_path / "library" / "Book Two" / "b.mp3"
names = [name for _, name in zip_entry_names([str(first), str(second)])]
assert names == ["Book One/a.mp3", "Book Two/b.mp3"]