Feature: Add Download counts to search result displays and Download sidebar (#1336)

Having the download counts from AA as an input on choosing which of the
many search results to pick for downloading is useful. This PR makes the
downloads numbers available on all the search result pages and also on
the Download sidebar after the user presses a download button for a
Direct Download.

I have also included a SKILL.md and associated download_books.py that
can be used just as a reference or with an LLM harness for automation.
The Downloads info is used by the script to pick which search result to
download out of the many available.

Since a picture is worth a thousand words:

<img width="1225" height="812" alt="search-results-with-downloads"
src="https://github.com/user-attachments/assets/e108ebe2-4cad-45e2-bb6a-d4f49b502de9"
/>

<img width="443" height="267" alt="download-sidebar-with-downloads"
src="https://github.com/user-attachments/assets/9ce9e6c4-dcf5-4f7c-ab22-71850398b534"
/>

Coded with llama.cpp and 🤖
This commit is contained in:
Paul Rimmer
2026-09-17 15:49:57 -04:00
committed by GitHub
parent c6b70a6844
commit 2ed2e9a5d4
21 changed files with 729 additions and 26 deletions
+7 -3
View File
@@ -207,6 +207,7 @@ class DownloadHistoryService:
"content_type": row.get("content_type"),
"source": row.get("source"),
"source_display_name": row.get("source_display_name"),
"downloads": row.get("downloads"),
"status_message": row.get("status_message"),
"download_path": DownloadHistoryService._resolve_existing_download_path(
row.get("download_path")
@@ -272,6 +273,7 @@ class DownloadHistoryService:
size: str | None,
preview: str | None,
content_type: str | None,
downloads: int | None,
origin: str,
retry_payload: dict[str, Any] | None = None,
) -> None:
@@ -307,15 +309,16 @@ class DownloadHistoryService:
title, author, format, size, preview, content_type,
origin, final_status,
status_message, download_path, retry_payload,
queued_at, terminal_at
queued_at, terminal_at, downloads
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', NULL, NULL, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', NULL, NULL, ?, ?, ?, ?)
ON CONFLICT(task_id) DO UPDATE SET
final_status = 'active',
status_message = NULL,
download_path = NULL,
retry_payload = excluded.retry_payload,
terminal_at = ?
terminal_at = ?,
downloads = excluded.downloads
""",
(
normalized_task_id,
@@ -334,6 +337,7 @@ class DownloadHistoryService:
normalized_retry_payload,
recorded_at,
recorded_at,
downloads,
recorded_at,
),
)
+1
View File
@@ -97,6 +97,7 @@ class DownloadTask:
year: str | None = None
format: str | None = None
size: str | None = None
downloads: int | None = None # Download count from source
preview: str | None = None
content_type: str | None = None # "book (fiction)", "audiobook", "magazine", etc.
source_url: str | None = None # Original release URL used by source-specific handlers
+10 -1
View File
@@ -87,7 +87,8 @@ CREATE TABLE IF NOT EXISTS download_history (
download_path TEXT,
retry_payload TEXT,
queued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
terminal_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
terminal_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
downloads INTEGER
);
CREATE INDEX IF NOT EXISTS idx_download_history_user_status
@@ -203,6 +204,7 @@ class UserDB:
self._migrate_request_delivery_columns(conn)
self._migrate_download_history_queued_at(conn)
self._migrate_download_history_retry_payload(conn)
self._migrate_download_history_downloads(conn)
conn.commit()
# WAL mode must be changed outside an open transaction.
conn.execute("PRAGMA journal_mode=WAL")
@@ -270,6 +272,13 @@ class UserDB:
if "retry_payload" not in column_names:
conn.execute("ALTER TABLE download_history ADD COLUMN retry_payload TEXT")
def _migrate_download_history_downloads(self, conn: sqlite3.Connection) -> None:
"""Ensure download_history.downloads exists for download count persistence."""
columns = conn.execute("PRAGMA table_info(download_history)").fetchall()
column_names = {str(col["name"]) for col in columns}
if "downloads" not in column_names:
conn.execute("ALTER TABLE download_history ADD COLUMN downloads INTEGER")
def create_user(
self,
username: str,
+4 -1
View File
@@ -296,6 +296,7 @@ def queue_release(
year=year,
format=release_data.get("format"),
size=release_data.get("size"),
downloads=release_data.get("downloads") or extra.get("downloads"),
preview=preview,
content_type=content_type,
source_url=source_url,
@@ -319,7 +320,7 @@ def queue_release(
logger.info("Release already in queue: %s", task.title)
return False, "Release is already in the download queue"
logger.info("Release queued with priority %s: %s", priority, task.title)
logger.info("Release queued with priority %s: %s (downloads=%s, release_data.downloads=%s, extra=%s)", priority, task.title, task.downloads, release_data.get("downloads"), extra)
# Broadcast status update via WebSocket
if ws_manager:
@@ -521,6 +522,7 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None:
year=normalize_optional_text(payload.get("year")),
format=normalize_optional_text(payload.get("format")),
size=normalize_optional_text(payload.get("size")),
downloads=int(payload["downloads"]) if payload.get("downloads") is not None else None,
preview=normalize_optional_text(payload.get("preview")),
content_type=normalize_optional_text(payload.get("content_type")),
source_url=normalize_optional_text(payload.get("source_url")),
@@ -617,6 +619,7 @@ def _task_to_dict(
"author": task.author,
"format": task.format,
"size": task.size,
"downloads": task.downloads,
"preview": preview,
"content_type": task.content_type,
"source": task.source,
+3
View File
@@ -1073,6 +1073,8 @@ def api_download_release() -> Response | tuple[Response, int]:
release_payload = dict(data)
release_payload["content_type"] = resolved_content_type
logger.info("Download request received. keys=%s downloads=%s extra.downloads=%s", list(data.keys()), data.get("downloads"), data.get("extra", {}).get("downloads") if isinstance(data.get("extra"), dict) else None)
priority = data.get("priority", 0)
# Per-user download overrides
db_user_id = session.get("db_user_id")
@@ -1354,6 +1356,7 @@ def _record_download_queued(task_id: str, task: Any) -> None:
size=normalize_optional_text(getattr(task, "size", None)),
preview=normalize_optional_text(getattr(task, "preview", None)),
content_type=normalize_optional_text(getattr(task, "content_type", None)),
downloads=getattr(task, "downloads", None),
origin=origin,
retry_payload=backend.serialize_task_for_retry(task),
)
+1
View File
@@ -46,6 +46,7 @@ class BrowseRecord:
content: str | None = None
format: str | None = None
size: str | None = None
downloads: int | None = None
info: dict[str, list[str]] | None = None
description: str | None = None
download_urls: list[str] = field(default_factory=list)
@@ -1,5 +1,6 @@
"""Anna's Archive search, metadata parsing, and MD5 mirror download cascade."""
import concurrent.futures
import itertools
import json
import re
@@ -786,9 +787,60 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
)
)
# Fetch download counts for all results in batch
if books:
_enrich_search_results_with_downloads(books)
return books
def _fetch_download_count_inline(book_id: str) -> int | None:
"""Fetch the download count for a single book from Anna's Archive inline_info API."""
try:
url = f"{network.get_aa_base_url()}/dyn/md5/inline_info/{book_id}"
resp = requests.get(url, timeout=5, headers={"Accept": "application/json"})
if resp.status_code == 200:
data = resp.json()
count = data.get("downloads_total")
if count is not None:
return count
except Exception:
logger.debug("Failed to fetch download count for %s", book_id, exc_info=True)
return None
def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None:
"""Fetch download counts for search results in batch and add them to each record's info."""
if not books:
return
book_ids = [b.id for b in books if b.id]
if not book_ids:
return
# Fetch counts in parallel using the inline_info API (cheaper than summary)
counts: dict[str, int] = {}
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = {
executor.submit(_fetch_download_count_inline, bid): bid for bid in book_ids
}
for future in concurrent.futures.as_completed(futures):
bid = futures[future]
try:
count = future.result()
if count is not None:
counts[bid] = count
except Exception:
logger.debug("Failed to fetch download count for %s", bid, exc_info=True)
# Add counts to each record's info
for book in books:
if book.id in counts:
if book.info is None:
book.info = {}
book.info["Downloads"] = [str(counts[book.id])]
def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseRecord:
"""Get detailed information for a specific book.
@@ -37,6 +37,21 @@ if TYPE_CHECKING:
logger = setup_logger(__name__)
def _extract_downloads(record: BrowseRecord) -> int | None:
"""Extract download count from record info for Release.extra.downloads."""
downloads = None
if record.info and "Downloads" in record.info:
downloads_value = record.info["Downloads"]
if isinstance(downloads_value, list) and len(downloads_value) > 0:
try:
downloads = int(downloads_value[0])
except (ValueError, TypeError):
pass
elif isinstance(downloads_value, (int, float)):
downloads = int(downloads_value)
return downloads
def _browse_record_to_release(record: BrowseRecord) -> Release:
"""Convert a browse record to a Release object.
@@ -67,6 +82,7 @@ def _browse_record_to_release(record: BrowseRecord) -> Release:
"download_urls": record.download_urls,
"info": record.info,
"direct_download_provider": provider_id,
"downloads": _extract_downloads(record),
# Kept for older frontends and persisted request payloads.
"web_provider": provider_id if provider_id != "annas_archive" else None,
},
@@ -98,8 +114,8 @@ class DirectDownloadSource(ReleaseSource):
def get_column_config(self) -> ReleaseColumnConfig:
"""Column configuration for Direct Download source.
Shows language, format, and size badges for each release.
Language is hidden on mobile; format and size are shown.
Shows language, format, size, and downloads for each release.
Language, format, size, and downloads are all shown on mobile.
"""
return ReleaseColumnConfig(
columns=[
@@ -131,8 +147,16 @@ class DirectDownloadSource(ReleaseSource):
width="80px",
hide_mobile=False, # Size shown on mobile
),
ColumnSchema(
key="extra.downloads",
label="Downloads",
render_type=ColumnRenderType.NUMBER,
align=ColumnAlign.CENTER,
width="80px",
hide_mobile=False, # Downloads shown on mobile
),
],
grid_template="minmax(0,2fr) 60px 80px 80px",
grid_template="minmax(0,2fr) 60px 80px 80px 80px",
supported_filters=["format", "language"], # AA has reliable language metadata
)
+196
View File
@@ -0,0 +1,196 @@
---
name: shelfmark
description: Tool for downloading books.
license: Complete terms in LICENSE.txt
---
# Shelfmark Book Download Skill
Use this skill to search for and download books from a local Shelfmark instance using Playwright.
## Prerequisites
- Shelfmark must be running at `http://localhost:8084/`
- Use `playwright-cli` skill for browser automation capabilities
- Python 3.10+ with `playwright` package installed
## Quick Start
```bash
# Show help information for download script
python3 /home/username/.agents/skills/shelfmark/download_books.py -h
# Download a single book
python3 /home/username/.agents/skills/shelfmark/download_books.py '[{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}]'
# Download multiple books
python3 /home/username/.agents/skills/shelfmark/download_books.py '[{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, {"title": "Oliver Twist", "author": "Charles Dickens"}, {"title": "Frankenstein", "author": "Marry Shelley"}]'
# Check calibre database before downloading (skip if already present)
python3 /home/username/.agents/skills/shelfmark/download_books.py --check-calibre '[{"title": "Frankenstein", "author": "Marry Shelley"}]'
# Load books from a JSON file
python3 /home/username/.agents/skills/shelfmark/download_books.py --file books.json
```
The JSON file (`books.json`) should contain an array of book objects:
```json
[
{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"},
{"title": "Frankenstein", "author": "Mary Shelley"},
{"title": "Oliver Twist", "author": "Charles Dickens"}
]
```
Array format is also supported: `[["The Great Gatsby", "F. Scott Fitzgerald"], ["Oliver Twist", "Charles Dickens"], ["Frankenstein", "Marry Shelley"]]`
Title-only (no author) is supported: `["Frankenstein"]`
## Key Characteristics
- **Shelfmark is a React SPA** — raw HTML is a shell; JavaScript dynamically populates the DOM
- **Desktop viewport required** (`1280x900`) — download buttons use `hidden sm:flex` and won't render on mobile
- **Download count pattern** in HTML: `<span>•</span> <span>NUMBER</span> </div>` (the last number before the download button)
- **Download button selector**: `<button ... data-action="download" ...>Download</button>`
- **Search input**: `<input type="search" placeholder="Search Books">`
- **Results indicator**: `<span class="text-sm font-medium whitespace-nowrap">Most relevant</span>`
## How It Works
### 1. Search for a Book
The script navigates to the main page, enters the search query, and waits for "Most relevant" to appear:
```python
# Navigate to main page first to reset SPA state
page.goto('http://localhost:8084/')
page.wait_for_load_state('networkidle')
# Enter search query and submit
page.fill('input[type="search"]', f'{title} {author}')
page.press('input[type="search"]', 'Enter')
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
```
- Always navigate to the main page before each search to reset React SPA state
- Include both title and author in the search query
- Wait for `span.text-sm.font-medium:has-text("Most relevant")` to appear — this indicates search results are fully loaded
- **Do not use timers** to wait for results — always wait for a specific page element
### 2. Find and Parse Download Buttons
```python
def parse_books(page):
btns = page.query_selector_all('button[data-action="download"]')
books = []
for i, btn in enumerate(btns):
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
title = title_match.group(1).strip() if title_match else 'Unknown'
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
author = author_match.group(1).strip() if author_match else 'Unknown'
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
return books
```
### 3. Select and Download the Book with Most Downloads
```python
# Filter books matching the search criteria
matching_books = [b for b in books if matches_search(b['title'], b['author'], title, author)]
if matching_books:
best = max(matching_books, key=lambda x: x['downloads'])
# CRITICAL: Click on the article h3 to open detail view
h3s = page.query_selector_all('article h3')
if best['index'] < len(h3s):
h3s[best['index']].click()
page.wait_for_timeout(300)
# Then click the download button
btn = page.query_selector_all('button[data-action="download"]')[best['index']]
btn.click()
page.wait_for_timeout(500)
```
**Important**: The React SPA requires clicking on the article's `<h3>` element first to open the detail view. Simply clicking the download button directly often fails silently.
### 4. Wait for Download to Complete
```python
for i in range(60):
page.wait_for_timeout(5000)
activity_text = page.inner_text('aside')
if 'IN PROGRESS' in activity_text:
print("Download started!")
elif 'Complete' in activity_text or 'Saved' in activity_text:
print("Download complete!")
break
elif 'No activity' in activity_text:
print("Download not started")
break
```
### 5. Clear Completed Downloads
```python
# Click "Clear Completed" using JavaScript
page.evaluate('''
() => {
for (const b of document.querySelectorAll('button')) {
if (b.textContent.includes('Clear Completed')) {
b.click();
return;
}
}
}
''')
page.wait_for_timeout(2000)
```
### Checking Calibre Database
Use `--check-calibre` (or `-c`) to check if books are already in your calibre database before downloading:
```bash
python3 download_books.py --check-calibre '[{"title": "Frankenstein", "author": "Mary Shelley"}]'
```
Books found in calibre are skipped with a message. If all books are already present, the script exits early without launching the browser.
## Important Notes
- **Always click the article element first** before clicking the download button — the React SPA requires this to properly initialize the download workflow
- **Wait for "Most relevant" text** to appear after search — this indicates results are fully loaded (don't use timers)
- **Navigate to main page** (`http://localhost:8084/`) before each new search to reset React SPA state
- **Some books may have different authors listed** than what's in your source file — the script falls back to title-only search if author search fails
- **The download count** is the last number in the format `• NUMBER` before the download button
- **Book titles may include series info** in brackets, e.g., `(The Locked Tomb Trilogy)`
- **Use `page.evaluate_handle`** to get parent element HTML for parsing — the button's `parentElement.parentElement` contains the card content
- **Title matching** prefers exact matches over partial matches (e.g., "Yesteryear" matches "Yesteryear: A Novel" but not "The Piers of Yesteryear")
- **Books are passed as JSON** — use `--check-calibre` to optionally skip books already in your calibre database
## Common Issues
| Issue | Solution |
|-------|----------|
| No download buttons found | Use desktop viewport (1280x900), wait for `span.text-sm.font-medium:has-text("Most relevant")` |
| Download button click does nothing | Click the article's `<h3>` element first, then click the download button |
| Search returns no results | The script falls back to title-only search automatically |
| Download count shows 0 | The parsing regex may need adjustment — check the HTML structure |
| Sidebar shows "No activity" after click | Ensure you clicked the `<h3>` element first, and wait at least 2 seconds before checking |
| Search results don't update between books | Navigate to `http://localhost:8084/` before each new search to reset SPA state |
| Book downloaded is wrong title | The script prefers exact title matches — if the title is ambiguous, the author search will help narrow it down |
| Books passed incorrectly | Books must be valid JSON — use `{"title": "...", "author": "..."}` format, not `Title: Author` |
+353
View File
@@ -0,0 +1,353 @@
#!/usr/bin/env python3
"""Shelfmark Book Downloader - Downloads books from a local Shelfmark instance."""
from playwright.sync_api import sync_playwright
import re, time, sys, argparse, json
from urllib.parse import quote
SHELFMARK_URL = 'http://localhost:8084/'
def parse_books(page):
btns = page.query_selector_all('button[data-action="download"]')
books = []
for i, btn in enumerate(btns):
content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html()
title_match = re.search(r'<h3[^>]*>(.*?)</h3>', content, re.IGNORECASE | re.DOTALL)
title = title_match.group(1).strip() if title_match else 'Unknown'
author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL)
author = author_match.group(1).strip() if author_match else 'Unknown'
dl_match = re.search(r'<span>•</span>\s*<span>([\d,]+)</span>', content)
downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0
books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads})
return books
def do_search(page, title, author, search_type="author"):
"""Search with title+author, fall back to title only"""
# Navigate to main page first to reset SPA state
page.goto(SHELFMARK_URL)
page.wait_for_load_state('networkidle')
# Enter search query and submit
search_query = f'{title} {author}' if search_type == "author" else title
page.fill('input[type="search"]', search_query)
page.press('input[type="search"]', 'Enter')
# Wait for "Most relevant" to appear (indicates search results are fully rendered)
try:
page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000)
except:
if search_type == "author":
print(" -> Trying title only...")
sys.stdout.flush()
return do_search(page, title, author, search_type="title")
else:
print(" >> Timeout waiting for search results")
sys.stdout.flush()
return None
books = parse_books(page)
def clean_title_for_match(book_title, search_title):
"""Clean book title to check if it matches the search title"""
# Remove common subtitle patterns
clean = re.sub(r'\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$', '', book_title, flags=re.IGNORECASE)
clean = clean.strip()
# Remove trailing punctuation
clean = clean.rstrip(':,;.')
return clean.lower().strip() == search_title.lower().strip()
def matches_search(book_title, book_author, search_title, search_author):
"""Check if book matches search criteria more strictly"""
title_match = search_title.lower() in book_title.lower()
author_match = search_author.lower() in book_author.lower()
# For title+author search, ensure title starts with search title (not just contains it)
if title_match and author_match:
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
# Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann")
if title_match:
author_parts = search_author.lower().split()
if len(author_parts) >= 2:
reversed_author = f"{author_parts[-1]} {author_parts[0]}"
if reversed_author in book_author.lower() or book_author.lower().startswith(reversed_author):
return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower())
return False
if search_type == "author":
matching = [b for b in books if matches_search(b['title'], b['author'], title, author)]
print(f" With author: {len(books)} total, {len(matching)} matching")
for b in books[:3]:
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
sys.stdout.flush()
if matching:
# Check if any matching book is already in the download queue
try:
sidebar_text = page.inner_text('aside')
if 'IN PROGRESS' in sidebar_text:
# Get the title of the book currently downloading
current_download = sidebar_text.split('IN PROGRESS')[1].split('')[0].strip().split('\n')[0].strip()
# Check if the current download matches our search
if title.lower() in current_download.lower():
print(f" >> '{current_download}' already downloading, skipping")
sys.stdout.flush()
return None
except:
pass
if matching:
return matching, books
# Fall back to title only
print(" -> Trying title only...")
sys.stdout.flush()
return do_search(page, title, author, search_type="title")
# Title-only search - prefer exact matches first
matching = [b for b in books if title.lower() in b['title'].lower()]
exact_matches = [b for b in matching if clean_title_for_match(b['title'], title)]
if exact_matches:
matching = exact_matches
print(f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)")
for b in books[:5]:
print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})")
sys.stdout.flush()
if not matching:
return None
return (matching, books)
def download_book(page, title, author):
result = do_search(page, title, author)
if not result:
print("\n >>> NOT FOUND")
sys.stdout.flush()
return
matching, all_books = result
if not matching:
print("\n >>> NOT FOUND")
sys.stdout.flush()
return
# Find the best book that is not disabled
btns = page.query_selector_all('button[data-action="download"]')
best = None
for b in sorted(matching, key=lambda x: x['downloads'], reverse=True):
if b['index'] < len(btns) and not btns[b['index']].is_disabled():
best = b
break
if not best:
print("\n >>> All matching books are already in download queue")
sys.stdout.flush()
return
print(f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]")
sys.stdout.flush()
# Click on the article h3 to open detail view (required for download to work)
h3s = page.query_selector_all('article h3')
if best['index'] < len(h3s):
h3s[best['index']].click()
page.wait_for_timeout(300)
else:
print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)")
sys.stdout.flush()
# Click the download button
btn = btns[best['index']]
btn.click()
# Wait a moment for the click to register
page.wait_for_timeout(500)
print(f" >> Click sent")
sys.stdout.flush()
# Check sidebar immediately
time.sleep(2)
try:
txt = page.inner_text('aside')
if 'IN PROGRESS' in txt:
print(f" >> Download started!")
sys.stdout.flush()
else:
print(f" >> Sidebar after 2s: {txt[:150]}")
sys.stdout.flush()
except:
print(f" >> Sidebar not visible after 2s")
sys.stdout.flush()
# Wait for download to complete - check sidebar every 2 seconds, max 5 minutes
last_state = None
for attempt in range(150):
time.sleep(2)
try:
txt = page.inner_text('aside')
if 'IN PROGRESS' in txt:
state = 'downloading'
elif 'Complete' in txt or 'Saved' in txt or 'No activity' in txt:
state = 'done'
else:
state = 'other'
if state != last_state:
if state == 'downloading':
print(f" >> Download started!")
elif state == 'done':
if 'No activity' in txt:
print(f" >> Download not started (no activity)")
else:
print(f" >> Download complete!")
last_state = state
if state == 'done':
break
except:
if last_state is None:
print(f" >> Sidebar not visible yet")
last_state = 'no_sidebar'
sys.stdout.flush()
else:
if last_state != 'done':
print(" >> Timeout waiting for download")
sys.stdout.flush()
def clear_completed_downloads(page):
"""Clear completed downloads from previous sessions"""
page.evaluate('''
() => {
for (const b of document.querySelectorAll('button')) {
if (b.textContent.includes('Clear Completed')) { b.click(); return; }
}
}
''')
page.wait_for_timeout(2000)
def check_calibre(title):
"""Check if a book is already in the calibre database."""
import subprocess
cmd = f'calibredb list --search title:="{title}"'
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
output = result.stdout.strip()
# calibredb output has header line "id title authors", count non-header lines
lines = [l.strip() for l in output.split('\n') if l.strip()]
# Remove header if present (first line starts with 'id')
if lines and lines[0].startswith('id'):
lines = lines[1:]
return len(lines) > 0
def load_books_from_json(json_str, source):
"""Load books from a JSON string or file path."""
try:
data = json.loads(json_str)
except json.JSONDecodeError as e:
print(f"Error: Invalid JSON in {source}: {e}")
sys.exit(1)
if not isinstance(data, list):
print(f"Error: JSON in {source} must be an array of book objects")
sys.exit(1)
books = []
for item in data:
if isinstance(item, dict):
if 'title' not in item or 'author' not in item:
print(f"Error: Each book object must have 'title' and 'author' keys: {item}")
sys.exit(1)
books.append((item['title'].strip(), item['author'].strip()))
elif isinstance(item, list) and len(item) >= 2:
books.append((str(item[0]).strip(), str(item[1]).strip()))
elif isinstance(item, str):
# Support plain strings as title-only (author will be searched separately)
books.append((item.strip(), ""))
else:
print(f"Warning: Skipping unrecognized book entry: {item}")
return books
def main():
parser = argparse.ArgumentParser(description='Download books from Shelfmark')
parser.add_argument('books', nargs='*', help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects')
parser.add_argument('--file', '-f', help='Path to JSON file containing an array of books')
parser.add_argument('-c', '--check-calibre', action='store_true', help='Check calibre database before downloading, skip if already present')
args = parser.parse_args()
# Load books from --file, positional JSON arg, or both
BOOKS = []
if args.file:
try:
with open(args.file, 'r') as f:
BOOKS = load_books_from_json(f.read(), f"file '{args.file}'")
except FileNotFoundError:
print(f"Error: File not found: {args.file}")
sys.exit(1)
except IOError as e:
print(f"Error reading file: {e}")
sys.exit(1)
for book_arg in args.books:
BOOKS.extend(load_books_from_json(book_arg, 'command line argument'))
if not BOOKS:
print("Error: No books specified. Use positional JSON args or --file <path>")
sys.exit(1)
# Check calibre database if requested
if args.check_calibre:
print("\nChecking calibre database...")
sys.stdout.flush()
remaining = []
for title, author in BOOKS:
if check_calibre(title):
print(f" >> '{title}' already in calibre, skipping")
sys.stdout.flush()
else:
remaining.append((title, author))
BOOKS = remaining
if not BOOKS:
print("All books already in calibre, nothing to download.")
sys.stdout.flush()
return
print(f"\n{'='*60}")
print(f"Downloading {len(BOOKS)} book(s) from Shelfmark")
print('='*60)
sys.stdout.flush()
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page(viewport={'width': 1280, 'height': 900})
page.goto(SHELFMARK_URL)
page.wait_for_load_state('networkidle')
# Clear completed downloads at start of batch
print(f"\n{'='*60}")
print("Clearing completed downloads from previous sessions...")
print('='*60)
sys.stdout.flush()
clear_completed_downloads(page)
for i, (title, author) in enumerate(BOOKS, 1):
print(f"\n{'='*60}")
print(f"[{i}/{len(BOOKS)}] '{title}' by {author}")
print('='*60)
sys.stdout.flush()
download_book(page, title, author)
browser.close()
print("\nDone!")
sys.stdout.flush()
if __name__ == '__main__':
main()
@@ -1,4 +1,5 @@
import type { Book, RequestRecord, StatusData } from '../../types';
import { getDownloadsCount } from '../../types';
import { STATUS_LABELS, isActiveDownloadStatus } from './activityStyles.js';
import type { ActivityItem, ActivityVisualStatus } from './activityTypes';
@@ -87,10 +88,13 @@ export const downloadToActivityItem = (book: Book, statusKey: DownloadStatusKey)
typeof book.request_id === 'number' && Number.isFinite(book.request_id) && book.request_id > 0
? Math.trunc(book.request_id)
: undefined;
const downloadsCount = getDownloadsCount(book);
const downloadsText = downloadsCount != null ? `${downloadsCount.toLocaleString()} downloads` : undefined;
const metaLine = joinMetaParts([
toOptionalText(book.format)?.toUpperCase(),
toOptionalText(book.size),
toOptionalText(book.source_display_name) || toSourceLabel(book.source),
downloadsText,
toOptionalText(book.username),
]);
const progress = getDownloadProgress(visualStatus, book.progress);
@@ -115,6 +119,7 @@ export const downloadToActivityItem = (book: Book, statusKey: DownloadStatusKey)
downloadRetryAvailable,
downloadPath: toOptionalText(book.download_path),
sizeRaw: toOptionalText(book.size),
downloads: downloadsCount ?? undefined,
requestId,
};
};
@@ -32,6 +32,7 @@ export interface ActivityItem {
progress?: number;
progressAnimated?: boolean;
sizeRaw?: string;
downloads?: number;
timestamp: number;
username?: string;
@@ -2,6 +2,7 @@ import { useState } from 'react';
import { useSearchMode } from '../../contexts/SearchModeContext';
import type { Book, ButtonStateInfo } from '../../types';
import { getDownloadsCount } from '../../types';
import { bookSupportsTargets } from '../../utils/bookTargetLoader';
import { BookActionButton } from '../BookActionButton';
import { BookTargetDropdown } from '../BookTargetDropdown';
@@ -210,6 +211,7 @@ export const CardView = ({
<span>{book.size}</span>
</>
)}
{searchMode !== 'universal' && (() => { const d = getDownloadsCount(book); return d != null && d > 0 ? <> <span></span> <span>{d.toLocaleString()}</span> </> : null; })()}
</div>
)}
</div>
@@ -2,6 +2,7 @@ import { useState } from 'react';
import { useSearchMode } from '../../contexts/SearchModeContext';
import type { Book, ButtonStateInfo } from '../../types';
import { getDownloadsCount } from '../../types';
import { bookSupportsTargets } from '../../utils/bookTargetLoader';
import { BookActionButton } from '../BookActionButton';
import { BookTargetDropdown } from '../BookTargetDropdown';
@@ -221,6 +222,7 @@ export const CompactView = ({
<span>{book.size}</span>
</>
)}
{(() => { const d = getDownloadsCount(book); return d != null && d > 0 ? <> <span></span> <span>{d.toLocaleString()}</span> </> : null; })()}
</div>
)}
@@ -2,6 +2,7 @@ import { useState } from 'react';
import { useSearchMode } from '../../contexts/SearchModeContext';
import type { Book, ButtonStateInfo, DisplayField } from '../../types';
import { getDownloadsCount } from '../../types';
import { bookSupportsTargets } from '../../utils/bookTargetLoader';
import { getFormatColor, getLanguageColor } from '../../utils/colorMaps';
import { BookActionButton } from '../BookActionButton';
@@ -167,7 +168,7 @@ export const ListView = ({
className={`grid w-full items-center gap-2 sm:gap-x-0.5 sm:gap-y-1 ${
searchMode === 'universal'
? 'grid-cols-[auto_minmax(0,1fr)_auto_auto] sm:grid-cols-[auto_minmax(0,2fr)_minmax(50px,0.25fr)_minmax(90px,0.5fr)_minmax(90px,0.5fr)_minmax(120px,0.7fr)_auto]'
: 'grid-cols-[auto_minmax(0,1fr)_auto_auto] sm:grid-cols-[auto_minmax(0,2fr)_minmax(50px,0.25fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_auto]'
: 'grid-cols-[auto_minmax(0,1fr)_auto_auto] sm:grid-cols-[auto_minmax(0,2fr)_minmax(50px,0.25fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_minmax(70px,0.35fr)_auto]'
}`}
>
{/* Thumbnail */}
@@ -287,6 +288,13 @@ export const ListView = ({
</div>
)}
{/* Direct mode: Downloads - Desktop only */}
{searchMode !== 'universal' && (
<div className="hidden justify-center text-xs text-gray-700 sm:flex dark:text-gray-200">
{(() => { const d = getDownloadsCount(book); return d != null && d > 0 ? d.toLocaleString() : '-'; })()}
</div>
)}
{/* Action Buttons */}
<div className="flex flex-row justify-end gap-0.5 sm:gap-1 sm:pr-3">
{bookSupportsTargets(book) && targetProvider && targetBookId && (
+1
View File
@@ -103,6 +103,7 @@ const parseHistoryBook = (value: unknown): Book | null => {
...(typeof value.retry_available === 'boolean'
? { retry_available: value.retry_available }
: {}),
...(typeof value.downloads === 'number' ? { downloads: value.downloads } : {}),
};
};
+1
View File
@@ -540,6 +540,7 @@ export type DownloadReleasePayload = {
format?: string;
size?: string;
size_bytes?: number;
downloads?: number;
download_url?: string;
protocol?: string;
indexer?: string;
+17
View File
@@ -56,6 +56,23 @@ export interface Book {
titles_by_language?: Record<string, string>;
username?: string;
retry_available?: boolean;
downloads?: number;
extra?: Record<string, unknown>;
}
/**
* Extract download count from a book's data.
* Checks both the direct `downloads` field and the `extra.downloads` fallback.
*/
export function getDownloadsCount(book: Book): number | null {
if (book.downloads != null && book.downloads > 0) {
return book.downloads;
}
const extraDownloads = book.extra?.downloads;
if (extraDownloads != null && typeof extraDownloads === 'number' && extraDownloads > 0) {
return extraDownloads;
}
return null;
}
// Status response types
@@ -62,6 +62,7 @@ interface SourceBackedBookData {
language?: unknown;
format?: unknown;
size?: unknown;
downloads?: unknown;
preview?: unknown;
publisher?: unknown;
info?: Record<string, string | string[]>;
@@ -157,6 +158,7 @@ const transformSourceBackedDataToBook = (data: SourceBackedBookData): Book => {
language: toOptionalText(data.language),
format: toOptionalText(data.format),
size: toOptionalText(data.size),
downloads: typeof data.downloads === 'number' ? data.downloads : undefined,
preview: toOptionalText(data.preview),
publisher: toOptionalText(data.publisher),
info: data.info,
@@ -181,6 +183,7 @@ export function transformReleaseToDirectBook(release: Release): Book {
language: release.language || extra.language,
format: release.format,
size: release.size,
downloads: extra.downloads,
preview: extra.preview,
publisher: extra.publisher,
info: parseBookInfo(extra.info),
+10
View File
@@ -30,6 +30,16 @@ export function buildReleaseDownloadPayload(
format: release.format,
size: release.size,
size_bytes: release.size_bytes,
downloads:
typeof release.extra?.downloads === 'number'
? release.extra.downloads
: typeof book.downloads === 'number' && book.downloads > 0
? book.downloads
: typeof book.extra?.downloads === 'number'
? book.extra.downloads
: Array.isArray(book.info?.Downloads) && book.info.Downloads.length > 0
? Number(book.info.Downloads[0]) || 0
: 0,
download_url: release.download_url,
protocol: release.protocol,
indexer: release.indexer,
+24 -17
View File
@@ -95,23 +95,30 @@ export const buildReleaseDataFromMetadataRelease = (
};
};
export const buildReleaseDataFromDirectBook = (book: Book) => {
const source = getBrowseSource(book);
return {
source,
source_id: book.id,
title: book.title || 'Unknown title',
author: book.author,
year: book.year,
format: book.format,
size: book.size,
preview: book.preview,
content_type: 'ebook' as const,
// Browsing a source directly means the book record IS the release record.
language: book.language,
search_mode: 'direct' as const,
};
};
export const buildReleaseDataFromDirectBook = (book: Book) => {
const source = getBrowseSource(book);
const downloads =
typeof book.downloads === 'number' && book.downloads > 0
? book.downloads
: Array.isArray(book.info?.Downloads) && book.info.Downloads.length > 0
? Number(book.info.Downloads[0]) || 0
: 0;
return {
source,
source_id: book.id,
title: book.title || 'Unknown title',
author: book.author,
year: book.year,
format: book.format,
size: book.size,
downloads,
preview: book.preview,
content_type: 'ebook' as const,
// Browsing a source directly means the book record IS the release record.
language: book.language,
search_mode: 'direct' as const,
};
};
export const buildDirectRequestPayload = (book: Book): CreateRequestPayload => {
const bookData = buildDirectBookRequestData(book);