diff --git a/shelfmark/core/download_history_service.py b/shelfmark/core/download_history_service.py index 1ffc121..a3087e7 100644 --- a/shelfmark/core/download_history_service.py +++ b/shelfmark/core/download_history_service.py @@ -207,6 +207,7 @@ class DownloadHistoryService: "content_type": row.get("content_type"), "source": row.get("source"), "source_display_name": row.get("source_display_name"), + "downloads": row.get("downloads"), "status_message": row.get("status_message"), "download_path": DownloadHistoryService._resolve_existing_download_path( row.get("download_path") @@ -272,6 +273,7 @@ class DownloadHistoryService: size: str | None, preview: str | None, content_type: str | None, + downloads: int | None, origin: str, retry_payload: dict[str, Any] | None = None, ) -> None: @@ -307,15 +309,16 @@ class DownloadHistoryService: title, author, format, size, preview, content_type, origin, final_status, status_message, download_path, retry_payload, - queued_at, terminal_at + queued_at, terminal_at, downloads ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', NULL, NULL, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', NULL, NULL, ?, ?, ?, ?) ON CONFLICT(task_id) DO UPDATE SET final_status = 'active', status_message = NULL, download_path = NULL, retry_payload = excluded.retry_payload, - terminal_at = ? + terminal_at = ?, + downloads = excluded.downloads """, ( normalized_task_id, @@ -334,6 +337,7 @@ class DownloadHistoryService: normalized_retry_payload, recorded_at, recorded_at, + downloads, recorded_at, ), ) diff --git a/shelfmark/core/models.py b/shelfmark/core/models.py index 20fd337..a596c6c 100644 --- a/shelfmark/core/models.py +++ b/shelfmark/core/models.py @@ -97,6 +97,7 @@ class DownloadTask: year: str | None = None format: str | None = None size: str | None = None + downloads: int | None = None # Download count from source preview: str | None = None content_type: str | None = None # "book (fiction)", "audiobook", "magazine", etc. source_url: str | None = None # Original release URL used by source-specific handlers diff --git a/shelfmark/core/user_db.py b/shelfmark/core/user_db.py index d431b6b..0df517d 100644 --- a/shelfmark/core/user_db.py +++ b/shelfmark/core/user_db.py @@ -87,7 +87,8 @@ CREATE TABLE IF NOT EXISTS download_history ( download_path TEXT, retry_payload TEXT, queued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - terminal_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + terminal_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + downloads INTEGER ); CREATE INDEX IF NOT EXISTS idx_download_history_user_status @@ -203,6 +204,7 @@ class UserDB: self._migrate_request_delivery_columns(conn) self._migrate_download_history_queued_at(conn) self._migrate_download_history_retry_payload(conn) + self._migrate_download_history_downloads(conn) conn.commit() # WAL mode must be changed outside an open transaction. conn.execute("PRAGMA journal_mode=WAL") @@ -270,6 +272,13 @@ class UserDB: if "retry_payload" not in column_names: conn.execute("ALTER TABLE download_history ADD COLUMN retry_payload TEXT") + def _migrate_download_history_downloads(self, conn: sqlite3.Connection) -> None: + """Ensure download_history.downloads exists for download count persistence.""" + columns = conn.execute("PRAGMA table_info(download_history)").fetchall() + column_names = {str(col["name"]) for col in columns} + if "downloads" not in column_names: + conn.execute("ALTER TABLE download_history ADD COLUMN downloads INTEGER") + def create_user( self, username: str, diff --git a/shelfmark/download/orchestrator.py b/shelfmark/download/orchestrator.py index cc8a6f7..6714156 100644 --- a/shelfmark/download/orchestrator.py +++ b/shelfmark/download/orchestrator.py @@ -296,6 +296,7 @@ def queue_release( year=year, format=release_data.get("format"), size=release_data.get("size"), + downloads=release_data.get("downloads") or extra.get("downloads"), preview=preview, content_type=content_type, source_url=source_url, @@ -319,7 +320,7 @@ def queue_release( logger.info("Release already in queue: %s", task.title) return False, "Release is already in the download queue" - logger.info("Release queued with priority %s: %s", priority, task.title) + logger.info("Release queued with priority %s: %s (downloads=%s, release_data.downloads=%s, extra=%s)", priority, task.title, task.downloads, release_data.get("downloads"), extra) # Broadcast status update via WebSocket if ws_manager: @@ -521,6 +522,7 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None: year=normalize_optional_text(payload.get("year")), format=normalize_optional_text(payload.get("format")), size=normalize_optional_text(payload.get("size")), + downloads=int(payload["downloads"]) if payload.get("downloads") is not None else None, preview=normalize_optional_text(payload.get("preview")), content_type=normalize_optional_text(payload.get("content_type")), source_url=normalize_optional_text(payload.get("source_url")), @@ -617,6 +619,7 @@ def _task_to_dict( "author": task.author, "format": task.format, "size": task.size, + "downloads": task.downloads, "preview": preview, "content_type": task.content_type, "source": task.source, diff --git a/shelfmark/main.py b/shelfmark/main.py index 9638390..2af333a 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -1073,6 +1073,8 @@ def api_download_release() -> Response | tuple[Response, int]: release_payload = dict(data) release_payload["content_type"] = resolved_content_type + logger.info("Download request received. keys=%s downloads=%s extra.downloads=%s", list(data.keys()), data.get("downloads"), data.get("extra", {}).get("downloads") if isinstance(data.get("extra"), dict) else None) + priority = data.get("priority", 0) # Per-user download overrides db_user_id = session.get("db_user_id") @@ -1354,6 +1356,7 @@ def _record_download_queued(task_id: str, task: Any) -> None: size=normalize_optional_text(getattr(task, "size", None)), preview=normalize_optional_text(getattr(task, "preview", None)), content_type=normalize_optional_text(getattr(task, "content_type", None)), + downloads=getattr(task, "downloads", None), origin=origin, retry_payload=backend.serialize_task_for_retry(task), ) diff --git a/shelfmark/release_sources/__init__.py b/shelfmark/release_sources/__init__.py index 6495c50..bedd7b3 100644 --- a/shelfmark/release_sources/__init__.py +++ b/shelfmark/release_sources/__init__.py @@ -46,6 +46,7 @@ class BrowseRecord: content: str | None = None format: str | None = None size: str | None = None + downloads: int | None = None info: dict[str, list[str]] | None = None description: str | None = None download_urls: list[str] = field(default_factory=list) diff --git a/shelfmark/release_sources/direct_download/annas_archive.py b/shelfmark/release_sources/direct_download/annas_archive.py index 4bd1b65..ff4e371 100644 --- a/shelfmark/release_sources/direct_download/annas_archive.py +++ b/shelfmark/release_sources/direct_download/annas_archive.py @@ -1,5 +1,6 @@ """Anna's Archive search, metadata parsing, and MD5 mirror download cascade.""" +import concurrent.futures import itertools import json import re @@ -786,9 +787,60 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]: ) ) + # Fetch download counts for all results in batch + if books: + _enrich_search_results_with_downloads(books) + return books +def _fetch_download_count_inline(book_id: str) -> int | None: + """Fetch the download count for a single book from Anna's Archive inline_info API.""" + try: + url = f"{network.get_aa_base_url()}/dyn/md5/inline_info/{book_id}" + resp = requests.get(url, timeout=5, headers={"Accept": "application/json"}) + if resp.status_code == 200: + data = resp.json() + count = data.get("downloads_total") + if count is not None: + return count + except Exception: + logger.debug("Failed to fetch download count for %s", book_id, exc_info=True) + return None + + +def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None: + """Fetch download counts for search results in batch and add them to each record's info.""" + if not books: + return + + book_ids = [b.id for b in books if b.id] + if not book_ids: + return + + # Fetch counts in parallel using the inline_info API (cheaper than summary) + counts: dict[str, int] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + futures = { + executor.submit(_fetch_download_count_inline, bid): bid for bid in book_ids + } + for future in concurrent.futures.as_completed(futures): + bid = futures[future] + try: + count = future.result() + if count is not None: + counts[bid] = count + except Exception: + logger.debug("Failed to fetch download count for %s", bid, exc_info=True) + + # Add counts to each record's info + for book in books: + if book.id in counts: + if book.info is None: + book.info = {} + book.info["Downloads"] = [str(counts[book.id])] + + def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseRecord: """Get detailed information for a specific book. diff --git a/shelfmark/release_sources/direct_download/source.py b/shelfmark/release_sources/direct_download/source.py index 022cd14..1f88576 100644 --- a/shelfmark/release_sources/direct_download/source.py +++ b/shelfmark/release_sources/direct_download/source.py @@ -37,6 +37,21 @@ if TYPE_CHECKING: logger = setup_logger(__name__) +def _extract_downloads(record: BrowseRecord) -> int | None: + """Extract download count from record info for Release.extra.downloads.""" + downloads = None + if record.info and "Downloads" in record.info: + downloads_value = record.info["Downloads"] + if isinstance(downloads_value, list) and len(downloads_value) > 0: + try: + downloads = int(downloads_value[0]) + except (ValueError, TypeError): + pass + elif isinstance(downloads_value, (int, float)): + downloads = int(downloads_value) + return downloads + + def _browse_record_to_release(record: BrowseRecord) -> Release: """Convert a browse record to a Release object. @@ -67,6 +82,7 @@ def _browse_record_to_release(record: BrowseRecord) -> Release: "download_urls": record.download_urls, "info": record.info, "direct_download_provider": provider_id, + "downloads": _extract_downloads(record), # Kept for older frontends and persisted request payloads. "web_provider": provider_id if provider_id != "annas_archive" else None, }, @@ -98,8 +114,8 @@ class DirectDownloadSource(ReleaseSource): def get_column_config(self) -> ReleaseColumnConfig: """Column configuration for Direct Download source. - Shows language, format, and size badges for each release. - Language is hidden on mobile; format and size are shown. + Shows language, format, size, and downloads for each release. + Language, format, size, and downloads are all shown on mobile. """ return ReleaseColumnConfig( columns=[ @@ -131,8 +147,16 @@ class DirectDownloadSource(ReleaseSource): width="80px", hide_mobile=False, # Size shown on mobile ), + ColumnSchema( + key="extra.downloads", + label="Downloads", + render_type=ColumnRenderType.NUMBER, + align=ColumnAlign.CENTER, + width="80px", + hide_mobile=False, # Downloads shown on mobile + ), ], - grid_template="minmax(0,2fr) 60px 80px 80px", + grid_template="minmax(0,2fr) 60px 80px 80px 80px", supported_filters=["format", "language"], # AA has reliable language metadata ) diff --git a/skills/shelfmark/SKILL.md b/skills/shelfmark/SKILL.md new file mode 100644 index 0000000..120bd70 --- /dev/null +++ b/skills/shelfmark/SKILL.md @@ -0,0 +1,196 @@ +--- +name: shelfmark +description: Tool for downloading books. +license: Complete terms in LICENSE.txt +--- + +# Shelfmark Book Download Skill + +Use this skill to search for and download books from a local Shelfmark instance using Playwright. + +## Prerequisites + +- Shelfmark must be running at `http://localhost:8084/` +- Use `playwright-cli` skill for browser automation capabilities +- Python 3.10+ with `playwright` package installed + +## Quick Start + +```bash +# Show help information for download script +python3 /home/username/.agents/skills/shelfmark/download_books.py -h + +# Download a single book +python3 /home/username/.agents/skills/shelfmark/download_books.py '[{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}]' + +# Download multiple books +python3 /home/username/.agents/skills/shelfmark/download_books.py '[{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, {"title": "Oliver Twist", "author": "Charles Dickens"}, {"title": "Frankenstein", "author": "Marry Shelley"}]' + +# Check calibre database before downloading (skip if already present) +python3 /home/username/.agents/skills/shelfmark/download_books.py --check-calibre '[{"title": "Frankenstein", "author": "Marry Shelley"}]' + +# Load books from a JSON file +python3 /home/username/.agents/skills/shelfmark/download_books.py --file books.json +``` + +The JSON file (`books.json`) should contain an array of book objects: + +```json +[ + {"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, + {"title": "Frankenstein", "author": "Mary Shelley"}, + {"title": "Oliver Twist", "author": "Charles Dickens"} +] +``` + +Array format is also supported: `[["The Great Gatsby", "F. Scott Fitzgerald"], ["Oliver Twist", "Charles Dickens"], ["Frankenstein", "Marry Shelley"]]` + +Title-only (no author) is supported: `["Frankenstein"]` + +## Key Characteristics + +- **Shelfmark is a React SPA** — raw HTML is a shell; JavaScript dynamically populates the DOM +- **Desktop viewport required** (`1280x900`) — download buttons use `hidden sm:flex` and won't render on mobile +- **Download count pattern** in HTML: `• NUMBER ` (the last number before the download button) +- **Download button selector**: `` +- **Search input**: `` +- **Results indicator**: `Most relevant` + +## How It Works + +### 1. Search for a Book + +The script navigates to the main page, enters the search query, and waits for "Most relevant" to appear: + +```python +# Navigate to main page first to reset SPA state +page.goto('http://localhost:8084/') +page.wait_for_load_state('networkidle') + +# Enter search query and submit +page.fill('input[type="search"]', f'{title} {author}') +page.press('input[type="search"]', 'Enter') + +# Wait for "Most relevant" to appear (indicates search results are fully rendered) +page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000) +``` + +- Always navigate to the main page before each search to reset React SPA state +- Include both title and author in the search query +- Wait for `span.text-sm.font-medium:has-text("Most relevant")` to appear — this indicates search results are fully loaded +- **Do not use timers** to wait for results — always wait for a specific page element + +### 2. Find and Parse Download Buttons + +```python +def parse_books(page): + btns = page.query_selector_all('button[data-action="download"]') + books = [] + for i, btn in enumerate(btns): + content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html() + + title_match = re.search(r']*>(.*?)', content, re.IGNORECASE | re.DOTALL) + title = title_match.group(1).strip() if title_match else 'Unknown' + + author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL) + author = author_match.group(1).strip() if author_match else 'Unknown' + + dl_match = re.search(r'•\s*([\d,]+)', content) + downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0 + + books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads}) + return books +``` + +### 3. Select and Download the Book with Most Downloads + +```python +# Filter books matching the search criteria +matching_books = [b for b in books if matches_search(b['title'], b['author'], title, author)] + +if matching_books: + best = max(matching_books, key=lambda x: x['downloads']) + + # CRITICAL: Click on the article h3 to open detail view + h3s = page.query_selector_all('article h3') + if best['index'] < len(h3s): + h3s[best['index']].click() + page.wait_for_timeout(300) + + # Then click the download button + btn = page.query_selector_all('button[data-action="download"]')[best['index']] + btn.click() + page.wait_for_timeout(500) +``` + +**Important**: The React SPA requires clicking on the article's `

` element first to open the detail view. Simply clicking the download button directly often fails silently. + +### 4. Wait for Download to Complete + +```python +for i in range(60): + page.wait_for_timeout(5000) + activity_text = page.inner_text('aside') + + if 'IN PROGRESS' in activity_text: + print("Download started!") + elif 'Complete' in activity_text or 'Saved' in activity_text: + print("Download complete!") + break + elif 'No activity' in activity_text: + print("Download not started") + break +``` + +### 5. Clear Completed Downloads + +```python +# Click "Clear Completed" using JavaScript +page.evaluate(''' + () => { + for (const b of document.querySelectorAll('button')) { + if (b.textContent.includes('Clear Completed')) { + b.click(); + return; + } + } + } +''') + +page.wait_for_timeout(2000) +``` + +### Checking Calibre Database + +Use `--check-calibre` (or `-c`) to check if books are already in your calibre database before downloading: + +```bash +python3 download_books.py --check-calibre '[{"title": "Frankenstein", "author": "Mary Shelley"}]' +``` + +Books found in calibre are skipped with a message. If all books are already present, the script exits early without launching the browser. + +## Important Notes + +- **Always click the article element first** before clicking the download button — the React SPA requires this to properly initialize the download workflow +- **Wait for "Most relevant" text** to appear after search — this indicates results are fully loaded (don't use timers) +- **Navigate to main page** (`http://localhost:8084/`) before each new search to reset React SPA state +- **Some books may have different authors listed** than what's in your source file — the script falls back to title-only search if author search fails +- **The download count** is the last number in the format `• NUMBER` before the download button +- **Book titles may include series info** in brackets, e.g., `(The Locked Tomb Trilogy)` +- **Use `page.evaluate_handle`** to get parent element HTML for parsing — the button's `parentElement.parentElement` contains the card content +- **Title matching** prefers exact matches over partial matches (e.g., "Yesteryear" matches "Yesteryear: A Novel" but not "The Piers of Yesteryear") +- **Books are passed as JSON** — use `--check-calibre` to optionally skip books already in your calibre database + +## Common Issues + +| Issue | Solution | +|-------|----------| +| No download buttons found | Use desktop viewport (1280x900), wait for `span.text-sm.font-medium:has-text("Most relevant")` | +| Download button click does nothing | Click the article's `

` element first, then click the download button | +| Search returns no results | The script falls back to title-only search automatically | +| Download count shows 0 | The parsing regex may need adjustment — check the HTML structure | +| Sidebar shows "No activity" after click | Ensure you clicked the `

` element first, and wait at least 2 seconds before checking | +| Search results don't update between books | Navigate to `http://localhost:8084/` before each new search to reset SPA state | +| Book downloaded is wrong title | The script prefers exact title matches — if the title is ambiguous, the author search will help narrow it down | +| Books passed incorrectly | Books must be valid JSON — use `{"title": "...", "author": "..."}` format, not `Title: Author` | diff --git a/skills/shelfmark/download_books.py b/skills/shelfmark/download_books.py new file mode 100755 index 0000000..1d3deed --- /dev/null +++ b/skills/shelfmark/download_books.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""Shelfmark Book Downloader - Downloads books from a local Shelfmark instance.""" + +from playwright.sync_api import sync_playwright +import re, time, sys, argparse, json +from urllib.parse import quote + +SHELFMARK_URL = 'http://localhost:8084/' + +def parse_books(page): + btns = page.query_selector_all('button[data-action="download"]') + books = [] + for i, btn in enumerate(btns): + content = btn.evaluate_handle('el => el.parentElement.parentElement').inner_html() + + title_match = re.search(r']*>(.*?)

', content, re.IGNORECASE | re.DOTALL) + title = title_match.group(1).strip() if title_match else 'Unknown' + + author_match = re.search(r'class="min-w-0 truncate text-xs[^"]*"[^>]*>(.*?)<', content, re.IGNORECASE | re.DOTALL) + author = author_match.group(1).strip() if author_match else 'Unknown' + + dl_match = re.search(r'•\s*([\d,]+)', content) + downloads = int(dl_match.group(1).replace(',', '')) if dl_match else 0 + + books.append({'index': i, 'title': title, 'author': author, 'downloads': downloads}) + return books + +def do_search(page, title, author, search_type="author"): + """Search with title+author, fall back to title only""" + # Navigate to main page first to reset SPA state + page.goto(SHELFMARK_URL) + page.wait_for_load_state('networkidle') + + # Enter search query and submit + search_query = f'{title} {author}' if search_type == "author" else title + page.fill('input[type="search"]', search_query) + page.press('input[type="search"]', 'Enter') + + # Wait for "Most relevant" to appear (indicates search results are fully rendered) + try: + page.wait_for_selector('span.text-sm.font-medium:has-text("Most relevant")', timeout=60000) + except: + if search_type == "author": + print(" -> Trying title only...") + sys.stdout.flush() + return do_search(page, title, author, search_type="title") + else: + print(" >> Timeout waiting for search results") + sys.stdout.flush() + return None + + books = parse_books(page) + + def clean_title_for_match(book_title, search_title): + """Clean book title to check if it matches the search title""" + # Remove common subtitle patterns + clean = re.sub(r'\s*[:–—]\s*(A Novel|Reese\'s Book Club.*?|The Hilarious.*?|A GMA Book Club Pick.*?|Movie Tie-In.*?|eBook.*?|\[.*?\].*?)$', '', book_title, flags=re.IGNORECASE) + clean = clean.strip() + # Remove trailing punctuation + clean = clean.rstrip(':,;.') + return clean.lower().strip() == search_title.lower().strip() + + def matches_search(book_title, book_author, search_title, search_author): + """Check if book matches search criteria more strictly""" + title_match = search_title.lower() in book_title.lower() + author_match = search_author.lower() in book_author.lower() + + # For title+author search, ensure title starts with search title (not just contains it) + if title_match and author_match: + return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower()) + + # Also check if author name appears in reverse order (e.g., "Grann, David" matches "David Grann") + if title_match: + author_parts = search_author.lower().split() + if len(author_parts) >= 2: + reversed_author = f"{author_parts[-1]} {author_parts[0]}" + if reversed_author in book_author.lower() or book_author.lower().startswith(reversed_author): + return clean_title_for_match(book_title, search_title) or book_title.lower().startswith(search_title.lower()) + + return False + + if search_type == "author": + matching = [b for b in books if matches_search(b['title'], b['author'], title, author)] + print(f" With author: {len(books)} total, {len(matching)} matching") + for b in books[:3]: + print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})") + sys.stdout.flush() + + if matching: + # Check if any matching book is already in the download queue + try: + sidebar_text = page.inner_text('aside') + if 'IN PROGRESS' in sidebar_text: + # Get the title of the book currently downloading + current_download = sidebar_text.split('IN PROGRESS')[1].split('—')[0].strip().split('\n')[0].strip() + # Check if the current download matches our search + if title.lower() in current_download.lower(): + print(f" >> '{current_download}' already downloading, skipping") + sys.stdout.flush() + return None + except: + pass + + if matching: + return matching, books + + # Fall back to title only + print(" -> Trying title only...") + sys.stdout.flush() + return do_search(page, title, author, search_type="title") + + # Title-only search - prefer exact matches first + matching = [b for b in books if title.lower() in b['title'].lower()] + exact_matches = [b for b in matching if clean_title_for_match(b['title'], title)] + + if exact_matches: + matching = exact_matches + + print(f" Title-only: {len(books)} total, {len(matching)} matching ({len(exact_matches)} exact)") + for b in books[:5]: + print(f" [{b['index']}] '{b['title']}' by {b['author']} ({b['downloads']})") + sys.stdout.flush() + + if not matching: + return None + + return (matching, books) + +def download_book(page, title, author): + result = do_search(page, title, author) + if not result: + print("\n >>> NOT FOUND") + sys.stdout.flush() + return + + matching, all_books = result + if not matching: + print("\n >>> NOT FOUND") + sys.stdout.flush() + return + + # Find the best book that is not disabled + btns = page.query_selector_all('button[data-action="download"]') + best = None + for b in sorted(matching, key=lambda x: x['downloads'], reverse=True): + if b['index'] < len(btns) and not btns[b['index']].is_disabled(): + best = b + break + + if not best: + print("\n >>> All matching books are already in download queue") + sys.stdout.flush() + return + + print(f"\n >>> DOWNLOADING: '{best['title']}' by {best['author']} ({best['downloads']} dl) [idx={best['index']}]") + sys.stdout.flush() + + # Click on the article h3 to open detail view (required for download to work) + h3s = page.query_selector_all('article h3') + if best['index'] < len(h3s): + h3s[best['index']].click() + page.wait_for_timeout(300) + else: + print(f" >> Warning: h3 index {best['index']} out of range ({len(h3s)} h3s)") + sys.stdout.flush() + + # Click the download button + btn = btns[best['index']] + btn.click() + + # Wait a moment for the click to register + page.wait_for_timeout(500) + print(f" >> Click sent") + sys.stdout.flush() + + # Check sidebar immediately + time.sleep(2) + try: + txt = page.inner_text('aside') + if 'IN PROGRESS' in txt: + print(f" >> Download started!") + sys.stdout.flush() + else: + print(f" >> Sidebar after 2s: {txt[:150]}") + sys.stdout.flush() + except: + print(f" >> Sidebar not visible after 2s") + sys.stdout.flush() + + # Wait for download to complete - check sidebar every 2 seconds, max 5 minutes + last_state = None + for attempt in range(150): + time.sleep(2) + try: + txt = page.inner_text('aside') + if 'IN PROGRESS' in txt: + state = 'downloading' + elif 'Complete' in txt or 'Saved' in txt or 'No activity' in txt: + state = 'done' + else: + state = 'other' + + if state != last_state: + if state == 'downloading': + print(f" >> Download started!") + elif state == 'done': + if 'No activity' in txt: + print(f" >> Download not started (no activity)") + else: + print(f" >> Download complete!") + last_state = state + + if state == 'done': + break + except: + if last_state is None: + print(f" >> Sidebar not visible yet") + last_state = 'no_sidebar' + sys.stdout.flush() + else: + if last_state != 'done': + print(" >> Timeout waiting for download") + sys.stdout.flush() + +def clear_completed_downloads(page): + """Clear completed downloads from previous sessions""" + page.evaluate(''' + () => { + for (const b of document.querySelectorAll('button')) { + if (b.textContent.includes('Clear Completed')) { b.click(); return; } + } + } + ''') + page.wait_for_timeout(2000) + +def check_calibre(title): + """Check if a book is already in the calibre database.""" + import subprocess + cmd = f'calibredb list --search title:="{title}"' + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + output = result.stdout.strip() + # calibredb output has header line "id title authors", count non-header lines + lines = [l.strip() for l in output.split('\n') if l.strip()] + # Remove header if present (first line starts with 'id') + if lines and lines[0].startswith('id'): + lines = lines[1:] + return len(lines) > 0 + +def load_books_from_json(json_str, source): + """Load books from a JSON string or file path.""" + try: + data = json.loads(json_str) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in {source}: {e}") + sys.exit(1) + + if not isinstance(data, list): + print(f"Error: JSON in {source} must be an array of book objects") + sys.exit(1) + + books = [] + for item in data: + if isinstance(item, dict): + if 'title' not in item or 'author' not in item: + print(f"Error: Each book object must have 'title' and 'author' keys: {item}") + sys.exit(1) + books.append((item['title'].strip(), item['author'].strip())) + elif isinstance(item, list) and len(item) >= 2: + books.append((str(item[0]).strip(), str(item[1]).strip())) + elif isinstance(item, str): + # Support plain strings as title-only (author will be searched separately) + books.append((item.strip(), "")) + else: + print(f"Warning: Skipping unrecognized book entry: {item}") + return books + +def main(): + parser = argparse.ArgumentParser(description='Download books from Shelfmark') + parser.add_argument('books', nargs='*', help='Books as JSON array of [title, author] pairs or {"title": ..., "author": ...} objects') + parser.add_argument('--file', '-f', help='Path to JSON file containing an array of books') + parser.add_argument('-c', '--check-calibre', action='store_true', help='Check calibre database before downloading, skip if already present') + args = parser.parse_args() + + # Load books from --file, positional JSON arg, or both + BOOKS = [] + + if args.file: + try: + with open(args.file, 'r') as f: + BOOKS = load_books_from_json(f.read(), f"file '{args.file}'") + except FileNotFoundError: + print(f"Error: File not found: {args.file}") + sys.exit(1) + except IOError as e: + print(f"Error reading file: {e}") + sys.exit(1) + + for book_arg in args.books: + BOOKS.extend(load_books_from_json(book_arg, 'command line argument')) + + if not BOOKS: + print("Error: No books specified. Use positional JSON args or --file ") + sys.exit(1) + + # Check calibre database if requested + if args.check_calibre: + print("\nChecking calibre database...") + sys.stdout.flush() + remaining = [] + for title, author in BOOKS: + if check_calibre(title): + print(f" >> '{title}' already in calibre, skipping") + sys.stdout.flush() + else: + remaining.append((title, author)) + BOOKS = remaining + if not BOOKS: + print("All books already in calibre, nothing to download.") + sys.stdout.flush() + return + + print(f"\n{'='*60}") + print(f"Downloading {len(BOOKS)} book(s) from Shelfmark") + print('='*60) + sys.stdout.flush() + + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page(viewport={'width': 1280, 'height': 900}) + page.goto(SHELFMARK_URL) + page.wait_for_load_state('networkidle') + + # Clear completed downloads at start of batch + print(f"\n{'='*60}") + print("Clearing completed downloads from previous sessions...") + print('='*60) + sys.stdout.flush() + clear_completed_downloads(page) + + for i, (title, author) in enumerate(BOOKS, 1): + print(f"\n{'='*60}") + print(f"[{i}/{len(BOOKS)}] '{title}' by {author}") + print('='*60) + sys.stdout.flush() + + download_book(page, title, author) + + browser.close() + print("\nDone!") + sys.stdout.flush() + +if __name__ == '__main__': + main() diff --git a/src/frontend/src/components/activity/activityMappers.ts b/src/frontend/src/components/activity/activityMappers.ts index 2f25a9e..01cccaa 100644 --- a/src/frontend/src/components/activity/activityMappers.ts +++ b/src/frontend/src/components/activity/activityMappers.ts @@ -1,4 +1,5 @@ import type { Book, RequestRecord, StatusData } from '../../types'; +import { getDownloadsCount } from '../../types'; import { STATUS_LABELS, isActiveDownloadStatus } from './activityStyles.js'; import type { ActivityItem, ActivityVisualStatus } from './activityTypes'; @@ -87,10 +88,13 @@ export const downloadToActivityItem = (book: Book, statusKey: DownloadStatusKey) typeof book.request_id === 'number' && Number.isFinite(book.request_id) && book.request_id > 0 ? Math.trunc(book.request_id) : undefined; + const downloadsCount = getDownloadsCount(book); + const downloadsText = downloadsCount != null ? `${downloadsCount.toLocaleString()} downloads` : undefined; const metaLine = joinMetaParts([ toOptionalText(book.format)?.toUpperCase(), toOptionalText(book.size), toOptionalText(book.source_display_name) || toSourceLabel(book.source), + downloadsText, toOptionalText(book.username), ]); const progress = getDownloadProgress(visualStatus, book.progress); @@ -115,6 +119,7 @@ export const downloadToActivityItem = (book: Book, statusKey: DownloadStatusKey) downloadRetryAvailable, downloadPath: toOptionalText(book.download_path), sizeRaw: toOptionalText(book.size), + downloads: downloadsCount ?? undefined, requestId, }; }; diff --git a/src/frontend/src/components/activity/activityTypes.ts b/src/frontend/src/components/activity/activityTypes.ts index 608aa09..62ab294 100644 --- a/src/frontend/src/components/activity/activityTypes.ts +++ b/src/frontend/src/components/activity/activityTypes.ts @@ -32,6 +32,7 @@ export interface ActivityItem { progress?: number; progressAnimated?: boolean; sizeRaw?: string; + downloads?: number; timestamp: number; username?: string; diff --git a/src/frontend/src/components/resultsViews/CardView.tsx b/src/frontend/src/components/resultsViews/CardView.tsx index ecc44bf..ef2f453 100644 --- a/src/frontend/src/components/resultsViews/CardView.tsx +++ b/src/frontend/src/components/resultsViews/CardView.tsx @@ -2,6 +2,7 @@ import { useState } from 'react'; import { useSearchMode } from '../../contexts/SearchModeContext'; import type { Book, ButtonStateInfo } from '../../types'; +import { getDownloadsCount } from '../../types'; import { bookSupportsTargets } from '../../utils/bookTargetLoader'; import { BookActionButton } from '../BookActionButton'; import { BookTargetDropdown } from '../BookTargetDropdown'; @@ -210,6 +211,7 @@ export const CardView = ({ {book.size} )} + {searchMode !== 'universal' && (() => { const d = getDownloadsCount(book); return d != null && d > 0 ? <> • {d.toLocaleString()} : null; })()} )} diff --git a/src/frontend/src/components/resultsViews/CompactView.tsx b/src/frontend/src/components/resultsViews/CompactView.tsx index 500315c..72eca0b 100644 --- a/src/frontend/src/components/resultsViews/CompactView.tsx +++ b/src/frontend/src/components/resultsViews/CompactView.tsx @@ -2,6 +2,7 @@ import { useState } from 'react'; import { useSearchMode } from '../../contexts/SearchModeContext'; import type { Book, ButtonStateInfo } from '../../types'; +import { getDownloadsCount } from '../../types'; import { bookSupportsTargets } from '../../utils/bookTargetLoader'; import { BookActionButton } from '../BookActionButton'; import { BookTargetDropdown } from '../BookTargetDropdown'; @@ -221,6 +222,7 @@ export const CompactView = ({ {book.size} )} + {(() => { const d = getDownloadsCount(book); return d != null && d > 0 ? <> • {d.toLocaleString()} : null; })()} )} diff --git a/src/frontend/src/components/resultsViews/ListView.tsx b/src/frontend/src/components/resultsViews/ListView.tsx index 6252acb..c1d90a3 100644 --- a/src/frontend/src/components/resultsViews/ListView.tsx +++ b/src/frontend/src/components/resultsViews/ListView.tsx @@ -2,6 +2,7 @@ import { useState } from 'react'; import { useSearchMode } from '../../contexts/SearchModeContext'; import type { Book, ButtonStateInfo, DisplayField } from '../../types'; +import { getDownloadsCount } from '../../types'; import { bookSupportsTargets } from '../../utils/bookTargetLoader'; import { getFormatColor, getLanguageColor } from '../../utils/colorMaps'; import { BookActionButton } from '../BookActionButton'; @@ -167,7 +168,7 @@ export const ListView = ({ className={`grid w-full items-center gap-2 sm:gap-x-0.5 sm:gap-y-1 ${ searchMode === 'universal' ? 'grid-cols-[auto_minmax(0,1fr)_auto_auto] sm:grid-cols-[auto_minmax(0,2fr)_minmax(50px,0.25fr)_minmax(90px,0.5fr)_minmax(90px,0.5fr)_minmax(120px,0.7fr)_auto]' - : 'grid-cols-[auto_minmax(0,1fr)_auto_auto] sm:grid-cols-[auto_minmax(0,2fr)_minmax(50px,0.25fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_auto]' + : 'grid-cols-[auto_minmax(0,1fr)_auto_auto] sm:grid-cols-[auto_minmax(0,2fr)_minmax(50px,0.25fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_minmax(60px,0.3fr)_minmax(70px,0.35fr)_auto]' }`} > {/* Thumbnail */} @@ -287,6 +288,13 @@ export const ListView = ({ )} + {/* Direct mode: Downloads - Desktop only */} + {searchMode !== 'universal' && ( +
+ {(() => { const d = getDownloadsCount(book); return d != null && d > 0 ? d.toLocaleString() : '-'; })()} +
+ )} + {/* Action Buttons */}
{bookSupportsTargets(book) && targetProvider && targetBookId && ( diff --git a/src/frontend/src/hooks/useActivity.ts b/src/frontend/src/hooks/useActivity.ts index 3fb1185..0f10805 100644 --- a/src/frontend/src/hooks/useActivity.ts +++ b/src/frontend/src/hooks/useActivity.ts @@ -103,6 +103,7 @@ const parseHistoryBook = (value: unknown): Book | null => { ...(typeof value.retry_available === 'boolean' ? { retry_available: value.retry_available } : {}), + ...(typeof value.downloads === 'number' ? { downloads: value.downloads } : {}), }; }; diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index 520a719..46dc8a6 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -540,6 +540,7 @@ export type DownloadReleasePayload = { format?: string; size?: string; size_bytes?: number; + downloads?: number; download_url?: string; protocol?: string; indexer?: string; diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts index 9ee4ea8..e67f988 100644 --- a/src/frontend/src/types/index.ts +++ b/src/frontend/src/types/index.ts @@ -56,6 +56,23 @@ export interface Book { titles_by_language?: Record; username?: string; retry_available?: boolean; + downloads?: number; + extra?: Record; +} + +/** + * Extract download count from a book's data. + * Checks both the direct `downloads` field and the `extra.downloads` fallback. + */ +export function getDownloadsCount(book: Book): number | null { + if (book.downloads != null && book.downloads > 0) { + return book.downloads; + } + const extraDownloads = book.extra?.downloads; + if (extraDownloads != null && typeof extraDownloads === 'number' && extraDownloads > 0) { + return extraDownloads; + } + return null; } // Status response types diff --git a/src/frontend/src/utils/bookTransformers.ts b/src/frontend/src/utils/bookTransformers.ts index f82c8f7..1279f73 100644 --- a/src/frontend/src/utils/bookTransformers.ts +++ b/src/frontend/src/utils/bookTransformers.ts @@ -62,6 +62,7 @@ interface SourceBackedBookData { language?: unknown; format?: unknown; size?: unknown; + downloads?: unknown; preview?: unknown; publisher?: unknown; info?: Record; @@ -157,6 +158,7 @@ const transformSourceBackedDataToBook = (data: SourceBackedBookData): Book => { language: toOptionalText(data.language), format: toOptionalText(data.format), size: toOptionalText(data.size), + downloads: typeof data.downloads === 'number' ? data.downloads : undefined, preview: toOptionalText(data.preview), publisher: toOptionalText(data.publisher), info: data.info, @@ -181,6 +183,7 @@ export function transformReleaseToDirectBook(release: Release): Book { language: release.language || extra.language, format: release.format, size: release.size, + downloads: extra.downloads, preview: extra.preview, publisher: extra.publisher, info: parseBookInfo(extra.info), diff --git a/src/frontend/src/utils/releasePayload.ts b/src/frontend/src/utils/releasePayload.ts index 9d4874b..da7a08c 100644 --- a/src/frontend/src/utils/releasePayload.ts +++ b/src/frontend/src/utils/releasePayload.ts @@ -30,6 +30,16 @@ export function buildReleaseDownloadPayload( format: release.format, size: release.size, size_bytes: release.size_bytes, + downloads: + typeof release.extra?.downloads === 'number' + ? release.extra.downloads + : typeof book.downloads === 'number' && book.downloads > 0 + ? book.downloads + : typeof book.extra?.downloads === 'number' + ? book.extra.downloads + : Array.isArray(book.info?.Downloads) && book.info.Downloads.length > 0 + ? Number(book.info.Downloads[0]) || 0 + : 0, download_url: release.download_url, protocol: release.protocol, indexer: release.indexer, diff --git a/src/frontend/src/utils/requestPayload.ts b/src/frontend/src/utils/requestPayload.ts index 6cb1715..0bf9432 100644 --- a/src/frontend/src/utils/requestPayload.ts +++ b/src/frontend/src/utils/requestPayload.ts @@ -95,23 +95,30 @@ export const buildReleaseDataFromMetadataRelease = ( }; }; -export const buildReleaseDataFromDirectBook = (book: Book) => { - const source = getBrowseSource(book); - return { - source, - source_id: book.id, - title: book.title || 'Unknown title', - author: book.author, - year: book.year, - format: book.format, - size: book.size, - preview: book.preview, - content_type: 'ebook' as const, - // Browsing a source directly means the book record IS the release record. - language: book.language, - search_mode: 'direct' as const, - }; -}; + export const buildReleaseDataFromDirectBook = (book: Book) => { + const source = getBrowseSource(book); + const downloads = + typeof book.downloads === 'number' && book.downloads > 0 + ? book.downloads + : Array.isArray(book.info?.Downloads) && book.info.Downloads.length > 0 + ? Number(book.info.Downloads[0]) || 0 + : 0; + return { + source, + source_id: book.id, + title: book.title || 'Unknown title', + author: book.author, + year: book.year, + format: book.format, + size: book.size, + downloads, + preview: book.preview, + content_type: 'ebook' as const, + // Browsing a source directly means the book record IS the release record. + language: book.language, + search_mode: 'direct' as const, + }; + }; export const buildDirectRequestPayload = (book: Book): CreateRequestPayload => { const bookData = buildDirectBookRequestData(book);