+ Posted: 01 Jan 2024
Format: M4B
File Size: 100 MBs
diff --git a/release-notes-v1.1.0.md b/release-notes-v1.1.0.md new file mode 100644 index 0000000..4015e05 --- /dev/null +++ b/release-notes-v1.1.0.md @@ -0,0 +1,47 @@ +## New Features + +### OIDC Authentication (#606, #612) +- **OIDC login** with PKCE flow, auto-discovery, and group-based admin mapping +- **Auto-provisioning** of OIDC users (configurable) and email-based account linking +- **Password fallback** when OIDC is enabled to prevent admin lockout +- Backwards compatible with all existing auth modes (no-auth, builtin, proxy, CWA) + +### Multi-User Support (#606, #612, #613) +- **User management** -create, edit, and delete users with admin/user roles +- **Per-user settings** -custom download destinations, BookLore library/path, email recipients, and `{User}` template variable +- **Per-user download visibility** -non-admins only see their own downloads + +### Multi-User Request System (#615, #617, #620) +- **Book request workflow** -users can request books with notes; admins review, approve, and fulfil requests +- **Policy-based configuration** -set download/request/block policies per content type or per source (e.g. allow direct downloads, set Prowlarr to request-only) +- **Per-user policy overrides** for tailored access control +- **New Activity Sidebar** -replaces downloads sidebar, combining active downloads with requests; sidebar can now be pinned +- Request retry support and admin-level request management + +### Notification Support (#618) +- **Apprise-based notifications** for request events and download completions +- Configurable globally or per user, with full customization of events and notification services +- Expanded activity cards with detailed request info and file management + +### AudiobookBay Release Source (#619, #621, #623) +- **New release source** -search AudiobookBay for audiobook torrents directly from the UI +- Results include title, language, format, and size +- Downloads via configured torrent client with audiobook-specific category support +- Configurable hostname, max search pages, and rate limit delay + +### Email Output Mode (#603, #604) +- **Email delivery** as an alternative output mode for downloaded books +- Per-user email recipient configuration + +## Improvements +- Admin-configurable visibility for self-settings options (delivery preferences, notifications) (#625) +- BookLore Bookdrop API destination support as an alternative to specific library selection (#625) +- Download path options for all torrent clients (#625) +- Add tag support to qBittorrent downloads (#610 by @dawescc) +- Add threading to file system operations for improved performance (#602) +- Enhanced custom scripting -JSON download info, more consistent activation, decoupled from staging (#591) +- Hardlink-before-move optimization for file transfers (#591) +- New BookLore API file formats (#591) +- Improved login cookie naming for reverse proxy compatibility (#591) +- Fix Transmission URL parsing (#591) +- Fix healthcheck starvation during large file processing (#591) diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index 82907ee..a31291d 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -179,6 +179,7 @@ def html_get_page( allow_bypasser_fallback: bool = True, include_response_url: bool = False, success_delay: float = 1.0, + session: Optional[requests.Session] = None, ) -> str | tuple[str, str]: """Fetch HTML content from a URL with retry mechanism. @@ -252,7 +253,8 @@ def html_get_page( while True: # Try with CF cookies/UA if available (from previous bypass) cookies = _apply_cf_bypass(current_url, headers) - response = requests.get( + request_client = session or requests + response = request_client.get( current_url, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, diff --git a/shelfmark/release_sources/audiobookbay/scraper.py b/shelfmark/release_sources/audiobookbay/scraper.py index 12841ec..6c8573a 100644 --- a/shelfmark/release_sources/audiobookbay/scraper.py +++ b/shelfmark/release_sources/audiobookbay/scraper.py @@ -5,6 +5,7 @@ import time from typing import List, Optional, Dict from urllib.parse import quote +import requests from bs4 import BeautifulSoup from shelfmark.core.config import config @@ -26,6 +27,7 @@ DEFAULT_TRACKERS = [ # ABB request behavior tuning SEARCH_PAGE_RETRY_ATTEMPTS = 2 DETAIL_PAGE_RETRY_ATTEMPTS = 2 +FIRST_PAGE_SESSION_REFRESH_ATTEMPTS = 2 # Legacy search parameter used by older ABB flows LEGACY_CATEGORY_QUERY = "undefined%2Cundefined" @@ -47,7 +49,11 @@ def _build_search_url( include_legacy_category: bool = False, ) -> str: """Build an ABB search URL, optionally including legacy category params.""" - url = f"https://{hostname}/page/{page}/?s={query_encoded}" + # Page 1 uses ABB's root search endpoint; pagination continues via /page/{n}/. + if page <= 1: + url = f"https://{hostname}/?s={query_encoded}" + else: + url = f"https://{hostname}/page/{page}/?s={query_encoded}" if include_legacy_category: return f"{url}&cat={LEGACY_CATEGORY_QUERY}" return url @@ -69,6 +75,37 @@ def _encode_search_query(query: str, exact_phrase: bool) -> str: return search_query.replace('"', "%22").replace(" ", "+") +def _normalize_result_url(url: str, hostname: str) -> str: + """Normalize ABB result URLs to absolute HTTPS URLs.""" + normalized_url = (url or "").strip() + if not normalized_url: + return "" + if normalized_url.startswith(("http://", "https://")): + return normalized_url + if normalized_url.startswith("//"): + return f"https:{normalized_url}" + if normalized_url.startswith("/"): + return f"https://{hostname}{normalized_url}" + return f"https://{hostname}/{normalized_url.lstrip('/')}" + + +def _bootstrap_abb_session( + hostname: str, + session: requests.Session, + retry_attempts: int, +) -> None: + """Warm up ABB session cookies (best effort).""" + downloader.html_get_page( + f"https://{hostname}/", + retry=retry_attempts, + use_bypasser=False, + allow_bypasser_fallback=False, + include_response_url=True, + success_delay=0, + session=session, + ) + + def search_audiobookbay( query: str, max_pages: int = 1, @@ -88,13 +125,24 @@ def search_audiobookbay( """ results = [] rate_limit_delay = config.get("ABB_RATE_LIMIT_DELAY", 1.0) + session = requests.Session() + + # Bootstrap ABB session cookie (PHPSESSID). ABB increasingly serves reliable + # search/detail pages only after session initialization, similar to browsers. + _bootstrap_abb_session(hostname, session, SEARCH_PAGE_RETRY_ATTEMPTS) # Iterate through pages for page in range(1, max_pages + 1): # Construct URL - use + for spaces (matching audiobookbay-automated implementation) # This avoids aggressive encoding that PHP-based sites may reject. query_encoded = _encode_search_query(query, exact_phrase) - primary_url = _build_search_url(hostname, page, query_encoded) + # ABB search expects the legacy category query parameter. + primary_url = _build_search_url( + hostname, + page, + query_encoded, + include_legacy_category=True, + ) try: # Reuse shared HTTP fetch logic (without bypasser) @@ -105,35 +153,41 @@ def search_audiobookbay( allow_bypasser_fallback=False, include_response_url=True, success_delay=0, + session=session, ) - # Legacy compatibility fallback: retry once with legacy category param if the - # first request fails/redirects unexpectedly. - if not page_html or _is_homepage_redirect(final_url, hostname): - legacy_url = _build_search_url( - hostname, - page, - query_encoded, - include_legacy_category=True, - ) - legacy_html, legacy_final_url = downloader.html_get_page( - legacy_url, - retry=SEARCH_PAGE_RETRY_ATTEMPTS, - use_bypasser=False, - allow_bypasser_fallback=False, - include_response_url=True, - success_delay=0, - ) - if legacy_html: - page_html = legacy_html - final_url = legacy_final_url - + was_home_redirect = _is_homepage_redirect(final_url, hostname) + + # ABB can intermittently fail even with a valid URL. + # If page 1 fails, refresh the session and retry the exact same URL. + if page == 1 and (not page_html or was_home_redirect): + for refresh_attempt in range(1, FIRST_PAGE_SESSION_REFRESH_ATTEMPTS + 1): + session = requests.Session() + _bootstrap_abb_session(hostname, session, SEARCH_PAGE_RETRY_ATTEMPTS) + page_html, final_url = downloader.html_get_page( + primary_url, + retry=SEARCH_PAGE_RETRY_ATTEMPTS, + use_bypasser=False, + allow_bypasser_fallback=False, + include_response_url=True, + success_delay=0, + session=session, + ) + was_home_redirect = _is_homepage_redirect(final_url, hostname) + if page_html and not was_home_redirect: + break + logger.debug( + "ABB page 1 session refresh %d/%d failed", + refresh_attempt, + FIRST_PAGE_SESSION_REFRESH_ATTEMPTS, + ) + if not page_html: logger.warning(f"Failed to fetch page {page}") break # Check if we were redirected to the homepage (search was rejected/blocked) - if _is_homepage_redirect(final_url, hostname): + if was_home_redirect: # Search was redirected to homepage - this means the search failed # This can happen due to geo-blocking, rate limiting, or invalid query format if page == 1: @@ -163,18 +217,15 @@ def search_audiobookbay( if not href: continue - if href.startswith('http'): - link = href - else: - link = f"https://{hostname}{href}" + link = _normalize_result_url(href, hostname) + if not link: + continue # Extract cover image (try .postContent .center img first, then fallback to any img) cover = None cover_elem = post.select_one('.postContent .center img') or post.select_one('img') if cover_elem: - cover = cover_elem.get('src', '') - if cover and not cover.startswith('http'): - cover = f"https://{hostname}{cover}" + cover = _normalize_result_url(cover_elem.get('src', ''), hostname) or None # Extract language from .postInfo language = None @@ -259,6 +310,9 @@ def extract_magnet_link( Magnet link, or None if extraction fails """ try: + session = requests.Session() + _bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS) + # Fetch detail page detail_html = downloader.html_get_page( details_url, @@ -266,7 +320,20 @@ def extract_magnet_link( use_bypasser=False, allow_bypasser_fallback=False, success_delay=0, + session=session, ) + + if not detail_html: + session = requests.Session() + _bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS) + detail_html = downloader.html_get_page( + details_url, + retry=DETAIL_PAGE_RETRY_ATTEMPTS, + use_bypasser=False, + allow_bypasser_fallback=False, + success_delay=0, + session=session, + ) if not detail_html: logger.warning("Failed to fetch details page") diff --git a/shelfmark/release_sources/audiobookbay/source.py b/shelfmark/release_sources/audiobookbay/source.py index 6e81145..751c138 100644 --- a/shelfmark/release_sources/audiobookbay/source.py +++ b/shelfmark/release_sources/audiobookbay/source.py @@ -170,42 +170,73 @@ class AudiobookBaySource(ReleaseSource): max_pages = config.get("ABB_PAGE_LIMIT", 1) exact_phrase = bool(config.get("ABB_EXACT_PHRASE", False)) - # Build search query from plan + # Build search query candidates from plan. + query_candidates: list[str] = [] if plan.manual_query: - query = plan.manual_query + query_candidates.append(plan.manual_query.strip()) elif plan.title_variants: - # Use first title variant with author variant = plan.title_variants[0] - query = f"{variant.title} {variant.author}".strip() - else: - query = book.title or "" - - if not query: + combined_query = f"{variant.title} {variant.author}".strip() + title_only_query = (variant.title or "").strip() + if combined_query: + query_candidates.append(combined_query) + if title_only_query and title_only_query.lower() != combined_query.lower(): + query_candidates.append(title_only_query) + elif book.title: + query_candidates.append(book.title.strip()) + + # Remove empty and duplicate queries while preserving order. + deduped_queries: list[str] = [] + seen_queries: set[str] = set() + for candidate in query_candidates: + normalized = candidate.strip() + if not normalized: + continue + key = normalized.lower() + if key in seen_queries: + continue + seen_queries.add(key) + deduped_queries.append(normalized) + + if not deduped_queries: logger.debug("No search query available") return [] - - # Convert to lowercase (matching audiobookbay-automated implementation) - query_lower = query.lower() - logger.info(f"Searching AudiobookBay for: {query_lower}") + + results = [] + query_lower = deduped_queries[0].lower() try: - # Search AudiobookBay - results = scraper.search_audiobookbay( - query=query_lower, - max_pages=max_pages, - hostname=hostname, - exact_phrase=exact_phrase, - ) + for index, query in enumerate(deduped_queries): + query_lower = query.lower() + logger.info(f"Searching AudiobookBay for: {query_lower}") - # For auto-generated queries, fallback to broad matching if exact phrase returns nothing. - if exact_phrase and not results and not plan.manual_query: - logger.info("No exact phrase results, retrying AudiobookBay search without quotes") + # Search AudiobookBay results = scraper.search_audiobookbay( query=query_lower, max_pages=max_pages, hostname=hostname, - exact_phrase=False, + exact_phrase=exact_phrase, ) + + # For auto-generated queries, fallback to broad matching if exact phrase returns nothing. + if exact_phrase and not results and not plan.manual_query: + logger.info("No exact phrase results, retrying AudiobookBay search without quotes") + results = scraper.search_audiobookbay( + query=query_lower, + max_pages=max_pages, + hostname=hostname, + exact_phrase=False, + ) + + if results: + break + + if index < len(deduped_queries) - 1: + logger.info( + "No AudiobookBay results for '%s', retrying with '%s'", + query_lower, + deduped_queries[index + 1].lower(), + ) # Extract query words for relevance checking query_words = set(word.lower() for word in query_lower.split() if len(word) > 2) diff --git a/tests/audiobookbay/test_scraper.py b/tests/audiobookbay/test_scraper.py index f47dd08..beb7b37 100644 --- a/tests/audiobookbay/test_scraper.py +++ b/tests/audiobookbay/test_scraper.py @@ -129,6 +129,7 @@ class TestSearchAudiobookbay: """Test pagination through multiple pages.""" mock_config_get.return_value = 0.0 # No delay for faster tests mock_html_get.side_effect = [ + (EMPTY_SEARCH_HTML, "https://audiobookbay.lu/"), # Session bootstrap (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/page/1/?s=test&cat=undefined%2Cundefined"), (EMPTY_SEARCH_HTML, "https://audiobookbay.lu/page/2/?s=test&cat=undefined%2Cundefined"), ] @@ -136,7 +137,46 @@ class TestSearchAudiobookbay: results = scraper.search_audiobookbay("test", max_pages=2, hostname="audiobookbay.lu") assert len(results) == 2 # Only from first page + assert mock_html_get.call_count == 3 + + @patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page') + @patch('shelfmark.release_sources.audiobookbay.scraper.config.get') + def test_search_audiobookbay_page_one_uses_root_search_endpoint(self, mock_config_get, mock_html_get): + """Test page 1 search uses ABB root endpoint instead of /page/1/.""" + mock_config_get.return_value = 0.0 + mock_html_get.return_value = ( + SAMPLE_SEARCH_HTML, + "https://audiobookbay.lu/?s=test", + ) + + results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu") + + assert len(results) == 2 + requested_url = mock_html_get.call_args.args[0] + assert requested_url.startswith("https://audiobookbay.lu/?s=test") + assert "/page/1/" not in requested_url + assert "cat=undefined%2Cundefined" in requested_url + + @patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page') + @patch('shelfmark.release_sources.audiobookbay.scraper.config.get') + def test_search_audiobookbay_bootstraps_and_reuses_session(self, mock_config_get, mock_html_get): + """Test ABB search initializes and reuses a request session for cookie continuity.""" + mock_config_get.return_value = 0.0 + mock_html_get.side_effect = [ + ("", "https://audiobookbay.lu/"), # Bootstrap attempt + (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined"), + ] + + results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu") + + assert len(results) == 2 assert mock_html_get.call_count == 2 + bootstrap_call = mock_html_get.call_args_list[0] + search_call = mock_html_get.call_args_list[1] + assert bootstrap_call.args[0] == "https://audiobookbay.lu/" + assert search_call.args[0].startswith("https://audiobookbay.lu/?s=test") + assert bootstrap_call.kwargs["session"] is not None + assert search_call.kwargs["session"] is bootstrap_call.kwargs["session"] @patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page') @patch('shelfmark.release_sources.audiobookbay.scraper.config.get') @@ -214,6 +254,36 @@ class TestSearchAudiobookbay: assert len(results) == 1 assert results[0]['link'] == "https://audiobookbay.lu/abss/relative-link/" + @patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page') + @patch('shelfmark.release_sources.audiobookbay.scraper.config.get') + def test_search_audiobookbay_protocol_relative_links(self, mock_config_get, mock_html_get): + """Test protocol-relative links are normalized without duplicating hostname.""" + mock_config_get.return_value = 0.0 + + html_with_protocol_relative_links = """ +
+ Posted: 01 Jan 2024
Format: M4B
File Size: 100 MBs