mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 20:50:28 +01:00
Fix HTTP request behavior (#626)
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
## New Features
|
||||
|
||||
### OIDC Authentication (#606, #612)
|
||||
- **OIDC login** with PKCE flow, auto-discovery, and group-based admin mapping
|
||||
- **Auto-provisioning** of OIDC users (configurable) and email-based account linking
|
||||
- **Password fallback** when OIDC is enabled to prevent admin lockout
|
||||
- Backwards compatible with all existing auth modes (no-auth, builtin, proxy, CWA)
|
||||
|
||||
### Multi-User Support (#606, #612, #613)
|
||||
- **User management** -create, edit, and delete users with admin/user roles
|
||||
- **Per-user settings** -custom download destinations, BookLore library/path, email recipients, and `{User}` template variable
|
||||
- **Per-user download visibility** -non-admins only see their own downloads
|
||||
|
||||
### Multi-User Request System (#615, #617, #620)
|
||||
- **Book request workflow** -users can request books with notes; admins review, approve, and fulfil requests
|
||||
- **Policy-based configuration** -set download/request/block policies per content type or per source (e.g. allow direct downloads, set Prowlarr to request-only)
|
||||
- **Per-user policy overrides** for tailored access control
|
||||
- **New Activity Sidebar** -replaces downloads sidebar, combining active downloads with requests; sidebar can now be pinned
|
||||
- Request retry support and admin-level request management
|
||||
|
||||
### Notification Support (#618)
|
||||
- **Apprise-based notifications** for request events and download completions
|
||||
- Configurable globally or per user, with full customization of events and notification services
|
||||
- Expanded activity cards with detailed request info and file management
|
||||
|
||||
### AudiobookBay Release Source (#619, #621, #623)
|
||||
- **New release source** -search AudiobookBay for audiobook torrents directly from the UI
|
||||
- Results include title, language, format, and size
|
||||
- Downloads via configured torrent client with audiobook-specific category support
|
||||
- Configurable hostname, max search pages, and rate limit delay
|
||||
|
||||
### Email Output Mode (#603, #604)
|
||||
- **Email delivery** as an alternative output mode for downloaded books
|
||||
- Per-user email recipient configuration
|
||||
|
||||
## Improvements
|
||||
- Admin-configurable visibility for self-settings options (delivery preferences, notifications) (#625)
|
||||
- BookLore Bookdrop API destination support as an alternative to specific library selection (#625)
|
||||
- Download path options for all torrent clients (#625)
|
||||
- Add tag support to qBittorrent downloads (#610 by @dawescc)
|
||||
- Add threading to file system operations for improved performance (#602)
|
||||
- Enhanced custom scripting -JSON download info, more consistent activation, decoupled from staging (#591)
|
||||
- Hardlink-before-move optimization for file transfers (#591)
|
||||
- New BookLore API file formats (#591)
|
||||
- Improved login cookie naming for reverse proxy compatibility (#591)
|
||||
- Fix Transmission URL parsing (#591)
|
||||
- Fix healthcheck starvation during large file processing (#591)
|
||||
@@ -179,6 +179,7 @@ def html_get_page(
|
||||
allow_bypasser_fallback: bool = True,
|
||||
include_response_url: bool = False,
|
||||
success_delay: float = 1.0,
|
||||
session: Optional[requests.Session] = None,
|
||||
) -> str | tuple[str, str]:
|
||||
"""Fetch HTML content from a URL with retry mechanism.
|
||||
|
||||
@@ -252,7 +253,8 @@ def html_get_page(
|
||||
while True:
|
||||
# Try with CF cookies/UA if available (from previous bypass)
|
||||
cookies = _apply_cf_bypass(current_url, headers)
|
||||
response = requests.get(
|
||||
request_client = session or requests
|
||||
response = request_client.get(
|
||||
current_url,
|
||||
proxies=get_proxies(current_url),
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from typing import List, Optional, Dict
|
||||
from urllib.parse import quote
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from shelfmark.core.config import config
|
||||
@@ -26,6 +27,7 @@ DEFAULT_TRACKERS = [
|
||||
# ABB request behavior tuning
|
||||
SEARCH_PAGE_RETRY_ATTEMPTS = 2
|
||||
DETAIL_PAGE_RETRY_ATTEMPTS = 2
|
||||
FIRST_PAGE_SESSION_REFRESH_ATTEMPTS = 2
|
||||
|
||||
# Legacy search parameter used by older ABB flows
|
||||
LEGACY_CATEGORY_QUERY = "undefined%2Cundefined"
|
||||
@@ -47,7 +49,11 @@ def _build_search_url(
|
||||
include_legacy_category: bool = False,
|
||||
) -> str:
|
||||
"""Build an ABB search URL, optionally including legacy category params."""
|
||||
url = f"https://{hostname}/page/{page}/?s={query_encoded}"
|
||||
# Page 1 uses ABB's root search endpoint; pagination continues via /page/{n}/.
|
||||
if page <= 1:
|
||||
url = f"https://{hostname}/?s={query_encoded}"
|
||||
else:
|
||||
url = f"https://{hostname}/page/{page}/?s={query_encoded}"
|
||||
if include_legacy_category:
|
||||
return f"{url}&cat={LEGACY_CATEGORY_QUERY}"
|
||||
return url
|
||||
@@ -69,6 +75,37 @@ def _encode_search_query(query: str, exact_phrase: bool) -> str:
|
||||
return search_query.replace('"', "%22").replace(" ", "+")
|
||||
|
||||
|
||||
def _normalize_result_url(url: str, hostname: str) -> str:
|
||||
"""Normalize ABB result URLs to absolute HTTPS URLs."""
|
||||
normalized_url = (url or "").strip()
|
||||
if not normalized_url:
|
||||
return ""
|
||||
if normalized_url.startswith(("http://", "https://")):
|
||||
return normalized_url
|
||||
if normalized_url.startswith("//"):
|
||||
return f"https:{normalized_url}"
|
||||
if normalized_url.startswith("/"):
|
||||
return f"https://{hostname}{normalized_url}"
|
||||
return f"https://{hostname}/{normalized_url.lstrip('/')}"
|
||||
|
||||
|
||||
def _bootstrap_abb_session(
|
||||
hostname: str,
|
||||
session: requests.Session,
|
||||
retry_attempts: int,
|
||||
) -> None:
|
||||
"""Warm up ABB session cookies (best effort)."""
|
||||
downloader.html_get_page(
|
||||
f"https://{hostname}/",
|
||||
retry=retry_attempts,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
include_response_url=True,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
|
||||
|
||||
def search_audiobookbay(
|
||||
query: str,
|
||||
max_pages: int = 1,
|
||||
@@ -88,13 +125,24 @@ def search_audiobookbay(
|
||||
"""
|
||||
results = []
|
||||
rate_limit_delay = config.get("ABB_RATE_LIMIT_DELAY", 1.0)
|
||||
session = requests.Session()
|
||||
|
||||
# Bootstrap ABB session cookie (PHPSESSID). ABB increasingly serves reliable
|
||||
# search/detail pages only after session initialization, similar to browsers.
|
||||
_bootstrap_abb_session(hostname, session, SEARCH_PAGE_RETRY_ATTEMPTS)
|
||||
|
||||
# Iterate through pages
|
||||
for page in range(1, max_pages + 1):
|
||||
# Construct URL - use + for spaces (matching audiobookbay-automated implementation)
|
||||
# This avoids aggressive encoding that PHP-based sites may reject.
|
||||
query_encoded = _encode_search_query(query, exact_phrase)
|
||||
primary_url = _build_search_url(hostname, page, query_encoded)
|
||||
# ABB search expects the legacy category query parameter.
|
||||
primary_url = _build_search_url(
|
||||
hostname,
|
||||
page,
|
||||
query_encoded,
|
||||
include_legacy_category=True,
|
||||
)
|
||||
|
||||
try:
|
||||
# Reuse shared HTTP fetch logic (without bypasser)
|
||||
@@ -105,35 +153,41 @@ def search_audiobookbay(
|
||||
allow_bypasser_fallback=False,
|
||||
include_response_url=True,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
|
||||
# Legacy compatibility fallback: retry once with legacy category param if the
|
||||
# first request fails/redirects unexpectedly.
|
||||
if not page_html or _is_homepage_redirect(final_url, hostname):
|
||||
legacy_url = _build_search_url(
|
||||
hostname,
|
||||
page,
|
||||
query_encoded,
|
||||
include_legacy_category=True,
|
||||
)
|
||||
legacy_html, legacy_final_url = downloader.html_get_page(
|
||||
legacy_url,
|
||||
retry=SEARCH_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
include_response_url=True,
|
||||
success_delay=0,
|
||||
)
|
||||
if legacy_html:
|
||||
page_html = legacy_html
|
||||
final_url = legacy_final_url
|
||||
|
||||
was_home_redirect = _is_homepage_redirect(final_url, hostname)
|
||||
|
||||
# ABB can intermittently fail even with a valid URL.
|
||||
# If page 1 fails, refresh the session and retry the exact same URL.
|
||||
if page == 1 and (not page_html or was_home_redirect):
|
||||
for refresh_attempt in range(1, FIRST_PAGE_SESSION_REFRESH_ATTEMPTS + 1):
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, SEARCH_PAGE_RETRY_ATTEMPTS)
|
||||
page_html, final_url = downloader.html_get_page(
|
||||
primary_url,
|
||||
retry=SEARCH_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
include_response_url=True,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
was_home_redirect = _is_homepage_redirect(final_url, hostname)
|
||||
if page_html and not was_home_redirect:
|
||||
break
|
||||
logger.debug(
|
||||
"ABB page 1 session refresh %d/%d failed",
|
||||
refresh_attempt,
|
||||
FIRST_PAGE_SESSION_REFRESH_ATTEMPTS,
|
||||
)
|
||||
|
||||
if not page_html:
|
||||
logger.warning(f"Failed to fetch page {page}")
|
||||
break
|
||||
|
||||
# Check if we were redirected to the homepage (search was rejected/blocked)
|
||||
if _is_homepage_redirect(final_url, hostname):
|
||||
if was_home_redirect:
|
||||
# Search was redirected to homepage - this means the search failed
|
||||
# This can happen due to geo-blocking, rate limiting, or invalid query format
|
||||
if page == 1:
|
||||
@@ -163,18 +217,15 @@ def search_audiobookbay(
|
||||
if not href:
|
||||
continue
|
||||
|
||||
if href.startswith('http'):
|
||||
link = href
|
||||
else:
|
||||
link = f"https://{hostname}{href}"
|
||||
link = _normalize_result_url(href, hostname)
|
||||
if not link:
|
||||
continue
|
||||
|
||||
# Extract cover image (try .postContent .center img first, then fallback to any img)
|
||||
cover = None
|
||||
cover_elem = post.select_one('.postContent .center img') or post.select_one('img')
|
||||
if cover_elem:
|
||||
cover = cover_elem.get('src', '')
|
||||
if cover and not cover.startswith('http'):
|
||||
cover = f"https://{hostname}{cover}"
|
||||
cover = _normalize_result_url(cover_elem.get('src', ''), hostname) or None
|
||||
|
||||
# Extract language from .postInfo
|
||||
language = None
|
||||
@@ -259,6 +310,9 @@ def extract_magnet_link(
|
||||
Magnet link, or None if extraction fails
|
||||
"""
|
||||
try:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
|
||||
# Fetch detail page
|
||||
detail_html = downloader.html_get_page(
|
||||
details_url,
|
||||
@@ -266,7 +320,20 @@ def extract_magnet_link(
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
|
||||
if not detail_html:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
detail_html = downloader.html_get_page(
|
||||
details_url,
|
||||
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
|
||||
if not detail_html:
|
||||
logger.warning("Failed to fetch details page")
|
||||
|
||||
@@ -170,42 +170,73 @@ class AudiobookBaySource(ReleaseSource):
|
||||
max_pages = config.get("ABB_PAGE_LIMIT", 1)
|
||||
exact_phrase = bool(config.get("ABB_EXACT_PHRASE", False))
|
||||
|
||||
# Build search query from plan
|
||||
# Build search query candidates from plan.
|
||||
query_candidates: list[str] = []
|
||||
if plan.manual_query:
|
||||
query = plan.manual_query
|
||||
query_candidates.append(plan.manual_query.strip())
|
||||
elif plan.title_variants:
|
||||
# Use first title variant with author
|
||||
variant = plan.title_variants[0]
|
||||
query = f"{variant.title} {variant.author}".strip()
|
||||
else:
|
||||
query = book.title or ""
|
||||
|
||||
if not query:
|
||||
combined_query = f"{variant.title} {variant.author}".strip()
|
||||
title_only_query = (variant.title or "").strip()
|
||||
if combined_query:
|
||||
query_candidates.append(combined_query)
|
||||
if title_only_query and title_only_query.lower() != combined_query.lower():
|
||||
query_candidates.append(title_only_query)
|
||||
elif book.title:
|
||||
query_candidates.append(book.title.strip())
|
||||
|
||||
# Remove empty and duplicate queries while preserving order.
|
||||
deduped_queries: list[str] = []
|
||||
seen_queries: set[str] = set()
|
||||
for candidate in query_candidates:
|
||||
normalized = candidate.strip()
|
||||
if not normalized:
|
||||
continue
|
||||
key = normalized.lower()
|
||||
if key in seen_queries:
|
||||
continue
|
||||
seen_queries.add(key)
|
||||
deduped_queries.append(normalized)
|
||||
|
||||
if not deduped_queries:
|
||||
logger.debug("No search query available")
|
||||
return []
|
||||
|
||||
# Convert to lowercase (matching audiobookbay-automated implementation)
|
||||
query_lower = query.lower()
|
||||
logger.info(f"Searching AudiobookBay for: {query_lower}")
|
||||
|
||||
results = []
|
||||
query_lower = deduped_queries[0].lower()
|
||||
|
||||
try:
|
||||
# Search AudiobookBay
|
||||
results = scraper.search_audiobookbay(
|
||||
query=query_lower,
|
||||
max_pages=max_pages,
|
||||
hostname=hostname,
|
||||
exact_phrase=exact_phrase,
|
||||
)
|
||||
for index, query in enumerate(deduped_queries):
|
||||
query_lower = query.lower()
|
||||
logger.info(f"Searching AudiobookBay for: {query_lower}")
|
||||
|
||||
# For auto-generated queries, fallback to broad matching if exact phrase returns nothing.
|
||||
if exact_phrase and not results and not plan.manual_query:
|
||||
logger.info("No exact phrase results, retrying AudiobookBay search without quotes")
|
||||
# Search AudiobookBay
|
||||
results = scraper.search_audiobookbay(
|
||||
query=query_lower,
|
||||
max_pages=max_pages,
|
||||
hostname=hostname,
|
||||
exact_phrase=False,
|
||||
exact_phrase=exact_phrase,
|
||||
)
|
||||
|
||||
# For auto-generated queries, fallback to broad matching if exact phrase returns nothing.
|
||||
if exact_phrase and not results and not plan.manual_query:
|
||||
logger.info("No exact phrase results, retrying AudiobookBay search without quotes")
|
||||
results = scraper.search_audiobookbay(
|
||||
query=query_lower,
|
||||
max_pages=max_pages,
|
||||
hostname=hostname,
|
||||
exact_phrase=False,
|
||||
)
|
||||
|
||||
if results:
|
||||
break
|
||||
|
||||
if index < len(deduped_queries) - 1:
|
||||
logger.info(
|
||||
"No AudiobookBay results for '%s', retrying with '%s'",
|
||||
query_lower,
|
||||
deduped_queries[index + 1].lower(),
|
||||
)
|
||||
|
||||
# Extract query words for relevance checking
|
||||
query_words = set(word.lower() for word in query_lower.split() if len(word) > 2)
|
||||
|
||||
@@ -129,6 +129,7 @@ class TestSearchAudiobookbay:
|
||||
"""Test pagination through multiple pages."""
|
||||
mock_config_get.return_value = 0.0 # No delay for faster tests
|
||||
mock_html_get.side_effect = [
|
||||
(EMPTY_SEARCH_HTML, "https://audiobookbay.lu/"), # Session bootstrap
|
||||
(SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/page/1/?s=test&cat=undefined%2Cundefined"),
|
||||
(EMPTY_SEARCH_HTML, "https://audiobookbay.lu/page/2/?s=test&cat=undefined%2Cundefined"),
|
||||
]
|
||||
@@ -136,7 +137,46 @@ class TestSearchAudiobookbay:
|
||||
results = scraper.search_audiobookbay("test", max_pages=2, hostname="audiobookbay.lu")
|
||||
|
||||
assert len(results) == 2 # Only from first page
|
||||
assert mock_html_get.call_count == 3
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
|
||||
def test_search_audiobookbay_page_one_uses_root_search_endpoint(self, mock_config_get, mock_html_get):
|
||||
"""Test page 1 search uses ABB root endpoint instead of /page/1/."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.return_value = (
|
||||
SAMPLE_SEARCH_HTML,
|
||||
"https://audiobookbay.lu/?s=test",
|
||||
)
|
||||
|
||||
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
|
||||
|
||||
assert len(results) == 2
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert requested_url.startswith("https://audiobookbay.lu/?s=test")
|
||||
assert "/page/1/" not in requested_url
|
||||
assert "cat=undefined%2Cundefined" in requested_url
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
|
||||
def test_search_audiobookbay_bootstraps_and_reuses_session(self, mock_config_get, mock_html_get):
|
||||
"""Test ABB search initializes and reuses a request session for cookie continuity."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.side_effect = [
|
||||
("", "https://audiobookbay.lu/"), # Bootstrap attempt
|
||||
(SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined"),
|
||||
]
|
||||
|
||||
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
|
||||
|
||||
assert len(results) == 2
|
||||
assert mock_html_get.call_count == 2
|
||||
bootstrap_call = mock_html_get.call_args_list[0]
|
||||
search_call = mock_html_get.call_args_list[1]
|
||||
assert bootstrap_call.args[0] == "https://audiobookbay.lu/"
|
||||
assert search_call.args[0].startswith("https://audiobookbay.lu/?s=test")
|
||||
assert bootstrap_call.kwargs["session"] is not None
|
||||
assert search_call.kwargs["session"] is bootstrap_call.kwargs["session"]
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
|
||||
@@ -214,6 +254,36 @@ class TestSearchAudiobookbay:
|
||||
assert len(results) == 1
|
||||
assert results[0]['link'] == "https://audiobookbay.lu/abss/relative-link/"
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
|
||||
def test_search_audiobookbay_protocol_relative_links(self, mock_config_get, mock_html_get):
|
||||
"""Test protocol-relative links are normalized without duplicating hostname."""
|
||||
mock_config_get.return_value = 0.0
|
||||
|
||||
html_with_protocol_relative_links = """
|
||||
<div class="post">
|
||||
<div class="postTitle"><h2><a href="//audiobookbay.lu/abss/protocol-relative/">Protocol Relative Book</a></h2></div>
|
||||
<div class="postInfo">Language: English</div>
|
||||
<div class="postContent">
|
||||
<div class="center">
|
||||
<img src="//audiobookbay.lu/wp-content/uploads/cover.jpg" alt="Cover" />
|
||||
</div>
|
||||
<p style="text-align:center;">Posted: 01 Jan 2024<br>Format: M4B<br>File Size: 100 MBs</p>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
mock_html_get.return_value = (
|
||||
html_with_protocol_relative_links,
|
||||
"https://audiobookbay.lu/?s=test",
|
||||
)
|
||||
|
||||
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["link"] == "https://audiobookbay.lu/abss/protocol-relative/"
|
||||
assert results[0]["cover"] == "https://audiobookbay.lu/wp-content/uploads/cover.jpg"
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
|
||||
def test_search_audiobookbay_exact_phrase_query(self, mock_config_get, mock_html_get):
|
||||
@@ -234,26 +304,26 @@ class TestSearchAudiobookbay:
|
||||
assert len(results) == 2
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert "s=%22test+query%22" in requested_url
|
||||
assert "cat=undefined%2Cundefined" not in requested_url
|
||||
assert "cat=undefined%2Cundefined" in requested_url
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
|
||||
def test_search_audiobookbay_legacy_category_fallback(self, mock_config_get, mock_html_get):
|
||||
"""Test fallback to legacy category query when primary search request fails."""
|
||||
def test_search_audiobookbay_always_uses_legacy_category_query(self, mock_config_get, mock_html_get):
|
||||
"""Test ABB search always includes legacy category query and does not fallback."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.side_effect = [
|
||||
("", "https://audiobookbay.lu/page/1/?s=test"), # Primary fetch failed
|
||||
(SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/page/1/?s=test&cat=undefined%2Cundefined"),
|
||||
]
|
||||
mock_html_get.return_value = ("", "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined")
|
||||
|
||||
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
|
||||
|
||||
assert len(results) == 2
|
||||
assert mock_html_get.call_count == 2
|
||||
first_url = mock_html_get.call_args_list[0].args[0]
|
||||
second_url = mock_html_get.call_args_list[1].args[0]
|
||||
assert "cat=undefined%2Cundefined" not in first_url
|
||||
assert "cat=undefined%2Cundefined" in second_url
|
||||
assert len(results) == 0
|
||||
assert mock_html_get.call_count >= 2
|
||||
search_urls = [
|
||||
call.args[0]
|
||||
for call in mock_html_get.call_args_list
|
||||
if "?s=test" in call.args[0]
|
||||
]
|
||||
assert search_urls
|
||||
assert all(url == "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined" for url in search_urls)
|
||||
|
||||
|
||||
class TestExtractMagnetLink:
|
||||
@@ -262,7 +332,10 @@ class TestExtractMagnetLink:
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
def test_extract_magnet_link_success(self, mock_html_get):
|
||||
"""Test successful magnet link extraction."""
|
||||
mock_html_get.return_value = SAMPLE_DETAIL_HTML
|
||||
mock_html_get.side_effect = [
|
||||
("", "https://audiobookbay.lu/"), # Bootstrap attempt
|
||||
SAMPLE_DETAIL_HTML,
|
||||
]
|
||||
|
||||
magnet_link = scraper.extract_magnet_link(
|
||||
"https://audiobookbay.lu/abss/test-book/",
|
||||
@@ -274,6 +347,27 @@ class TestExtractMagnetLink:
|
||||
assert "ABC123DEF456GHI789JKL012MNO345PQR678STU" in magnet_link
|
||||
assert "udp%3A//tracker.openbittorrent.com%3A80" in magnet_link
|
||||
assert "http%3A//tracker.example.com%3A8080" in magnet_link
|
||||
assert mock_html_get.call_count == 2
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
def test_extract_magnet_link_reuses_bootstrap_session(self, mock_html_get):
|
||||
"""Test detail page fetch reuses the bootstrap session for ABB cookies."""
|
||||
mock_html_get.side_effect = [
|
||||
("", "https://audiobookbay.lu/"),
|
||||
SAMPLE_DETAIL_HTML,
|
||||
]
|
||||
|
||||
scraper.extract_magnet_link(
|
||||
"https://audiobookbay.lu/abss/test-book/",
|
||||
hostname="audiobookbay.lu"
|
||||
)
|
||||
|
||||
assert mock_html_get.call_count == 2
|
||||
bootstrap_call = mock_html_get.call_args_list[0]
|
||||
detail_call = mock_html_get.call_args_list[1]
|
||||
assert bootstrap_call.args[0] == "https://audiobookbay.lu/"
|
||||
assert detail_call.args[0] == "https://audiobookbay.lu/abss/test-book/"
|
||||
assert detail_call.kwargs["session"] is bootstrap_call.kwargs["session"]
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
|
||||
def test_extract_magnet_link_fallback(self, mock_html_get):
|
||||
|
||||
@@ -185,7 +185,7 @@ class TestAudiobookBaySource:
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay')
|
||||
def test_search_query_generation_from_variants(self, mock_search):
|
||||
"""Test search query generation from title variants."""
|
||||
"""Test search query generation from title variants with title-only retry."""
|
||||
mock_search.return_value = []
|
||||
|
||||
source = AudiobookBaySource()
|
||||
@@ -205,9 +205,11 @@ class TestAudiobookBaySource:
|
||||
|
||||
source.search(book, plan, content_type="audiobook")
|
||||
|
||||
mock_search.assert_called_once()
|
||||
call_args = mock_search.call_args
|
||||
assert call_args.kwargs['query'] == "test book test author"
|
||||
assert mock_search.call_count == 2
|
||||
first_call = mock_search.call_args_list[0]
|
||||
second_call = mock_search.call_args_list[1]
|
||||
assert first_call.kwargs['query'] == "test book test author"
|
||||
assert second_call.kwargs['query'] == "test book"
|
||||
|
||||
@patch('shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay')
|
||||
def test_search_query_generation_from_title_only(self, mock_search):
|
||||
|
||||
Reference in New Issue
Block a user