Fix HTTP request behavior (#626)

This commit is contained in:
Alex
2026-02-20 10:58:20 +00:00
committed by GitHub
parent 554f5fcbe7
commit 05115f7b41
6 changed files with 316 additions and 73 deletions
+47
View File
@@ -0,0 +1,47 @@
## New Features
### OIDC Authentication (#606, #612)
- **OIDC login** with PKCE flow, auto-discovery, and group-based admin mapping
- **Auto-provisioning** of OIDC users (configurable) and email-based account linking
- **Password fallback** when OIDC is enabled to prevent admin lockout
- Backwards compatible with all existing auth modes (no-auth, builtin, proxy, CWA)
### Multi-User Support (#606, #612, #613)
- **User management** -create, edit, and delete users with admin/user roles
- **Per-user settings** -custom download destinations, BookLore library/path, email recipients, and `{User}` template variable
- **Per-user download visibility** -non-admins only see their own downloads
### Multi-User Request System (#615, #617, #620)
- **Book request workflow** -users can request books with notes; admins review, approve, and fulfil requests
- **Policy-based configuration** -set download/request/block policies per content type or per source (e.g. allow direct downloads, set Prowlarr to request-only)
- **Per-user policy overrides** for tailored access control
- **New Activity Sidebar** -replaces downloads sidebar, combining active downloads with requests; sidebar can now be pinned
- Request retry support and admin-level request management
### Notification Support (#618)
- **Apprise-based notifications** for request events and download completions
- Configurable globally or per user, with full customization of events and notification services
- Expanded activity cards with detailed request info and file management
### AudiobookBay Release Source (#619, #621, #623)
- **New release source** -search AudiobookBay for audiobook torrents directly from the UI
- Results include title, language, format, and size
- Downloads via configured torrent client with audiobook-specific category support
- Configurable hostname, max search pages, and rate limit delay
### Email Output Mode (#603, #604)
- **Email delivery** as an alternative output mode for downloaded books
- Per-user email recipient configuration
## Improvements
- Admin-configurable visibility for self-settings options (delivery preferences, notifications) (#625)
- BookLore Bookdrop API destination support as an alternative to specific library selection (#625)
- Download path options for all torrent clients (#625)
- Add tag support to qBittorrent downloads (#610 by @dawescc)
- Add threading to file system operations for improved performance (#602)
- Enhanced custom scripting -JSON download info, more consistent activation, decoupled from staging (#591)
- Hardlink-before-move optimization for file transfers (#591)
- New BookLore API file formats (#591)
- Improved login cookie naming for reverse proxy compatibility (#591)
- Fix Transmission URL parsing (#591)
- Fix healthcheck starvation during large file processing (#591)
+3 -1
View File
@@ -179,6 +179,7 @@ def html_get_page(
allow_bypasser_fallback: bool = True,
include_response_url: bool = False,
success_delay: float = 1.0,
session: Optional[requests.Session] = None,
) -> str | tuple[str, str]:
"""Fetch HTML content from a URL with retry mechanism.
@@ -252,7 +253,8 @@ def html_get_page(
while True:
# Try with CF cookies/UA if available (from previous bypass)
cookies = _apply_cf_bypass(current_url, headers)
response = requests.get(
request_client = session or requests
response = request_client.get(
current_url,
proxies=get_proxies(current_url),
timeout=REQUEST_TIMEOUT,
@@ -5,6 +5,7 @@ import time
from typing import List, Optional, Dict
from urllib.parse import quote
import requests
from bs4 import BeautifulSoup
from shelfmark.core.config import config
@@ -26,6 +27,7 @@ DEFAULT_TRACKERS = [
# ABB request behavior tuning
SEARCH_PAGE_RETRY_ATTEMPTS = 2
DETAIL_PAGE_RETRY_ATTEMPTS = 2
FIRST_PAGE_SESSION_REFRESH_ATTEMPTS = 2
# Legacy search parameter used by older ABB flows
LEGACY_CATEGORY_QUERY = "undefined%2Cundefined"
@@ -47,7 +49,11 @@ def _build_search_url(
include_legacy_category: bool = False,
) -> str:
"""Build an ABB search URL, optionally including legacy category params."""
url = f"https://{hostname}/page/{page}/?s={query_encoded}"
# Page 1 uses ABB's root search endpoint; pagination continues via /page/{n}/.
if page <= 1:
url = f"https://{hostname}/?s={query_encoded}"
else:
url = f"https://{hostname}/page/{page}/?s={query_encoded}"
if include_legacy_category:
return f"{url}&cat={LEGACY_CATEGORY_QUERY}"
return url
@@ -69,6 +75,37 @@ def _encode_search_query(query: str, exact_phrase: bool) -> str:
return search_query.replace('"', "%22").replace(" ", "+")
def _normalize_result_url(url: str, hostname: str) -> str:
"""Normalize ABB result URLs to absolute HTTPS URLs."""
normalized_url = (url or "").strip()
if not normalized_url:
return ""
if normalized_url.startswith(("http://", "https://")):
return normalized_url
if normalized_url.startswith("//"):
return f"https:{normalized_url}"
if normalized_url.startswith("/"):
return f"https://{hostname}{normalized_url}"
return f"https://{hostname}/{normalized_url.lstrip('/')}"
def _bootstrap_abb_session(
hostname: str,
session: requests.Session,
retry_attempts: int,
) -> None:
"""Warm up ABB session cookies (best effort)."""
downloader.html_get_page(
f"https://{hostname}/",
retry=retry_attempts,
use_bypasser=False,
allow_bypasser_fallback=False,
include_response_url=True,
success_delay=0,
session=session,
)
def search_audiobookbay(
query: str,
max_pages: int = 1,
@@ -88,13 +125,24 @@ def search_audiobookbay(
"""
results = []
rate_limit_delay = config.get("ABB_RATE_LIMIT_DELAY", 1.0)
session = requests.Session()
# Bootstrap ABB session cookie (PHPSESSID). ABB increasingly serves reliable
# search/detail pages only after session initialization, similar to browsers.
_bootstrap_abb_session(hostname, session, SEARCH_PAGE_RETRY_ATTEMPTS)
# Iterate through pages
for page in range(1, max_pages + 1):
# Construct URL - use + for spaces (matching audiobookbay-automated implementation)
# This avoids aggressive encoding that PHP-based sites may reject.
query_encoded = _encode_search_query(query, exact_phrase)
primary_url = _build_search_url(hostname, page, query_encoded)
# ABB search expects the legacy category query parameter.
primary_url = _build_search_url(
hostname,
page,
query_encoded,
include_legacy_category=True,
)
try:
# Reuse shared HTTP fetch logic (without bypasser)
@@ -105,35 +153,41 @@ def search_audiobookbay(
allow_bypasser_fallback=False,
include_response_url=True,
success_delay=0,
session=session,
)
# Legacy compatibility fallback: retry once with legacy category param if the
# first request fails/redirects unexpectedly.
if not page_html or _is_homepage_redirect(final_url, hostname):
legacy_url = _build_search_url(
hostname,
page,
query_encoded,
include_legacy_category=True,
)
legacy_html, legacy_final_url = downloader.html_get_page(
legacy_url,
retry=SEARCH_PAGE_RETRY_ATTEMPTS,
use_bypasser=False,
allow_bypasser_fallback=False,
include_response_url=True,
success_delay=0,
)
if legacy_html:
page_html = legacy_html
final_url = legacy_final_url
was_home_redirect = _is_homepage_redirect(final_url, hostname)
# ABB can intermittently fail even with a valid URL.
# If page 1 fails, refresh the session and retry the exact same URL.
if page == 1 and (not page_html or was_home_redirect):
for refresh_attempt in range(1, FIRST_PAGE_SESSION_REFRESH_ATTEMPTS + 1):
session = requests.Session()
_bootstrap_abb_session(hostname, session, SEARCH_PAGE_RETRY_ATTEMPTS)
page_html, final_url = downloader.html_get_page(
primary_url,
retry=SEARCH_PAGE_RETRY_ATTEMPTS,
use_bypasser=False,
allow_bypasser_fallback=False,
include_response_url=True,
success_delay=0,
session=session,
)
was_home_redirect = _is_homepage_redirect(final_url, hostname)
if page_html and not was_home_redirect:
break
logger.debug(
"ABB page 1 session refresh %d/%d failed",
refresh_attempt,
FIRST_PAGE_SESSION_REFRESH_ATTEMPTS,
)
if not page_html:
logger.warning(f"Failed to fetch page {page}")
break
# Check if we were redirected to the homepage (search was rejected/blocked)
if _is_homepage_redirect(final_url, hostname):
if was_home_redirect:
# Search was redirected to homepage - this means the search failed
# This can happen due to geo-blocking, rate limiting, or invalid query format
if page == 1:
@@ -163,18 +217,15 @@ def search_audiobookbay(
if not href:
continue
if href.startswith('http'):
link = href
else:
link = f"https://{hostname}{href}"
link = _normalize_result_url(href, hostname)
if not link:
continue
# Extract cover image (try .postContent .center img first, then fallback to any img)
cover = None
cover_elem = post.select_one('.postContent .center img') or post.select_one('img')
if cover_elem:
cover = cover_elem.get('src', '')
if cover and not cover.startswith('http'):
cover = f"https://{hostname}{cover}"
cover = _normalize_result_url(cover_elem.get('src', ''), hostname) or None
# Extract language from .postInfo
language = None
@@ -259,6 +310,9 @@ def extract_magnet_link(
Magnet link, or None if extraction fails
"""
try:
session = requests.Session()
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
# Fetch detail page
detail_html = downloader.html_get_page(
details_url,
@@ -266,7 +320,20 @@ def extract_magnet_link(
use_bypasser=False,
allow_bypasser_fallback=False,
success_delay=0,
session=session,
)
if not detail_html:
session = requests.Session()
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
detail_html = downloader.html_get_page(
details_url,
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
use_bypasser=False,
allow_bypasser_fallback=False,
success_delay=0,
session=session,
)
if not detail_html:
logger.warning("Failed to fetch details page")
@@ -170,42 +170,73 @@ class AudiobookBaySource(ReleaseSource):
max_pages = config.get("ABB_PAGE_LIMIT", 1)
exact_phrase = bool(config.get("ABB_EXACT_PHRASE", False))
# Build search query from plan
# Build search query candidates from plan.
query_candidates: list[str] = []
if plan.manual_query:
query = plan.manual_query
query_candidates.append(plan.manual_query.strip())
elif plan.title_variants:
# Use first title variant with author
variant = plan.title_variants[0]
query = f"{variant.title} {variant.author}".strip()
else:
query = book.title or ""
if not query:
combined_query = f"{variant.title} {variant.author}".strip()
title_only_query = (variant.title or "").strip()
if combined_query:
query_candidates.append(combined_query)
if title_only_query and title_only_query.lower() != combined_query.lower():
query_candidates.append(title_only_query)
elif book.title:
query_candidates.append(book.title.strip())
# Remove empty and duplicate queries while preserving order.
deduped_queries: list[str] = []
seen_queries: set[str] = set()
for candidate in query_candidates:
normalized = candidate.strip()
if not normalized:
continue
key = normalized.lower()
if key in seen_queries:
continue
seen_queries.add(key)
deduped_queries.append(normalized)
if not deduped_queries:
logger.debug("No search query available")
return []
# Convert to lowercase (matching audiobookbay-automated implementation)
query_lower = query.lower()
logger.info(f"Searching AudiobookBay for: {query_lower}")
results = []
query_lower = deduped_queries[0].lower()
try:
# Search AudiobookBay
results = scraper.search_audiobookbay(
query=query_lower,
max_pages=max_pages,
hostname=hostname,
exact_phrase=exact_phrase,
)
for index, query in enumerate(deduped_queries):
query_lower = query.lower()
logger.info(f"Searching AudiobookBay for: {query_lower}")
# For auto-generated queries, fallback to broad matching if exact phrase returns nothing.
if exact_phrase and not results and not plan.manual_query:
logger.info("No exact phrase results, retrying AudiobookBay search without quotes")
# Search AudiobookBay
results = scraper.search_audiobookbay(
query=query_lower,
max_pages=max_pages,
hostname=hostname,
exact_phrase=False,
exact_phrase=exact_phrase,
)
# For auto-generated queries, fallback to broad matching if exact phrase returns nothing.
if exact_phrase and not results and not plan.manual_query:
logger.info("No exact phrase results, retrying AudiobookBay search without quotes")
results = scraper.search_audiobookbay(
query=query_lower,
max_pages=max_pages,
hostname=hostname,
exact_phrase=False,
)
if results:
break
if index < len(deduped_queries) - 1:
logger.info(
"No AudiobookBay results for '%s', retrying with '%s'",
query_lower,
deduped_queries[index + 1].lower(),
)
# Extract query words for relevance checking
query_words = set(word.lower() for word in query_lower.split() if len(word) > 2)
+108 -14
View File
@@ -129,6 +129,7 @@ class TestSearchAudiobookbay:
"""Test pagination through multiple pages."""
mock_config_get.return_value = 0.0 # No delay for faster tests
mock_html_get.side_effect = [
(EMPTY_SEARCH_HTML, "https://audiobookbay.lu/"), # Session bootstrap
(SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/page/1/?s=test&cat=undefined%2Cundefined"),
(EMPTY_SEARCH_HTML, "https://audiobookbay.lu/page/2/?s=test&cat=undefined%2Cundefined"),
]
@@ -136,7 +137,46 @@ class TestSearchAudiobookbay:
results = scraper.search_audiobookbay("test", max_pages=2, hostname="audiobookbay.lu")
assert len(results) == 2 # Only from first page
assert mock_html_get.call_count == 3
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
def test_search_audiobookbay_page_one_uses_root_search_endpoint(self, mock_config_get, mock_html_get):
"""Test page 1 search uses ABB root endpoint instead of /page/1/."""
mock_config_get.return_value = 0.0
mock_html_get.return_value = (
SAMPLE_SEARCH_HTML,
"https://audiobookbay.lu/?s=test",
)
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
assert len(results) == 2
requested_url = mock_html_get.call_args.args[0]
assert requested_url.startswith("https://audiobookbay.lu/?s=test")
assert "/page/1/" not in requested_url
assert "cat=undefined%2Cundefined" in requested_url
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
def test_search_audiobookbay_bootstraps_and_reuses_session(self, mock_config_get, mock_html_get):
"""Test ABB search initializes and reuses a request session for cookie continuity."""
mock_config_get.return_value = 0.0
mock_html_get.side_effect = [
("", "https://audiobookbay.lu/"), # Bootstrap attempt
(SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined"),
]
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
assert len(results) == 2
assert mock_html_get.call_count == 2
bootstrap_call = mock_html_get.call_args_list[0]
search_call = mock_html_get.call_args_list[1]
assert bootstrap_call.args[0] == "https://audiobookbay.lu/"
assert search_call.args[0].startswith("https://audiobookbay.lu/?s=test")
assert bootstrap_call.kwargs["session"] is not None
assert search_call.kwargs["session"] is bootstrap_call.kwargs["session"]
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
@@ -214,6 +254,36 @@ class TestSearchAudiobookbay:
assert len(results) == 1
assert results[0]['link'] == "https://audiobookbay.lu/abss/relative-link/"
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
def test_search_audiobookbay_protocol_relative_links(self, mock_config_get, mock_html_get):
"""Test protocol-relative links are normalized without duplicating hostname."""
mock_config_get.return_value = 0.0
html_with_protocol_relative_links = """
<div class="post">
<div class="postTitle"><h2><a href="//audiobookbay.lu/abss/protocol-relative/">Protocol Relative Book</a></h2></div>
<div class="postInfo">Language: English</div>
<div class="postContent">
<div class="center">
<img src="//audiobookbay.lu/wp-content/uploads/cover.jpg" alt="Cover" />
</div>
<p style="text-align:center;">Posted: 01 Jan 2024<br>Format: M4B<br>File Size: 100 MBs</p>
</div>
</div>
"""
mock_html_get.return_value = (
html_with_protocol_relative_links,
"https://audiobookbay.lu/?s=test",
)
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
assert len(results) == 1
assert results[0]["link"] == "https://audiobookbay.lu/abss/protocol-relative/"
assert results[0]["cover"] == "https://audiobookbay.lu/wp-content/uploads/cover.jpg"
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
def test_search_audiobookbay_exact_phrase_query(self, mock_config_get, mock_html_get):
@@ -234,26 +304,26 @@ class TestSearchAudiobookbay:
assert len(results) == 2
requested_url = mock_html_get.call_args.args[0]
assert "s=%22test+query%22" in requested_url
assert "cat=undefined%2Cundefined" not in requested_url
assert "cat=undefined%2Cundefined" in requested_url
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
@patch('shelfmark.release_sources.audiobookbay.scraper.config.get')
def test_search_audiobookbay_legacy_category_fallback(self, mock_config_get, mock_html_get):
"""Test fallback to legacy category query when primary search request fails."""
def test_search_audiobookbay_always_uses_legacy_category_query(self, mock_config_get, mock_html_get):
"""Test ABB search always includes legacy category query and does not fallback."""
mock_config_get.return_value = 0.0
mock_html_get.side_effect = [
("", "https://audiobookbay.lu/page/1/?s=test"), # Primary fetch failed
(SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/page/1/?s=test&cat=undefined%2Cundefined"),
]
mock_html_get.return_value = ("", "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined")
results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
assert len(results) == 2
assert mock_html_get.call_count == 2
first_url = mock_html_get.call_args_list[0].args[0]
second_url = mock_html_get.call_args_list[1].args[0]
assert "cat=undefined%2Cundefined" not in first_url
assert "cat=undefined%2Cundefined" in second_url
assert len(results) == 0
assert mock_html_get.call_count >= 2
search_urls = [
call.args[0]
for call in mock_html_get.call_args_list
if "?s=test" in call.args[0]
]
assert search_urls
assert all(url == "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined" for url in search_urls)
class TestExtractMagnetLink:
@@ -262,7 +332,10 @@ class TestExtractMagnetLink:
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
def test_extract_magnet_link_success(self, mock_html_get):
"""Test successful magnet link extraction."""
mock_html_get.return_value = SAMPLE_DETAIL_HTML
mock_html_get.side_effect = [
("", "https://audiobookbay.lu/"), # Bootstrap attempt
SAMPLE_DETAIL_HTML,
]
magnet_link = scraper.extract_magnet_link(
"https://audiobookbay.lu/abss/test-book/",
@@ -274,6 +347,27 @@ class TestExtractMagnetLink:
assert "ABC123DEF456GHI789JKL012MNO345PQR678STU" in magnet_link
assert "udp%3A//tracker.openbittorrent.com%3A80" in magnet_link
assert "http%3A//tracker.example.com%3A8080" in magnet_link
assert mock_html_get.call_count == 2
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
def test_extract_magnet_link_reuses_bootstrap_session(self, mock_html_get):
"""Test detail page fetch reuses the bootstrap session for ABB cookies."""
mock_html_get.side_effect = [
("", "https://audiobookbay.lu/"),
SAMPLE_DETAIL_HTML,
]
scraper.extract_magnet_link(
"https://audiobookbay.lu/abss/test-book/",
hostname="audiobookbay.lu"
)
assert mock_html_get.call_count == 2
bootstrap_call = mock_html_get.call_args_list[0]
detail_call = mock_html_get.call_args_list[1]
assert bootstrap_call.args[0] == "https://audiobookbay.lu/"
assert detail_call.args[0] == "https://audiobookbay.lu/abss/test-book/"
assert detail_call.kwargs["session"] is bootstrap_call.kwargs["session"]
@patch('shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page')
def test_extract_magnet_link_fallback(self, mock_html_get):
+6 -4
View File
@@ -185,7 +185,7 @@ class TestAudiobookBaySource:
@patch('shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay')
def test_search_query_generation_from_variants(self, mock_search):
"""Test search query generation from title variants."""
"""Test search query generation from title variants with title-only retry."""
mock_search.return_value = []
source = AudiobookBaySource()
@@ -205,9 +205,11 @@ class TestAudiobookBaySource:
source.search(book, plan, content_type="audiobook")
mock_search.assert_called_once()
call_args = mock_search.call_args
assert call_args.kwargs['query'] == "test book test author"
assert mock_search.call_count == 2
first_call = mock_search.call_args_list[0]
second_call = mock_search.call_args_list[1]
assert first_call.kwargs['query'] == "test book test author"
assert second_call.kwargs['query'] == "test book"
@patch('shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay')
def test_search_query_generation_from_title_only(self, mock_search):