From 1aa9846f22a0f8ece2d02522e68bfe835a4be0b0 Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Wed, 21 Oct 2026 11:19:48 -0400 Subject: [PATCH] fix(download): trust a pinned AA mirror and repair the full e2e profile (#1518) The destination guard from the oceanofpdf/SSRF fix (f837506) trusts only hosts in the mirror lists and refuses any other host that resolves to a private address. The full e2e profile has failed on every PR since then: its AA mirror was mock-aa, but the slow-download links pointed at cf.mock.test (172.30.0.11). The guard refused the slow-download page before the internal Chrome bypasser ever started, so the download ended with "All download sources failed". Adding cf.mock.test as a second AA mirror would not help: AAMirrorSelector.rewrite would then move the gated link onto mock-aa, and the test would pass without Chrome solving anything. The profile now uses the gate as its only AA mirror. The cloudflare mock takes CF_GATED_PATHS, and the full profile challenges only /slow_download/, so search, detail and the final file pass through while the download still needs Chrome. The unused SLOW_DOWNLOAD_BASE indirection is gone. The same allowlist missed a supported setup: Primary Mirror (AA_BASE_URL) may pin a URL that is not in the mirror list, and network uses it as the active AA base. With it on a LAN address, every AA request was refused. The guard now trusts the pinned URL too (mirrors.get_pinned_aa_base_url), and has_aa_mirror_configuration reuses the same helper. download_url also ran the guard a second time outside its try block, which repeated the DNS lookup and could raise instead of returning None. The duplicate call is gone. --- shelfmark/core/mirrors.py | 17 +++++++--- shelfmark/download/http.py | 1 - shelfmark/download/network.py | 2 ++ .../src/components/AdvancedFilters.tsx | 5 +-- .../test_network_destination_guard.py | 21 ++++++++++++ tests/e2e/platform/README.md | 15 +++++---- tests/e2e/platform/docker-compose.e2e.yml | 17 +++++----- tests/e2e/platform/env/full.env | 24 ++++++++------ .../platform/mocks/fixtures/aa_detail.html | 2 +- tests/e2e/platform/mocks/mock_services.py | 32 ++++++++----------- .../suite/test_cluster_full_pipeline.py | 6 ++-- tests/frontend/test_search_menu_dismiss.py | 18 ++++------- 12 files changed, 92 insertions(+), 68 deletions(-) diff --git a/shelfmark/core/mirrors.py b/shelfmark/core/mirrors.py index 15fe783c..257637ec 100644 --- a/shelfmark/core/mirrors.py +++ b/shelfmark/core/mirrors.py @@ -107,17 +107,24 @@ def get_aa_mirrors() -> list[str]: return _normalize_configured_urls(config.get("AA_ADDITIONAL_URLS", "")) -def has_aa_mirror_configuration() -> bool: - """Return True when direct-download search has at least one AA base URL to use.""" - if get_aa_mirrors(): - return True +def get_pinned_aa_base_url() -> str | None: + """Return the AA URL the Primary Mirror setting pins, or None when it is Auto. + The pinned URL need not be in the mirror list. + """ configured_base_url = normalize_http_url( _string_config_value(_get_config().get("AA_BASE_URL", "auto")), default_scheme="https", allow_special=("auto",), ) - return bool(configured_base_url and configured_base_url != "auto") + if configured_base_url and configured_base_url != "auto": + return configured_base_url + return None + + +def has_aa_mirror_configuration() -> bool: + """Return True when direct-download search has at least one AA base URL to use.""" + return bool(get_aa_mirrors()) or get_pinned_aa_base_url() is not None def get_libgen_mirrors() -> list[str]: diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index 585428cd..c9e821da 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -890,7 +890,6 @@ def download_url( except network.UnsafeDestinationError as e: logger.warning("Blocked download: %s", e) return None - network.ensure_public_destination(current_url) # Build headers with optional referer headers = DOWNLOAD_HEADERS.copy() diff --git a/shelfmark/download/network.py b/shelfmark/download/network.py index 298b13e0..11427813 100644 --- a/shelfmark/download/network.py +++ b/shelfmark/download/network.py @@ -592,8 +592,10 @@ def _configured_mirror_hosts() -> set[str]: """Hostnames of every mirror the operator configured, which are trusted as given.""" from shelfmark.core import mirrors + pinned_aa_url = mirrors.get_pinned_aa_base_url() urls = [ *mirrors.get_aa_mirrors(), + *([pinned_aa_url] if pinned_aa_url else []), *mirrors.get_libgen_mirrors(), *mirrors.get_zlib_mirrors(), *mirrors.get_welib_mirrors(), diff --git a/src/frontend/src/components/AdvancedFilters.tsx b/src/frontend/src/components/AdvancedFilters.tsx index f9f7f733..87024b9f 100644 --- a/src/frontend/src/components/AdvancedFilters.tsx +++ b/src/frontend/src/components/AdvancedFilters.tsx @@ -131,10 +131,7 @@ export const AdvancedFilters = ({ const wrapperClassName = formClassName ? 'px-2' : 'px-2 lg:ml-16 lg:w-[calc(50vw+4rem)]'; const settingsForm = ( -
e.stopPropagation()} - > +
e.stopPropagation()}> {onClose && (