mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-17 22:20:30 +00:00
The destination guard from the oceanofpdf/SSRF fix (f837506) trusts only
hosts in
the mirror lists and refuses any other host that resolves to a private
address.
The full e2e profile has failed on every PR since then: its AA mirror
was mock-aa,
but the slow-download links pointed at cf.mock.test (172.30.0.11). The
guard refused
the slow-download page before the internal Chrome bypasser ever started,
so the
download ended with "All download sources failed". Adding cf.mock.test
as a second
AA mirror would not help: AAMirrorSelector.rewrite would then move the
gated link
onto mock-aa, and the test would pass without Chrome solving anything.
The profile
now uses the gate as its only AA mirror. The cloudflare mock takes
CF_GATED_PATHS,
and the full profile challenges only /slow_download/, so search, detail
and the
final file pass through while the download still needs Chrome. The
unused
SLOW_DOWNLOAD_BASE indirection is gone.
The same allowlist missed a supported setup: Primary Mirror
(AA_BASE_URL) may pin a
URL that is not in the mirror list, and network uses it as the active AA
base. With
it on a LAN address, every AA request was refused. The guard now trusts
the pinned
URL too (mirrors.get_pinned_aa_base_url), and
has_aa_mirror_configuration reuses
the same helper.
download_url also ran the guard a second time outside its try block,
which repeated
the DNS lookup and could raise instead of returning None. The duplicate
call is gone.
290 lines
8.5 KiB
Python
290 lines
8.5 KiB
Python
"""Centralized mirror configuration for direct-download sources."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
from shelfmark.core.utils import normalize_http_url
|
|
|
|
if TYPE_CHECKING:
|
|
from shelfmark.core.config import Config
|
|
|
|
_config_module = None
|
|
|
|
|
|
def _get_config() -> Config:
|
|
"""Lazy import of config module to avoid circular imports."""
|
|
global _config_module
|
|
if _config_module is None:
|
|
from shelfmark.core.config import config
|
|
|
|
_config_module = config
|
|
return _config_module
|
|
|
|
|
|
# Mirror URLs are intentionally user-supplied only.
|
|
DEFAULT_AA_MIRRORS: list[str] = []
|
|
DEFAULT_LIBGEN_MIRRORS: list[str] = []
|
|
DEFAULT_ZLIB_MIRRORS: list[str] = []
|
|
DEFAULT_WELIB_MIRRORS: list[str] = []
|
|
|
|
_DOWNLOAD_SOURCE_MIRROR_LABELS = {
|
|
"aa-fast": "Anna's Archive",
|
|
"aa-slow": "Anna's Archive",
|
|
"aa-slow-nowait": "Anna's Archive",
|
|
"aa-slow-wait": "Anna's Archive",
|
|
"libgen": "LibGen",
|
|
"zlib": "Z-Library",
|
|
"welib": "Welib",
|
|
"oceanofpdf": "OceanofPDF",
|
|
}
|
|
|
|
|
|
def _normalize_mirror_url(url: str) -> str:
|
|
return normalize_http_url(url, default_scheme="https")
|
|
|
|
|
|
def _string_config_value(value: object) -> str:
|
|
"""Normalize mirror-related config values to strings."""
|
|
return value if isinstance(value, str) else str(value or "")
|
|
|
|
|
|
def _normalize_configured_urls(value: object) -> list[str]:
|
|
"""Normalize list or comma-separated mirror config into unique URLs."""
|
|
if isinstance(value, list):
|
|
parts = value
|
|
elif isinstance(value, str) and value.strip():
|
|
parts = value.split(",")
|
|
else:
|
|
return []
|
|
|
|
normalized_urls: list[str] = []
|
|
for raw_url in parts:
|
|
normalized = _normalize_mirror_url(str(raw_url))
|
|
if normalized and normalized not in normalized_urls:
|
|
normalized_urls.append(normalized)
|
|
return normalized_urls
|
|
|
|
|
|
def _get_primary_mirror_url(key: str) -> str | None:
|
|
"""Return a configured primary mirror URL, if present."""
|
|
config = _get_config()
|
|
primary = _normalize_mirror_url(_string_config_value(config.get(key, "")))
|
|
return primary or None
|
|
|
|
|
|
def _build_primary_and_additional_mirrors(primary_key: str, additional_key: str) -> list[str]:
|
|
"""Build an ordered mirror list from primary + additional config values."""
|
|
config = _get_config()
|
|
mirrors: list[str] = []
|
|
|
|
primary = _get_primary_mirror_url(primary_key)
|
|
if primary:
|
|
mirrors.append(primary)
|
|
|
|
for url in _normalize_configured_urls(config.get(additional_key, "")):
|
|
if url not in mirrors:
|
|
mirrors.append(url)
|
|
|
|
return mirrors
|
|
|
|
|
|
def get_aa_mirrors() -> list[str]:
|
|
"""Get Anna's Archive mirrors.
|
|
|
|
Returns:
|
|
Ordered list of user-configured AA mirror URLs.
|
|
|
|
Notes:
|
|
- The list is used to populate the AA mirror dropdown in Settings.
|
|
- When AA_BASE_URL is set to 'auto', mirrors are tried in the order listed.
|
|
|
|
"""
|
|
config = _get_config()
|
|
configured_list = _normalize_configured_urls(config.get("AA_MIRROR_URLS", None))
|
|
if configured_list:
|
|
return configured_list
|
|
return _normalize_configured_urls(config.get("AA_ADDITIONAL_URLS", ""))
|
|
|
|
|
|
def get_pinned_aa_base_url() -> str | None:
|
|
"""Return the AA URL the Primary Mirror setting pins, or None when it is Auto.
|
|
|
|
The pinned URL need not be in the mirror list.
|
|
"""
|
|
configured_base_url = normalize_http_url(
|
|
_string_config_value(_get_config().get("AA_BASE_URL", "auto")),
|
|
default_scheme="https",
|
|
allow_special=("auto",),
|
|
)
|
|
if configured_base_url and configured_base_url != "auto":
|
|
return configured_base_url
|
|
return None
|
|
|
|
|
|
def has_aa_mirror_configuration() -> bool:
|
|
"""Return True when direct-download search has at least one AA base URL to use."""
|
|
return bool(get_aa_mirrors()) or get_pinned_aa_base_url() is not None
|
|
|
|
|
|
def get_libgen_mirrors() -> list[str]:
|
|
"""Get user-configured LibGen mirrors.
|
|
|
|
Returns:
|
|
List of LibGen mirror URLs.
|
|
|
|
"""
|
|
config = _get_config()
|
|
configured_list = _normalize_configured_urls(config.get("LIBGEN_MIRROR_URLS", None))
|
|
if configured_list:
|
|
return configured_list
|
|
return _normalize_configured_urls(config.get("LIBGEN_ADDITIONAL_URLS", ""))
|
|
|
|
|
|
def has_libgen_mirror_configuration() -> bool:
|
|
"""Return True when at least one LibGen mirror URL is configured."""
|
|
return bool(get_libgen_mirrors())
|
|
|
|
|
|
def get_zlib_mirrors() -> list[str]:
|
|
"""Get user-configured Z-Library mirrors, with primary first.
|
|
|
|
Returns:
|
|
List of Z-Library mirror URLs, primary first.
|
|
|
|
"""
|
|
config = _get_config()
|
|
configured_list = _normalize_configured_urls(config.get("ZLIB_MIRROR_URLS", None))
|
|
if configured_list:
|
|
return configured_list
|
|
return _build_primary_and_additional_mirrors("ZLIB_PRIMARY_URL", "ZLIB_ADDITIONAL_URLS")
|
|
|
|
|
|
def has_zlib_mirror_configuration() -> bool:
|
|
"""Return True when at least one Z-Library mirror URL is configured."""
|
|
return bool(get_zlib_mirrors())
|
|
|
|
|
|
def get_zlib_primary_url() -> str | None:
|
|
"""Get the primary Z-Library mirror URL.
|
|
|
|
Returns:
|
|
Primary Z-Library mirror URL, if configured.
|
|
|
|
"""
|
|
mirrors = get_zlib_mirrors()
|
|
return mirrors[0] if mirrors else None
|
|
|
|
|
|
def get_zlib_url_template() -> str | None:
|
|
"""Get Z-Library URL template using configured primary mirror.
|
|
|
|
Returns:
|
|
URL template with {md5} placeholder, if configured.
|
|
|
|
"""
|
|
primary = get_zlib_primary_url()
|
|
return f"{primary}/md5/{{md5}}" if primary else None
|
|
|
|
|
|
def get_welib_mirrors() -> list[str]:
|
|
"""Get user-configured Welib mirrors, with primary first.
|
|
|
|
Returns:
|
|
List of Welib mirror URLs, primary first.
|
|
|
|
"""
|
|
config = _get_config()
|
|
configured_list = _normalize_configured_urls(config.get("WELIB_MIRROR_URLS", None))
|
|
if configured_list:
|
|
return configured_list
|
|
return _build_primary_and_additional_mirrors("WELIB_PRIMARY_URL", "WELIB_ADDITIONAL_URLS")
|
|
|
|
|
|
def has_welib_mirror_configuration() -> bool:
|
|
"""Return True when at least one Welib mirror URL is configured."""
|
|
return bool(get_welib_mirrors())
|
|
|
|
|
|
def get_oceanofpdf_mirrors() -> list[str]:
|
|
"""Get user-configured OceanofPDF mirrors, in the order they were entered."""
|
|
return _normalize_configured_urls(_get_config().get("OCEANOFPDF_MIRROR_URLS", None))
|
|
|
|
|
|
def get_oceanofpdf_base_url() -> str | None:
|
|
"""Get the OceanofPDF mirror in use. Only the first configured one is used."""
|
|
mirrors = get_oceanofpdf_mirrors()
|
|
return mirrors[0] if mirrors else None
|
|
|
|
|
|
def has_oceanofpdf_mirror_configuration() -> bool:
|
|
"""Return True when at least one OceanofPDF mirror URL is configured."""
|
|
return bool(get_oceanofpdf_mirrors())
|
|
|
|
|
|
def has_download_source_mirror_configuration(source_id: str) -> bool:
|
|
"""Return True when the requested direct-download source has mirror config."""
|
|
if source_id in {"aa-fast", "aa-slow", "aa-slow-nowait", "aa-slow-wait"}:
|
|
return has_aa_mirror_configuration()
|
|
if source_id == "libgen":
|
|
return has_libgen_mirror_configuration()
|
|
if source_id == "zlib":
|
|
return has_zlib_mirror_configuration()
|
|
if source_id == "welib":
|
|
return has_welib_mirror_configuration()
|
|
if source_id == "oceanofpdf":
|
|
return has_oceanofpdf_mirror_configuration()
|
|
return False
|
|
|
|
|
|
def get_download_source_missing_mirror_reason(source_id: str) -> str | None:
|
|
"""Return a user-facing reason when a direct-download source has no mirror config."""
|
|
if has_download_source_mirror_configuration(source_id):
|
|
return None
|
|
|
|
label = _DOWNLOAD_SOURCE_MIRROR_LABELS.get(source_id)
|
|
if not label:
|
|
return None
|
|
|
|
return f"Add at least one {label} mirror in Mirrors"
|
|
|
|
|
|
def get_welib_primary_url() -> str | None:
|
|
"""Get the primary Welib mirror URL.
|
|
|
|
Returns:
|
|
Primary Welib mirror URL, if configured.
|
|
|
|
"""
|
|
mirrors = get_welib_mirrors()
|
|
return mirrors[0] if mirrors else None
|
|
|
|
|
|
def get_welib_url_template() -> str | None:
|
|
"""Get Welib URL template using configured primary mirror.
|
|
|
|
Returns:
|
|
URL template with {md5} placeholder, if configured.
|
|
|
|
"""
|
|
primary = get_welib_primary_url()
|
|
return f"{primary}/md5/{{md5}}" if primary else None
|
|
|
|
|
|
def get_zlib_cookie_domains() -> set:
|
|
"""Get set of Z-Library domains that need full cookie handling.
|
|
|
|
Used by internal_bypasser for CF bypass cookie management.
|
|
|
|
Returns:
|
|
Set of domain strings (without protocol).
|
|
|
|
"""
|
|
domains = set()
|
|
|
|
for url in get_zlib_mirrors():
|
|
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
|
|
domains.add(domain)
|
|
|
|
return domains
|