From f84fb082ad12d060d72cff0e650a278d958d9ea4 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 6 Feb 2026 10:04:31 +0000 Subject: [PATCH] Fix: AA mirror behavior (#589) - Refreshed available AA URLs - Fixed potential redirect from AA itself causing mirror cache errors - Added fully customizable mirror list in UI - Segmented rotation behavior to Auto mode only Fixes #588 --- docs/environment-variables.md | 22 ++- shelfmark/config/settings.py | 73 ++++++++- shelfmark/core/mirrors.py | 43 +++-- shelfmark/core/settings_registry.py | 116 +++++++++++++- shelfmark/download/http.py | 81 +++++++++- shelfmark/download/network.py | 38 ++++- .../src/components/OnboardingModal.tsx | 11 ++ .../components/settings/SettingsContent.tsx | 11 ++ .../settings/fields/TagListField.tsx | 149 ++++++++++++++++++ .../src/components/settings/fields/index.ts | 1 + src/frontend/src/types/settings.ts | 8 + .../config/test_mirror_settings_live_apply.py | 28 ++++ tests/config/test_mirror_settings_options.py | 11 ++ tests/core/test_mirrors_config.py | 41 +++++ tests/download/test_http_aa_redirects.py | 142 +++++++++++++++++ 15 files changed, 741 insertions(+), 34 deletions(-) create mode 100644 src/frontend/src/components/settings/fields/TagListField.tsx create mode 100644 tests/config/test_mirror_settings_live_apply.py create mode 100644 tests/config/test_mirror_settings_options.py create mode 100644 tests/core/test_mirrors_config.py create mode 100644 tests/download/test_http_aa_redirects.py diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 31a0d136..e4e3854a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -1138,8 +1138,9 @@ Timeout for external bypasser requests in milliseconds. | Variable | Description | Type | Default | |----------|-------------|------|---------| -| `AA_BASE_URL` | Select 'Auto' to probe mirrors on startup, or choose a specific mirror. | string (choice) | `auto` | -| `AA_ADDITIONAL_URLS` | Comma-separated list of custom mirror URLs. | string | _none_ | +| `AA_BASE_URL` | Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback). | string (choice) | `auto` | +| `AA_MIRROR_URLS` | Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation | string | `https://annas-archive.gl,https://annas-archive.li` | +| `AA_ADDITIONAL_URLS` | Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables. | string | _none_ | | `LIBGEN_ADDITIONAL_URLS` | Comma-separated list of custom LibGen mirrors to add to the defaults. | string | _none_ | | `ZLIB_PRIMARY_URL` | Z-Library mirror to use for downloads. | string (choice) | `https://z-lib.fm` | | `ZLIB_ADDITIONAL_URLS` | Comma-separated list of custom Z-Library mirror URLs. | string | _none_ | @@ -1153,17 +1154,26 @@ Timeout for external bypasser requests in milliseconds. **Primary Mirror** -Select 'Auto' to probe mirrors on startup, or choose a specific mirror. +Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback). - **Type:** string (choice) - **Default:** `auto` -- **Options:** `auto` (Auto (Recommended)), `https://annas-archive.se` (annas-archive.se), `https://annas-archive.li` (annas-archive.li), `https://annas-archive.pm` (annas-archive.pm), `https://annas-archive.in` (annas-archive.in) +- **Options:** `auto` (Auto (Recommended)), `https://annas-archive.gl` (annas-archive.gl), `https://annas-archive.li` (annas-archive.li) + +#### `AA_MIRROR_URLS` + +**Mirrors** + +Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation + +- **Type:** string +- **Default:** `https://annas-archive.gl,https://annas-archive.li` #### `AA_ADDITIONAL_URLS` -**Additional Mirrors** +**Additional Mirrors (Legacy)** -Comma-separated list of custom mirror URLs. +Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables. - **Type:** string - **Default:** _none_ diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index 1e633bfc..eaf6e00d 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -125,6 +125,7 @@ from shelfmark.core.settings_registry import ( CheckboxField, SelectField, MultiSelectField, + TagListField, OrderableListField, TableField, HeadingField, @@ -225,16 +226,30 @@ _LANGUAGE_OPTIONS = [{"value": lang["code"], "label": lang["language"]} for lang def _get_aa_base_url_options(): """Build AA URL options dynamically, including additional mirrors from config.""" from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS, get_aa_mirrors + from shelfmark.core.config import config + from shelfmark.core.utils import normalize_http_url options = [{"value": "auto", "label": "Auto (Recommended)"}] # Get all mirrors (defaults + custom) all_mirrors = get_aa_mirrors() + # If AA_BASE_URL is configured to a custom mirror that isn't present in the + # defaults/additional list, include it so the UI can display the active value. + configured_url = normalize_http_url( + config.get("AA_BASE_URL", "auto"), + default_scheme="https", + allow_special=("auto",), + ) + if configured_url and configured_url != "auto" and configured_url not in all_mirrors: + all_mirrors = [configured_url] + all_mirrors + for url in all_mirrors: domain = url.replace("https://", "").replace("http://", "") is_custom = url not in DEFAULT_AA_MIRRORS label = f"{domain} (custom)" if is_custom else domain + if configured_url and url == configured_url and is_custom: + label = f"{domain} (configured)" options.append({"value": url, "label": label}) return options @@ -1103,30 +1118,76 @@ def cloudflare_bypass_settings(): ), ] +def _on_save_mirrors(values: Dict[str, Any]) -> Dict[str, Any]: + """Normalize mirror list settings before persisting.""" + from shelfmark.core.logger import setup_logger + from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS + from shelfmark.core.utils import normalize_http_url + + logger = setup_logger(__name__) + + raw_urls = values.get("AA_MIRROR_URLS") + if raw_urls is None: + return {"error": False, "values": values} + + if isinstance(raw_urls, str): + parts = [p.strip() for p in raw_urls.split(",") if p.strip()] + elif isinstance(raw_urls, list): + parts = [str(p).strip() for p in raw_urls if str(p).strip()] + else: + parts = [] + + normalized: list[str] = [] + for url in parts: + if url.lower() == "auto": + continue + norm = normalize_http_url(url, default_scheme="https") + if norm and norm not in normalized: + normalized.append(norm) + + if not normalized: + logger.warning("AA_MIRROR_URLS saved empty/invalid; falling back to defaults") + normalized = [normalize_http_url(url, default_scheme="https") for url in DEFAULT_AA_MIRRORS] + normalized = [url for url in normalized if url] + + values["AA_MIRROR_URLS"] = normalized + return {"error": False, "values": values} + +# Register the on_save handler for this tab +register_on_save("mirrors", _on_save_mirrors) + @register_settings("mirrors", "Mirrors", icon="globe", order=23, group="direct_download") def mirror_settings(): """Configure download source mirrors.""" - from shelfmark.core.mirrors import DEFAULT_ZLIB_MIRRORS, DEFAULT_WELIB_MIRRORS + from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS, DEFAULT_ZLIB_MIRRORS, DEFAULT_WELIB_MIRRORS return [ # === PRIMARY SOURCE === HeadingField( key="aa_mirrors_heading", - title="Primary Source", - description="Primary mirror with auto-probe on startup. Additional mirrors used as fallback.", + title="Anna's Archive", + description="Choose a primary mirror, or use Auto to try mirrors from your list below. The mirror list controls which options appear in the dropdown and the order used in Auto mode.", ), SelectField( key="AA_BASE_URL", label="Primary Mirror", - description="Select 'Auto' to probe mirrors on startup, or choose a specific mirror.", + description="Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback).", options=_get_aa_base_url_options, default="auto", ), + TagListField( + key="AA_MIRROR_URLS", + label="Mirrors", + description="Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation", + placeholder="https://annas-archive.gl", + default=DEFAULT_AA_MIRRORS, + ), TextField( key="AA_ADDITIONAL_URLS", - label="Additional Mirrors", - description="Comma-separated list of custom mirror URLs.", + label="Additional Mirrors (Legacy)", + description="Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables.", + show_when={"field": "AA_ADDITIONAL_URLS", "notEmpty": True}, ), # === LIBGEN === diff --git a/shelfmark/core/mirrors.py b/shelfmark/core/mirrors.py index ac98822c..8a2223ea 100644 --- a/shelfmark/core/mirrors.py +++ b/shelfmark/core/mirrors.py @@ -19,10 +19,8 @@ def _get_config(): # Default mirror lists (hardcoded fallbacks) DEFAULT_AA_MIRRORS = [ - "https://annas-archive.se", + "https://annas-archive.gl", "https://annas-archive.li", - "https://annas-archive.pm", - "https://annas-archive.in", ] DEFAULT_LIBGEN_MIRRORS = [ @@ -52,22 +50,47 @@ def _normalize_mirror_url(url: str) -> str: def get_aa_mirrors() -> List[str]: """ - Get Anna's Archive mirrors from config + defaults. + Get Anna's Archive mirrors. Returns: - List of AA mirror URLs, starting with defaults then custom additions. + Ordered list of AA mirror URLs. + + If AA_MIRROR_URLS is configured, it is treated as the full list. + Otherwise, defaults are used and AA_ADDITIONAL_URLS (legacy) is appended. + + Notes: + - The list is used to populate the AA mirror dropdown in Settings. + - When AA_BASE_URL is set to 'auto', mirrors are tried in the order listed. """ - mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS] - mirrors = [url for url in mirrors if url] config = _get_config() - additional = config.get("AA_ADDITIONAL_URLS", "") - if additional: - for url in additional.split(","): + mirrors: list[str] = [] + + configured_list = config.get("AA_MIRROR_URLS", None) + if isinstance(configured_list, list): + for url in configured_list: + normalized = _normalize_mirror_url(str(url)) + if normalized and normalized not in mirrors: + mirrors.append(normalized) + elif isinstance(configured_list, str) and configured_list.strip(): + # Allow comma-separated env/manual configs. + for url in configured_list.split(","): normalized = _normalize_mirror_url(url) if normalized and normalized not in mirrors: mirrors.append(normalized) + if not mirrors: + mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS] + mirrors = [url for url in mirrors if url] + + # Backwards-compatible append-only behavior for legacy configs/env. + additional = config.get("AA_ADDITIONAL_URLS", "") + if additional: + for url in additional.split(","): + normalized = _normalize_mirror_url(url) + if normalized and normalized not in mirrors: + mirrors.append(normalized) + return mirrors diff --git a/shelfmark/core/settings_registry.py b/shelfmark/core/settings_registry.py index 282c37de..5167ddb3 100644 --- a/shelfmark/core/settings_registry.py +++ b/shelfmark/core/settings_registry.py @@ -83,6 +83,13 @@ class MultiSelectField(FieldBase): variant: str = "pills" # "pills" (default) or "dropdown" for checkbox dropdown style +@dataclass +class TagListField(FieldBase): + """Editable list of free-form string values (tag/chip input).""" + placeholder: str = "" + default: List[str] = field(default_factory=list) + + @dataclass class OrderableListField(FieldBase): # Options can be a list or a callable that returns a list (for lazy evaluation) @@ -145,7 +152,19 @@ class HeadingField: # Type alias for all field types -SettingsField = Union[TextField, PasswordField, NumberField, CheckboxField, SelectField, MultiSelectField, OrderableListField, ActionButton, HeadingField] +SettingsField = Union[ + TextField, + PasswordField, + NumberField, + CheckboxField, + SelectField, + MultiSelectField, + TagListField, + OrderableListField, + TableField, + ActionButton, + HeadingField, +] @dataclass @@ -398,6 +417,61 @@ def sync_env_to_config() -> None: logger.debug(f"Synced {len(values_to_sync)} ENV values to {tab.name} config: {list(values_to_sync.keys())}") migrate_legacy_settings() + migrate_mirror_settings() + + +def migrate_mirror_settings() -> None: + """ + Migrate legacy AA mirror config into the new editable mirror list setting. + + Legacy: + - AA_ADDITIONAL_URLS: comma-separated extra URLs appended to defaults + + New: + - AA_MIRROR_URLS: full ordered list of available mirrors (used for Auto mode and for Settings options) + """ + mirrors_config = load_config_file("mirrors") + + from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS + from shelfmark.core.utils import normalize_http_url + + raw_list = mirrors_config.get("AA_MIRROR_URLS") + raw_additional = mirrors_config.get("AA_ADDITIONAL_URLS", "") + + def _normalize_list(values: list[str]) -> list[str]: + out: list[str] = [] + for item in values: + if str(item).strip().lower() == "auto": + continue + norm = normalize_http_url(str(item), default_scheme="https") + if norm and norm not in out: + out.append(norm) + return out + + # If already a proper list, just ensure it's non-empty. + if isinstance(raw_list, list): + normalized = _normalize_list([str(v) for v in raw_list]) + if normalized: + return + save_config_file("mirrors", {"AA_MIRROR_URLS": _normalize_list(DEFAULT_AA_MIRRORS)}) + return + + # If saved as a string, convert to list. + if isinstance(raw_list, str) and raw_list.strip(): + parts = [p.strip() for p in raw_list.split(",") if p.strip()] + normalized = _normalize_list(parts) + if normalized: + save_config_file("mirrors", {"AA_MIRROR_URLS": normalized}) + return + save_config_file("mirrors", {"AA_MIRROR_URLS": _normalize_list(DEFAULT_AA_MIRRORS)}) + return + + # If there's legacy additional mirrors, seed the full list so the UI reflects reality. + if isinstance(raw_additional, str) and raw_additional.strip(): + additional_parts = [p.strip() for p in raw_additional.split(",") if p.strip()] + combined = _normalize_list(DEFAULT_AA_MIRRORS + additional_parts) + if combined: + save_config_file("mirrors", {"AA_MIRROR_URLS": combined}) def migrate_legacy_settings() -> None: @@ -542,6 +616,8 @@ def _parse_env_value(value: str, field: SettingsField) -> Any: return field.default elif isinstance(field, MultiSelectField): return [v.strip() for v in value.split(',') if v.strip()] + elif isinstance(field, TagListField): + return [v.strip() for v in value.split(',') if v.strip()] elif isinstance(field, OrderableListField): # Parse JSON array: [{"id": "...", "enabled": true}, ...] try: @@ -643,6 +719,8 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T options = field.options() if callable(field.options) else field.options result["options"] = options result["variant"] = field.variant + elif isinstance(field, TagListField): + result["placeholder"] = field.placeholder elif isinstance(field, OrderableListField): # Support callable options for lazy evaluation (avoids circular imports) options = field.options() if callable(field.options) else field.options @@ -674,6 +752,16 @@ def serialize_field(field: SettingsField, tab_name: str, include_value: bool = T value = [v.strip() for v in value.split(",") if v.strip()] else: value = [] + elif isinstance(field, TagListField): + if value is None: + value = [] + elif isinstance(value, list): + value = [str(v) for v in value] + elif isinstance(value, str): + # Support legacy/manual configs where lists were saved as comma-separated strings. + value = [v.strip() for v in value.split(",") if v.strip()] + else: + value = [] elif isinstance(field, TableField): if value is None: value = [] @@ -801,6 +889,23 @@ def _apply_dns_settings(config) -> None: except Exception as e: logger.warning(f"Failed to apply DNS settings: {e}") +def _apply_aa_mirror_settings(config) -> None: + """ + Apply AA mirror settings changes to the network module. + + This ensures AA_BASE_URL / AA_ADDITIONAL_URLS changes take effect immediately + without requiring a container restart. + """ + try: + from shelfmark.download import network + + # Reload AA mirror list and configured base URL from refreshed config. + network.init_aa(force=True) + except ImportError: + pass # Network module not available + except Exception as e: + logger.warning(f"Failed to apply AA mirror settings: {e}") + def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]: tab = get_settings_tab(tab_name) @@ -885,6 +990,15 @@ def update_settings(tab_name: str, values: Dict[str, Any]) -> Dict[str, Any]: ): _apply_dns_settings(config_obj) + # Apply AA mirror settings changes live (mirrors tab) + aa_keys = {"AA_BASE_URL", "AA_MIRROR_URLS", "AA_ADDITIONAL_URLS"} + if ( + config_obj is not None + and tab_name == "mirrors" + and aa_keys.intersection(values_to_save.keys()) + ): + _apply_aa_mirror_settings(config_obj) + # Sync metadata provider selection when a provider's enabled state changes tab = get_settings_tab(tab_name) if tab and tab.group == "metadata_providers": diff --git a/shelfmark/download/http.py b/shelfmark/download/http.py index 11d2caaf..2e03df74 100644 --- a/shelfmark/download/http.py +++ b/shelfmark/download/http.py @@ -5,7 +5,7 @@ import time from io import BytesIO from threading import Event, Thread from typing import Callable, Optional -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin import requests from tqdm import tqdm @@ -228,13 +228,78 @@ def html_get_page( heartbeat_thread.join(timeout=1) logger.debug(f"GET: {current_url}") - # Try with CF cookies/UA if available (from previous bypass) - headers = {} - cookies = _apply_cf_bypass(current_url, headers) - response = requests.get(current_url, proxies=get_proxies(current_url), timeout=REQUEST_TIMEOUT, cookies=cookies, headers=headers) - response.raise_for_status() - time.sleep(1) - return response.text + + # Use a browser-like UA by default (AA can behave differently for python-requests UA). + headers = {"User-Agent": DOWNLOAD_HEADERS["User-Agent"]} + + # AA mirrors sometimes redirect to other (seized/dead) mirror domains. If we let + # requests follow those redirects, the request fails on DNS and we rotate away + # from an otherwise working mirror. Handle AA redirects manually instead. + is_aa_url = network.should_rotate_dns_for_url(current_url) + allow_redirects = not is_aa_url + + redirects_followed = 0 + while True: + # Try with CF cookies/UA if available (from previous bypass) + cookies = _apply_cf_bypass(current_url, headers) + response = requests.get( + current_url, + proxies=get_proxies(current_url), + timeout=REQUEST_TIMEOUT, + cookies=cookies, + headers=headers, + allow_redirects=allow_redirects, + ) + + if is_aa_url and response.is_redirect: + location = response.headers.get("Location", "") + if not location: + raise requests.exceptions.TooManyRedirects(f"Redirect with no Location header: {current_url}") + + redirect_url = urljoin(current_url, location) + current_host = urlparse(current_url).hostname or "" + redirect_host = urlparse(redirect_url).hostname or "" + + # If an AA mirror redirects to a different hostname, treat that as a mirror + # failure and rotate rather than following the redirect (auto mode only). + if current_host and redirect_host and current_host != redirect_host: + if not network.is_aa_auto_mode(): + logger.warning( + "AA mirror locked to %s but redirected to %s: %s", + current_host, + redirect_host, + current_url, + ) + return "" + + new_url = _try_rotation(original_url, current_url, selector) + if new_url: + current_url = new_url + # Reset per-request state for the new host. + headers = {"User-Agent": DOWNLOAD_HEADERS["User-Agent"]} + is_aa_url = network.should_rotate_dns_for_url(current_url) + allow_redirects = not is_aa_url + redirects_followed = 0 + continue + + logger.warning( + "AA redirect from %s to %s but mirrors exhausted: %s", + current_host, + redirect_host, + current_url, + ) + return "" + + # Same-host redirect (relative or absolute) - follow manually. + redirects_followed += 1 + if redirects_followed > 5: + raise requests.exceptions.TooManyRedirects(f"Too many redirects for {current_url}") + current_url = redirect_url + continue + + response.raise_for_status() + time.sleep(1) + return response.text except Exception as e: status = _get_status_code(e) diff --git a/shelfmark/download/network.py b/shelfmark/download/network.py index 9022be7d..ca3dbce2 100644 --- a/shelfmark/download/network.py +++ b/shelfmark/download/network.py @@ -738,11 +738,22 @@ def rotate_dns_and_reset_aa() -> bool: if not configured_url: configured_url = "auto" - if configured_url == "auto" or _aa_base_url in _aa_urls: + if configured_url == "auto": + # Auto mode always resets to the first mirror to restart the cascade _current_aa_url_index = 0 - _aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.se" + _aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.gl" logger.info(f"After DNS switch, resetting AA URL to: {_aa_base_url}") _save_state(aa_url=_aa_base_url) + else: + # Keep the user's configured primary mirror (if it exists in the list), + # otherwise keep the configured URL as-is (custom/env). + if configured_url in _aa_urls: + _current_aa_url_index = _aa_urls.index(configured_url) + else: + _current_aa_url_index = 0 + _aa_base_url = configured_url + logger.info(f"After DNS switch, keeping configured AA URL: {_aa_base_url}") + _save_state(aa_url=_aa_base_url) return True def set_dns_provider(provider: str, manual_servers: list[str] | None = None, use_doh: bool | None = None) -> bool: @@ -916,6 +927,11 @@ def _initialize_aa_state() -> None: if not configured_url: configured_url = "auto" + # If AA_BASE_URL is pinned to a custom URL that's not in the mirror list, we still + # want to treat it as the active base (and rewrite known mirror links to it). + if configured_url != "auto" and configured_url not in _aa_urls: + _aa_urls = [configured_url] + _aa_urls + if configured_url == "auto": if state.get('aa_base_url') and state['aa_base_url'] in _aa_urls: _current_aa_url_index = _aa_urls.index(state['aa_base_url']) @@ -1031,6 +1047,17 @@ def get_aa_base_url(): _ensure_initialized() return _aa_base_url +def is_aa_auto_mode() -> bool: + """Return True when AA_BASE_URL is set to 'auto' (mirror failover enabled).""" + configured_url = normalize_http_url( + app_config.get("AA_BASE_URL", "auto"), + default_scheme="https", + allow_special=("auto",), + ) + if not configured_url: + configured_url = "auto" + return configured_url == "auto" + def get_available_aa_urls(): """Get list of configured AA URLs (copy).""" _ensure_initialized() @@ -1082,12 +1109,17 @@ class AAMirrorSelector: Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'. """ self.attempts_this_dns += 1 - if self.attempts_this_dns >= len(self.aa_urls): + max_attempts = len(self.aa_urls) if is_aa_auto_mode() else 1 + if self.attempts_this_dns >= max_attempts: if allow_dns and rotate_dns_and_reset_aa(): self._ensure_fresh_state(reset_attempts=True) return self.current_base, "dns" return None, "exhausted" + if not is_aa_auto_mode(): + # Mirror is explicitly configured; do not fail over to other mirrors. + return None, "exhausted" + next_index = (self._index + 1) % len(self.aa_urls) set_aa_url_index(next_index) self._ensure_fresh_state(reset_attempts=False) diff --git a/src/frontend/src/components/OnboardingModal.tsx b/src/frontend/src/components/OnboardingModal.tsx index 85b1c03b..8ee9df96 100644 --- a/src/frontend/src/components/OnboardingModal.tsx +++ b/src/frontend/src/components/OnboardingModal.tsx @@ -13,6 +13,7 @@ import { CheckboxFieldConfig, SelectFieldConfig, MultiSelectFieldConfig, + TagListFieldConfig, HeadingFieldConfig, ActionButtonConfig, ActionResult, @@ -25,6 +26,7 @@ import { CheckboxField, SelectField, MultiSelectField, + TagListField, HeadingField, ActionButton, } from './settings/fields'; @@ -137,6 +139,15 @@ const renderField = ( disabled={isDisabled} /> ); + case 'TagListField': + return ( + onChange(v)} + disabled={isDisabled} + /> + ); case 'ActionButton': return ( ); + case 'TagListField': + return ( + onChange(v)} + disabled={isDisabled} + /> + ); case 'OrderableListField': return ( void; + disabled?: boolean; +} + +function normalizeTag(raw: string): string { + let s = raw.trim(); + if (!s) return ''; + + // Strip wrapping quotes from pasted env/JSON style strings. + if ( + (s.startsWith('"') && s.endsWith('"')) || + (s.startsWith("'") && s.endsWith("'")) + ) { + s = s.slice(1, -1).trim(); + } + + // Avoid special sentinel values. + if (s.toLowerCase() === 'auto') return ''; + + // Basic URL normalization to keep UX friendly; backend also normalizes on save. + if (!s.includes('://') && !s.startsWith('/')) { + s = `https://${s}`; + } + s = s.replace(/\/+$/, ''); + return s.trim(); +} + +export const TagListField = ({ field, value, onChange, disabled }: TagListFieldProps) => { + const isDisabled = disabled ?? false; + const inputRef = useRef(null); + const [draft, setDraft] = useState(''); + + const tags = useMemo(() => (value ?? []).map(String).filter((t) => t.trim() !== ''), [value]); + + const addFromRaw = (raw: string) => { + if (isDisabled) return; + const parts = raw + .split(/[\n,]+/) + .map((p) => p.trim()) + .filter(Boolean); + if (parts.length === 0) return; + + const next = [...tags]; + for (const part of parts) { + const normalized = normalizeTag(part); + if (!normalized) continue; + if (next.includes(normalized)) continue; + next.push(normalized); + } + + if (next.length !== tags.length) { + onChange(next); + } + }; + + const removeAt = (idx: number) => { + if (isDisabled) return; + onChange(tags.filter((_, i) => i !== idx)); + }; + + const commitDraft = () => { + const raw = draft; + if (!raw.trim()) return; + addFromRaw(raw); + setDraft(''); + }; + + return ( +
{ + if (isDisabled) return; + inputRef.current?.focus(); + }} + > +
+ {tags.map((tag, idx) => ( + + {tag} + {!isDisabled && ( + + )} + + ))} + + {!isDisabled && ( + setDraft(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + commitDraft(); + return; + } + + if (e.key === 'Backspace' && !draft && tags.length > 0) { + removeAt(tags.length - 1); + } + }} + onBlur={() => commitDraft()} + placeholder={tags.length === 0 ? field.placeholder : ''} + className="flex-1 min-w-[12rem] bg-transparent outline-none py-1" + /> + )} + + {isDisabled && tags.length === 0 && ( + {field.placeholder || 'No values'} + )} +
+
+ ); +}; + diff --git a/src/frontend/src/components/settings/fields/index.ts b/src/frontend/src/components/settings/fields/index.ts index ed3e7013..5af804ce 100644 --- a/src/frontend/src/components/settings/fields/index.ts +++ b/src/frontend/src/components/settings/fields/index.ts @@ -4,6 +4,7 @@ export { NumberField } from './NumberField'; export { CheckboxField } from './CheckboxField'; export { SelectField } from './SelectField'; export { MultiSelectField } from './MultiSelectField'; +export { TagListField } from './TagListField'; export { OrderableListField } from './OrderableListField'; export { ActionButton } from './ActionButton'; export { TableField } from './TableField'; diff --git a/src/frontend/src/types/settings.ts b/src/frontend/src/types/settings.ts index 1d05f89b..b3d5e362 100644 --- a/src/frontend/src/types/settings.ts +++ b/src/frontend/src/types/settings.ts @@ -7,6 +7,7 @@ export type FieldType = | 'CheckboxField' | 'SelectField' | 'MultiSelectField' + | 'TagListField' | 'OrderableListField' | 'TableField' | 'ActionButton' @@ -93,6 +94,12 @@ export interface MultiSelectFieldConfig extends BaseField { variant?: 'pills' | 'dropdown'; // 'pills' (default) or 'dropdown' for checkbox dropdown style } +export interface TagListFieldConfig extends BaseField { + type: 'TagListField'; + value: string[]; + placeholder?: string; +} + // OrderableListField types - generic drag-and-drop reorderable list export interface OrderableListItem { id: string; @@ -163,6 +170,7 @@ export type SettingsField = | CheckboxFieldConfig | SelectFieldConfig | MultiSelectFieldConfig + | TagListFieldConfig | OrderableListFieldConfig | TableFieldConfig | ActionButtonConfig diff --git a/tests/config/test_mirror_settings_live_apply.py b/tests/config/test_mirror_settings_live_apply.py new file mode 100644 index 00000000..9e710d22 --- /dev/null +++ b/tests/config/test_mirror_settings_live_apply.py @@ -0,0 +1,28 @@ +def test_update_settings_mirrors_applies_aa_changes_live(monkeypatch): + # Ensure settings tabs are registered (mirrors tab lives here). + import shelfmark.config.settings # noqa: F401 + + from shelfmark.core.config import config as config_obj + from shelfmark.core.settings_registry import update_settings + + monkeypatch.delenv("AA_BASE_URL", raising=False) + monkeypatch.delenv("AA_ADDITIONAL_URLS", raising=False) + + # Avoid writing to disk and avoid forcing a full config refresh in this unit test. + monkeypatch.setattr("shelfmark.core.settings_registry.save_config_file", lambda _tab, _values: True) + monkeypatch.setattr(config_obj, "refresh", lambda: None) + + called: dict[str, object] = {} + + def fake_init_aa(*, force: bool = False) -> None: + called["force"] = force + + import shelfmark.download.network as network + + monkeypatch.setattr(network, "init_aa", fake_init_aa) + + result = update_settings("mirrors", {"AA_BASE_URL": "https://annas-archive.li"}) + + assert result["success"] is True + assert called["force"] is True + diff --git a/tests/config/test_mirror_settings_options.py b/tests/config/test_mirror_settings_options.py new file mode 100644 index 00000000..a3544cfe --- /dev/null +++ b/tests/config/test_mirror_settings_options.py @@ -0,0 +1,11 @@ +def test_aa_base_url_options_include_configured_custom_url(monkeypatch): + import shelfmark.config.settings as settings + from shelfmark.core.config import config as config_obj + + # Use a custom mirror not present in defaults/additional. + monkeypatch.setenv("AA_BASE_URL", "https://custom-aa.example") + config_obj.refresh() + + options = settings._get_aa_base_url_options() + assert any(opt["value"] == "https://custom-aa.example" for opt in options) + diff --git a/tests/core/test_mirrors_config.py b/tests/core/test_mirrors_config.py new file mode 100644 index 00000000..335575d8 --- /dev/null +++ b/tests/core/test_mirrors_config.py @@ -0,0 +1,41 @@ +from shelfmark.core import mirrors + + +class _DummyConfig: + def __init__(self, values: dict): + self._values = values + + def get(self, key: str, default=None): + return self._values.get(key, default) + + +def test_get_aa_mirrors_prefers_full_configured_list(monkeypatch): + dummy = _DummyConfig( + { + "AA_MIRROR_URLS": ["annas-archive.gl/", "https://annas-archive.li"], + "AA_ADDITIONAL_URLS": "https://should-not-be-appended.example", + } + ) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + assert mirrors.get_aa_mirrors() == [ + "https://annas-archive.gl", + "https://annas-archive.li", + ] + + +def test_get_aa_mirrors_falls_back_to_defaults_and_legacy_additional(monkeypatch): + dummy = _DummyConfig( + { + "AA_MIRROR_URLS": [], + "AA_ADDITIONAL_URLS": "extra.example, https://extra2.example/", + } + ) + monkeypatch.setattr(mirrors, "_get_config", lambda: dummy) + + aa = mirrors.get_aa_mirrors() + assert "https://annas-archive.gl" in aa + assert "https://annas-archive.li" in aa + assert "https://extra.example" in aa + assert "https://extra2.example" in aa + diff --git a/tests/download/test_http_aa_redirects.py b/tests/download/test_http_aa_redirects.py new file mode 100644 index 00000000..a5a4df77 --- /dev/null +++ b/tests/download/test_http_aa_redirects.py @@ -0,0 +1,142 @@ +import requests + + +class _FakeResponse: + def __init__(self, status_code: int, *, headers: dict | None = None, text: str = "") -> None: + self.status_code = status_code + self.headers = headers or {} + self.text = text + + @property + def is_redirect(self) -> bool: # requests.Response compatibility + return self.status_code in (301, 302, 303, 307, 308) and bool(self.headers.get("Location")) + + def raise_for_status(self) -> None: # requests.Response compatibility + if self.status_code >= 400: + raise requests.exceptions.HTTPError(response=self) + + +class _DummySelector: + """Minimal AA selector stub for unit testing http.html_get_page().""" + + def __init__(self, bases: list[str]) -> None: + self._bases = bases + self._index = 0 + self.current_base = bases[0] + self.attempts_this_dns = 0 + + def rewrite(self, url: str) -> str: + for base in self._bases: + if url.startswith(base): + return url.replace(base, self.current_base, 1) + return url + + def next_mirror_or_rotate_dns(self, allow_dns: bool = True) -> tuple[str | None, str]: + self.attempts_this_dns += 1 + self._index = (self._index + 1) % len(self._bases) + self.current_base = self._bases[self._index] + return self.current_base, "mirror" + + +def test_html_get_page_aa_cross_host_redirect_rotates_mirror(monkeypatch): + import shelfmark.download.http as http + + # Avoid bypasser imports in unit tests. + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: False) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http.time, "sleep", lambda _s: None) + monkeypatch.setattr(http.network, "get_aa_base_url", lambda: "https://annas-archive.li") + monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: True) + + calls: list[dict] = [] + + def fake_get(url: str, **kwargs): + calls.append({"url": url, "allow_redirects": kwargs.get("allow_redirects")}) + if url.startswith("https://annas-archive.li/"): + return _FakeResponse(302, headers={"Location": "https://annas-archive.pm/search?q=test"}) + if url.startswith("https://annas-archive.gl/"): + return _FakeResponse(200, text="OK") + raise AssertionError(f"Unexpected URL: {url}") + + monkeypatch.setattr(http.requests, "get", fake_get) + + selector = _DummySelector(["https://annas-archive.li", "https://annas-archive.gl"]) + html = http.html_get_page( + "https://annas-archive.li/search?q=test", + selector=selector, + retry=2, + allow_bypasser_fallback=False, + ) + + assert html == "OK" + assert calls[0]["allow_redirects"] is False # AA redirects handled manually + assert calls[0]["url"].startswith("https://annas-archive.li/") + assert calls[1]["url"].startswith("https://annas-archive.gl/") # rotated away from redirect target + + +def test_html_get_page_aa_same_host_redirect_is_followed(monkeypatch): + import shelfmark.download.http as http + + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: False) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http.time, "sleep", lambda _s: None) + monkeypatch.setattr(http.network, "get_aa_base_url", lambda: "https://annas-archive.li") + monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: True) + + calls: list[dict] = [] + + def fake_get(url: str, **kwargs): + calls.append({"url": url, "allow_redirects": kwargs.get("allow_redirects")}) + if url == "https://annas-archive.li/search?q=test": + return _FakeResponse(302, headers={"Location": "/search?q=test&page=1"}) + if url == "https://annas-archive.li/search?q=test&page=1": + return _FakeResponse(200, text="OK2") + raise AssertionError(f"Unexpected URL: {url}") + + monkeypatch.setattr(http.requests, "get", fake_get) + + selector = _DummySelector(["https://annas-archive.li"]) + html = http.html_get_page( + "https://annas-archive.li/search?q=test", + selector=selector, + retry=1, + allow_bypasser_fallback=False, + ) + + assert html == "OK2" + assert [c["url"] for c in calls] == [ + "https://annas-archive.li/search?q=test", + "https://annas-archive.li/search?q=test&page=1", + ] + assert all(c["allow_redirects"] is False for c in calls) + + +def test_html_get_page_locked_aa_does_not_fail_over_on_cross_host_redirect(monkeypatch): + import shelfmark.download.http as http + + monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: False) + monkeypatch.setattr(http, "get_proxies", lambda _url: {}) + monkeypatch.setattr(http.time, "sleep", lambda _s: None) + monkeypatch.setattr(http.network, "get_aa_base_url", lambda: "https://annas-archive.li") + monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: False) + + calls: list[str] = [] + + def fake_get(url: str, **kwargs): + calls.append(url) + if url.startswith("https://annas-archive.li/"): + return _FakeResponse(302, headers={"Location": "https://annas-archive.pm/search?q=test"}) + raise AssertionError(f"Unexpected URL: {url}") + + monkeypatch.setattr(http.requests, "get", fake_get) + + selector = _DummySelector(["https://annas-archive.li", "https://annas-archive.gl"]) + html = http.html_get_page( + "https://annas-archive.li/search?q=test", + selector=selector, + retry=2, + allow_bypasser_fallback=False, + ) + + assert html == "" + assert calls == ["https://annas-archive.li/search?q=test"]