feat(newznab): support multiple named indexers (#1271)

## Summary

- add a named Newznab indexer table with per-indexer URL and API key
settings
- search every configured indexer and retain the originating indexer
name on each result
- namespace cached release IDs across connections and isolate individual
indexer failures
- preserve the legacy single-indexer settings as a fallback
- support masked API-key cells and trusted SABnzbd prefetching for named
indexers

## Validation

- 121 Newznab and SABnzbd backend tests passed on Python 3.14
- Ruff passed for all changed Python files
- frontend TypeScript and strict lint checks passed
- all 134 frontend unit tests passed
- frontend formatting check passed

## Compatibility

Existing `NEWZNAB_URL` and `NEWZNAB_API_KEY` configurations continue to
work whenever `NEWZNAB_INDEXERS` is empty.

Co-authored-by: Ryan <zab1996@users.noreply.github.com>
This commit is contained in:
zab1996
2026-08-27 00:29:58 -04:00
committed by GitHub
co-authored by Ryan
parent ff06a1a581
commit 02b7e9d958
9 changed files with 480 additions and 48 deletions
+23 -7
View File
@@ -1315,8 +1315,9 @@ Apply per-indexer seed time and ratio preferences from Prowlarr when sending tor
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `NEWZNAB_ENABLED` | Enable searching for books via a Newznab-compatible indexer | boolean | `false` |
| `NEWZNAB_URL` | Base URL of your Newznab indexer or aggregator | string | _none_ |
| `NEWZNAB_API_KEY` | Your Newznab API key (leave blank if not required) | string (secret) | _none_ |
| `NEWZNAB_INDEXERS` | Named Newznab connections. Each row accepts `name`, `url`, and `api_key`. | JSON array | `[]` |
| `NEWZNAB_URL` | Legacy single-indexer URL, used when `NEWZNAB_INDEXERS` is empty | string | _none_ |
| `NEWZNAB_API_KEY` | Legacy single-indexer API key | string (secret) | _none_ |
| `NEWZNAB_EBOOK_CATEGORIES` | Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000. | string (comma-separated) | `7000` |
| `NEWZNAB_AUDIOBOOK_CATEGORIES` | Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030. | string (comma-separated) | `3030` |
| `NEWZNAB_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
@@ -1333,21 +1334,36 @@ Enable searching for books via a Newznab-compatible indexer
- **Type:** boolean
- **Default:** `false`
#### `NEWZNAB_INDEXERS`
**Named Indexers**
Configure multiple named Newznab-compatible indexers. The name is shown beside each search result. For environment-based configuration, provide a JSON array:
```json
[
{"name":"NZBGeek","url":"https://api.nzbgeek.info","api_key":"..."},
{"name":"DrunkenSlug","url":"https://drunkenslug.com","api_key":"..."}
]
```
- **Type:** JSON array
- **Default:** `[]`
#### `NEWZNAB_URL`
**Newznab URL**
**Legacy Newznab URL**
Base URL of your Newznab indexer or aggregator
Single-indexer fallback used only when `NEWZNAB_INDEXERS` is empty.
- **Type:** string
- **Default:** _none_
- **Required:** Yes
#### `NEWZNAB_API_KEY`
**API Key**
**Legacy API Key**
Your Newznab API key (leave blank if not required)
API key for the legacy Newznab URL.
- **Type:** string (secret)
- **Default:** _none_
+9
View File
@@ -248,6 +248,15 @@ class SABnzbdClient(DownloadClient):
if trusted_url and _url_origin(trusted_url) == target_origin:
return True
named_indexers = config.get("NEWZNAB_INDEXERS", [])
if isinstance(named_indexers, list):
for row in named_indexers:
if not isinstance(row, dict):
continue
trusted_url = normalize_http_config_url(row.get("url"))
if trusted_url and _url_origin(trusted_url) == target_origin:
return True
return False
def _get_prowlarr_headers(self, url: str) -> dict:
+68 -8
View File
@@ -8,6 +8,7 @@ from shelfmark.core.settings_registry import (
HeadingField,
PasswordField,
SettingsField,
TableField,
TagListField,
TextField,
register_settings,
@@ -16,12 +17,36 @@ from shelfmark.core.utils import normalize_http_url
def _test_newznab_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
"""Test the Newznab connection using current form values."""
"""Test all named Newznab connections, or the legacy connection as fallback."""
from shelfmark.core.config import config
from shelfmark.release_sources.newznab.api import NewznabClient
from shelfmark.release_sources.newznab.source import _parse_indexer_rows
current_values = current_values or {}
raw_indexers = current_values.get("NEWZNAB_INDEXERS")
if raw_indexers is None:
raw_indexers = config.get("NEWZNAB_INDEXERS", [])
indexers = _parse_indexer_rows(raw_indexers)
if indexers:
details: list[str] = []
all_successful = True
for name, url, api_key in indexers:
try:
success, message = NewznabClient(url, api_key).test_connection()
except Exception as e: # noqa: BLE001 — surface unexpected errors to the UI
success, message = False, f"Connection failed: {e!s}"
all_successful = all_successful and success
details.append(f"{name}: {message}")
summary = (
f"Connected to all {len(indexers)} indexers"
if all_successful
else "One or more Newznab indexers failed"
)
return {"success": all_successful, "message": summary, "details": details}
raw_url = str(current_values.get("NEWZNAB_URL") or config.get("NEWZNAB_URL", "") or "")
api_key = str(current_values.get("NEWZNAB_API_KEY") or config.get("NEWZNAB_API_KEY", "") or "")
@@ -64,25 +89,60 @@ def newznab_config_settings() -> list[SettingsField]:
default=False,
description="Enable searching for books via a Newznab-compatible indexer",
),
TableField(
key="NEWZNAB_INDEXERS",
label="Named Indexers",
description=(
"Add each Newznab-compatible indexer separately. The configured name is shown "
"beside every result from that indexer."
),
columns=[
{
"key": "name",
"label": "Name",
"type": "text",
"placeholder": "NZBGeek",
},
{
"key": "url",
"label": "URL",
"type": "text",
"placeholder": "https://api.nzbgeek.info",
},
{
"key": "api_key",
"label": "API Key",
"type": "password",
"placeholder": "Optional",
},
],
default=[],
add_label="Add Indexer",
empty_message=(
"No named indexers configured. The legacy single-indexer fields below are used "
"as a fallback."
),
show_when={"field": "NEWZNAB_ENABLED", "value": True},
),
TextField(
key="NEWZNAB_URL",
label="Newznab URL",
description="Base URL of your Newznab indexer or aggregator",
label="Legacy Newznab URL",
description="Used only when the named indexer list is empty",
placeholder="http://nzbhydra:5076",
required=True,
required=False,
show_when={"field": "NEWZNAB_ENABLED", "value": True},
),
PasswordField(
key="NEWZNAB_API_KEY",
label="API Key",
description="Your Newznab API key (leave blank if not required)",
label="Legacy API Key",
description="Used only with the legacy Newznab URL",
required=False,
show_when={"field": "NEWZNAB_ENABLED", "value": True},
),
ActionButton(
key="test_newznab",
label="Test Connection",
description="Verify your Newznab configuration",
label="Test Connections",
description="Verify every named indexer, or the legacy connection when the list is empty",
style="primary",
callback=_test_newznab_connection,
show_when={"field": "NEWZNAB_ENABLED", "value": True},
+125 -30
View File
@@ -4,7 +4,10 @@ from __future__ import annotations
import re
import time
from dataclasses import dataclass
from hashlib import sha256
from typing import TYPE_CHECKING, ClassVar
from urllib.parse import urlparse
if TYPE_CHECKING:
from shelfmark.core.search_plan import ReleaseSearchPlan
@@ -48,6 +51,50 @@ _DEFAULT_BOOK_CATS = [7000]
NEWZNAB_SEARCH_TIMEOUT_SECONDS = _SEARCH_TIMEOUT
@dataclass(frozen=True)
class _NamedClient:
"""A configured Newznab connection and its stable cache namespace."""
name: str
connection_id: str
client: NewznabClient
def _parse_indexer_rows(raw: object) -> list[tuple[str, str, str]]:
"""Normalize structured Newznab indexer settings.
Invalid/incomplete rows are ignored so one partially edited row cannot disable
the other configured indexers.
"""
if not isinstance(raw, list):
return []
indexers: list[tuple[str, str, str]] = []
seen_connections: set[tuple[str, str]] = set()
for row in raw:
if not isinstance(row, dict):
continue
raw_url = str(row.get("url") or "").strip()
url = normalize_http_url(raw_url)
if not url:
if raw_url:
logger.warning("Newznab: ignoring indexer row with invalid URL '%s'", raw_url)
continue
api_key = str(row.get("api_key") or "").strip()
connection_key = (url, api_key)
if connection_key in seen_connections:
continue
seen_connections.add(connection_key)
configured_name = str(row.get("name") or "").strip()
hostname = urlparse(url).hostname or ""
name = configured_name or hostname or "Newznab"
indexers.append((name, url, api_key))
return indexers
def _parse_category_ids(raw: object) -> list[int]:
"""Parse a configured category setting into Newznab category IDs.
@@ -146,8 +193,11 @@ def _newznab_result_to_release(
else None
)
# Build source_id from GUID
source_id = result.get("guid") or f"newznab:{hash(raw_title)}"
# Namespace IDs from named connections so identical GUIDs returned by two
# indexers cannot overwrite one another in the private release cache.
raw_source_id = result.get("guid") or f"newznab:{hash(raw_title)}"
connection_id = str(result.get("_newznab_connection_id") or "").strip()
source_id = f"newznab:{connection_id}:{raw_source_id}" if connection_id else raw_source_id
# Cache the raw result for the handler
cache_release(source_id, result)
@@ -272,6 +322,7 @@ class NewznabSource(ReleaseSource):
)
def _get_client(self) -> NewznabClient | None:
"""Build the legacy single-indexer client."""
raw_url = str(config.get("NEWZNAB_URL", "") or "")
api_key = str(config.get("NEWZNAB_API_KEY", "") or "")
@@ -284,6 +335,28 @@ class NewznabSource(ReleaseSource):
return NewznabClient(url, api_key or "")
def _get_clients(self) -> list[_NamedClient]:
"""Build named clients, falling back to the legacy single connection."""
configured = _parse_indexer_rows(config.get("NEWZNAB_INDEXERS", []))
if configured:
clients: list[_NamedClient] = []
for name, url, api_key in configured:
digest = sha256(f"{name}\0{url}\0{api_key}".encode()).hexdigest()[:16]
clients.append(
_NamedClient(
name=name,
connection_id=digest,
client=NewznabClient(url, api_key),
)
)
return clients
legacy_client = self._get_client()
if legacy_client is None:
return []
legacy_name = str(config.get("NEWZNAB_NAME", "") or "").strip() or "Newznab"
return [_NamedClient(name=legacy_name, connection_id="legacy", client=legacy_client)]
def search(
self,
book: BookMetadata,
@@ -293,8 +366,8 @@ class NewznabSource(ReleaseSource):
content_type: str = "ebook",
) -> list[Release]:
"""Search the Newznab indexer for releases matching the book."""
client = self._get_client()
if not client:
clients = self._get_clients()
if not clients:
logger.warning("Newznab not configured - skipping search")
return []
@@ -324,40 +397,60 @@ class NewznabSource(ReleaseSource):
all_results: list[dict] = []
try:
for idx, query in enumerate(queries, start=1):
_check_timeout()
if len(queries) > 1:
logger.debug("Newznab query %d/%d: '%s'", idx, len(queries), query)
for connection in clients:
try:
for idx, query in enumerate(queries, start=1):
_check_timeout()
if len(queries) > 1:
logger.debug(
"Newznab [%s] query %d/%d: '%s'",
connection.name,
idx,
len(queries),
query,
)
raw = client.search(query=query, categories=categories)
raw = connection.client.search(query=query, categories=categories)
# Auto-expand: retry without category filter if no results
if not raw and categories and auto_expand:
_check_timeout()
logger.info(
"Newznab: no results for '%s' with category filter, auto-expanding",
query,
)
raw = client.search(query=query, categories=None)
# Auto-expand: retry without category filter if no results
if not raw and categories and auto_expand:
_check_timeout()
logger.info(
"Newznab [%s]: no results for '%s' with category filter, "
"auto-expanding",
connection.name,
query,
)
raw = connection.client.search(query=query, categories=None)
for r in raw:
key = (
r.get("guid")
or r.get("downloadUrl")
or f"{r.get('indexer')}:{r.get('title')}"
)
if key in seen_keys:
continue
seen_keys.add(key)
all_results.append(r)
for raw_result in raw:
r = dict(raw_result)
# Aggregators can identify the underlying indexer. Plain feeds
# generally cannot, so use the user-configured connection name.
r["indexer"] = r.get("indexer") or connection.name
r["_newznab_connection_id"] = connection.connection_id
key = (
connection.connection_id,
r.get("guid")
or r.get("downloadUrl")
or f"{r.get('indexer')}:{r.get('title')}",
)
if key in seen_keys:
continue
seen_keys.add(key)
all_results.append(r)
except TimeoutError:
raise
except Exception:
logger.exception("Newznab search failed for %s", connection.name)
except TimeoutError as e:
logger.warning("Newznab search timed out: %s", e)
except Exception:
logger.exception("Newznab search failed")
return []
results = [_newznab_result_to_release(r, content_type, categories) for r in all_results]
if plan.indexers:
selected_indexers = set(plan.indexers)
results = [r for r in results if r.indexer in selected_indexers]
if results:
nzb_count = sum(1 for r in results if r.protocol == ReleaseProtocol.NZB)
@@ -379,5 +472,7 @@ class NewznabSource(ReleaseSource):
def is_available(self) -> bool:
if not config.get("NEWZNAB_ENABLED", False):
return False
if _parse_indexer_rows(config.get("NEWZNAB_INDEXERS", [])):
return True
url = normalize_http_url(str(config.get("NEWZNAB_URL", "") or ""))
return bool(url)
@@ -353,12 +353,12 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
);
}
// text/path
// text/password/path
return (
<div key={col.key} className="flex min-w-0 flex-col gap-1">
{mobileLabel}
<input
type="text"
type={col.type === 'password' ? 'password' : 'text'}
value={toPrimitiveString(cellValue)}
onChange={(e) => updateCell(rowIndex, col.key, e.target.value)}
placeholder={col.placeholder}
+7 -1
View File
@@ -146,7 +146,13 @@ export interface TableFieldColumnOption {
childOf?: string;
}
export type TableFieldColumnType = 'text' | 'select' | 'multiselect' | 'checkbox' | 'path';
export type TableFieldColumnType =
| 'text'
| 'password'
| 'select'
| 'multiselect'
| 'checkbox'
| 'path';
export interface TableFieldColumn {
key: string;
+78
View File
@@ -0,0 +1,78 @@
"""Tests for named Newznab indexer settings."""
from shelfmark.core.settings_registry import TableField
from shelfmark.release_sources.newznab.settings import (
_test_newznab_connection,
newznab_config_settings,
)
def test_settings_include_named_indexer_table():
field = next(
field
for field in newznab_config_settings()
if getattr(field, "key", None) == "NEWZNAB_INDEXERS"
)
assert isinstance(field, TableField)
assert [column["key"] for column in field.columns] == ["name", "url", "api_key"]
assert field.columns[2]["type"] == "password"
def test_connection_action_tests_every_named_indexer(monkeypatch):
import shelfmark.release_sources.newznab.api as api_module
tested: list[tuple[str, str]] = []
class FakeClient:
def __init__(self, url, api_key):
tested.append((url, api_key))
def test_connection(self):
return True, "Connected"
monkeypatch.setattr(api_module, "NewznabClient", FakeClient)
result = _test_newznab_connection(
{
"NEWZNAB_INDEXERS": [
{"name": "NZBGeek", "url": "https://geek.example", "api_key": "one"},
{"name": "DrunkenSlug", "url": "https://slug.example", "api_key": "two"},
]
}
)
assert result == {
"success": True,
"message": "Connected to all 2 indexers",
"details": ["NZBGeek: Connected", "DrunkenSlug: Connected"],
}
assert tested == [("https://geek.example", "one"), ("https://slug.example", "two")]
def test_connection_action_reports_each_failure(monkeypatch):
import shelfmark.release_sources.newznab.api as api_module
class FakeClient:
def __init__(self, url, _api_key):
self.url = url
def test_connection(self):
if "down" in self.url:
return False, "Could not connect"
return True, "Connected"
monkeypatch.setattr(api_module, "NewznabClient", FakeClient)
result = _test_newznab_connection(
{
"NEWZNAB_INDEXERS": [
{"name": "Working", "url": "https://working.example"},
{"name": "Unavailable", "url": "https://down.example"},
]
}
)
assert result["success"] is False
assert result["message"] == "One or more Newznab indexers failed"
assert result["details"] == ["Working: Connected", "Unavailable: Could not connect"]
+134
View File
@@ -1,5 +1,6 @@
"""Unit tests for the Newznab release source."""
from dataclasses import replace
from unittest.mock import MagicMock
import pytest
@@ -11,6 +12,7 @@ from shelfmark.release_sources.newznab.source import (
NewznabSource,
_newznab_result_to_release,
_parse_category_ids,
_parse_indexer_rows,
)
# ── fixtures / helpers ─────────────────────────────────────────────────────────
@@ -221,6 +223,19 @@ class TestIsAvailable:
monkeypatch.setattr(mod.config, "get", self._config(NEWZNAB_URL=""))
assert NewznabSource().is_available() is False
def test_available_with_named_indexer_and_no_legacy_url(self, monkeypatch):
import shelfmark.release_sources.newznab.source as mod
monkeypatch.setattr(
mod.config,
"get",
self._config(
NEWZNAB_URL="",
NEWZNAB_INDEXERS=[{"name": "NZBGeek", "url": "https://geek.example"}],
),
)
assert NewznabSource().is_available() is True
# ── category parsing ───────────────────────────────────────────────────────────
@@ -247,6 +262,42 @@ class TestParseCategoryIds:
assert _parse_category_ids(" ") == []
class TestParseIndexerRows:
def test_parses_named_connections(self):
assert _parse_indexer_rows(
[
{
"name": "NZBGeek",
"url": "https://api.nzbgeek.info/",
"api_key": "geek-key",
},
{
"name": "DrunkenSlug",
"url": "drunkenslug.com",
"api_key": "slug-key",
},
]
) == [
("NZBGeek", "https://api.nzbgeek.info", "geek-key"),
("DrunkenSlug", "http://drunkenslug.com", "slug-key"),
]
def test_uses_hostname_when_name_is_blank(self):
assert _parse_indexer_rows([{"url": "https://indexer.example.com"}]) == [
("indexer.example.com", "https://indexer.example.com", "")
]
def test_ignores_invalid_and_duplicate_connections(self):
assert _parse_indexer_rows(
[
None,
{"name": "Incomplete", "url": ""},
{"name": "First", "url": "https://indexer.example.com", "api_key": "key"},
{"name": "Duplicate", "url": "https://indexer.example.com", "api_key": "key"},
]
) == [("First", "https://indexer.example.com", "key")]
# ── NewznabSource.search ───────────────────────────────────────────────────────
@@ -449,6 +500,89 @@ class TestSearch:
call_kwargs = client.search.call_args
assert call_kwargs[1]["query"] == "9780441013593"
def test_searches_all_named_indexers_and_labels_plain_feed_results(self, monkeypatch):
import shelfmark.release_sources.newznab.source as mod
rows = [
{"name": "NZBGeek", "url": "https://geek.example", "api_key": "one"},
{"name": "DrunkenSlug", "url": "https://slug.example", "api_key": "two"},
]
monkeypatch.setattr(
mod.config,
"get",
self._fake_config(NEWZNAB_INDEXERS=rows),
)
clients = {}
def client_factory(url, api_key):
client = MagicMock()
client.search.return_value = [
_make_result(
guid="shared-guid",
downloadUrl=f"{url}/download?apikey={api_key}",
indexer=None,
)
]
clients[url] = client
return client
monkeypatch.setattr(mod, "NewznabClient", client_factory)
results = NewznabSource().search(_make_book(), _make_plan(_make_book()))
assert {release.indexer for release in results} == {"NZBGeek", "DrunkenSlug"}
assert len({release.source_id for release in results}) == 2
assert all(release.source_id.startswith("newznab:") for release in results)
assert set(clients) == {"https://geek.example", "https://slug.example"}
def test_indexer_filter_is_applied_to_named_results(self, monkeypatch):
import shelfmark.release_sources.newznab.source as mod
rows = [
{"name": "NZBGeek", "url": "https://geek.example"},
{"name": "DrunkenSlug", "url": "https://slug.example"},
]
monkeypatch.setattr(mod.config, "get", self._fake_config(NEWZNAB_INDEXERS=rows))
def client_factory(url, _api_key):
client = MagicMock()
client.search.return_value = [
_make_result(guid=f"{url}/guid", indexer=None)
]
return client
monkeypatch.setattr(mod, "NewznabClient", client_factory)
book = _make_book()
plan = replace(_make_plan(book), indexers=["DrunkenSlug"])
results = NewznabSource().search(book, plan)
assert [release.indexer for release in results] == ["DrunkenSlug"]
def test_one_named_indexer_failure_does_not_hide_other_results(self, monkeypatch):
import shelfmark.release_sources.newznab.source as mod
rows = [
{"name": "Unavailable", "url": "https://down.example"},
{"name": "Working", "url": "https://working.example"},
]
monkeypatch.setattr(mod.config, "get", self._fake_config(NEWZNAB_INDEXERS=rows))
def client_factory(url, _api_key):
client = MagicMock()
if "down" in url:
client.search.side_effect = RuntimeError("offline")
else:
client.search.return_value = [_make_result(indexer=None)]
return client
monkeypatch.setattr(mod, "NewznabClient", client_factory)
results = NewznabSource().search(_make_book(), _make_plan(_make_book()))
assert [release.indexer for release in results] == ["Working"]
def test_exception_in_client_returns_empty(self, monkeypatch):
client = MagicMock()
client.search.side_effect = RuntimeError("boom")
+34
View File
@@ -654,6 +654,40 @@ class TestSABnzbdClientAddDownload:
assert "https://attacker.example/download.nzb" not in called_urls
assert called_urls == ["http://localhost:8080/api"]
def test_add_download_prefetches_named_newznab_indexer_url(self, monkeypatch):
"""Named Newznab origins should receive the same backend prefetch as legacy URLs."""
config_values = {
"SABNZBD_URL": "http://localhost:8080",
"SABNZBD_API_KEY": "abc123",
"SABNZBD_CATEGORY": "books",
"NEWZNAB_INDEXERS": [
{"name": "NZBGeek", "url": "https://geek.example", "api_key": "secret"}
],
}
monkeypatch.setattr(
"shelfmark.download.clients.sabnzbd.config.get",
lambda key, default="": config_values.get(key, default),
)
from shelfmark.download.clients.sabnzbd import SABnzbdClient
with (
patch.object(SABnzbdClient, "_fetch_nzb_content", return_value=b"nzbdata") as fetch,
patch.object(
SABnzbdClient,
"_api_post_file",
return_value={"status": True, "nzo_ids": ["SABnzbd_nzo_named"]},
),
):
client = SABnzbdClient()
result = client.add_download(
"https://geek.example/download.nzb?apikey=secret",
"Test Book",
)
assert result == "SABnzbd_nzo_named"
fetch.assert_called_once_with("https://geek.example/download.nzb?apikey=secret")
class TestSABnzbdClientRemove:
"""Tests for SABnzbdClient.remove()."""