From 2b8b35bb521801239abd79ca8664d524186008a7 Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Sat, 15 Aug 2026 11:48:10 -0400 Subject: [PATCH] fix(newznab): make indexer book categories configurable (#1214) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newznab searches hardcoded category 7000 for ebooks and 3030 for audiobooks, so indexers using custom IDs returned no results or the wrong ones. Add NEWZNAB_EBOOK_CATEGORIES and NEWZNAB_AUDIOBOOK_CATEGORIES (tag lists, defaulting to 7000 and 3030) and resolve the search categories from config. Values are parsed leniently — list or comma/whitespace separated, non-numeric entries skipped, duplicates dropped — and fall back to the standard IDs when empty, so a cleared field can't silently widen the search to every category. NEWZNAB_AUTO_EXPAND remains the way to do that on purpose. Results carrying a custom ID outside the standard 7000-7999 / 3030 ranges were typed as "other", which routed custom-category audiobooks as ebooks. Trust the searched content type when a result carries a category we explicitly asked for. Also drop the unused NEWZNAB_BOOKS / NEWZNAB_AUDIOBOOKS constants from api.py — a third copy of the same hardcoding. Closes #1208 --- docs/environment-variables.md | 20 ++++ shelfmark/release_sources/newznab/api.py | 4 - shelfmark/release_sources/newznab/settings.py | 25 +++++ shelfmark/release_sources/newznab/source.py | 97 ++++++++++++++++--- tests/newznab/test_source.py | 76 +++++++++++++++ 5 files changed, 206 insertions(+), 16 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 205b0524..9682fdf9 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -1304,6 +1304,8 @@ Apply per-indexer seed time and ratio preferences from Prowlarr when sending tor | `NEWZNAB_ENABLED` | Enable searching for books via a Newznab-compatible indexer | boolean | `false` | | `NEWZNAB_URL` | Base URL of your Newznab indexer or aggregator | string | _none_ | | `NEWZNAB_API_KEY` | Your Newznab API key (leave blank if not required) | string (secret) | _none_ | +| `NEWZNAB_EBOOK_CATEGORIES` | Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000. | string (comma-separated) | `7000` | +| `NEWZNAB_AUDIOBOOK_CATEGORIES` | Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030. | string (comma-separated) | `3030` | | `NEWZNAB_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
@@ -1337,6 +1339,24 @@ Your Newznab API key (leave blank if not required) - **Type:** string (secret) - **Default:** _none_ +#### `NEWZNAB_EBOOK_CATEGORIES` + +**Ebook Categories** + +Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000. + +- **Type:** string (comma-separated) +- **Default:** `7000` + +#### `NEWZNAB_AUDIOBOOK_CATEGORIES` + +**Audiobook Categories** + +Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030. + +- **Type:** string (comma-separated) +- **Default:** `3030` + #### `NEWZNAB_AUTO_EXPAND` **Auto-expand search on no results** diff --git a/shelfmark/release_sources/newznab/api.py b/shelfmark/release_sources/newznab/api.py index 0a02bf3b..be4166a6 100644 --- a/shelfmark/release_sources/newznab/api.py +++ b/shelfmark/release_sources/newznab/api.py @@ -14,10 +14,6 @@ from shelfmark.release_sources.prowlarr.torznab import parse_torznab_xml logger = setup_logger(__name__) -# Newznab standard book category IDs -NEWZNAB_BOOKS = 7000 -NEWZNAB_AUDIOBOOKS = 3030 - class NewznabClient: """Client for any Newznab-compatible indexer API.""" diff --git a/shelfmark/release_sources/newznab/settings.py b/shelfmark/release_sources/newznab/settings.py index 1d580076..3450781f 100644 --- a/shelfmark/release_sources/newznab/settings.py +++ b/shelfmark/release_sources/newznab/settings.py @@ -8,6 +8,7 @@ from shelfmark.core.settings_registry import ( HeadingField, PasswordField, SettingsField, + TagListField, TextField, register_settings, ) @@ -86,6 +87,30 @@ def newznab_config_settings() -> list[SettingsField]: callback=_test_newznab_connection, show_when={"field": "NEWZNAB_ENABLED", "value": True}, ), + TagListField( + key="NEWZNAB_EBOOK_CATEGORIES", + label="Ebook Categories", + description=( + "Newznab category IDs searched for ebooks. Most indexers use the standard 7000, " + "but some use custom IDs. Leave empty to use 7000." + ), + placeholder="7000", + default=["7000"], + normalize_urls=False, + show_when={"field": "NEWZNAB_ENABLED", "value": True}, + ), + TagListField( + key="NEWZNAB_AUDIOBOOK_CATEGORIES", + label="Audiobook Categories", + description=( + "Newznab category IDs searched for audiobooks. Most indexers use the standard " + "3030, but some use custom IDs. Leave empty to use 3030." + ), + placeholder="3030", + default=["3030"], + normalize_urls=False, + show_when={"field": "NEWZNAB_ENABLED", "value": True}, + ), CheckboxField( key="NEWZNAB_AUTO_EXPAND", label="Auto-expand search on no results", diff --git a/shelfmark/release_sources/newznab/source.py b/shelfmark/release_sources/newznab/source.py index 966d6d0d..2e2da599 100644 --- a/shelfmark/release_sources/newznab/source.py +++ b/shelfmark/release_sources/newznab/source.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re import time from typing import TYPE_CHECKING, ClassVar @@ -39,15 +40,92 @@ from shelfmark.release_sources.prowlarr.source import ( logger = setup_logger(__name__) -# Newznab category IDs -_AUDIOBOOK_CATS = [3030] -_BOOK_CATS = [7000] +# Standard Newznab category IDs, used when the indexer's categories aren't configured. +_DEFAULT_AUDIOBOOK_CATS = [3030] +_DEFAULT_BOOK_CATS = [7000] # Reuse the same timeout constant as Prowlarr. NEWZNAB_SEARCH_TIMEOUT_SECONDS = _SEARCH_TIMEOUT -def _newznab_result_to_release(result: dict, content_type: str = "ebook") -> Release: +def _parse_category_ids(raw: object) -> list[int]: + """Parse a configured category setting into Newznab category IDs. + + Accepts a list of values or a comma/whitespace separated string. Entries that + aren't positive integers are skipped, and duplicates are dropped. + """ + if raw is None: + return [] + + values = list(raw) if isinstance(raw, (list, tuple)) else [raw] + + category_ids: list[int] = [] + for value in values: + for token in re.split(r"[,\s]+", str(value).strip()): + if not token: + continue + try: + category_id = int(token) + except ValueError: + logger.warning("Newznab: ignoring invalid category ID '%s'", token) + continue + if category_id > 0 and category_id not in category_ids: + category_ids.append(category_id) + + return category_ids + + +def _configured_categories(content_type: str) -> list[int]: + """Return the categories to search for a content type, falling back to defaults.""" + if content_type == "audiobook": + key, defaults = "NEWZNAB_AUDIOBOOK_CATEGORIES", _DEFAULT_AUDIOBOOK_CATS + else: + key, defaults = "NEWZNAB_EBOOK_CATEGORIES", _DEFAULT_BOOK_CATS + + return _parse_category_ids(config.get(key, None)) or list(defaults) + + +def _result_category_ids(categories: object) -> set[int]: + """Extract numeric category IDs from a result's categories field.""" + if not isinstance(categories, (list, tuple)): + return set() + + category_ids: set[int] = set() + for cat in categories: + raw = cat.get("id") if isinstance(cat, dict) else cat + try: + category_ids.add(int(raw)) # type: ignore[arg-type] + except TypeError, ValueError: + continue + return category_ids + + +def _resolve_content_type( + categories: object, + content_type: str, + searched_categories: list[int] | None, +) -> str: + """Resolve a result's content type, honouring custom indexer categories. + + Indexers using non-standard IDs (e.g. 7100 for ebooks) fall outside the standard + ranges, so trust the searched content type when the result carries a category we + explicitly asked for. + """ + detected = _detect_content_type_from_categories(categories, content_type) + if ( + detected == "other" + and searched_categories + and _result_category_ids(categories) & set(searched_categories) + ): + return "audiobook" if content_type == "audiobook" else "book" + return detected + + +def _newznab_result_to_release( + result: dict, + content_type: str = "ebook", + searched_categories: list[int] | None = None, +) -> Release: """Convert a parsed Newznab XML result dict to a Release object.""" raw_title = result.get("title", "Unknown") size_bytes = result.get("size") @@ -125,7 +203,7 @@ def _newznab_result_to_release(result: dict, content_type: str = "ebook") -> Rel indexer=indexer, seeders=seeders if is_torrent else None, peers=peers_display, - content_type=_detect_content_type_from_categories(categories, content_type), + content_type=_resolve_content_type(categories, content_type, searched_categories), extra={ "publish_date": result.get("publishDate"), "categories": categories, @@ -230,12 +308,7 @@ class NewznabSource(ReleaseSource): return [] # Category selection — omit categories when expanding search - if expand_search: - categories = None - elif content_type == "audiobook": - categories = [3030] - else: - categories = [7000] + categories = None if expand_search else _configured_categories(content_type) auto_expand = config.get("NEWZNAB_AUTO_EXPAND", False) deadline = time.monotonic() + NEWZNAB_SEARCH_TIMEOUT_SECONDS @@ -283,7 +356,7 @@ class NewznabSource(ReleaseSource): logger.exception("Newznab search failed") return [] - results = [_newznab_result_to_release(r, content_type) for r in all_results] + results = [_newznab_result_to_release(r, content_type, categories) for r in all_results] if results: nzb_count = sum(1 for r in results if r.protocol == ReleaseProtocol.NZB) diff --git a/tests/newznab/test_source.py b/tests/newznab/test_source.py index a50720cd..9c05cf31 100644 --- a/tests/newznab/test_source.py +++ b/tests/newznab/test_source.py @@ -10,6 +10,7 @@ from shelfmark.release_sources import ReleaseProtocol from shelfmark.release_sources.newznab.source import ( NewznabSource, _newznab_result_to_release, + _parse_category_ids, ) # ── fixtures / helpers ───────────────────────────────────────────────────────── @@ -100,6 +101,18 @@ class TestResultToRelease: r = _newznab_result_to_release(_make_result(categories=[]), "audiobook") assert r.content_type == "audiobook" + def test_custom_searched_category_treated_as_book(self): + r = _newznab_result_to_release(_make_result(categories=[8010]), "ebook", [8010]) + assert r.content_type == "book" + + def test_custom_searched_category_treated_as_audiobook(self): + r = _newznab_result_to_release(_make_result(categories=[{"id": 3040}]), "audiobook", [3040]) + assert r.content_type == "audiobook" + + def test_unsearched_out_of_range_category_stays_other(self): + r = _newznab_result_to_release(_make_result(categories=[2000]), "ebook", [8010]) + assert r.content_type == "other" + def test_freeleech_flag_detected_via_download_volume(self): r = _newznab_result_to_release(_make_result(downloadVolumeFactor=0.0)) assert r.extra["freeleech"] is True @@ -209,6 +222,31 @@ class TestIsAvailable: assert NewznabSource().is_available() is False +# ── category parsing ─────────────────────────────────────────────────────────── + + +class TestParseCategoryIds: + def test_parses_list_of_strings(self): + assert _parse_category_ids(["7100", "7120"]) == [7100, 7120] + + def test_parses_comma_separated_string(self): + assert _parse_category_ids("7100, 7120") == [7100, 7120] + + def test_parses_comma_separated_entry_inside_list(self): + assert _parse_category_ids(["7100,7120", "8010"]) == [7100, 7120, 8010] + + def test_drops_duplicates_and_keeps_order(self): + assert _parse_category_ids(["7120", "7100", "7120"]) == [7120, 7100] + + def test_skips_non_numeric_and_non_positive_values(self): + assert _parse_category_ids(["books", "0", "-7000", "7100"]) == [7100] + + def test_returns_empty_for_unset_or_blank(self): + assert _parse_category_ids(None) == [] + assert _parse_category_ids([]) == [] + assert _parse_category_ids(" ") == [] + + # ── NewznabSource.search ─────────────────────────────────────────────────────── @@ -276,6 +314,44 @@ class TestSearch: _, kwargs = client.search.call_args assert kwargs["categories"] == [3030] + def test_searches_with_configured_ebook_categories(self, monkeypatch): + client = MagicMock() + client.search.return_value = [] + src = self._patched_source( + monkeypatch, client, {"NEWZNAB_EBOOK_CATEGORIES": ["7100", "7120"]} + ) + book = _make_book() + src.search(book, _make_plan(book), content_type="ebook") + _, kwargs = client.search.call_args + assert kwargs["categories"] == [7100, 7120] + + def test_searches_with_configured_audiobook_categories(self, monkeypatch): + client = MagicMock() + client.search.return_value = [] + src = self._patched_source(monkeypatch, client, {"NEWZNAB_AUDIOBOOK_CATEGORIES": "3040"}) + book = _make_book() + src.search(book, _make_plan(book), content_type="audiobook") + _, kwargs = client.search.call_args + assert kwargs["categories"] == [3040] + + def test_falls_back_to_default_when_categories_cleared(self, monkeypatch): + client = MagicMock() + client.search.return_value = [] + src = self._patched_source(monkeypatch, client, {"NEWZNAB_EBOOK_CATEGORIES": []}) + book = _make_book() + src.search(book, _make_plan(book), content_type="ebook") + _, kwargs = client.search.call_args + assert kwargs["categories"] == [7000] + + def test_ebook_categories_do_not_affect_audiobook_search(self, monkeypatch): + client = MagicMock() + client.search.return_value = [] + src = self._patched_source(monkeypatch, client, {"NEWZNAB_EBOOK_CATEGORIES": ["7100"]}) + book = _make_book() + src.search(book, _make_plan(book), content_type="audiobook") + _, kwargs = client.search.call_args + assert kwargs["categories"] == [3030] + def test_expand_search_removes_categories(self, monkeypatch): client = MagicMock() client.search.return_value = []