From c45d342931e892c3e2c41e7712b51ae14b3e1f63 Mon Sep 17 00:00:00 2001 From: Jorge Lima <5619521+jfmlima@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:49:33 +0100 Subject: [PATCH] fix(prowlarr): skip indexers in Prowlarr failure back-off (#1324) ## What Read `/api/v1/indexerstatus` once per search and skip indexers whose `disabledTill` is still ahead. Skipped is neither attempted nor failed. One client method, one counter on `_IndexerSearchOutcome`, ten tests. ## Why Prowlarr's own search leaves out an indexer it has disabled after repeated failures. Shelfmark queries each indexer through its Torznab endpoint, which answers 429 instead: ``` Prowlarr Torznab error response: Prowlarr: 1 of 5 indexer searches failed (indexer 2 search failed: 429 Client Error: Too Many Requests ...) Release search failed for source prowlarr: 1 of 5 indexer searches failed (...) ``` That counted as a failed search, so with one indexer in back-off and the other four answering empty, `/api/releases?source=prowlarr` returned 503 for every book for the length of the back-off (one hour here). `/api/v1/indexerstatus` on Prowlarr 2.5.2: ```json [{"indexerId": 2, "disabledTill": "2026-09-09T15:00:34Z", "mostRecentFailure": "2026-09-09T14:00:34Z", "initialFailure": "2026-09-09T14:00:34Z"}] ``` ## Behaviour | indexers | before | after | |---|---|---| | 1 in back-off, 4 answer empty | 503 "1 of 5 indexer searches failed" | "No releases found" | | 1 in back-off, 1 answers with releases | releases | releases, one Torznab call fewer | | 1 in back-off, 1 times out, 3 answer empty | "1 of 5 failed" | "1 of 4 failed" | | every indexer in back-off | 503 "5 of 5 failed" | "every indexer is disabled by Prowlarr after recent failures (until ...)" | | status endpoint unreachable | n/a | as before, nothing skipped | Auto-expand no longer retries a pass in which nothing was asked. ## Tests `uv run pytest tests/prowlarr`: 563 passed, 42 skipped. `ruff check` and `ruff format` clean. --- shelfmark/release_sources/prowlarr/api.py | 29 +++ shelfmark/release_sources/prowlarr/source.py | 28 +++ tests/prowlarr/test_disabled_indexers.py | 184 +++++++++++++++++++ 3 files changed, 241 insertions(+) create mode 100644 tests/prowlarr/test_disabled_indexers.py diff --git a/shelfmark/release_sources/prowlarr/api.py b/shelfmark/release_sources/prowlarr/api.py index b131f122..5b9bb92a 100644 --- a/shelfmark/release_sources/prowlarr/api.py +++ b/shelfmark/release_sources/prowlarr/api.py @@ -2,6 +2,7 @@ from collections.abc import Mapping from contextlib import suppress +from datetime import UTC, datetime from http import HTTPStatus from typing import Any, TypedDict @@ -235,6 +236,34 @@ class ProwlarrClient: logger.exception("Failed to get indexers") return [] + def get_disabled_indexers(self, *, now: datetime | None = None) -> dict[int, str]: + """Get indexers in failure back-off, keyed by ID with their disabledTill value.""" + try: + entries = _normalize_json_object_list( + self._request("GET", "/api/v1/indexerstatus"), + context="Prowlarr indexer status", + ) + except _PROWLARR_CLIENT_ERRORS: + logger.exception("Failed to get indexer status") + return {} + + current = now or datetime.now(UTC) + disabled: dict[int, str] = {} + for entry in entries: + indexer_id = coerce_int_like(entry.get("indexerId")) + disabled_till_raw = entry.get("disabledTill") + if indexer_id is None or not disabled_till_raw: + continue + try: + disabled_till = datetime.fromisoformat(str(disabled_till_raw)) + except ValueError: + continue + if disabled_till.tzinfo is None: + disabled_till = disabled_till.replace(tzinfo=UTC) + if disabled_till > current: + disabled[indexer_id] = str(disabled_till_raw) + return disabled + def get_enabled_indexers_detailed( self, *, raise_on_error: bool = False ) -> list[dict[str, Any]]: diff --git a/shelfmark/release_sources/prowlarr/source.py b/shelfmark/release_sources/prowlarr/source.py index e0025dc7..a7c23921 100644 --- a/shelfmark/release_sources/prowlarr/source.py +++ b/shelfmark/release_sources/prowlarr/source.py @@ -297,6 +297,7 @@ class _IndexerSearchOutcome: results: list[dict] attempted: int = 0 failed: int = 0 + skipped: int = 0 last_error: str | None = None @@ -1010,6 +1011,22 @@ class ProwlarrSource(ReleaseSource): if time.monotonic() > deadline: _raise_timeout_error(f"Prowlarr search timed out after {int(search_budget)}s") + # Prowlarr's own search skips an indexer in failure back-off; the + # per-indexer Torznab endpoint answers 429 instead. + try: + disabled_indexers = client.get_disabled_indexers() + except _PROWLARR_SOURCE_ERRORS as e: + logger.warning("Failed to load Prowlarr indexer status: %s", e) + disabled_indexers = {} + if disabled_indexers: + logger.info( + "Prowlarr: skipping indexer(s) in failure back-off: %s", + ", ".join( + f"{indexer_id} (till {till})" + for indexer_id, till in sorted(disabled_indexers.items()) + ), + ) + def search_indexers(query: str, cats: list[int] | None) -> _IndexerSearchOutcome: """Search indexers with given categories via Torznab/Newznab. @@ -1027,6 +1044,9 @@ class ProwlarrSource(ReleaseSource): return outcome for indexer_id in target_indexer_ids: + if indexer_id in disabled_indexers: + outcome.skipped += 1 + continue _check_timeout() outcome.attempted += 1 try: @@ -1052,6 +1072,7 @@ class ProwlarrSource(ReleaseSource): all_results: list[dict] = [] attempted_searches = 0 failed_searches = 0 + skipped_searches = 0 last_search_error: str | None = None for idx, variant in enumerate(variants, start=1): @@ -1070,6 +1091,7 @@ class ProwlarrSource(ReleaseSource): if ( not outcome.results and not outcome.failed + and outcome.attempted and categories and auto_expand_enabled ): @@ -1082,11 +1104,13 @@ class ProwlarrSource(ReleaseSource): outcome.results = expanded.results outcome.attempted += expanded.attempted outcome.failed += expanded.failed + outcome.skipped += expanded.skipped outcome.last_error = expanded.last_error or outcome.last_error self.last_search_type = "expanded" attempted_searches += outcome.attempted failed_searches += outcome.failed + skipped_searches += outcome.skipped last_search_error = outcome.last_error or last_search_error for r in outcome.results: @@ -1192,6 +1216,10 @@ class ProwlarrSource(ReleaseSource): f"({last_search_error})" ) raise SourceUnavailableError(msg) + if not results and not attempted_searches and skipped_searches: + until = max(disabled_indexers.values(), default="later") + msg = f"every indexer is disabled by Prowlarr after recent failures (until {until})" + raise SourceUnavailableError(msg) return results def is_available(self) -> bool: diff --git a/tests/prowlarr/test_disabled_indexers.py b/tests/prowlarr/test_disabled_indexers.py new file mode 100644 index 00000000..c724104a --- /dev/null +++ b/tests/prowlarr/test_disabled_indexers.py @@ -0,0 +1,184 @@ +"""Indexers in Prowlarr failure back-off are skipped, not counted as failed searches.""" + +from datetime import UTC, datetime + +import pytest +import requests + +from shelfmark.metadata_providers import BookMetadata +from shelfmark.release_sources import SourceUnavailableError +from shelfmark.release_sources.prowlarr.api import ProwlarrClient, ProwlarrSearchError +from shelfmark.release_sources.prowlarr.source import ProwlarrSource + +_NOW = datetime(2026, 9, 9, 14, 30, tzinfo=UTC) + + +class _Client: + indexer_timeout = 90 + + def __init__(self, results, disabled=None, failing=()): + self.results = results + self.disabled = disabled or {} + self.failing = set(failing) + self.calls: list[tuple[int, object]] = [] + + def get_enabled_indexers_detailed(self, *, raise_on_error=False): + del raise_on_error + return [ + { + "id": indexer_id, + "enable": True, + "capabilities": {"categories": [{"id": 7000, "subCategories": []}]}, + } + for indexer_id in sorted(self.results) + ] + + def get_disabled_indexers(self): + return dict(self.disabled) + + def torznab_search(self, *, indexer_id, query, categories=None, search_type="book", **_): + del query, search_type + self.calls.append((indexer_id, categories)) + if indexer_id in self.failing: + msg = f"indexer {indexer_id} did not respond within 90s" + raise ProwlarrSearchError(msg) + return self.results[indexer_id] + + def get_enriched_indexer_ids(self, restrict_to=None, indexers=None): + del restrict_to, indexers + return [] + + def get_indexer_seed_settings(self, restrict_to=None): + del restrict_to + return {} + + +def _release(indexer_id: int) -> dict: + return { + "guid": f"g{indexer_id}", + "title": "Dune", + "indexerId": indexer_id, + "indexer": f"indexer-{indexer_id}", + "protocol": "torrent", + "size": 1048576, + "seeders": 5, + } + + +def _search(monkeypatch, client, config_values=None): + import shelfmark.release_sources.prowlarr.source as prowlarr_source + from shelfmark.core.search_plan import build_release_search_plan + + values = {"PROWLARR_INDEXERS": "", "PROWLARR_AUTO_EXPAND": False} + values.update(config_values or {}) + monkeypatch.setattr( + prowlarr_source.config, "get", lambda key, default=None: values.get(key, default) + ) + + source = ProwlarrSource() + monkeypatch.setattr(source, "_get_client", lambda: client) + + book = BookMetadata( + provider="hardcover", provider_id="123", title="Dune", authors=["Frank Herbert"] + ) + plan = build_release_search_plan(book, languages=["en"]) + return source.search(book, plan) + + +class TestDisabledIndexersAreSkipped: + def test_disabled_indexer_is_not_queried_and_empty_answers_stay_no_results(self, monkeypatch): + client = _Client({1: [], 2: []}, disabled={1: "2026-09-09T15:00:34Z"}) + + assert _search(monkeypatch, client) == [] + assert client.calls == [(2, [7000])] + + def test_disabled_indexer_does_not_hide_the_others_results(self, monkeypatch): + client = _Client({1: [], 2: [_release(2)]}, disabled={1: "2026-09-09T15:00:34Z"}) + + releases = _search(monkeypatch, client) + + assert [r.indexer for r in releases] == ["indexer-2"] + assert client.calls == [(2, [7000])] + + def test_every_indexer_disabled_reports_that_instead_of_no_results(self, monkeypatch): + client = _Client( + {1: [], 2: []}, + disabled={1: "2026-09-09T15:00:34Z", 2: "2026-09-09T15:10:00Z"}, + ) + + with pytest.raises(SourceUnavailableError) as excinfo: + _search(monkeypatch, client) + + assert "disabled by Prowlarr" in str(excinfo.value) + assert "2026-09-09T15:10:00Z" in str(excinfo.value) + assert client.calls == [] + + def test_a_real_failure_elsewhere_is_still_reported(self, monkeypatch): + client = _Client({1: [], 2: [], 3: []}, disabled={1: "2026-09-09T15:00:34Z"}, failing={2}) + + with pytest.raises(SourceUnavailableError) as excinfo: + _search(monkeypatch, client) + + assert "1 of 2 indexer searches failed" in str(excinfo.value) + + def test_auto_expand_skips_the_disabled_indexer_on_the_retry_too(self, monkeypatch): + client = _Client({1: [], 2: []}, disabled={1: "2026-09-09T15:00:34Z"}) + + assert _search(monkeypatch, client, {"PROWLARR_AUTO_EXPAND": True}) == [] + assert client.calls == [(2, [7000]), (2, None)] + + def test_auto_expand_does_not_retry_when_nothing_was_asked(self, monkeypatch): + client = _Client({1: []}, disabled={1: "2026-09-09T15:00:34Z"}) + + with pytest.raises(SourceUnavailableError): + _search(monkeypatch, client, {"PROWLARR_AUTO_EXPAND": True}) + + assert client.calls == [] + + def test_status_lookup_failure_fails_open(self, monkeypatch): + class _NoStatusClient(_Client): + def get_disabled_indexers(self): + raise requests.exceptions.ConnectionError("status unavailable") + + client = _NoStatusClient({1: [], 2: []}) + + assert _search(monkeypatch, client) == [] + assert client.calls == [(1, [7000]), (2, [7000])] + + +class TestGetDisabledIndexers: + def _client(self, monkeypatch, payload): + client = ProwlarrClient("http://prowlarr:9696", "key") + monkeypatch.setattr(client, "_request", lambda *args, **kwargs: payload) + return client + + def test_reports_indexers_whose_back_off_has_not_ended(self, monkeypatch): + client = self._client( + monkeypatch, + [ + {"indexerId": 2, "disabledTill": "2026-09-09T15:00:34Z"}, + {"indexerId": 3, "disabledTill": "2026-09-09T14:00:00Z"}, + {"indexerId": 4}, + {"indexerId": "x", "disabledTill": "2026-09-09T15:00:34Z"}, + {"indexerId": 5, "disabledTill": "not a date"}, + ], + ) + + assert client.get_disabled_indexers(now=_NOW) == {2: "2026-09-09T15:00:34Z"} + + def test_naive_timestamps_are_read_as_utc(self, monkeypatch): + client = self._client( + monkeypatch, [{"indexerId": 2, "disabledTill": "2026-09-09T15:00:34"}] + ) + + assert client.get_disabled_indexers(now=_NOW) == {2: "2026-09-09T15:00:34"} + + def test_request_failure_reports_nothing_disabled(self, monkeypatch): + client = ProwlarrClient("http://prowlarr:9696", "key") + + def _boom(*args, **kwargs): + raise requests.exceptions.ConnectionError("connection refused") + + monkeypatch.setattr(client, "_request", _boom) + + assert client.get_disabled_indexers(now=_NOW) == {}