fix(prowlarr): skip indexers in Prowlarr failure back-off (#1324)

## What Read `/api/v1/indexerstatus` once per search and skip indexers whose `disabledTill` is still ahead. Skipped is neither attempted nor failed. One client method, one counter on `_IndexerSearchOutcome`, ten tests. ## Why Prowlarr's own search leaves out an indexer it has disabled after repeated failures. Shelfmark queries each indexer through its Torznab endpoint, which answers 429 instead: ``` Prowlarr Torznab error response: <error code="429" description="Indexer is disabled till 09/09/2026 15:00:34 due to recent failures." /> Prowlarr: 1 of 5 indexer searches failed (indexer 2 search failed: 429 Client Error: Too Many Requests ...) Release search failed for source prowlarr: 1 of 5 indexer searches failed (...) ``` That counted as a failed search, so with one indexer in back-off and the other four answering empty, `/api/releases?source=prowlarr` returned 503 for every book for the length of the back-off (one hour here). `/api/v1/indexerstatus` on Prowlarr 2.5.2: ```json [{"indexerId": 2, "disabledTill": "2026-09-09T15:00:34Z", "mostRecentFailure": "2026-09-09T14:00:34Z", "initialFailure": "2026-09-09T14:00:34Z"}] ``` ## Behaviour | indexers | before | after | |---|---|---| | 1 in back-off, 4 answer empty | 503 "1 of 5 indexer searches failed" | "No releases found" | | 1 in back-off, 1 answers with releases | releases | releases, one Torznab call fewer | | 1 in back-off, 1 times out, 3 answer empty | "1 of 5 failed" | "1 of 4 failed" | | every indexer in back-off | 503 "5 of 5 failed" | "every indexer is disabled by Prowlarr after recent failures (until ...)" | | status endpoint unreachable | n/a | as before, nothing skipped | Auto-expand no longer retries a pass in which nothing was asked. ## Tests `uv run pytest tests/prowlarr`: 563 passed, 42 skipped. `ruff check` and `ruff format` clean.
2026-10-05 12:11:12 +01:00 · 2026-09-11 00:49:33 -04:00
parent 1e3fd48b8b
commit c45d342931
3 changed files with 241 additions and 0 deletions
@@ -2,6 +2,7 @@

 from collections.abc import Mapping
 from contextlib import suppress
+from datetime import UTC, datetime
 from http import HTTPStatus
 from typing import Any, TypedDict

@@ -235,6 +236,34 @@ class ProwlarrClient:
            logger.exception("Failed to get indexers")
            return []

+    def get_disabled_indexers(self, *, now: datetime | None = None) -> dict[int, str]:
+        """Get indexers in failure back-off, keyed by ID with their disabledTill value."""
+        try:
+            entries = _normalize_json_object_list(
+                self._request("GET", "/api/v1/indexerstatus"),
+                context="Prowlarr indexer status",
+            )
+        except _PROWLARR_CLIENT_ERRORS:
+            logger.exception("Failed to get indexer status")
+            return {}
+
+        current = now or datetime.now(UTC)
+        disabled: dict[int, str] = {}
+        for entry in entries:
+            indexer_id = coerce_int_like(entry.get("indexerId"))
+            disabled_till_raw = entry.get("disabledTill")
+            if indexer_id is None or not disabled_till_raw:
+                continue
+            try:
+                disabled_till = datetime.fromisoformat(str(disabled_till_raw))
+            except ValueError:
+                continue
+            if disabled_till.tzinfo is None:
+                disabled_till = disabled_till.replace(tzinfo=UTC)
+            if disabled_till > current:
+                disabled[indexer_id] = str(disabled_till_raw)
+        return disabled
+
    def get_enabled_indexers_detailed(
        self, *, raise_on_error: bool = False
    ) -> list[dict[str, Any]]:
@@ -297,6 +297,7 @@ class _IndexerSearchOutcome:
    results: list[dict]
    attempted: int = 0
    failed: int = 0
+    skipped: int = 0
    last_error: str | None = None


@@ -1010,6 +1011,22 @@ class ProwlarrSource(ReleaseSource):
                if time.monotonic() > deadline:
                    _raise_timeout_error(f"Prowlarr search timed out after {int(search_budget)}s")

+            # Prowlarr's own search skips an indexer in failure back-off; the
+            # per-indexer Torznab endpoint answers 429 instead.
+            try:
+                disabled_indexers = client.get_disabled_indexers()
+            except _PROWLARR_SOURCE_ERRORS as e:
+                logger.warning("Failed to load Prowlarr indexer status: %s", e)
+                disabled_indexers = {}
+            if disabled_indexers:
+                logger.info(
+                    "Prowlarr: skipping indexer(s) in failure back-off: %s",
+                    ", ".join(
+                        f"{indexer_id} (till {till})"
+                        for indexer_id, till in sorted(disabled_indexers.items())
+                    ),
+                )
+
            def search_indexers(query: str, cats: list[int] | None) -> _IndexerSearchOutcome:
                """Search indexers with given categories via Torznab/Newznab.

@@ -1027,6 +1044,9 @@ class ProwlarrSource(ReleaseSource):
                    return outcome

                for indexer_id in target_indexer_ids:
+                    if indexer_id in disabled_indexers:
+                        outcome.skipped += 1
+                        continue
                    _check_timeout()
                    outcome.attempted += 1
                    try:
@@ -1052,6 +1072,7 @@ class ProwlarrSource(ReleaseSource):
            all_results: list[dict] = []
            attempted_searches = 0
            failed_searches = 0
+            skipped_searches = 0
            last_search_error: str | None = None

            for idx, variant in enumerate(variants, start=1):
@@ -1070,6 +1091,7 @@ class ProwlarrSource(ReleaseSource):
                if (
                    not outcome.results
                    and not outcome.failed
+                    and outcome.attempted
                    and categories
                    and auto_expand_enabled
                ):
@@ -1082,11 +1104,13 @@ class ProwlarrSource(ReleaseSource):
                    outcome.results = expanded.results
                    outcome.attempted += expanded.attempted
                    outcome.failed += expanded.failed
+                    outcome.skipped += expanded.skipped
                    outcome.last_error = expanded.last_error or outcome.last_error
                    self.last_search_type = "expanded"

                attempted_searches += outcome.attempted
                failed_searches += outcome.failed
+                skipped_searches += outcome.skipped
                last_search_error = outcome.last_error or last_search_error

                for r in outcome.results:
@@ -1192,6 +1216,10 @@ class ProwlarrSource(ReleaseSource):
                    f"({last_search_error})"
                )
                raise SourceUnavailableError(msg)
+            if not results and not attempted_searches and skipped_searches:
+                until = max(disabled_indexers.values(), default="later")
+                msg = f"every indexer is disabled by Prowlarr after recent failures (until {until})"
+                raise SourceUnavailableError(msg)
            return results

    def is_available(self) -> bool:
@@ -0,0 +1,184 @@
+"""Indexers in Prowlarr failure back-off are skipped, not counted as failed searches."""
+
+from datetime import UTC, datetime
+
+import pytest
+import requests
+
+from shelfmark.metadata_providers import BookMetadata
+from shelfmark.release_sources import SourceUnavailableError
+from shelfmark.release_sources.prowlarr.api import ProwlarrClient, ProwlarrSearchError
+from shelfmark.release_sources.prowlarr.source import ProwlarrSource
+
+_NOW = datetime(2026, 9, 9, 14, 30, tzinfo=UTC)
+
+
+class _Client:
+    indexer_timeout = 90
+
+    def __init__(self, results, disabled=None, failing=()):
+        self.results = results
+        self.disabled = disabled or {}
+        self.failing = set(failing)
+        self.calls: list[tuple[int, object]] = []
+
+    def get_enabled_indexers_detailed(self, *, raise_on_error=False):
+        del raise_on_error
+        return [
+            {
+                "id": indexer_id,
+                "enable": True,
+                "capabilities": {"categories": [{"id": 7000, "subCategories": []}]},
+            }
+            for indexer_id in sorted(self.results)
+        ]
+
+    def get_disabled_indexers(self):
+        return dict(self.disabled)
+
+    def torznab_search(self, *, indexer_id, query, categories=None, search_type="book", **_):
+        del query, search_type
+        self.calls.append((indexer_id, categories))
+        if indexer_id in self.failing:
+            msg = f"indexer {indexer_id} did not respond within 90s"
+            raise ProwlarrSearchError(msg)
+        return self.results[indexer_id]
+
+    def get_enriched_indexer_ids(self, restrict_to=None, indexers=None):
+        del restrict_to, indexers
+        return []
+
+    def get_indexer_seed_settings(self, restrict_to=None):
+        del restrict_to
+        return {}
+
+
+def _release(indexer_id: int) -> dict:
+    return {
+        "guid": f"g{indexer_id}",
+        "title": "Dune",
+        "indexerId": indexer_id,
+        "indexer": f"indexer-{indexer_id}",
+        "protocol": "torrent",
+        "size": 1048576,
+        "seeders": 5,
+    }
+
+
+def _search(monkeypatch, client, config_values=None):
+    import shelfmark.release_sources.prowlarr.source as prowlarr_source
+    from shelfmark.core.search_plan import build_release_search_plan
+
+    values = {"PROWLARR_INDEXERS": "", "PROWLARR_AUTO_EXPAND": False}
+    values.update(config_values or {})
+    monkeypatch.setattr(
+        prowlarr_source.config, "get", lambda key, default=None: values.get(key, default)
+    )
+
+    source = ProwlarrSource()
+    monkeypatch.setattr(source, "_get_client", lambda: client)
+
+    book = BookMetadata(
+        provider="hardcover", provider_id="123", title="Dune", authors=["Frank Herbert"]
+    )
+    plan = build_release_search_plan(book, languages=["en"])
+    return source.search(book, plan)
+
+
+class TestDisabledIndexersAreSkipped:
+    def test_disabled_indexer_is_not_queried_and_empty_answers_stay_no_results(self, monkeypatch):
+        client = _Client({1: [], 2: []}, disabled={1: "2026-09-09T15:00:34Z"})
+
+        assert _search(monkeypatch, client) == []
+        assert client.calls == [(2, [7000])]
+
+    def test_disabled_indexer_does_not_hide_the_others_results(self, monkeypatch):
+        client = _Client({1: [], 2: [_release(2)]}, disabled={1: "2026-09-09T15:00:34Z"})
+
+        releases = _search(monkeypatch, client)
+
+        assert [r.indexer for r in releases] == ["indexer-2"]
+        assert client.calls == [(2, [7000])]
+
+    def test_every_indexer_disabled_reports_that_instead_of_no_results(self, monkeypatch):
+        client = _Client(
+            {1: [], 2: []},
+            disabled={1: "2026-09-09T15:00:34Z", 2: "2026-09-09T15:10:00Z"},
+        )
+
+        with pytest.raises(SourceUnavailableError) as excinfo:
+            _search(monkeypatch, client)
+
+        assert "disabled by Prowlarr" in str(excinfo.value)
+        assert "2026-09-09T15:10:00Z" in str(excinfo.value)
+        assert client.calls == []
+
+    def test_a_real_failure_elsewhere_is_still_reported(self, monkeypatch):
+        client = _Client({1: [], 2: [], 3: []}, disabled={1: "2026-09-09T15:00:34Z"}, failing={2})
+
+        with pytest.raises(SourceUnavailableError) as excinfo:
+            _search(monkeypatch, client)
+
+        assert "1 of 2 indexer searches failed" in str(excinfo.value)
+
+    def test_auto_expand_skips_the_disabled_indexer_on_the_retry_too(self, monkeypatch):
+        client = _Client({1: [], 2: []}, disabled={1: "2026-09-09T15:00:34Z"})
+
+        assert _search(monkeypatch, client, {"PROWLARR_AUTO_EXPAND": True}) == []
+        assert client.calls == [(2, [7000]), (2, None)]
+
+    def test_auto_expand_does_not_retry_when_nothing_was_asked(self, monkeypatch):
+        client = _Client({1: []}, disabled={1: "2026-09-09T15:00:34Z"})
+
+        with pytest.raises(SourceUnavailableError):
+            _search(monkeypatch, client, {"PROWLARR_AUTO_EXPAND": True})
+
+        assert client.calls == []
+
+    def test_status_lookup_failure_fails_open(self, monkeypatch):
+        class _NoStatusClient(_Client):
+            def get_disabled_indexers(self):
+                raise requests.exceptions.ConnectionError("status unavailable")
+
+        client = _NoStatusClient({1: [], 2: []})
+
+        assert _search(monkeypatch, client) == []
+        assert client.calls == [(1, [7000]), (2, [7000])]
+
+
+class TestGetDisabledIndexers:
+    def _client(self, monkeypatch, payload):
+        client = ProwlarrClient("http://prowlarr:9696", "key")
+        monkeypatch.setattr(client, "_request", lambda *args, **kwargs: payload)
+        return client
+
+    def test_reports_indexers_whose_back_off_has_not_ended(self, monkeypatch):
+        client = self._client(
+            monkeypatch,
+            [
+                {"indexerId": 2, "disabledTill": "2026-09-09T15:00:34Z"},
+                {"indexerId": 3, "disabledTill": "2026-09-09T14:00:00Z"},
+                {"indexerId": 4},
+                {"indexerId": "x", "disabledTill": "2026-09-09T15:00:34Z"},
+                {"indexerId": 5, "disabledTill": "not a date"},
+            ],
+        )
+
+        assert client.get_disabled_indexers(now=_NOW) == {2: "2026-09-09T15:00:34Z"}
+
+    def test_naive_timestamps_are_read_as_utc(self, monkeypatch):
+        client = self._client(
+            monkeypatch, [{"indexerId": 2, "disabledTill": "2026-09-09T15:00:34"}]
+        )
+
+        assert client.get_disabled_indexers(now=_NOW) == {2: "2026-09-09T15:00:34"}
+
+    def test_request_failure_reports_nothing_disabled(self, monkeypatch):
+        client = ProwlarrClient("http://prowlarr:9696", "key")
+
+        def _boom(*args, **kwargs):
+            raise requests.exceptions.ConnectionError("connection refused")
+
+        monkeypatch.setattr(client, "_request", _boom)
+
+        assert client.get_disabled_indexers(now=_NOW) == {}