fix(library): back off after a failed index instead of retrying for every result (#1430)

Kept small on purpose so it's easy to review.

`library_index._entries_for` retries `fetch_entries` on every call while
the index is failing. The lookup runs once per search result, so with
the Calibre path unreadable a page of 40 results makes 40 attempts and
logs 40 warnings. The entrypoint runs one gevent worker and the Calibre
read is synchronous sqlite, so each attempt also holds up every other
request.

After a failure it now answers from the stale cache (or an empty list)
for 60 seconds before trying again, and a successful index clears that.
The new tests cover 40 lookups against a failing library with a stale
cache (one failed attempt, one warning), a failing library with no cache
(tried once), and a retry once the 60 seconds have passed. The first and
second fail on current `main`.

I left a few other things in this module out to keep this one small:
re-indexing blocks the worker for as long as a big library takes (I
haven't measured how long), `test_connection` stores entries from the
unsaved form path under the shared cache key, and a title with a
collection word in it, like "Books of Blood", shows as "In a
collection". I can send any of those separately.

Ruff and basedpyright are clean and `tests/core` passes.

The one red check is the existing clock-dependent test
`test_a_queued_torrent_asks_for_a_grace_once`, which fails on any runner
that booted recently. It is fixed in #1426 and is not caused by this
change.
This commit is contained in:
splitsec2 authored and GitHub committed 2026-10-20 00:58:49 -04:00
1 parent 47707b827a
commit e8f6505bea
2 files changed
+58

No files matched your search

+11
View File
@@ -37,6 +37,9 @@ if TYPE_CHECKING:
logger = setup_logger(__name__)
_CACHE_TTL_SECONDS = 600 # Re-index a library at most every 10 minutes unless it changed.
# After a failed index, answer from the stale cache for this long before trying again, so a
# search page with dozens of results costs one failed attempt and one warning, not one each.
_FAILURE_BACKOFF_SECONDS = 60
@dataclass
@@ -44,6 +47,7 @@ class _CacheSlot:
entries: list[LibraryEntry] | None = None
fetched_at: float = 0.0
fingerprint: object | None = None
failed_at: float | None = None
_lock = threading.Lock()
@@ -61,11 +65,17 @@ def _store(provider_name: str, entries: list[LibraryEntry], fingerprint: object
slot.entries = entries
slot.fetched_at = time.monotonic()
slot.fingerprint = fingerprint
slot.failed_at = None
def _entries_for(provider: LibraryProvider) -> list[LibraryEntry]:
"""Cached entries for one provider, re-indexed past the TTL or when the library changed."""
slot = _slot(provider.name)
with _lock:
if slot.failed_at is not None and time.monotonic() - slot.failed_at < (
_FAILURE_BACKOFF_SECONDS
):
return slot.entries or []
try:
fingerprint = provider.fingerprint()
with _lock:
@@ -78,6 +88,7 @@ def _entries_for(provider: LibraryProvider) -> list[LibraryEntry]:
except Exception as exc: # noqa: BLE001 - any failure must fail open
logger.warning("library check: %s unavailable (%s); failing open", provider.describe(), exc)
with _lock:
slot.failed_at = time.monotonic()
return slot.entries or [] # Use the stale cache if we have one.
_store(provider.name, entries, fingerprint)
+47
View File
@@ -234,6 +234,53 @@ def test_provider_error_keeps_answering_from_the_stale_cache(
assert len(warnings) == 1
def test_a_failing_library_is_tried_once_per_search_not_once_per_result(
providers: list[_Provider], monkeypatch: pytest.MonkeyPatch, clock: list[float]
) -> None:
provider = _Provider("ebook", {"ebook"}, [_DCC_ENTRY])
providers.append(provider)
assert library_index.is_in_library(_book(), "ebook") is True
provider.error = RuntimeError("boom")
clock[0] += library_index._CACHE_TTL_SECONDS + 1
warnings = _warnings(monkeypatch)
for _ in range(40): # one search page, every result asks
assert library_index.is_in_library(_book(), "ebook") is True
assert provider.fetches == 2 # the first index, then one failed attempt
assert len(warnings) == 1
def test_a_failing_library_with_no_cache_is_not_retried_for_every_result(
providers: list[_Provider], clock: list[float]
) -> None:
provider = _Provider("ebook", {"ebook"}, error=OSError("no such file"))
providers.append(provider)
for _ in range(10):
assert library_index.is_in_library(_book(), "ebook") is False
assert provider.fetches == 1
def test_a_failing_library_is_retried_once_the_back_off_has_passed(
providers: list[_Provider], clock: list[float]
) -> None:
provider = _Provider("ebook", {"ebook"}, [_DCC_ENTRY], error=OSError("down"))
providers.append(provider)
assert library_index.is_in_library(_book(), "ebook") is False
provider.error = None
clock[0] += library_index._FAILURE_BACKOFF_SECONDS - 1
assert library_index.is_in_library(_book(), "ebook") is False
assert provider.fetches == 1
clock[0] += 2
assert library_index.is_in_library(_book(), "ebook") is True
assert provider.fetches == 2
def test_entries_are_cached_until_the_ttl_expires(
providers: list[_Provider], clock: list[float]
) -> None: