Files
shelfmark/tests/metadata/test_hardcover_rate_limit.py
T
splitsec2andCaliBrain 45548687b7 fix(hardcover): defensive rate limiting and 429 backoff (#1439)
Kept as small as I could so it's easy to review. It is still about 140
lines of code plus tests, in two commits so the second can be dropped on
its own. Same kind of fix as #1429, so see the question there about
separate PRs versus combined.

This is a defensive change. I found the problem on my own deployment,
which has a more automated ingestion on top of this provider: Hardcover
want-to-read items turn into requests, and each pending request gets one
`get_book` lookup. One pass made around 38 lookups back to back, which
is more than Hardcover's burst allows, and a few came back as 429.
`_execute_query` swallowed the 429 and returned `None`, so `get_book`
returned `None`, and the caller read that as "no such book". Nothing in
the logs said a lookup had been throttled.

**What it does**
- Every POST goes through one shared token bucket (burst 5, 0.8 requests
per second), set under the documented Free limits. The limit is per user
across all of the account's apps, so I kept well below it.
- A 429 is retried up to 3 attempts in total. It waits for `Retry-After`
when Hardcover sends one, otherwise 2^attempt seconds plus up to a
second of jitter, and the wait holds back other threads too. A
`Retry-After` over 30 seconds (a spent daily quota) is not waited out.
- When the retries run out it raises `HardcoverRateLimitError`, a
`RuntimeError`, instead of returning `None`.
- The second commit makes `get_book` ask for that error, so a throttled
or failed lookup is no longer taken for a missing book. GraphQL
rejections still return `None`.

**Behaviour change in the second commit.** `/api/metadata/book` already
turns a `RuntimeError` into a 503, so a timeout or HTTP error there now
returns 503 instead of 404. In `/api/releases` the same error goes
through the operational errors handler and returns 500 with the message,
where it used to return 404 "Book not found". If you'd rather keep
returning `None` and only have the pacing and backoff, I can drop the
second commit.

**Limits.** I read them from the `hardcover-docs` repo
(`api/Getting-Started.mdx`, Rate Limits), because the docs site blocks
my headless browser: Free is 60 per minute, a burst of 10 and 5,000 a
day, with `Retry-After` on a 429. Personal Access Tokens get double the
burst. I didn't parse the `RateLimit-Policy` headers, so the bucket
constants are fixed.

**Testing.** The tests use a fake clock and never sleep. I broke each
rule in turn (pacing, `Retry-After`, the 30 second cap, the shared
hold-back, and `get_book` raising) and each break failed a test. Ruff,
basedpyright, `tests/metadata` and `tests/core` pass. I haven't seen a
real 429 from Hardcover with this change running, so the `Retry-After`
handling comes from the docs.

The one red check is the existing clock-dependent test
`test_a_queued_torrent_asks_for_a_grace_once`, which fails on any runner
that booted recently. It is fixed in #1426 and is not caused by this
change.

---------

Co-authored-by: CaliBrain <calibrain@l4n.xyz>
2026-10-20 01:06:56 -04:00

273 lines
8.6 KiB
Python

"""Hardcover allows a 10 request burst refilling at 60/min and answers over-limit
calls with HTTP 429 plus a Retry-After header. Every request goes through
``_execute_query``, so that is where the client paces itself and backs off."""
from typing import Any
import pytest
import requests
from shelfmark.core.cache import get_metadata_cache
from shelfmark.metadata_providers import hardcover
from shelfmark.metadata_providers.hardcover import (
HardcoverProvider,
HardcoverRateLimitError,
_TokenBucket,
)
PAT = "hc_pat_" + "a" * 32
BOOK = {
"books": [
{
"id": 7,
"title": "Dune",
"contributions": [{"author": {"name": "Frank Herbert"}}],
}
]
}
class FakeClock:
"""Monotonic clock whose sleep just moves time forward."""
def __init__(self) -> None:
self.now = 1000.0
self.sleeps: list[float] = []
def __call__(self) -> float:
return self.now
def sleep(self, seconds: float) -> None:
self.sleeps.append(seconds)
self.now += seconds
class FakeResponse:
def __init__(self, status: int, body: dict | None = None, headers: dict | None = None):
self.status_code = status
self._body = body if body is not None else {}
self.headers = headers or {}
def raise_for_status(self) -> None:
if self.status_code >= 400:
raise requests.HTTPError(f"{self.status_code} error", response=self)
def json(self) -> dict:
return self._body
class FakeSession:
def __init__(self, responses: list[FakeResponse]) -> None:
self.responses = list(responses)
self.posts = 0
def post(self, *_args: Any, **_kwargs: Any) -> FakeResponse:
self.posts += 1
return self.responses.pop(0) if len(self.responses) > 1 else self.responses[0]
def ok(data: dict | None = None) -> FakeResponse:
return FakeResponse(200, {"data": data if data is not None else BOOK})
def throttled(retry_after: str | None = None) -> FakeResponse:
headers = {"Retry-After": retry_after} if retry_after is not None else {}
return FakeResponse(429, {"error": "Too Many Requests"}, headers)
@pytest.fixture(autouse=True)
def _fresh_metadata_cache():
"""get_book is cached by id; keep one test's book from answering the next."""
get_metadata_cache().clear()
yield
get_metadata_cache().clear()
@pytest.fixture
def clock(monkeypatch) -> FakeClock:
"""Swap the shared limiter for a fake clock; nothing really sleeps."""
fake = FakeClock()
monkeypatch.setattr(hardcover, "_rate_limiter", _TokenBucket(5, 1.0, fake, fake.sleep))
monkeypatch.setattr(hardcover, "_retry_jitter", lambda: 0.0)
return fake
def provider_with(*responses: FakeResponse) -> tuple[HardcoverProvider, FakeSession]:
provider = HardcoverProvider(api_key=PAT)
session = FakeSession(list(responses))
provider.session = session # type: ignore[assignment]
return provider, session
class TestTokenBucket:
def test_allows_a_burst_without_waiting(self):
fake = FakeClock()
bucket = _TokenBucket(5, 1.0, fake, fake.sleep)
for _ in range(5):
bucket.acquire()
assert fake.sleeps == []
def test_waits_for_a_token_once_the_burst_is_spent(self):
fake = FakeClock()
bucket = _TokenBucket(5, 0.5, fake, fake.sleep)
for _ in range(5):
bucket.acquire()
bucket.acquire()
assert sum(fake.sleeps) == pytest.approx(2.0)
def test_refills_with_time(self):
fake = FakeClock()
bucket = _TokenBucket(5, 1.0, fake, fake.sleep)
for _ in range(5):
bucket.acquire()
fake.now += 3.0
for _ in range(3):
bucket.acquire()
assert fake.sleeps == []
def test_idle_time_does_not_bank_more_than_the_burst(self):
fake = FakeClock()
bucket = _TokenBucket(5, 1.0, fake, fake.sleep)
fake.now += 10_000.0
for _ in range(5):
bucket.acquire()
assert fake.sleeps == []
bucket.acquire()
assert sum(fake.sleeps) == pytest.approx(1.0)
def test_sustained_rate_never_exceeds_the_refill_rate(self):
fake = FakeClock()
bucket = _TokenBucket(5, 1.0, fake, fake.sleep)
start = fake.now
for _ in range(65):
bucket.acquire()
# 5 free, then one per second.
assert fake.now - start == pytest.approx(60.0)
def test_penalize_blocks_the_next_acquire(self):
fake = FakeClock()
bucket = _TokenBucket(5, 1.0, fake, fake.sleep)
bucket.penalize(12.0)
bucket.acquire()
assert sum(fake.sleeps) >= 12.0
class TestExecuteQueryThrottling:
def test_every_request_takes_a_token(self, clock):
provider, session = provider_with(ok())
for _ in range(8):
provider._execute_query("query { me { id } }", {})
assert session.posts == 8
# 5 burst + 3 paced at one per second.
assert sum(clock.sleeps) == pytest.approx(3.0)
def test_429_is_retried_after_the_retry_after_delay(self, clock):
provider, session = provider_with(throttled("7"), ok())
result = provider._execute_query("query { me { id } }", {})
assert result == BOOK
assert session.posts == 2
assert sum(clock.sleeps) >= 7.0
def test_429_without_retry_after_backs_off_and_grows(self, clock):
provider, session = provider_with(throttled(), throttled(), ok())
result = provider._execute_query("query { me { id } }", {})
assert result == BOOK
assert session.posts == 3
waits = [s for s in clock.sleeps if s > 0]
assert len(waits) == 2
assert waits[1] > waits[0] > 0
def test_unparseable_retry_after_falls_back_to_backoff(self, clock):
provider, session = provider_with(throttled("Wed, 21 Oct 2026 07:28:00 GMT"), ok())
assert provider._execute_query("query { me { id } }", {}) == BOOK
assert session.posts == 2
assert sum(clock.sleeps) > 0
def test_retries_are_bounded(self, clock):
provider, session = provider_with(throttled("1"))
with pytest.raises(HardcoverRateLimitError):
provider._execute_query("query { me { id } }", {}, raise_on_error=True)
assert session.posts == hardcover.HARDCOVER_RATE_LIMIT_ATTEMPTS
def test_exhausted_429_returns_none_without_raise_on_error(self, clock):
provider, session = provider_with(throttled("1"))
assert provider._execute_query("query { me { id } }", {}) is None
assert session.posts == hardcover.HARDCOVER_RATE_LIMIT_ATTEMPTS
def test_long_retry_after_is_not_waited_out(self, clock):
"""A spent daily quota says to wait until midnight UTC; don't hold a worker for that."""
provider, session = provider_with(throttled("3600"))
with pytest.raises(HardcoverRateLimitError):
provider._execute_query("query { me { id } }", {}, raise_on_error=True)
assert session.posts == 1
assert all(s < 60 for s in clock.sleeps)
def test_a_429_holds_back_other_callers_too(self, clock):
provider, _session = provider_with(throttled("20"), ok())
provider._execute_query("query { me { id } }", {})
clock.sleeps.clear()
before = clock.now
hardcover._rate_limiter.acquire()
# The retry already waited 20s; the shared bucket starts empty afterwards.
assert clock.now - before <= 1.0 + 1e-9
@pytest.mark.parametrize("status", [401, 403, 500])
def test_other_http_errors_are_not_retried(self, clock, status):
provider, session = provider_with(FakeResponse(status, {"error": "no"}))
assert provider._execute_query("query { me { id } }", {}) is None
assert session.posts == 1
class TestGetBookDuringThrottle:
def test_get_book_recovers_from_a_single_429(self, clock):
provider, session = provider_with(throttled("1"), ok())
book = provider.get_book("7")
assert book is not None
assert book.title == "Dune"
assert session.posts == 2
def test_get_book_raises_instead_of_reporting_not_found(self, clock):
provider, _session = provider_with(throttled("1"))
with pytest.raises(HardcoverRateLimitError):
provider.get_book("7")
def test_get_book_still_returns_none_when_graphql_rejects_it(self, clock):
provider, _session = provider_with(FakeResponse(200, {"errors": [{"message": "nope"}]}))
assert provider.get_book("7") is None
def test_get_book_still_returns_none_for_a_missing_book(self, clock):
provider, _session = provider_with(ok({"books": []}))
assert provider.get_book("7") is None