diff --git a/Dockerfile b/Dockerfile index 8baf0e80..8f71af6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,6 +59,10 @@ ENV FLASK_PORT=8084 # Configure locale, timezone, and perform initial cleanup in a single layer RUN apt-get update && \ apt-get install -y --no-install-recommends \ + # For building C-extensions (cffi, gevent, etc.) + gcc \ + libffi-dev \ + python3-dev \ # For locale locales tzdata \ # For healthcheck diff --git a/readme.md b/readme.md index e431c982..ffb15589 100644 --- a/readme.md +++ b/readme.md @@ -44,6 +44,7 @@ Works great alongside the following library tools, with support for automatic im ### Prerequisites - Docker & Docker Compose +- At least 2 GB of RAM available to the container when using the standard image — see [Memory Requirements](#memory-requirements) ### Installation @@ -138,6 +139,17 @@ docker compose up -d The full-featured image with all network capabilities included. +#### Memory Requirements + +The standard image ships a real Chromium browser, which it launches to solve Cloudflare challenges for Direct Download. Chromium needs room to run: + +- **2 GB of RAM available to the container** is a safe minimum; 1 GB or less is where problems usually start +- Only relevant if you use Direct Download. Prowlarr, IRC and audiobook sources don't start the browser + +When the container is starved of memory, Chromium fails to start and every Direct Download fails with unrelated-looking errors — repeated `403 detected; switching to bypasser` followed by `No download URL found`, and downloads that never complete. If you're seeing that, check the container's memory limit and the host's free memory before suspecting your ISP or DNS. + +If you can't spare the memory, use the [Lite](#lite) image with an external resolver (e.g. FlareSolverr) running elsewhere. + #### Tor Routing Optional Tor support for network privacy: ```bash @@ -175,6 +187,7 @@ A lighter image without the built-in browser automation. Ideal for: - **External services** - Already running FlareSolverr or similar for other applications - **Alternative sources** - Using Prowlarr, IRC, or other configured sources - **Audiobooks** - Using Shelfmark primarily for audiobooks +- **Constrained hosts** - No bundled browser, so it runs comfortably below the standard image's [memory requirements](#memory-requirements) ```bash curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml diff --git a/shelfmark/metadata_providers/hardcover.py b/shelfmark/metadata_providers/hardcover.py index 06a2a6e6..2bc9537f 100644 --- a/shelfmark/metadata_providers/hardcover.py +++ b/shelfmark/metadata_providers/hardcover.py @@ -1,6 +1,7 @@ """Hardcover.app metadata provider. Requires API key.""" import re +import time from contextlib import suppress from dataclasses import dataclass from datetime import UTC, datetime @@ -553,6 +554,31 @@ TITLE_SUGGESTION_FIELDS = "title,alternative_titles" TITLE_SUGGESTION_WEIGHTS = "5,2" TITLE_SUGGESTION_SORT = "_text_match:desc,users_count:desc" +# Hardcover forwards `sort` to Typesense's `sort_by` and rejects the whole search +# if it does not like the value -- an unknown field, a bare field name with no +# direction, more than three keys. A rejected search comes back as HTTP 200 with +# no GraphQL errors and a null `results` body, which is otherwise indistinguishable +# from "nothing matched". An empty sort is always accepted, so fall back to it and +# keep the fallback sticky for a while rather than paying for a doomed request on +# every search. +SORT_FALLBACK = "" +SORT_FALLBACK_TTL = 900.0 +_sort_fallback_until = 0.0 + + +def _search_payload_rejected(result: dict[str, Any] | None) -> bool: + """Report whether Hardcover answered a search with a null results body. + + A search that genuinely matched nothing still returns a results object with + ``found: 0``; only a rejected search nulls it out entirely. + """ + if not isinstance(result, dict): + return False + root = result.get("search", result) + if not isinstance(root, dict) or "results" not in root: + return False + return root["results"] is None + def _combine_headline_description(headline: str | None, description: str | None) -> str | None: """Combine headline (tagline) and description into a single description.""" @@ -1200,7 +1226,7 @@ class HardcoverProvider(MetadataProvider): if not self.api_key or len(normalized_query) < HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH: return [] - result = self._execute_query( + result = self._execute_search_query( SEARCH_FIELD_OPTIONS_QUERY, { "query": normalized_query, @@ -1431,7 +1457,7 @@ class HardcoverProvider(MetadataProvider): logger.debug("Invalid Hardcover series id field value: %s", normalized_value) return None - result = self._execute_query( + result = self._execute_search_query( SEARCH_FIELD_OPTIONS_QUERY, { "query": normalized_value, @@ -2386,7 +2412,7 @@ class HardcoverProvider(MetadataProvider): variables["weights"] = search_weights try: - result = self._execute_query(graphql_query, variables) + result = self._execute_search_query(graphql_query, variables) if not result: logger.debug("Hardcover search: No result from API") return SearchResult(books=[], page=options.page, total_found=0, has_more=False) @@ -2654,6 +2680,45 @@ class HardcoverProvider(MetadataProvider): raise RuntimeError(msg) from e return None + def _execute_search_query(self, query: str, variables: dict[str, Any]) -> dict | None: + """Execute a search query, retrying without ``sort`` if Hardcover rejects it. + + Returns None when the search was rejected, so callers report an empty + result rather than silently treating a failure as "nothing matched". + """ + global _sort_fallback_until + + sort = variables.get("sort") + if sort and time.monotonic() < _sort_fallback_until: + variables = {**variables, "sort": SORT_FALLBACK} + sort = None + + result = self._execute_query(query, variables) + if not _search_payload_rejected(result): + return result + + if not sort: + logger.error( + "Hardcover rejected this search (query_type=%s, fields=%s) and returned " + "no result body", + variables.get("queryType", "Book"), + variables.get("fields"), + ) + return None + + logger.warning( + "Hardcover rejected sort '%s'; retrying searches without a sort order for %ss", + sort, + int(SORT_FALLBACK_TTL), + ) + _sort_fallback_until = time.monotonic() + SORT_FALLBACK_TTL + + retry = self._execute_query(query, {**variables, "sort": SORT_FALLBACK}) + if _search_payload_rejected(retry): + logger.error("Hardcover rejected this search even without a sort order") + return None + return retry + def _parse_search_result(self, item: dict) -> BookMetadata | None: """Parse a search result item into BookMetadata.""" try: diff --git a/tests/metadata/test_hardcover_sort_fallback.py b/tests/metadata/test_hardcover_sort_fallback.py new file mode 100644 index 00000000..65c99b13 --- /dev/null +++ b/tests/metadata/test_hardcover_sort_fallback.py @@ -0,0 +1,113 @@ +from typing import Any + +import pytest + +from shelfmark.metadata_providers import MetadataSearchOptions +from shelfmark.metadata_providers.hardcover import HardcoverProvider + +# Hardcover answers a rejected search with HTTP 200, no GraphQL errors, and a +# null results body. A search that genuinely matched nothing still returns a +# results object with found: 0. +REJECTED = {"search": {"results": None}} +EMPTY = {"search": {"results": {"hits": [], "found": 0}}} +ONE_HIT = {"search": {"results": {"hits": [{"document": {"id": 7, "title": "Dune"}}], "found": 1}}} + + +@pytest.fixture(autouse=True) +def _reset_sort_fallback(monkeypatch): + """Keep the process-wide sort fallback from leaking between tests.""" + monkeypatch.setattr("shelfmark.metadata_providers.hardcover._sort_fallback_until", 0.0) + + +def _reject_sorted(calls: list[dict[str, Any]], *, success=ONE_HIT): + """Build an _execute_query stand-in that rejects any request carrying a sort.""" + + def fake_execute(query: str, variables): + calls.append(dict(variables)) + return REJECTED if variables.get("sort") else success + + return fake_execute + + +class TestHardcoverSortFallback: + def test_retries_without_sort_when_hardcover_rejects_the_sort(self, monkeypatch): + provider = HardcoverProvider(api_key="test-token") + calls: list[dict[str, Any]] = [] + monkeypatch.setattr(provider, "_execute_query", _reject_sorted(calls)) + + result = provider._execute_search_query("query", {"query": "dune", "sort": "relevance"}) + + assert result == ONE_HIT + assert [call["sort"] for call in calls] == ["relevance", ""] + + def test_treats_an_empty_result_set_as_success(self, monkeypatch): + provider = HardcoverProvider(api_key="test-token") + calls: list[dict[str, Any]] = [] + monkeypatch.setattr( + provider, "_execute_query", lambda query, variables: calls.append(variables) or EMPTY + ) + + result = provider._execute_search_query("query", {"query": "dune", "sort": "rating:desc"}) + + assert result == EMPTY + assert len(calls) == 1 + + def test_reports_failure_when_the_retry_is_also_rejected(self, monkeypatch): + provider = HardcoverProvider(api_key="test-token") + calls: list[dict[str, Any]] = [] + monkeypatch.setattr( + provider, "_execute_query", lambda query, variables: calls.append(variables) or REJECTED + ) + + result = provider._execute_search_query("query", {"query": "dune", "sort": "rating:desc"}) + + assert result is None + assert len(calls) == 2 + + def test_reports_failure_for_an_unsorted_rejection(self, monkeypatch): + provider = HardcoverProvider(api_key="test-token") + calls: list[dict[str, Any]] = [] + monkeypatch.setattr( + provider, "_execute_query", lambda query, variables: calls.append(variables) or REJECTED + ) + + result = provider._execute_search_query("query", {"query": "dune", "sort": ""}) + + assert result is None + assert len(calls) == 1 + + def test_skips_the_doomed_request_on_later_searches(self, monkeypatch): + provider = HardcoverProvider(api_key="test-token") + calls: list[dict[str, Any]] = [] + monkeypatch.setattr(provider, "_execute_query", _reject_sorted(calls)) + + provider._execute_search_query("query", {"query": "dune", "sort": "relevance"}) + provider._execute_search_query("query", {"query": "hyperion", "sort": "relevance"}) + + assert [call["sort"] for call in calls] == ["relevance", "", ""] + + def test_search_returns_results_despite_a_rejected_sort(self, monkeypatch): + provider = HardcoverProvider(api_key="test-token") + calls: list[dict[str, Any]] = [] + monkeypatch.setattr(provider, "_execute_query", _reject_sorted(calls)) + + result = provider.search_paginated( + MetadataSearchOptions(query="dune sort fallback", page=1, limit=25) + ) + + assert result.total_found == 1 + assert [book.title for book in result.books] == ["Dune"] + assert [call["sort"] for call in calls] == ["_text_match:desc,users_count:desc", ""] + + +class TestSearchPayloadRejection: + def test_distinguishes_a_null_body_from_an_empty_result_set(self): + from shelfmark.metadata_providers.hardcover import _search_payload_rejected + + assert _search_payload_rejected(REJECTED) is True + assert _search_payload_rejected(EMPTY) is False + assert _search_payload_rejected(ONE_HIT) is False + assert _search_payload_rejected(None) is False + assert _search_payload_rejected({}) is False + # Non-search payloads (list lookups, book fetches) must pass through. + assert _search_payload_rejected({"series": [{"id": 1}]}) is False