From c43da4537d657a053fb77cb49c39ce5758e8af21 Mon Sep 17 00:00:00 2001 From: dbdefault Date: Thu, 22 Oct 2026 14:47:43 +0800 Subject: [PATCH] fix(network): apply HTTP proxy to Open Library and Prowlarr requests (#1526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #1524 Open Library metadata searches and Prowlarr API calls used `get_ssl_verify` but never `get_proxies`, so the HTTP proxy set in Settings → Network was not applied to their HTTPS requests. Both now pass the configured proxies. Relying on `requests`' `trust_env` with only `HTTP_PROXY` set leaves the same gap for `https://` destinations. Verification (Linux, `uv run --frozen pytest`): - New tests `tests/metadata/test_openlibrary_proxy.py` and `tests/prowlarr/test_api_proxy.py`: 4 passed. With the source change reverted they fail (2 failed, 2 errors). - `tests/metadata` + `tests/prowlarr`: 949 passed, 42 skipped. - The full suite in my environment showed 61 failures and 26 errors outside these areas. I did not compare them against `main`, so I'm not claiming the full suite is green here; CI will show it. - `ruff check` and `ruff format --check` on the changed files pass. Prepared with AI assistance; reviewed and tested by me before submitting. Co-authored-by: dbdefault <246079841+dbdefault@users.noreply.github.com> Co-authored-by: CaliBrain --- shelfmark/metadata_providers/openlibrary.py | 7 +- shelfmark/release_sources/prowlarr/api.py | 4 +- tests/metadata/test_openlibrary_proxy.py | 84 +++++++++++++++++++++ tests/prowlarr/test_api_proxy.py | 67 ++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 tests/metadata/test_openlibrary_proxy.py create mode 100644 tests/prowlarr/test_api_proxy.py diff --git a/shelfmark/metadata_providers/openlibrary.py b/shelfmark/metadata_providers/openlibrary.py index f626739d..f52b94b5 100644 --- a/shelfmark/metadata_providers/openlibrary.py +++ b/shelfmark/metadata_providers/openlibrary.py @@ -22,7 +22,7 @@ from shelfmark.core.settings_registry import ( register_settings, ) from shelfmark.core.text_match import STOPWORDS -from shelfmark.download.network import get_ssl_verify +from shelfmark.download.network import get_proxies, get_ssl_verify from shelfmark.metadata_providers import ( BookMetadata, DisplayField, @@ -241,6 +241,7 @@ class OpenLibraryProvider(MetadataProvider): f"{OPENLIBRARY_BASE_URL}/search.json", params=params, timeout=15, + proxies=get_proxies(OPENLIBRARY_BASE_URL), verify=get_ssl_verify(OPENLIBRARY_BASE_URL), ) response.raise_for_status() @@ -285,6 +286,7 @@ class OpenLibraryProvider(MetadataProvider): response = self.session.get( f"{OPENLIBRARY_BASE_URL}/works/{book_id}.json", timeout=15, + proxies=get_proxies(OPENLIBRARY_BASE_URL), verify=get_ssl_verify(OPENLIBRARY_BASE_URL), ) response.raise_for_status() @@ -321,6 +323,7 @@ class OpenLibraryProvider(MetadataProvider): response = self.session.get( f"{OPENLIBRARY_BASE_URL}/isbn/{clean_isbn}.json", timeout=15, + proxies=get_proxies(OPENLIBRARY_BASE_URL), verify=get_ssl_verify(OPENLIBRARY_BASE_URL), ) response.raise_for_status() @@ -595,6 +598,7 @@ class OpenLibraryProvider(MetadataProvider): response = self.session.get( f"{OPENLIBRARY_BASE_URL}{author_key}.json", timeout=10, + proxies=get_proxies(OPENLIBRARY_BASE_URL), verify=get_ssl_verify(OPENLIBRARY_BASE_URL), ) response.raise_for_status() @@ -616,6 +620,7 @@ def _test_openlibrary_connection() -> dict[str, Any]: f"{OPENLIBRARY_BASE_URL}/search.json", params={"q": "test", "limit": 1}, timeout=10, + proxies=get_proxies(OPENLIBRARY_BASE_URL), verify=get_ssl_verify(OPENLIBRARY_BASE_URL), ) response.raise_for_status() diff --git a/shelfmark/release_sources/prowlarr/api.py b/shelfmark/release_sources/prowlarr/api.py index 34678c97..7540bb4c 100644 --- a/shelfmark/release_sources/prowlarr/api.py +++ b/shelfmark/release_sources/prowlarr/api.py @@ -11,7 +11,7 @@ import requests from shelfmark.core.config import config from shelfmark.core.logger import setup_logger from shelfmark.core.utils import normalize_http_url -from shelfmark.download.network import get_ssl_verify +from shelfmark.download.network import get_proxies, get_ssl_verify from shelfmark.release_sources.prowlarr.torznab import parse_torznab_xml from shelfmark.release_sources.prowlarr.utils import coerce_float_like, coerce_int_like @@ -178,6 +178,7 @@ class ProwlarrClient: params=params, json=json_data, timeout=self.timeout, + proxies=get_proxies(url), verify=get_ssl_verify(url), ) @@ -432,6 +433,7 @@ class ProwlarrClient: # Override the session default JSON accept header. "Accept": "application/rss+xml, application/xml;q=0.9, */*;q=0.8" }, + proxies=get_proxies(url), verify=get_ssl_verify(url), ) if not response.ok: diff --git a/tests/metadata/test_openlibrary_proxy.py b/tests/metadata/test_openlibrary_proxy.py new file mode 100644 index 00000000..356d52d7 --- /dev/null +++ b/tests/metadata/test_openlibrary_proxy.py @@ -0,0 +1,84 @@ +"""Open Library requests must honor Network HTTP proxy settings (#1524).""" + +import shelfmark.metadata_providers.openlibrary as openlibrary_module +from shelfmark.metadata_providers import MetadataSearchOptions, SearchType +from shelfmark.metadata_providers.openlibrary import OpenLibraryProvider + + +class _FakeResponse: + def __init__(self, payload=None, status_code=200): + self._payload = payload if payload is not None else {"docs": []} + self.status_code = status_code + self.ok = status_code < 400 + + def raise_for_status(self): + if not self.ok: + raise openlibrary_module.requests.HTTPError(f"{self.status_code}", response=self) + + def json(self): + return self._payload + + +def test_openlibrary_search_passes_configured_http_proxy(monkeypatch): + """Universal Open Library search must send traffic through get_proxies().""" + expected = { + "http": "http://10.0.80.3:8118", + "https": "http://10.0.80.3:8118", + } + monkeypatch.setattr( + openlibrary_module, + "get_proxies", + lambda url: expected if "openlibrary.org" in url else {}, + ) + monkeypatch.setattr(openlibrary_module, "get_ssl_verify", lambda _url: True) + monkeypatch.setattr(openlibrary_module._rate_limiter, "wait_if_needed", lambda: None) + + from shelfmark.core.config import config as app_config + + monkeypatch.setattr( + app_config, + "get", + lambda key, default=None: False if key == "METADATA_CACHE_ENABLED" else default, + ) + + provider = OpenLibraryProvider() + captured: dict = {} + + def fake_get(url, **kwargs): + captured["url"] = url + captured.update(kwargs) + return _FakeResponse({"docs": []}) + + monkeypatch.setattr(provider.session, "get", fake_get) + + provider.search(MetadataSearchOptions(query="Mistborn", search_type=SearchType.GENERAL)) + + assert captured.get("proxies") == expected + + +def test_openlibrary_connection_test_passes_configured_http_proxy(monkeypatch): + """Settings connection test for Open Library must also use the proxy.""" + expected = { + "http": "http://proxy.local:8118", + "https": "http://proxy.local:8118", + } + monkeypatch.setattr(openlibrary_module, "get_proxies", lambda _url: expected) + monkeypatch.setattr(openlibrary_module, "get_ssl_verify", lambda _url: True) + + captured: dict = {} + + def fake_get(url, **kwargs): + captured["url"] = url + captured.update(kwargs) + return _FakeResponse({"docs": [{"title": "ok"}]}) + + monkeypatch.setattr( + openlibrary_module.requests.Session, + "get", + lambda self, url, **kwargs: fake_get(url, **kwargs), + ) + + result = openlibrary_module._test_openlibrary_connection() + + assert result["success"] is True + assert captured.get("proxies") == expected diff --git a/tests/prowlarr/test_api_proxy.py b/tests/prowlarr/test_api_proxy.py new file mode 100644 index 00000000..f077e62a --- /dev/null +++ b/tests/prowlarr/test_api_proxy.py @@ -0,0 +1,67 @@ +"""Prowlarr API client must honor Network HTTP proxy settings (#1524).""" + +import pytest +import requests + +import shelfmark.release_sources.prowlarr.api as prowlarr_api +from shelfmark.release_sources.prowlarr.api import ProwlarrClient + +_TORZNAB_EMPTY = """ +""" + + +class _Response: + def __init__(self, text="", status_code=200, reason="OK", payload=None): + self.text = text + self.status_code = status_code + self.reason = reason + self.ok = status_code < 400 + self._payload = payload if payload is not None else {} + + def raise_for_status(self): + if not self.ok: + raise requests.exceptions.HTTPError(f"{self.status_code} {self.reason}", response=self) + + def json(self): + return self._payload + + +@pytest.fixture +def http_proxy(monkeypatch): + expected = { + "http": "http://10.0.80.3:8118", + "https": "http://10.0.80.3:8118", + } + monkeypatch.setattr(prowlarr_api, "get_proxies", lambda _url: expected) + monkeypatch.setattr(prowlarr_api, "get_ssl_verify", lambda _url: True) + return expected + + +def test_prowlarr_json_request_passes_configured_http_proxy(monkeypatch, http_proxy): + client = ProwlarrClient("http://prowlarr:9696", "apikey") + captured: dict[str, object] = {} + + def fake_request(**kwargs): + captured.update(kwargs) + return _Response(payload={"version": "1.0.0"}) + + monkeypatch.setattr(client._session, "request", fake_request) + + ok, _message = client.test_connection() + + assert ok is True + assert captured.get("proxies") == http_proxy + + +def test_prowlarr_torznab_search_passes_configured_http_proxy(monkeypatch, http_proxy): + client = ProwlarrClient("http://prowlarr:9696", "apikey", indexer_timeout=90) + captured: dict[str, object] = {} + + def fake_get(**kwargs): + captured.update(kwargs) + return _Response(_TORZNAB_EMPTY) + + monkeypatch.setattr(client._session, "get", fake_get) + + assert client.torznab_search(indexer_id=1, query="Dune") == [] + assert captured.get("proxies") == http_proxy