mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-16 10:38:11 +00:00
fix(network): apply HTTP proxy to Open Library and Prowlarr requests (#1526)
Fixes #1524 Open Library metadata searches and Prowlarr API calls used `get_ssl_verify` but never `get_proxies`, so the HTTP proxy set in Settings → Network was not applied to their HTTPS requests. Both now pass the configured proxies. Relying on `requests`' `trust_env` with only `HTTP_PROXY` set leaves the same gap for `https://` destinations. Verification (Linux, `uv run --frozen pytest`): - New tests `tests/metadata/test_openlibrary_proxy.py` and `tests/prowlarr/test_api_proxy.py`: 4 passed. With the source change reverted they fail (2 failed, 2 errors). - `tests/metadata` + `tests/prowlarr`: 949 passed, 42 skipped. - The full suite in my environment showed 61 failures and 26 errors outside these areas. I did not compare them against `main`, so I'm not claiming the full suite is green here; CI will show it. - `ruff check` and `ruff format --check` on the changed files pass. Prepared with AI assistance; reviewed and tested by me before submitting. Co-authored-by: dbdefault <246079841+dbdefault@users.noreply.github.com> Co-authored-by: CaliBrain <calibrain@l4n.xyz>
This commit is contained in:
4 files changed
+160
-2
No files matched your search
@@ -22,7 +22,7 @@ from shelfmark.core.settings_registry import (
|
||||
register_settings,
|
||||
)
|
||||
from shelfmark.core.text_match import STOPWORDS
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
from shelfmark.download.network import get_proxies, get_ssl_verify
|
||||
from shelfmark.metadata_providers import (
|
||||
BookMetadata,
|
||||
DisplayField,
|
||||
@@ -241,6 +241,7 @@ class OpenLibraryProvider(MetadataProvider):
|
||||
f"{OPENLIBRARY_BASE_URL}/search.json",
|
||||
params=params,
|
||||
timeout=15,
|
||||
proxies=get_proxies(OPENLIBRARY_BASE_URL),
|
||||
verify=get_ssl_verify(OPENLIBRARY_BASE_URL),
|
||||
)
|
||||
response.raise_for_status()
|
||||
@@ -285,6 +286,7 @@ class OpenLibraryProvider(MetadataProvider):
|
||||
response = self.session.get(
|
||||
f"{OPENLIBRARY_BASE_URL}/works/{book_id}.json",
|
||||
timeout=15,
|
||||
proxies=get_proxies(OPENLIBRARY_BASE_URL),
|
||||
verify=get_ssl_verify(OPENLIBRARY_BASE_URL),
|
||||
)
|
||||
response.raise_for_status()
|
||||
@@ -321,6 +323,7 @@ class OpenLibraryProvider(MetadataProvider):
|
||||
response = self.session.get(
|
||||
f"{OPENLIBRARY_BASE_URL}/isbn/{clean_isbn}.json",
|
||||
timeout=15,
|
||||
proxies=get_proxies(OPENLIBRARY_BASE_URL),
|
||||
verify=get_ssl_verify(OPENLIBRARY_BASE_URL),
|
||||
)
|
||||
response.raise_for_status()
|
||||
@@ -595,6 +598,7 @@ class OpenLibraryProvider(MetadataProvider):
|
||||
response = self.session.get(
|
||||
f"{OPENLIBRARY_BASE_URL}{author_key}.json",
|
||||
timeout=10,
|
||||
proxies=get_proxies(OPENLIBRARY_BASE_URL),
|
||||
verify=get_ssl_verify(OPENLIBRARY_BASE_URL),
|
||||
)
|
||||
response.raise_for_status()
|
||||
@@ -616,6 +620,7 @@ def _test_openlibrary_connection() -> dict[str, Any]:
|
||||
f"{OPENLIBRARY_BASE_URL}/search.json",
|
||||
params={"q": "test", "limit": 1},
|
||||
timeout=10,
|
||||
proxies=get_proxies(OPENLIBRARY_BASE_URL),
|
||||
verify=get_ssl_verify(OPENLIBRARY_BASE_URL),
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
@@ -11,7 +11,7 @@ import requests
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
from shelfmark.download.network import get_proxies, get_ssl_verify
|
||||
from shelfmark.release_sources.prowlarr.torznab import parse_torznab_xml
|
||||
from shelfmark.release_sources.prowlarr.utils import coerce_float_like, coerce_int_like
|
||||
|
||||
@@ -178,6 +178,7 @@ class ProwlarrClient:
|
||||
params=params,
|
||||
json=json_data,
|
||||
timeout=self.timeout,
|
||||
proxies=get_proxies(url),
|
||||
verify=get_ssl_verify(url),
|
||||
)
|
||||
|
||||
@@ -432,6 +433,7 @@ class ProwlarrClient:
|
||||
# Override the session default JSON accept header.
|
||||
"Accept": "application/rss+xml, application/xml;q=0.9, */*;q=0.8"
|
||||
},
|
||||
proxies=get_proxies(url),
|
||||
verify=get_ssl_verify(url),
|
||||
)
|
||||
if not response.ok:
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Open Library requests must honor Network HTTP proxy settings (#1524)."""
|
||||
|
||||
import shelfmark.metadata_providers.openlibrary as openlibrary_module
|
||||
from shelfmark.metadata_providers import MetadataSearchOptions, SearchType
|
||||
from shelfmark.metadata_providers.openlibrary import OpenLibraryProvider
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload=None, status_code=200):
|
||||
self._payload = payload if payload is not None else {"docs": []}
|
||||
self.status_code = status_code
|
||||
self.ok = status_code < 400
|
||||
|
||||
def raise_for_status(self):
|
||||
if not self.ok:
|
||||
raise openlibrary_module.requests.HTTPError(f"{self.status_code}", response=self)
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def test_openlibrary_search_passes_configured_http_proxy(monkeypatch):
|
||||
"""Universal Open Library search must send traffic through get_proxies()."""
|
||||
expected = {
|
||||
"http": "http://10.0.80.3:8118",
|
||||
"https": "http://10.0.80.3:8118",
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
openlibrary_module,
|
||||
"get_proxies",
|
||||
lambda url: expected if "openlibrary.org" in url else {},
|
||||
)
|
||||
monkeypatch.setattr(openlibrary_module, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(openlibrary_module._rate_limiter, "wait_if_needed", lambda: None)
|
||||
|
||||
from shelfmark.core.config import config as app_config
|
||||
|
||||
monkeypatch.setattr(
|
||||
app_config,
|
||||
"get",
|
||||
lambda key, default=None: False if key == "METADATA_CACHE_ENABLED" else default,
|
||||
)
|
||||
|
||||
provider = OpenLibraryProvider()
|
||||
captured: dict = {}
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
captured["url"] = url
|
||||
captured.update(kwargs)
|
||||
return _FakeResponse({"docs": []})
|
||||
|
||||
monkeypatch.setattr(provider.session, "get", fake_get)
|
||||
|
||||
provider.search(MetadataSearchOptions(query="Mistborn", search_type=SearchType.GENERAL))
|
||||
|
||||
assert captured.get("proxies") == expected
|
||||
|
||||
|
||||
def test_openlibrary_connection_test_passes_configured_http_proxy(monkeypatch):
|
||||
"""Settings connection test for Open Library must also use the proxy."""
|
||||
expected = {
|
||||
"http": "http://proxy.local:8118",
|
||||
"https": "http://proxy.local:8118",
|
||||
}
|
||||
monkeypatch.setattr(openlibrary_module, "get_proxies", lambda _url: expected)
|
||||
monkeypatch.setattr(openlibrary_module, "get_ssl_verify", lambda _url: True)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
captured["url"] = url
|
||||
captured.update(kwargs)
|
||||
return _FakeResponse({"docs": [{"title": "ok"}]})
|
||||
|
||||
monkeypatch.setattr(
|
||||
openlibrary_module.requests.Session,
|
||||
"get",
|
||||
lambda self, url, **kwargs: fake_get(url, **kwargs),
|
||||
)
|
||||
|
||||
result = openlibrary_module._test_openlibrary_connection()
|
||||
|
||||
assert result["success"] is True
|
||||
assert captured.get("proxies") == expected
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Prowlarr API client must honor Network HTTP proxy settings (#1524)."""
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
import shelfmark.release_sources.prowlarr.api as prowlarr_api
|
||||
from shelfmark.release_sources.prowlarr.api import ProwlarrClient
|
||||
|
||||
_TORZNAB_EMPTY = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0"><channel></channel></rss>"""
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, text="", status_code=200, reason="OK", payload=None):
|
||||
self.text = text
|
||||
self.status_code = status_code
|
||||
self.reason = reason
|
||||
self.ok = status_code < 400
|
||||
self._payload = payload if payload is not None else {}
|
||||
|
||||
def raise_for_status(self):
|
||||
if not self.ok:
|
||||
raise requests.exceptions.HTTPError(f"{self.status_code} {self.reason}", response=self)
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_proxy(monkeypatch):
|
||||
expected = {
|
||||
"http": "http://10.0.80.3:8118",
|
||||
"https": "http://10.0.80.3:8118",
|
||||
}
|
||||
monkeypatch.setattr(prowlarr_api, "get_proxies", lambda _url: expected)
|
||||
monkeypatch.setattr(prowlarr_api, "get_ssl_verify", lambda _url: True)
|
||||
return expected
|
||||
|
||||
|
||||
def test_prowlarr_json_request_passes_configured_http_proxy(monkeypatch, http_proxy):
|
||||
client = ProwlarrClient("http://prowlarr:9696", "apikey")
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_request(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _Response(payload={"version": "1.0.0"})
|
||||
|
||||
monkeypatch.setattr(client._session, "request", fake_request)
|
||||
|
||||
ok, _message = client.test_connection()
|
||||
|
||||
assert ok is True
|
||||
assert captured.get("proxies") == http_proxy
|
||||
|
||||
|
||||
def test_prowlarr_torznab_search_passes_configured_http_proxy(monkeypatch, http_proxy):
|
||||
client = ProwlarrClient("http://prowlarr:9696", "apikey", indexer_timeout=90)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_get(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _Response(_TORZNAB_EMPTY)
|
||||
|
||||
monkeypatch.setattr(client._session, "get", fake_get)
|
||||
|
||||
assert client.torznab_search(indexer_id=1, query="Dune") == []
|
||||
assert captured.get("proxies") == http_proxy
|
||||
Reference in new issue
Block a user