test(e2e): serve the mock Prowlarr .torrent from Prowlarr's own origin (#1602)

The nightly client-deluge profile has failed since 61c7ad3 with "Failed
to
add to deluge: Failed to fetch torrent file: torrent fetch refused:
address
is not public". Shelfmark now fetches a .torrent only from the
configured
Prowlarr/Newznab/Jackett origin or from a public address, and the mock
Prowlarr's release linked straight to http://mock-aa/payload.torrent, a
private address on another origin. qBittorrent and Transmission hid this
by
handing the URL to the client; Deluge needs the file and failed.

Real Prowlarr rewrites indexer links to its own /{id}/download endpoint.
The
mock now does the same: the torznab link uses the host Shelfmark reached
it
on, and /{id}/download fetches the .torrent from mock-aa. Every client
now
gets the file through Shelfmark's own fetch.

tests/download/test_prowlarr_mock_torrent_fetch.py runs the real
extract_torrent_info against the mock in normal CI, so the link cannot
drift
off the Prowlarr origin again unnoticed; it fails on the old mock with
the
same error.
This commit is contained in:
CaliBrain authored and GitHub committed 2026-10-27 11:25:49 -04:00
1 parent 16e20679b4
commit 947ce7bfe5
4 files changed
+111 -12

No files matched your search

@@ -0,0 +1,76 @@
"""Fetch the e2e platform's mock Prowlarr release through shelfmark's torrent fetch.
The client-* e2e profiles (nightly only) drive real torrent clients from the mock
Prowlarr's one release. Shelfmark fetches a .torrent only from the configured
Prowlarr origin or from public addresses, and every mock lives on a private
address, so a release link that leaves the Prowlarr origin is refused. Deluge
then fails, while qBittorrent and Transmission quietly hand the URL to the client
instead. This runs the real ``extract_torrent_info`` against the mock on localhost
in normal CI, so the link cannot drift off the Prowlarr origin unnoticed.
"""
from __future__ import annotations
import importlib.util
import threading
from pathlib import Path
from socketserver import ThreadingMixIn
from wsgiref.simple_server import WSGIServer, make_server
import pytest
import requests
from shelfmark.download.clients.torrent_utils import extract_torrent_info
from shelfmark.release_sources.prowlarr.torznab import parse_torznab_xml
MOCK_PATH = Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "mock_services.py"
class _ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
# The Prowlarr download handler fetches the .torrent from this same server.
daemon_threads = True
@pytest.fixture
def prowlarr_url(monkeypatch):
"""Serve the mock with every role, its Prowlarr reading .torrent files from itself."""
if not MOCK_PATH.exists():
pytest.skip(f"platform mock not found at {MOCK_PATH}")
server = make_server("127.0.0.1", 0, None, server_class=_ThreadingWSGIServer)
url = f"http://127.0.0.1:{server.server_port}"
# The module wires its routes and reads its upstream URL at import time. Another
# host name for this server stands in for mock-aa, a private non-Prowlarr origin.
monkeypatch.setenv("MOCK_ROLE", "all")
monkeypatch.setenv("AA_INTERNAL_URL", f"http://localhost:{server.server_port}")
spec = importlib.util.spec_from_file_location("mock_prowlarr_services", MOCK_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
server.set_app(module.app)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield url
finally:
server.shutdown()
def test_mock_prowlarr_release_torrent_is_fetchable(prowlarr_url, monkeypatch) -> None:
config_values = {"PROWLARR_URL": prowlarr_url}
monkeypatch.setattr(
"shelfmark.download.clients.torrent_utils.config.get",
lambda key, default="": config_values.get(key, default),
)
search = requests.get(
f"{prowlarr_url}/api/v1/indexer/1/newznab",
params={"t": "search", "q": "Moby Dick"},
timeout=10,
)
search.raise_for_status()
(release,) = parse_torznab_xml(search.text)
info = extract_torrent_info(release["downloadUrl"])
assert info.fetch_error is None
assert info.torrent_data
assert info.info_hash
+10 -4
View File
@@ -164,9 +164,15 @@ demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, an
- **DoH** on at boot.
- **Real qBittorrent download.** A mock Prowlarr (`/api/v1/system/status`,
`/api/v1/indexer`, torznab search) returns one release whose `.torrent` is a
**tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. A real
qBittorrent completes the download over HTTP (no tracker/peer/seeder), and
shelfmark's completion detection + file move lands the book in `/books`. The
**tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. Like real
Prowlarr, the release link is Prowlarr's own `/{id}/download` endpoint, which
fetches the `.torrent` from `mock-aa`: shelfmark fetches a `.torrent` only from
the configured Prowlarr origin or a public address, and a link straight to
`mock-aa` is refused (Deluge then fails; qBittorrent and Transmission hand the
URL to the client instead). `tests/download/test_prowlarr_mock_torrent_fetch.py`
checks this in normal CI. A real qBittorrent completes the download over HTTP
(no tracker/peer/seeder), and shelfmark's completion detection + file move
lands the book in `/books`. The
webseed torrent is generated by `mocks/make_webseed_torrent.py` (cross-checked
against shelfmark's own bencode/infohash parser in
`tests/download/test_webseed_torrent_generator.py`), and the whole
@@ -176,7 +182,7 @@ demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, an
| Path | Purpose |
|---|---|
| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`. |
| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`, which `prowlarr` re-serves from its own origin. |
| `mocks/make_webseed_torrent.py` | Stdlib bencode + BEP-19 webseed `.torrent` generator for the `full` real-client download. |
| `qbittorrent/qBittorrent.conf` | Real qBittorrent config (auth bypassed for the e2e subnet) for the `full` profile. |
| `env/full.env` | The heavy `full` profile: real Chrome bypasser + DoH + real qBittorrent. |
+5 -2
View File
@@ -208,8 +208,11 @@ services:
# ----- Mock Prowlarr indexer (full + client-* profiles) ----------------- #
# Minimal Prowlarr API returning one torrent release whose .torrent is a
# webseed pointing at mock-aa. Drives the real torrent-client download in the
# `full` (qBittorrent) and `client-*` (transmission/deluge/rtorrent) profiles.
# webseed pointing at mock-aa. Like real Prowlarr it serves that .torrent from
# its own origin (fetched from AA_INTERNAL_URL): shelfmark fetches a .torrent
# from a private host only when it is the configured Prowlarr. Drives the real
# torrent-client download in the `full` (qBittorrent) and `client-*`
# (transmission/deluge/rtorrent) profiles.
mock-prowlarr:
<<: *mock-build
container_name: e2e-mock-prowlarr
+20 -6
View File
@@ -356,8 +356,11 @@ def register_flaresolverr(flask_app: Flask) -> None:
# Role: prowlarr (minimal Prowlarr API for the `full` real-client pipeline)
# --------------------------------------------------------------------------- #
# Implements just the endpoints shelfmark's prowlarr client calls, returning one
# torrent release whose download is mock-aa's webseed .torrent. A real
# qBittorrent then completes the download over HTTP (no tracker/peer needed).
# torrent release whose download is mock-aa's webseed .torrent. Like real Prowlarr,
# the release link points at Prowlarr's own download endpoint, which fetches the
# indexer's file: shelfmark fetches .torrent files only from the configured
# Prowlarr origin or from public addresses, and mock-aa is neither. A real torrent
# client then completes the download over HTTP (no tracker/peer needed).
AA_INTERNAL_URL = os.environ.get("AA_INTERNAL_URL", "http://mock-aa")
@@ -390,12 +393,22 @@ def register_prowlarr(flask_app: Flask) -> None:
@flask_app.route("/api/v1/indexer/<int:indexer_id>/newznab")
def prowlarr_torznab(indexer_id: int) -> Response:
del indexer_id
t = request.args.get("t", "search")
if t == "caps":
return Response(_torznab_caps(), mimetype="application/xml")
query = request.args.get("q", "") or "E2E Mock Book"
return Response(_torznab_search(query), mimetype="application/xml")
return Response(_torznab_search(query, indexer_id), mimetype="application/xml")
@flask_app.route("/<int:indexer_id>/download")
def prowlarr_download(indexer_id: int) -> Response:
# Prowlarr's proxied download: fetch the indexer's .torrent and serve it.
del indexer_id
import requests as _rq
upstream = _rq.get(f"{AA_INTERNAL_URL}/payload.torrent", timeout=10)
resp = make_response(upstream.content, upstream.status_code)
resp.headers["Content-Type"] = "application/x-bittorrent"
return resp
def _torznab_caps() -> str:
@@ -410,8 +423,9 @@ def _torznab_caps() -> str:
)
def _torznab_search(query: str) -> str:
torrent_url = f"{AA_INTERNAL_URL}/payload.torrent"
def _torznab_search(query: str, indexer_id: int) -> str:
# Same origin shelfmark used to reach this mock, i.e. its PROWLARR_URL.
torrent_url = f"{request.host_url.rstrip('/')}/{indexer_id}/download"
size = len(_payload_bytes())
title = f"{query} - E2E Mock Book"
return (