mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-17 10:18:15 +00:00
test(e2e): serve the mock Prowlarr .torrent from Prowlarr's own origin (#1602)
The nightly client-deluge profile has failed since 61c7ad3 with "Failed
to
add to deluge: Failed to fetch torrent file: torrent fetch refused:
address
is not public". Shelfmark now fetches a .torrent only from the
configured
Prowlarr/Newznab/Jackett origin or from a public address, and the mock
Prowlarr's release linked straight to http://mock-aa/payload.torrent, a
private address on another origin. qBittorrent and Transmission hid this
by
handing the URL to the client; Deluge needs the file and failed.
Real Prowlarr rewrites indexer links to its own /{id}/download endpoint.
The
mock now does the same: the torznab link uses the host Shelfmark reached
it
on, and /{id}/download fetches the .torrent from mock-aa. Every client
now
gets the file through Shelfmark's own fetch.
tests/download/test_prowlarr_mock_torrent_fetch.py runs the real
extract_torrent_info against the mock in normal CI, so the link cannot
drift
off the Prowlarr origin again unnoticed; it fails on the old mock with
the
same error.
This commit is contained in:
1 parent
16e20679b4
commit
947ce7bfe5
4 files changed
+111
-12
No files matched your search
@@ -0,0 +1,76 @@
|
||||
"""Fetch the e2e platform's mock Prowlarr release through shelfmark's torrent fetch.
|
||||
|
||||
The client-* e2e profiles (nightly only) drive real torrent clients from the mock
|
||||
Prowlarr's one release. Shelfmark fetches a .torrent only from the configured
|
||||
Prowlarr origin or from public addresses, and every mock lives on a private
|
||||
address, so a release link that leaves the Prowlarr origin is refused. Deluge
|
||||
then fails, while qBittorrent and Transmission quietly hand the URL to the client
|
||||
instead. This runs the real ``extract_torrent_info`` against the mock on localhost
|
||||
in normal CI, so the link cannot drift off the Prowlarr origin unnoticed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from socketserver import ThreadingMixIn
|
||||
from wsgiref.simple_server import WSGIServer, make_server
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from shelfmark.download.clients.torrent_utils import extract_torrent_info
|
||||
from shelfmark.release_sources.prowlarr.torznab import parse_torznab_xml
|
||||
|
||||
MOCK_PATH = Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "mock_services.py"
|
||||
|
||||
|
||||
class _ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
|
||||
# The Prowlarr download handler fetches the .torrent from this same server.
|
||||
daemon_threads = True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prowlarr_url(monkeypatch):
|
||||
"""Serve the mock with every role, its Prowlarr reading .torrent files from itself."""
|
||||
if not MOCK_PATH.exists():
|
||||
pytest.skip(f"platform mock not found at {MOCK_PATH}")
|
||||
server = make_server("127.0.0.1", 0, None, server_class=_ThreadingWSGIServer)
|
||||
url = f"http://127.0.0.1:{server.server_port}"
|
||||
# The module wires its routes and reads its upstream URL at import time. Another
|
||||
# host name for this server stands in for mock-aa, a private non-Prowlarr origin.
|
||||
monkeypatch.setenv("MOCK_ROLE", "all")
|
||||
monkeypatch.setenv("AA_INTERNAL_URL", f"http://localhost:{server.server_port}")
|
||||
spec = importlib.util.spec_from_file_location("mock_prowlarr_services", MOCK_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
server.set_app(module.app)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield url
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_mock_prowlarr_release_torrent_is_fetchable(prowlarr_url, monkeypatch) -> None:
|
||||
config_values = {"PROWLARR_URL": prowlarr_url}
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.torrent_utils.config.get",
|
||||
lambda key, default="": config_values.get(key, default),
|
||||
)
|
||||
search = requests.get(
|
||||
f"{prowlarr_url}/api/v1/indexer/1/newznab",
|
||||
params={"t": "search", "q": "Moby Dick"},
|
||||
timeout=10,
|
||||
)
|
||||
search.raise_for_status()
|
||||
(release,) = parse_torznab_xml(search.text)
|
||||
|
||||
info = extract_torrent_info(release["downloadUrl"])
|
||||
|
||||
assert info.fetch_error is None
|
||||
assert info.torrent_data
|
||||
assert info.info_hash
|
||||
@@ -164,9 +164,15 @@ demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, an
|
||||
- **DoH** on at boot.
|
||||
- **Real qBittorrent download.** A mock Prowlarr (`/api/v1/system/status`,
|
||||
`/api/v1/indexer`, torznab search) returns one release whose `.torrent` is a
|
||||
**tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. A real
|
||||
qBittorrent completes the download over HTTP (no tracker/peer/seeder), and
|
||||
shelfmark's completion detection + file move lands the book in `/books`. The
|
||||
**tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. Like real
|
||||
Prowlarr, the release link is Prowlarr's own `/{id}/download` endpoint, which
|
||||
fetches the `.torrent` from `mock-aa`: shelfmark fetches a `.torrent` only from
|
||||
the configured Prowlarr origin or a public address, and a link straight to
|
||||
`mock-aa` is refused (Deluge then fails; qBittorrent and Transmission hand the
|
||||
URL to the client instead). `tests/download/test_prowlarr_mock_torrent_fetch.py`
|
||||
checks this in normal CI. A real qBittorrent completes the download over HTTP
|
||||
(no tracker/peer/seeder), and shelfmark's completion detection + file move
|
||||
lands the book in `/books`. The
|
||||
webseed torrent is generated by `mocks/make_webseed_torrent.py` (cross-checked
|
||||
against shelfmark's own bencode/infohash parser in
|
||||
`tests/download/test_webseed_torrent_generator.py`), and the whole
|
||||
@@ -176,7 +182,7 @@ demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, an
|
||||
|
||||
| Path | Purpose |
|
||||
|---|---|
|
||||
| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`. |
|
||||
| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`, which `prowlarr` re-serves from its own origin. |
|
||||
| `mocks/make_webseed_torrent.py` | Stdlib bencode + BEP-19 webseed `.torrent` generator for the `full` real-client download. |
|
||||
| `qbittorrent/qBittorrent.conf` | Real qBittorrent config (auth bypassed for the e2e subnet) for the `full` profile. |
|
||||
| `env/full.env` | The heavy `full` profile: real Chrome bypasser + DoH + real qBittorrent. |
|
||||
|
||||
@@ -208,8 +208,11 @@ services:
|
||||
|
||||
# ----- Mock Prowlarr indexer (full + client-* profiles) ----------------- #
|
||||
# Minimal Prowlarr API returning one torrent release whose .torrent is a
|
||||
# webseed pointing at mock-aa. Drives the real torrent-client download in the
|
||||
# `full` (qBittorrent) and `client-*` (transmission/deluge/rtorrent) profiles.
|
||||
# webseed pointing at mock-aa. Like real Prowlarr it serves that .torrent from
|
||||
# its own origin (fetched from AA_INTERNAL_URL): shelfmark fetches a .torrent
|
||||
# from a private host only when it is the configured Prowlarr. Drives the real
|
||||
# torrent-client download in the `full` (qBittorrent) and `client-*`
|
||||
# (transmission/deluge/rtorrent) profiles.
|
||||
mock-prowlarr:
|
||||
<<: *mock-build
|
||||
container_name: e2e-mock-prowlarr
|
||||
|
||||
@@ -356,8 +356,11 @@ def register_flaresolverr(flask_app: Flask) -> None:
|
||||
# Role: prowlarr (minimal Prowlarr API for the `full` real-client pipeline)
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Implements just the endpoints shelfmark's prowlarr client calls, returning one
|
||||
# torrent release whose download is mock-aa's webseed .torrent. A real
|
||||
# qBittorrent then completes the download over HTTP (no tracker/peer needed).
|
||||
# torrent release whose download is mock-aa's webseed .torrent. Like real Prowlarr,
|
||||
# the release link points at Prowlarr's own download endpoint, which fetches the
|
||||
# indexer's file: shelfmark fetches .torrent files only from the configured
|
||||
# Prowlarr origin or from public addresses, and mock-aa is neither. A real torrent
|
||||
# client then completes the download over HTTP (no tracker/peer needed).
|
||||
AA_INTERNAL_URL = os.environ.get("AA_INTERNAL_URL", "http://mock-aa")
|
||||
|
||||
|
||||
@@ -390,12 +393,22 @@ def register_prowlarr(flask_app: Flask) -> None:
|
||||
|
||||
@flask_app.route("/api/v1/indexer/<int:indexer_id>/newznab")
|
||||
def prowlarr_torznab(indexer_id: int) -> Response:
|
||||
del indexer_id
|
||||
t = request.args.get("t", "search")
|
||||
if t == "caps":
|
||||
return Response(_torznab_caps(), mimetype="application/xml")
|
||||
query = request.args.get("q", "") or "E2E Mock Book"
|
||||
return Response(_torznab_search(query), mimetype="application/xml")
|
||||
return Response(_torznab_search(query, indexer_id), mimetype="application/xml")
|
||||
|
||||
@flask_app.route("/<int:indexer_id>/download")
|
||||
def prowlarr_download(indexer_id: int) -> Response:
|
||||
# Prowlarr's proxied download: fetch the indexer's .torrent and serve it.
|
||||
del indexer_id
|
||||
import requests as _rq
|
||||
|
||||
upstream = _rq.get(f"{AA_INTERNAL_URL}/payload.torrent", timeout=10)
|
||||
resp = make_response(upstream.content, upstream.status_code)
|
||||
resp.headers["Content-Type"] = "application/x-bittorrent"
|
||||
return resp
|
||||
|
||||
|
||||
def _torznab_caps() -> str:
|
||||
@@ -410,8 +423,9 @@ def _torznab_caps() -> str:
|
||||
)
|
||||
|
||||
|
||||
def _torznab_search(query: str) -> str:
|
||||
torrent_url = f"{AA_INTERNAL_URL}/payload.torrent"
|
||||
def _torznab_search(query: str, indexer_id: int) -> str:
|
||||
# Same origin shelfmark used to reach this mock, i.e. its PROWLARR_URL.
|
||||
torrent_url = f"{request.host_url.rstrip('/')}/{indexer_id}/download"
|
||||
size = len(_payload_bytes())
|
||||
title = f"{query} - E2E Mock Book"
|
||||
return (
|
||||
|
||||
Reference in new issue
Block a user