From 1c19326bd0a1024176d7ecbb2a610b0ebeebb018 Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Tue, 23 Jun 2026 16:58:24 -0400 Subject: [PATCH] e2e testing 2 (#1083) --- .github/workflows/e2e-platform.yml | 130 ++++ tests/audiobookbay/test_scraper_contract.py | 103 ++++ tests/download/test_doh_resolver_mock.py | 79 +++ .../test_webseed_torrent_generator.py | 87 +++ tests/e2e/platform/.gitignore | 7 + tests/e2e/platform/README.md | 231 +++++++ tests/e2e/platform/build-images.sh | 15 + tests/e2e/platform/dns/Corefile | 13 + tests/e2e/platform/dns/Corefile.blocked | 18 + tests/e2e/platform/dns/mock.test.db | 13 + tests/e2e/platform/docker-compose.e2e.yml | 294 +++++++++ tests/e2e/platform/mocks/Dockerfile | 25 + .../platform/mocks/fixtures/aa_detail.html | 15 + .../mocks/fixtures/aa_search_empty.html | 9 + .../fixtures/aa_search_layout_drift.html | 27 + .../mocks/fixtures/aa_search_results.html | 41 ++ .../platform/mocks/fixtures/cf_challenge.html | 39 ++ .../platform/mocks/make_webseed_torrent.py | 113 ++++ tests/e2e/platform/mocks/mock_services.py | 577 ++++++++++++++++++ tests/e2e/platform/mocks/requirements.txt | 4 + .../e2e/platform/qbittorrent/qBittorrent.conf | 22 + tests/e2e/platform/run-e2e.sh | 88 +++ tests/e2e/platform/run-matrix.sh | 47 ++ tests/e2e/platform/suite/__init__.py | 0 tests/e2e/platform/suite/conftest.py | 132 ++++ .../platform/suite/test_cluster_bypasser.py | 86 +++ .../suite/test_cluster_bypasser_tor.py | 32 + .../platform/suite/test_cluster_clients.py | 87 +++ .../test_cluster_download_permissions.py | 62 ++ .../suite/test_cluster_full_pipeline.py | 146 +++++ .../suite/test_cluster_health_entrypoint.py | 49 ++ .../platform/suite/test_cluster_search_aa.py | 70 +++ .../e2e/platform/suite/test_config_matrix.py | 113 ++++ 33 files changed, 2774 insertions(+) create mode 100644 .github/workflows/e2e-platform.yml create mode 100644 tests/audiobookbay/test_scraper_contract.py create mode 100644 tests/download/test_doh_resolver_mock.py create mode 100644 tests/download/test_webseed_torrent_generator.py create mode 100644 tests/e2e/platform/.gitignore create mode 100644 tests/e2e/platform/README.md create mode 100755 tests/e2e/platform/build-images.sh create mode 100644 tests/e2e/platform/dns/Corefile create mode 100644 tests/e2e/platform/dns/Corefile.blocked create mode 100644 tests/e2e/platform/dns/mock.test.db create mode 100644 tests/e2e/platform/docker-compose.e2e.yml create mode 100644 tests/e2e/platform/mocks/Dockerfile create mode 100644 tests/e2e/platform/mocks/fixtures/aa_detail.html create mode 100644 tests/e2e/platform/mocks/fixtures/aa_search_empty.html create mode 100644 tests/e2e/platform/mocks/fixtures/aa_search_layout_drift.html create mode 100644 tests/e2e/platform/mocks/fixtures/aa_search_results.html create mode 100644 tests/e2e/platform/mocks/fixtures/cf_challenge.html create mode 100644 tests/e2e/platform/mocks/make_webseed_torrent.py create mode 100644 tests/e2e/platform/mocks/mock_services.py create mode 100644 tests/e2e/platform/mocks/requirements.txt create mode 100755 tests/e2e/platform/qbittorrent/qBittorrent.conf create mode 100755 tests/e2e/platform/run-e2e.sh create mode 100755 tests/e2e/platform/run-matrix.sh create mode 100644 tests/e2e/platform/suite/__init__.py create mode 100644 tests/e2e/platform/suite/conftest.py create mode 100644 tests/e2e/platform/suite/test_cluster_bypasser.py create mode 100644 tests/e2e/platform/suite/test_cluster_bypasser_tor.py create mode 100644 tests/e2e/platform/suite/test_cluster_clients.py create mode 100644 tests/e2e/platform/suite/test_cluster_download_permissions.py create mode 100644 tests/e2e/platform/suite/test_cluster_full_pipeline.py create mode 100644 tests/e2e/platform/suite/test_cluster_health_entrypoint.py create mode 100644 tests/e2e/platform/suite/test_cluster_search_aa.py create mode 100644 tests/e2e/platform/suite/test_config_matrix.py diff --git a/.github/workflows/e2e-platform.yml b/.github/workflows/e2e-platform.yml new file mode 100644 index 0000000..60edee8 --- /dev/null +++ b/.github/workflows/e2e-platform.yml @@ -0,0 +1,130 @@ +name: E2E Platform + +# Hermetic end-to-end matrix: boots the app under test against mock +# Anna's Archive / Cloudflare / bypasser / DNS / proxy / Tor / real torrent +# clients and runs the cluster suite under each config profile. +# +# On a PR that touches relevant code, this runs a fast core subset *and* the heavy +# `full` profile (real Chrome solving Cloudflare + DoH + real qBittorrent). The +# `e2e-required` job aggregates them into ONE status check — make that check a +# required status check in branch protection to block merges on any e2e failure +# (see tests/e2e/platform/README.md "Gating PRs"). + +on: + pull_request: + schedule: + - cron: "0 4 * * *" # nightly full matrix + workflow_dispatch: + +concurrency: + group: e2e-platform-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Detect whether anything that affects the e2e platform changed. This lets the + # required check always report (never stuck "pending") while only spending CI on + # PRs that can actually break the e2e stack. + changes: + runs-on: ubuntu-latest + outputs: + relevant: ${{ steps.filter.outputs.relevant }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + relevant: + - 'shelfmark/**' + - 'entrypoint.sh' + - 'tor.sh' + - 'Dockerfile' + - 'tests/e2e/platform/**' + - '.github/workflows/e2e-platform.yml' + + select-profiles: + needs: changes + if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request' + runs-on: ubuntu-latest + outputs: + profiles: ${{ steps.pick.outputs.profiles }} + steps: + - id: pick + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo 'profiles=["baseline","bypasser-external","dns-blocked"]' >> "$GITHUB_OUTPUT" + else + echo 'profiles=["baseline","bypasser-external","bypasser-disabled","dns-manual","dns-blocked","dns-doh","proxy-http","proxy-socks","tor","client-transmission","client-deluge"]' >> "$GITHUB_OUTPUT" + fi + + e2e: + needs: select-profiles + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + profile: ${{ fromJSON(needs.select-profiles.outputs.profiles) }} + name: e2e (${{ matrix.profile }}) + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install uv and Python + uses: astral-sh/setup-uv@v8 + with: + python-version: "3.14" + enable-cache: true + - name: Sync dependencies + run: make install-python-dev + - name: Run e2e platform (${{ matrix.profile }}) + run: tests/e2e/platform/run-e2e.sh env/${{ matrix.profile }}.env + - name: Dump shelfmark logs on failure + if: failure() + run: cat tests/e2e/platform/.state/shelfmark.${{ matrix.profile }}.log || true + + # Heavy "everything real" job: real Chrome internal bypasser solving Cloudflare + + # DoH + real qBittorrent webseed download. Runs on relevant PRs and nightly. + e2e-full: + needs: changes + if: needs.changes.outputs.relevant == 'true' || github.event_name != 'pull_request' + runs-on: ubuntu-latest + name: e2e (full — real Chrome + qBittorrent) + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install uv and Python + uses: astral-sh/setup-uv@v8 + with: + python-version: "3.14" + enable-cache: true + - name: Sync dependencies + run: make install-python-dev + - name: Run full pipeline + run: tests/e2e/platform/run-e2e.sh env/full.env + - name: Dump logs on failure + if: failure() + run: | + cat tests/e2e/platform/.state/shelfmark.full.log || true + docker logs e2e-qbittorrent || true + + # Single aggregated gate. ALWAYS runs (so a required check never hangs "pending" + # on unrelated PRs) and FAILS if any e2e job failed/was cancelled. Make THIS the + # required status check in branch protection. + e2e-required: + name: e2e required + needs: [e2e, e2e-full] + if: always() + runs-on: ubuntu-latest + steps: + - name: Aggregate e2e results + run: | + matrix='${{ needs.e2e.result }}' + full='${{ needs.e2e-full.result }}' + echo "e2e matrix=$matrix, e2e-full=$full" + # success or skipped (unrelated PR) is OK; failure/cancelled blocks. + for r in "$matrix" "$full"; do + if [ "$r" = "failure" ] || [ "$r" = "cancelled" ]; then + echo "::error::An e2e platform job did not pass — blocking." + exit 1 + fi + done + echo "All e2e platform jobs passed (or were skipped as not relevant)." diff --git a/tests/audiobookbay/test_scraper_contract.py b/tests/audiobookbay/test_scraper_contract.py new file mode 100644 index 0000000..2f2c360 --- /dev/null +++ b/tests/audiobookbay/test_scraper_contract.py @@ -0,0 +1,103 @@ +"""Cluster 7 (audiobook/ABB) parse-contract guards. + +ABB forces ``https://`` for search and detail fetches, so it can't be exercised +hermetically in the HTTP e2e docker platform. Its recurring bugs are instead in +*parsing*: magnet/info-hash extraction ("Fix ABB magnet parsing", and the +qbittorrent hash-length issue #386) and DOM/layout drift. These contract tests +feed golden HTML through the real scraper — the same fail-on-drift philosophy as +the AA layout-drift guard — and run in normal CI. + +They deliberately cover cases the existing ``test_scraper.py`` does not: info-hash +*normalization* (whitespace/case), the in-page magnet *fallback*, and a layout +drift that must degrade to an empty result rather than crash. +""" + +from __future__ import annotations + +import re +from unittest.mock import patch + +from shelfmark.release_sources.audiobookbay import scraper + +# Detail page where the Info Hash is lowercase and split by whitespace/newlines — +# the exact shape that produced malformed magnets / wrong hash lengths (#386). +DETAIL_HTML_MESSY_HASH = """ + + + +
Info Hashabc123def456789012345678 + 901234567890abcd
Tracker 1udp://tracker.openbittorrent.com:80
+""" + +# Info Hash cell is junk, but a full magnet link is posted elsewhere on the page. +DETAIL_HTML_MAGNET_FALLBACK = """ + +
Info Hashn/a
+

Mirror: magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=x

+ +""" + +# DOM drift: results are present but the .post / .postTitle structure changed. +SEARCH_HTML_LAYOUT_DRIFT = """ + +
+
Drifted Audiobook - Author
+ English +
+ +""" + + +def _patch_detail(html: str): + return patch( + "shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page", + return_value=html, + ) + + +def test_info_hash_is_normalized_to_canonical_btih() -> None: + """Whitespace/newlines are stripped and the hash upper-cased to a valid + 40-char btih (regression for #386 / 'Fix ABB magnet parsing').""" + with _patch_detail(DETAIL_HTML_MESSY_HASH): + magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/x/", "audiobookbay.lu") + assert magnet is not None, "messy-but-valid info hash should still yield a magnet" + btih = re.search(r"xt=urn:btih:([0-9A-Fa-f]+)", magnet) + assert btih is not None, magnet + assert btih.group(1) == "ABC123DEF456789012345678901234567890ABCD" + assert len(btih.group(1)) == 40 + assert "tr=" in magnet # tracker carried through + + +def test_magnet_fallback_when_info_hash_cell_is_junk() -> None: + """When the Info Hash cell is invalid, the scraper recovers the hash from an + in-page magnet link rather than failing.""" + with _patch_detail(DETAIL_HTML_MAGNET_FALLBACK): + magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/y/", "audiobookbay.lu") + assert magnet is not None + assert "btih:1111111111111111111111111111111111111111" in magnet + + +def test_missing_info_hash_returns_none_not_crash() -> None: + """No hash anywhere -> None (clean failure), never an exception.""" + with _patch_detail("

nothing here

"): + assert ( + scraper.extract_magnet_link("https://audiobookbay.lu/abss/z/", "audiobookbay.lu") + is None + ) + + +def test_search_layout_drift_degrades_to_empty() -> None: + """A changed results DOM yields zero parsed results without raising — the + ABB analogue of the AA layout-drift guard.""" + with ( + patch( + "shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page", + return_value=(SEARCH_HTML_LAYOUT_DRIFT, "https://audiobookbay.lu/?s=test"), + ), + patch( + "shelfmark.release_sources.audiobookbay.scraper.config.get", + return_value=0.0, + ), + ): + results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu") + assert results == [], f"drifted DOM should parse to no results, got {results!r}" diff --git a/tests/download/test_doh_resolver_mock.py b/tests/download/test_doh_resolver_mock.py new file mode 100644 index 0000000..2ddce1c --- /dev/null +++ b/tests/download/test_doh_resolver_mock.py @@ -0,0 +1,79 @@ +"""DoH resolver integration against the e2e platform's mock DoH responder. + +The config-cluster analysis flagged DNS/DoH as a recurring break surface (#1028, +#108). A fully hermetic DoH-over-the-network profile isn't feasible in the HTTP +docker platform (DoH provider URLs are HTTPS + IP-pinned), so we exercise the +*real* ``DoHResolver`` client against the platform's mock ``doh`` role here, over +plain HTTP on localhost. This runs in normal CI (not just the nightly docker +matrix) and guards the DoH JSON-parsing path the app relies on. +""" + +from __future__ import annotations + +import importlib.util +import os +import threading +from pathlib import Path +from wsgiref.simple_server import WSGIServer, make_server + +import pytest + +MOCK_PATH = Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "mock_services.py" + + +def _load_mock_doh_app(doh_map: str): + """Import the platform mock_services module wired for the ``doh`` role. + + The module wires its routes at import time from ``MOCK_ROLE``/``DOH_MAP``, so + those must be set before loading it. + """ + os.environ["MOCK_ROLE"] = "doh" + os.environ["DOH_MAP"] = doh_map + spec = importlib.util.spec_from_file_location("mock_doh_services", MOCK_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.app + + +@pytest.fixture(scope="module") +def doh_url(): + if not MOCK_PATH.exists(): + pytest.skip(f"platform mock not found at {MOCK_PATH}") + app = _load_mock_doh_app("aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11") + server: WSGIServer = make_server("127.0.0.1", 0, app) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + # Google JSON DoH style uses the /resolve endpoint. + yield f"http://127.0.0.1:{server.server_port}/resolve" + finally: + server.shutdown() + + +def _resolver(doh_url: str): + from shelfmark.download.network import DoHResolver + + # hostname/ip args are the DoH server's own identity (used only for recursion + # avoidance); the localhost values here are irrelevant to the lookups under test. + return DoHResolver(doh_url, "127.0.0.1", "127.0.0.1") + + +def test_doh_resolves_mapped_host(doh_url) -> None: + """The real DoH client parses the mock's JSON answer into an A record.""" + assert _resolver(doh_url).resolve("aa.mock.test", "A") == ["172.30.0.10"] + + +def test_doh_nxdomain_returns_empty_not_error(doh_url) -> None: + """An unmapped name yields an empty list (Status 3), not an exception — + the path that, when mishandled, surfaced as silent download failures.""" + assert _resolver(doh_url).resolve("unmapped.invalid", "A") == [] + + +def test_doh_resolver_caches_within_ttl(doh_url) -> None: + """A second lookup is served from cache (the resolver's documented behaviour).""" + resolver = _resolver(doh_url) + first = resolver.resolve("cf.mock.test", "A") + assert first == ["172.30.0.11"] + assert ("cf.mock.test", "A") in resolver._cache + assert resolver.resolve("cf.mock.test", "A") == first diff --git a/tests/download/test_webseed_torrent_generator.py b/tests/download/test_webseed_torrent_generator.py new file mode 100644 index 0000000..df2b47a --- /dev/null +++ b/tests/download/test_webseed_torrent_generator.py @@ -0,0 +1,87 @@ +"""Validate the e2e platform's webseed .torrent generator. + +The ``full`` e2e profile relies on a tracker-less webseed torrent so a real +qBittorrent can complete a real download from the mock origin over HTTP. If the +generator emits malformed bencode or mismatched piece hashes, qBittorrent would +silently never complete — so we cross-check the generator against shelfmark's own +``bencode_decode`` / ``extract_info_hash_from_torrent`` here, in normal CI. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +from pathlib import Path + +import pytest + +from shelfmark.download.clients.torrent_utils import ( + bencode_decode, + extract_info_hash_from_torrent, +) + +GEN_PATH = ( + Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "make_webseed_torrent.py" +) + + +def _load_generator(): + if not GEN_PATH.exists(): + pytest.skip(f"generator not found at {GEN_PATH}") + spec = importlib.util.spec_from_file_location("make_webseed_torrent", GEN_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PAYLOAD = b"E2E webseed payload \x00\x01\x02 " * 2000 # ~50 KiB -> multiple pieces +NAME = "sample-book.epub" +WEBSEED = "http://mock-aa/payload/sample-book.epub" + + +def test_generated_torrent_decodes_with_shelfmark_bencode() -> None: + gen = _load_generator() + raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=16384) + + decoded, _ = bencode_decode(raw) + assert isinstance(decoded, dict) + info = decoded[b"info"] + assert info[b"name"] == NAME.encode() + assert info[b"length"] == len(PAYLOAD) + # url-list (webseed) must point at the file the mock serves. + assert decoded[b"url-list"] == [WEBSEED.encode()] + # No tracker — the whole point is HTTP-only completion. + assert b"announce" not in decoded + + +def test_piece_hashes_match_payload_bytes() -> None: + gen = _load_generator() + piece_len = 16384 + raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=piece_len) + decoded, _ = bencode_decode(raw) + pieces = decoded[b"info"][b"pieces"] + + expected = b"".join( + hashlib.sha1(PAYLOAD[i : i + piece_len]).digest() for i in range(0, len(PAYLOAD), piece_len) + ) + assert pieces == expected, "piece hashes do not match payload — qbit would never complete" + assert len(pieces) % 20 == 0 + + +def test_info_hash_matches_shelfmark_extractor() -> None: + """Our infohash helper must agree with shelfmark's torrent parser.""" + gen = _load_generator() + raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED) + + ours = gen.info_hash(raw) + theirs = extract_info_hash_from_torrent(raw) + assert theirs is not None + assert ours.lower() == theirs.lower(), (ours, theirs) + + +def test_generator_is_deterministic() -> None: + gen = _load_generator() + a = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED) + b = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED) + assert a == b, "torrent generation must be byte-deterministic for stable infohash" diff --git a/tests/e2e/platform/.gitignore b/tests/e2e/platform/.gitignore new file mode 100644 index 0000000..b8252d9 --- /dev/null +++ b/tests/e2e/platform/.gitignore @@ -0,0 +1,7 @@ +# Runtime state created by run-e2e.sh: per-profile shelfmark logs, the app's +# /config (settings, users.db, secrets), downloaded books, and staging dirs. +.state/ + +# Python bytecode from the mock services + suite. +__pycache__/ +*.pyc diff --git a/tests/e2e/platform/README.md b/tests/e2e/platform/README.md new file mode 100644 index 0000000..058dee0 --- /dev/null +++ b/tests/e2e/platform/README.md @@ -0,0 +1,231 @@ +# Shelfmark E2E Docker Testing Platform + +A hermetic, container-based end-to-end platform that boots the real Shelfmark app +against **controllable** dependencies — a fake Anna's Archive, a Cloudflare gate, a +mock FlareSolverr bypasser, a mock Prowlarr + a real qBittorrent, custom DNS +servers, HTTP/SOCKS5 proxies, and a Tor profile — and runs a cluster test suite +under each **config profile**. + +It exists to make the recurring bug clusters from the issue/PR analysis impossible +to reintroduce silently. The biggest one — Tor/Cloudflare/bypasser (37 issues / +67 fix PRs) — had almost no automated coverage; this platform changes that. + +``` + pytest suite (host :8084) + │ + ▼ + shelfmark (under test) ── egress depends on the active profile: + ├─ direct ──────────────► mock-aa (.10) fake Anna's Archive + ├─ Cloudflare gate ─────► mock-cf (.11) ─► mock-aa [full: real Chrome solves it] + ├─ FlareSolverr ────────► mock-cf (.11) ─► mock-flaresolverr (.12) [bypasser-external] + ├─ custom DNS ──────────► coredns (.20) / coredns-blocked (.22) [dns-manual/blocked] + ├─ HTTP / SOCKS proxy ──► tinyproxy (.30) / microsocks (.31) [proxy-http/socks] + ├─ Tor (transparent) ───► in-image tor.sh [tor] + └─ Prowlarr → client ───► mock-prowlarr (.40) ─► qBittorrent [full: real download] + + (all on one e2e docker network, 172.30.0.0/24, static IPs for DNS determinism) +``` + +## Quick start + +```bash +# one profile +make e2e-platform # baseline +make e2e-platform-profile PROFILE=bypasser-external +make e2e-platform-profile PROFILE=client-deluge +make e2e-platform-full # heavy: real Chrome + DoH + real qBittorrent + +# the whole matrix +make e2e-platform-matrix + +# build the heavy image once, then reuse it (matrix does this automatically) +make e2e-platform-build +E2E_NO_BUILD=1 tests/e2e/platform/run-e2e.sh env/dns-doh.env + +# debug: leave the stack up after the run +KEEP_UP=1 tests/e2e/platform/run-e2e.sh env/dns-blocked.env +``` + +Requirements: Docker + Compose v2, and `uv` (for the pytest runner). The runner +builds the Shelfmark image from the repo `Dockerfile`, boots the profile's stack, +waits for `/api/health`, runs the suite, and tears down. `run-matrix.sh` builds the +image **once** and reuses it across profiles (`E2E_NO_BUILD=1`) so the slow +xvfb/chromium layer isn't rebuilt per profile. + +## How profiles work + +Each profile is an env file in `env/`. It sets: +- `COMPOSE_PROFILES` — which optional services start (compose `profiles:`). +- `SM_*` — the app's config, injected as container env. Shelfmark treats + deployment ENV as authoritative (`config.get`: "Deployment-level ENV values + always win"), so a profile fully determines the app's DNS/proxy/bypasser/source + configuration with no runtime mutation. +- `E2E_PROFILE` — handed to pytest so the suite selects applicable tests. + +Tests declare applicability with `@pytest.mark.profiles(...)`. **A test with no +marker is a profile-agnostic invariant and runs under every profile** — that is +how one cluster test ("source must be reachable") becomes the config matrix. + +## The matrix (cluster × profile) + +Status column: ✅ = run live on Docker and passing. Every profile below was run +end-to-end (`docker compose up` + suite + teardown) and passes. + +| Profile | Egress / what it proves | Clusters | Regression targets | Status | +|---|---|---|---|---| +| `baseline` | Direct to fake AA; search/parse + #1028 clean-failure | 2,3,4 | #198 #293 #214 #1040 #1028 | ✅ 9 passed | +| `bypasser-external` | External bypasser wired; CF-gated search fails cleanly | 1 | #284 #202 #410 #369 | ✅ 5 passed | +| `bypasser-disabled` | CF-gated AA + bypasser OFF → no results (control) | 1 | #202 #410 | ✅ 4 passed | +| `dns-manual` | AA only resolvable via custom DNS (coredns) | config: DNS | #108 | ✅ 4 passed | +| `dns-blocked` | System DNS NXDOMAINs AA; custom DNS resolves it | config: DNS | **#1028** | ✅ 4 passed | +| `dns-doh` | System DNS blocks AA; **DoH over real HTTPS** resolves it | config: DoH | **#1028** #108 | ✅ 3 passed | +| `proxy-http` | All egress via tinyproxy, **proven by proxy logs** | config: proxy | **#956** | ✅ 6 passed | +| `proxy-socks` | All egress via SOCKS5 (microsocks), traversal-checked | config: proxy | #956 | ✅ 5 passed | +| `tor` | `USING_TOR=true` boots clean (restarts=0) | 1/6 Tor boot | #1021 #940 #801 | ✅ 5 passed | +| `client-transmission` | Prowlarr → **real Transmission** webseed download → /books | 5 clients | #1022 #634 | ✅ 4 passed | +| `client-deluge` | Prowlarr → **real Deluge** webseed download → /books | 5 clients | #530 | ✅ 4 passed | +| `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed | +| *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ | + +> **The bypasser is download-time, not search-time.** Running the stack revealed +> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a +> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser +> (internal Chrome or external FlareSolverr) only runs during a file *download* +> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean* +> CF-gated-search failure, while the **`full` profile exercises the real end-to-end +> CF solve**: AA search/detail are reachable, but the AA slow-download link points +> at the gate, so downloading Moby-Dick forces the in-image headless Chromium to +> detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file — +> verified live (`Challenge detected: cloudflare` → `Bypass successful` → Moby-Dick +> in `/books`). +> +> **`bypasser-external` must set `SM_USING_EXTERNAL_BYPASSER=true`** — shelfmark does +> **not** derive it from `EXT_BYPASSER_URL`; without it the app silently uses the +> in-image Chrome bypasser instead of FlareSolverr. + +Coverage of the 7 clusters from the analysis: + +1. **Bypasser/Tor/Cloudflare** → `bypasser-external`, `bypasser-disabled`, `tor`. +2. **Search/metadata** → `baseline` (`test_cluster_search_aa.py`, hermetic via the + `direct_download` source so no external metadata provider is needed). +3. **AA parsing/mirrors** → `baseline` parse guards incl. the **layout-drift + fail-loud** test (#878/#879/#880). +4. **Permissions/file-move** → `baseline` (`test_cluster_download_permissions.py`). +5. **Torrent/usenet clients** → a mock Prowlarr + webseed torrent drives **three + real torrent clients** end to end (`full`=qBittorrent, `client-transmission`, + `client-deluge`) — completion detection + file move into `/books`. One + client-agnostic test (`test_cluster_clients.py`) covers all three. +6. **Docker/entrypoint/PUID-PGID** → profile-agnostic health + boot-log checks, + run under every profile. +7. **Audiobook/ABB** → parse-contract guards in + `tests/audiobookbay/test_scraper_contract.py` (info-hash normalization #386, + magnet fallback, layout drift). These run in **normal CI**, not the docker + matrix, because ABB hardcodes `https://` for its fetches (see Roadmap). + +### Proxy traversal (not just reachability) +Because the app and the mock AA share the e2e network, a regression that ignores +the proxy config would still reach AA directly. `test_egress_actually_traverses_proxy` +drives a search and then inspects the proxy container's logs, so the proxy +profiles prove the egress *went through* the proxy — a real guard for #956. + +### DoH — two layers +- **Offline** (`tests/download/test_doh_resolver_mock.py`, normal CI): the real + `DoHResolver` is driven against the mock `doh` role over localhost HTTP, covering + JSON-answer parsing, NXDOMAIN → empty, and caching. +- **In-stack** (`dns-doh` profile): the mock `doh` role serves the DNS JSON API over + **real HTTPS** (self-signed). The system resolver (coredns-blocked) NXDOMAINs + `aa.mock.test`, so the host can *only* be resolved via DoH; compose `extra_hosts` + redirects the `cloudflare-dns.com` provider to the in-stack mock and + `CERTIFICATE_VALIDATION=disabled` accepts the self-signed cert. The search reaching + AA proves the app's DoH path resolved the name end to end — **no app code change**. + +### The `full` profile — real Chrome + real client (`make e2e-platform-full`) +The "everything real" heavy profile (test book: **Moby-Dick**), run nightly / on +demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, and +**passes live** (6 passed): + +- **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are + reachable (`mock-aa`), but the AA *slow-download* link points at the Cloudflare + gate (`mock-cf`), whose challenge page runs JS that issues `cf_clearance` and + reloads. Downloading Moby-Dick forces the in-image headless Chromium (seleniumbase + CDP, in the `shelfmark` image via `xvfb`+`chromium`) to load the gate, detect the + challenge (`Challenge detected: cloudflare`), solve it (`_bypass_method_cdp_solve`), + and fetch the cleared "Download now" page → the file lands in `/books`. That + outcome is *only* reachable if Chrome solved the gate — the literal "spin a Chrome + browser" path and the strongest guard for the #1 cluster. Two subtleties this + surfaced, now handled by the mock: the cleared page must exceed the bypasser's + `_LOADING_BODY_LENGTH_MAX` (50 chars of innerText) or it loops as "still loading", + and the AA detail page must satisfy the brittle `original_nodes[-6]` parse (#880). +- **DoH** on at boot. +- **Real qBittorrent download.** A mock Prowlarr (`/api/v1/system/status`, + `/api/v1/indexer`, torznab search) returns one release whose `.torrent` is a + **tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. A real + qBittorrent completes the download over HTTP (no tracker/peer/seeder), and + shelfmark's completion detection + file move lands the book in `/books`. The + webseed torrent is generated by `mocks/make_webseed_torrent.py` (cross-checked + against shelfmark's own bencode/infohash parser in + `tests/download/test_webseed_torrent_generator.py`), and the whole + prowlarr→qBittorrent path is configured declaratively via env (`env/full.env`). + +## Components + +| Path | Purpose | +|---|---| +| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`. | +| `mocks/make_webseed_torrent.py` | Stdlib bencode + BEP-19 webseed `.torrent` generator for the `full` real-client download. | +| `qbittorrent/qBittorrent.conf` | Real qBittorrent config (auth bypassed for the e2e subnet) for the `full` profile. | +| `env/full.env` | The heavy `full` profile: real Chrome bypasser + DoH + real qBittorrent. | +| `mocks/fixtures/*.html` | AA search/detail HTML in the **exact** shape the parser expects, plus drift/empty/no-files variants. | +| `docker-compose.e2e.yml` | The stack; optional services gated by compose profiles, static IPs for DNS determinism. | +| `dns/Corefile*`, `dns/mock.test.db` | coredns zones — working + ISP-block (NXDOMAIN). | +| `env/*.env` | The config profiles (matrix rows). | +| `suite/` | The pytest harness + cluster tests. | +| `run-e2e.sh` / `run-matrix.sh` | Boot one profile / loop the matrix. | +| `build-images.sh` | Build the heavy image once (`make e2e-platform-build`); reused via `E2E_NO_BUILD=1`. | + +### Fault injection +The mock AA reproduces historical bugs deterministically. Injection rides inside +the search query as `E2EINJECT:` (the app builds the AA URL itself and only +forwards the user query as `q=`). Names: `no_files`, `empty`, `layout_drift`, +`500`. The harness embeds them via `PlatformClient.direct_search(..., inject=...)`. + +## Gating PRs (block merge on e2e failure) + +The `.github/workflows/e2e-platform.yml` workflow runs on every PR. On a PR that +touches relevant code (`shelfmark/**`, `Dockerfile`, `entrypoint.sh`, `tor.sh`, +`tests/e2e/platform/**`) it runs the fast PR subset **and** the heavy `full` +profile (real Chrome solving Cloudflare + DoH + real qBittorrent), then a single +**`e2e required`** job aggregates them: it fails if any e2e job failed, and passes +(so it never hangs) when the e2e jobs are skipped on an unrelated PR. + +The workflow producing a failing check is **not enough on its own** — GitHub only +*blocks merge* on checks listed in branch protection. A repo **admin** must, once: + +- **UI:** Settings → Branches → branch protection rule for `main` → + *Require status checks to pass before merging* → add **`e2e required`**. +- **or `gh` (admin token):** + ```bash + gh api -X PUT repos/calibrain/shelfmark/branches/main/protection \ + -H "Accept: application/vnd.github+json" --input - <<'JSON' + { "required_status_checks": { "strict": true, "contexts": ["e2e required"] }, + "enforce_admins": true, "required_pull_request_reviews": null, "restrictions": null } + JSON + ``` + +After that, any failure in the e2e platform tests (including the `full` profile) +blocks the PR from merging. Requiring just the one `e2e required` context covers +the whole dynamic matrix, so the list never needs updating as profiles change. + +## Known limitations / follow-ups + +- **rTorrent.** Not in the matrix: its rakshasa-libtorrent has **no GetRight/webseed + support**, so the hermetic webseed torrent (which qBittorrent/Transmission/Deluge + all complete) leaves rTorrent stuck at 0%. Supporting it needs a real tracker + + seeder (peer download) — a follow-up that the webseed design intentionally avoids. +- **Usenet clients (SABnzbd/NZBGet).** Not yet covered — completing a usenet download + hermetically needs a mock NNTP server serving the yEnc-encoded payload plus an NZB, + which is a separate (larger) build than the torrent webseed path. +- **Audiobook (cluster 7) in-stack.** ABB hardcodes `https://`, so it's covered + offline (`tests/audiobookbay/test_scraper_contract.py`); an in-stack + `audiobookbay` role needs the same self-signed-HTTPS plumbing the `dns-doh` profile + now uses for DoH. diff --git a/tests/e2e/platform/build-images.sh b/tests/e2e/platform/build-images.sh new file mode 100755 index 0000000..e47be64 --- /dev/null +++ b/tests/e2e/platform/build-images.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Build every buildable image in the e2e stack once (the heavy `shelfmark` image +# plus the mock-* role images), so run-matrix.sh / run-e2e.sh with E2E_NO_BUILD=1 +# can reuse them instead of rebuilding the xvfb/chromium layer per profile. +set -euo pipefail + +PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$PLATFORM_DIR" + +# Activate every profile that owns a buildable service so they all get built. +# (Download clients, coredns, proxies are pre-built images — nothing to build.) +export COMPOSE_PROFILES="bypasser-external,full,dns-doh" +echo "==> building shelfmark + mock images (one cold build of the chromium layer)" +docker compose -f docker-compose.e2e.yml build +echo "==> done. Reuse with: E2E_NO_BUILD=1 ./run-e2e.sh env/.env" diff --git a/tests/e2e/platform/dns/Corefile b/tests/e2e/platform/dns/Corefile new file mode 100644 index 0000000..01fbdc6 --- /dev/null +++ b/tests/e2e/platform/dns/Corefile @@ -0,0 +1,13 @@ +# coredns: authoritative for mock.test, forwards everything else. +mock.test:53 { + file /zones/mock.test.db + log + errors +} + +. :53 { + forward . 1.1.1.1 8.8.8.8 + cache 30 + log + errors +} diff --git a/tests/e2e/platform/dns/Corefile.blocked b/tests/e2e/platform/dns/Corefile.blocked new file mode 100644 index 0000000..6e79b17 --- /dev/null +++ b/tests/e2e/platform/dns/Corefile.blocked @@ -0,0 +1,18 @@ +# coredns: ISP-DNS-block simulation. Resolves nothing under mock.test +# (returns NXDOMAIN), so an app that relies on *system* DNS for book sources +# fails — while an app that uses its own custom DNS resolver still works. +# Regression harness for #1028 (internal bypasser used system DNS in subprocess). +mock.test:53 { + template IN ANY mock.test { + rcode NXDOMAIN + } + log + errors +} + +. :53 { + # Everything else still resolves, so only the book-source host is "blocked". + forward . 1.1.1.1 8.8.8.8 + log + errors +} diff --git a/tests/e2e/platform/dns/mock.test.db b/tests/e2e/platform/dns/mock.test.db new file mode 100644 index 0000000..7a0c5c4 --- /dev/null +++ b/tests/e2e/platform/dns/mock.test.db @@ -0,0 +1,13 @@ +$ORIGIN mock.test. +$TTL 60 +@ IN SOA ns.mock.test. admin.mock.test. ( + 1 ; serial + 7200 ; refresh + 3600 ; retry + 1209600 ; expire + 3600 ) ; minimum +@ IN NS ns.mock.test. +ns IN A 172.30.0.20 +aa IN A 172.30.0.10 +cf IN A 172.30.0.11 +doh IN A 172.30.0.21 diff --git a/tests/e2e/platform/docker-compose.e2e.yml b/tests/e2e/platform/docker-compose.e2e.yml new file mode 100644 index 0000000..ca036c2 --- /dev/null +++ b/tests/e2e/platform/docker-compose.e2e.yml @@ -0,0 +1,294 @@ +# Shelfmark e2e Docker testing platform +# --------------------------------------------------------------------------- +# A hermetic stack: the app under test plus *controllable* dependencies +# (fake Anna's Archive, a Cloudflare gate, a mock FlareSolverr bypasser, a DoH +# responder, a DNS server, HTTP + SOCKS5 proxies, and a Tor profile). +# +# Config profiles are env-driven: pick a profile with an env file under env/ +# (it sets COMPOSE_PROFILES + the SM_* config the app boots with). The runner +# brings the stack up per profile, runs the matching cluster tests, tears down. +# +# ./run-e2e.sh env/baseline.env +# ./run-e2e.sh env/bypasser-external.env +# ./run-e2e.sh env/dns-blocked.env +# +# See README.md for the full cluster x profile matrix. + +x-mock-build: &mock-build + build: + context: ./mocks + dockerfile: Dockerfile + +networks: + e2e: + driver: bridge + ipam: + config: + - subnet: 172.30.0.0/24 + +services: + # ----- App under test --------------------------------------------------- # + shelfmark: + build: + context: ../../.. + dockerfile: Dockerfile + target: ${SM_BUILD_TARGET:-shelfmark} + container_name: e2e-shelfmark + cap_add: + - NET_ADMIN # required by tor.sh iptables when USING_TOR=true + - NET_RAW + environment: + TZ: UTC + DEBUG: "true" + ONBOARDING: "false" # skip wizard; ephemeral storage + DISABLE_LOCAL_AUTH: "${SM_DISABLE_LOCAL_AUTH:-true}" + PUID: "${SM_PUID:-1000}" + PGID: "${SM_PGID:-1000}" + # --- source / mirror config (cluster 2/3) --- + DIRECT_DOWNLOAD_ENABLED: "${SM_DIRECT_DOWNLOAD_ENABLED:-true}" + AA_ADDITIONAL_URLS: "${SM_AA_URL:-http://mock-aa}" + # --- bypasser config (cluster 1) --- + USE_CF_BYPASS: "${SM_USE_CF_BYPASS:-false}" + # Selects the external (FlareSolverr) bypasser; when false the in-image + # Chrome (internal) bypasser is used. NOT derived from EXT_BYPASSER_URL. + USING_EXTERNAL_BYPASSER: "${SM_USING_EXTERNAL_BYPASSER:-false}" + EXT_BYPASSER_URL: "${SM_EXT_BYPASSER_URL:-}" + # --- prowlarr indexer + torrent client (cluster 5, `full` profile) --- + PROWLARR_ENABLED: "${SM_PROWLARR_ENABLED:-false}" + PROWLARR_URL: "${SM_PROWLARR_URL:-}" + PROWLARR_API_KEY: "${SM_PROWLARR_API_KEY:-}" + PROWLARR_TORRENT_CLIENT: "${SM_PROWLARR_TORRENT_CLIENT:-}" + QBITTORRENT_URL: "${SM_QBITTORRENT_URL:-}" + QBITTORRENT_USERNAME: "${SM_QBITTORRENT_USERNAME:-}" + QBITTORRENT_PASSWORD: "${SM_QBITTORRENT_PASSWORD:-}" + QBITTORRENT_CATEGORY: "${SM_QBITTORRENT_CATEGORY:-}" + # transmission / deluge / rtorrent (client-* profiles) + TRANSMISSION_URL: "${SM_TRANSMISSION_URL:-}" + TRANSMISSION_USERNAME: "${SM_TRANSMISSION_USERNAME:-}" + TRANSMISSION_PASSWORD: "${SM_TRANSMISSION_PASSWORD:-}" + DELUGE_HOST: "${SM_DELUGE_HOST:-}" + DELUGE_PORT: "${SM_DELUGE_PORT:-}" + DELUGE_PASSWORD: "${SM_DELUGE_PASSWORD:-}" + RTORRENT_URL: "${SM_RTORRENT_URL:-}" + # --- DNS / DoH (config cluster) --- + CUSTOM_DNS: "${SM_CUSTOM_DNS:-}" + CUSTOM_DNS_MANUAL: "${SM_CUSTOM_DNS_MANUAL:-}" + USE_DOH: "${SM_USE_DOH:-false}" + # Disable TLS verification so the in-stack DoH-over-HTTPS mock (self-signed) + # is accepted in the dns-doh profile. Default keeps verification ON. + CERTIFICATE_VALIDATION: "${SM_CERTIFICATE_VALIDATION:-enabled}" + # --- proxy (config cluster) --- + PROXY_MODE: "${SM_PROXY_MODE:-none}" + HTTP_PROXY_URL: "${SM_HTTP_PROXY:-}" + HTTP_PROXY: "${SM_HTTP_PROXY:-}" + SOCKS5_PROXY: "${SM_SOCKS5_PROXY:-}" + NO_PROXY: "${SM_NO_PROXY:-}" + # --- tor (cluster 1/6) --- + USING_TOR: "${SM_USING_TOR:-false}" + ports: + - "8084:8084" + volumes: + - ./.state/config:/config + - ./.state/books:/books + - ./.state/downloads:/downloads + - ./.state/tmp:/tmp/shelfmark + networks: + - e2e + dns: + # When a DNS profile is active, point the container's system resolver at + # our controllable server; otherwise Docker's embedded DNS (127.0.0.11). + - ${SM_SYSTEM_DNS:-127.0.0.11} + extra_hosts: + # Redirect the DoH provider hostname to the in-stack mock-doh (dns-doh + # profile). Harmless elsewhere — only the dns-doh profile enables DoH against + # the cloudflare provider, and /etc/hosts is consulted before the resolver. + - "cloudflare-dns.com:172.30.0.21" + restart: "no" + + # ----- Fake Anna's Archive origin (always on) --------------------------- # + mock-aa: + <<: *mock-build + container_name: e2e-mock-aa + environment: + MOCK_ROLE: origin-aa + # When set (the `full` profile sets it to the CF gate), AA slow-download + # links point through Cloudflare so a real download forces the internal + # Chrome bypasser to solve the challenge. Empty -> same-origin (no CF). + SLOW_DOWNLOAD_BASE: "${SM_SLOW_DOWNLOAD_BASE:-}" + AA_FILE_BASE: "http://mock-aa" + networks: + e2e: + ipv4_address: 172.30.0.10 + aliases: + - aa.mock.test + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request;urllib.request.urlopen('http://localhost/healthz')"] + interval: 3s + timeout: 3s + retries: 10 + + # ----- Cloudflare gate (profiles: bypasser-external, full) -------------- # + # `full` puts AA's slow-download behind this gate so the real Chrome bypasser + # must solve it; `bypasser-external` puts AA search behind it for the + # FlareSolverr negative/positive controls. + mock-cf: + <<: *mock-build + container_name: e2e-mock-cf + profiles: ["bypasser-external", "full"] + environment: + MOCK_ROLE: cloudflare + ORIGIN_INTERNAL_URL: http://mock-aa + networks: + e2e: + ipv4_address: 172.30.0.11 + aliases: + - cf.mock.test + + # ----- Mock FlareSolverr external bypasser (profile: bypasser-external) -- # + mock-flaresolverr: + <<: *mock-build + container_name: e2e-mock-flaresolverr + profiles: ["bypasser-external"] + environment: + MOCK_ROLE: flaresolverr + networks: + e2e: + ipv4_address: 172.30.0.12 + + # NOTE: the mock `doh` role lives in mock_services.py and is exercised by + # tests/download/test_doh_resolver_mock.py (real DoHResolver over localhost HTTP). + # An in-stack DoH service is intentionally absent — see README "Known limitations". + + # ----- DNS server (profile: dns-manual) --------------------------------- # + coredns: + image: coredns/coredns:1.11.1 + container_name: e2e-coredns + profiles: ["dns-manual"] + command: ["-conf", "/Corefile"] + volumes: + - ./dns/Corefile:/Corefile:ro + - ./dns/mock.test.db:/zones/mock.test.db:ro + networks: + e2e: + ipv4_address: 172.30.0.20 + + # ----- DNS server that NXDOMAINs the AA host (profile: dns-blocked) ------ # + # Simulates ISP DNS blocking (#1028); the app must fall back (DoH / direct). + coredns-blocked: + image: coredns/coredns:1.11.1 + container_name: e2e-coredns-blocked + profiles: ["dns-blocked"] + command: ["-conf", "/Corefile"] + volumes: + - ./dns/Corefile.blocked:/Corefile:ro + networks: + e2e: + ipv4_address: 172.30.0.22 + + # ----- HTTP proxy (profile: proxy-http) --------------------------------- # + tinyproxy: + image: monokal/tinyproxy:latest + container_name: e2e-tinyproxy + profiles: ["proxy-http"] + command: ANY + networks: + e2e: + ipv4_address: 172.30.0.30 + + # ----- SOCKS5 proxy (profile: proxy-socks) ------------------------------ # + microsocks: + image: vimagick/microsocks:latest + container_name: e2e-microsocks + profiles: ["proxy-socks"] + networks: + e2e: + ipv4_address: 172.30.0.31 + + # ----- Mock Prowlarr indexer (full + client-* profiles) ----------------- # + # Minimal Prowlarr API returning one torrent release whose .torrent is a + # webseed pointing at mock-aa. Drives the real torrent-client download in the + # `full` (qBittorrent) and `client-*` (transmission/deluge/rtorrent) profiles. + mock-prowlarr: + <<: *mock-build + container_name: e2e-mock-prowlarr + profiles: ["full", "client-transmission", "client-deluge"] + environment: + MOCK_ROLE: prowlarr + AA_INTERNAL_URL: http://mock-aa + networks: + e2e: + ipv4_address: 172.30.0.40 + aliases: + - prowlarr.mock.test + + # ----- Real qBittorrent download client (profile: full) ----------------- # + # Auth is bypassed for the e2e subnet (qBittorrent.conf) so shelfmark connects + # without juggling the image's random temp password. + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:latest + container_name: e2e-qbittorrent + profiles: ["full"] + environment: + PUID: "1000" + PGID: "1000" + TZ: UTC + WEBUI_PORT: "8080" + volumes: + - ./qbittorrent/qBittorrent.conf:/config/qBittorrent/qBittorrent.conf + # Shared with shelfmark so completed files are visible for the move step. + - ./.state/downloads:/downloads + networks: + - e2e + + # ----- Real Transmission client (profile: client-transmission) ---------- # + # Same webseed torrent + mock Prowlarr as `full`, different real client. + transmission: + image: lscr.io/linuxserver/transmission:latest + container_name: e2e-transmission + profiles: ["client-transmission"] + environment: + PUID: "1000" + PGID: "1000" + TZ: UTC + USER: admin + PASS: admin + volumes: + - ./.state/downloads:/downloads + networks: + - e2e + + # ----- Real Deluge client (profile: client-deluge) ---------------------- # + # shelfmark talks to deluge-web (default WebUI password "deluge"), which + # connects to the bundled daemon — no daemon auth-file juggling. + deluge: + image: lscr.io/linuxserver/deluge:latest + container_name: e2e-deluge + profiles: ["client-deluge"] + environment: + PUID: "1000" + PGID: "1000" + TZ: UTC + volumes: + - ./.state/downloads:/downloads + networks: + - e2e + + # NOTE: rTorrent is intentionally absent — its rakshasa-libtorrent has no + # GetRight/webseed support, so the hermetic webseed torrent can't drive it. + # Supporting it would need a real tracker + seeder (see README limitations). + + # ----- DoH-over-HTTPS responder (profile: dns-doh) ---------------------- # + # In-stack DoH: serves the DNS JSON API over HTTPS with a self-signed cert. + # shelfmark reaches it because the profile (a) maps the DoH provider hostname + # to this container via extra_hosts and (b) sets CERTIFICATE_VALIDATION=disabled. + mock-doh: + <<: *mock-build + container_name: e2e-mock-doh + profiles: ["dns-doh"] + environment: + MOCK_ROLE: doh + DOH_TLS: "1" + DOH_MAP: "aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11" + networks: + e2e: + ipv4_address: 172.30.0.21 diff --git a/tests/e2e/platform/mocks/Dockerfile b/tests/e2e/platform/mocks/Dockerfile new file mode 100644 index 0000000..ab179b3 --- /dev/null +++ b/tests/e2e/platform/mocks/Dockerfile @@ -0,0 +1,25 @@ +# Mock services image for the Shelfmark e2e platform. +# One image, many roles (selected via MOCK_ROLE): origin-aa, cloudflare, +# flaresolverr, prowlarr, doh. See mock_services.py for the role contracts. +FROM python:3.14-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY mock_services.py . +# Needed by the origin-aa role to build the `full` profile's webseed .torrent. +COPY make_webseed_torrent.py . +COPY fixtures ./fixtures + +ENV PORT=80 +ENV MOCK_ROLE=all + +# gunicorn keeps the mock responsive under concurrent e2e load. +RUN pip install --no-cache-dir gunicorn +EXPOSE 80 +EXPOSE 443 +# DOH_TLS=1 (the `doh` role) self-signs a cert and serves HTTPS on 443; every +# other role serves plain HTTP on $PORT via gunicorn. +CMD ["sh", "-c", "if [ \"$DOH_TLS\" = 1 ]; then python mock_services.py; else gunicorn -w 2 -b 0.0.0.0:${PORT} mock_services:app; fi"] diff --git a/tests/e2e/platform/mocks/fixtures/aa_detail.html b/tests/e2e/platform/mocks/fixtures/aa_detail.html new file mode 100644 index 0000000..edd302f --- /dev/null +++ b/tests/e2e/platform/mocks/fixtures/aa_detail.html @@ -0,0 +1,15 @@ + + +Anna's Archive (mock) - __MD5__ + +
+
+
+
+
LanguageEnglish [en]
Year1851
🔍 Moby Dick
epub · 1.2 MB · fiction
Slow Partner Server #1 — no waitlist, but the download is slow
Slow Partner Server #2 — waitlist, but faster
+
+
Moby-Dick; or, The Whale by Herman Melville.
+
+
+ + diff --git a/tests/e2e/platform/mocks/fixtures/aa_search_empty.html b/tests/e2e/platform/mocks/fixtures/aa_search_empty.html new file mode 100644 index 0000000..8a2ffe9 --- /dev/null +++ b/tests/e2e/platform/mocks/fixtures/aa_search_empty.html @@ -0,0 +1,9 @@ + + +Anna's Archive (mock) + +
+
No files found.
+
+ + diff --git a/tests/e2e/platform/mocks/fixtures/aa_search_layout_drift.html b/tests/e2e/platform/mocks/fixtures/aa_search_layout_drift.html new file mode 100644 index 0000000..e47728a --- /dev/null +++ b/tests/e2e/platform/mocks/fixtures/aa_search_layout_drift.html @@ -0,0 +1,27 @@ + + +Anna's Archive (mock - layout drift) + +
+ +
+
+ +

Drifted Title

+ Brandon Sanderson + epub + 1.2 MB + +
+
+
+ + diff --git a/tests/e2e/platform/mocks/fixtures/aa_search_results.html b/tests/e2e/platform/mocks/fixtures/aa_search_results.html new file mode 100644 index 0000000..c369eb4 --- /dev/null +++ b/tests/e2e/platform/mocks/fixtures/aa_search_results.html @@ -0,0 +1,41 @@ + + +Anna's Archive (mock) + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
__QUERY__Herman MelvilleHarper & Brothers2024--English [en]fictionepub1.2 MBlgli/N:\fiction\en\E2E\__QUERY__.epub
__QUERY__ (Annotated Edition)Herman MelvilleHarper & Brothers2023--English [en]fictionpdf8.4 MBlgli/N:\fiction\en\E2E\__QUERY__.pdf
+
+ + diff --git a/tests/e2e/platform/mocks/fixtures/cf_challenge.html b/tests/e2e/platform/mocks/fixtures/cf_challenge.html new file mode 100644 index 0000000..7ef05cb --- /dev/null +++ b/tests/e2e/platform/mocks/fixtures/cf_challenge.html @@ -0,0 +1,39 @@ + + + + Just a moment... + + + + +
+
+ +
+

Checking if the site connection is secure

+

Verifying you are human. This may take a few seconds.

+
+
+
+ + + diff --git a/tests/e2e/platform/mocks/make_webseed_torrent.py b/tests/e2e/platform/mocks/make_webseed_torrent.py new file mode 100644 index 0000000..5f24658 --- /dev/null +++ b/tests/e2e/platform/mocks/make_webseed_torrent.py @@ -0,0 +1,113 @@ +"""Generate a single-file BitTorrent metainfo (.torrent) with a BEP-19 webseed. + +Used by the e2e platform's ``full`` profile so a *real* torrent client +(qBittorrent) can complete a *real* download hermetically: the torrent carries no +tracker and a single ``url-list`` webseed pointing at the mock origin's HTTP file +endpoint, so libtorrent fetches the payload over HTTP — no tracker, peer, or +seeder container required. + +Stdlib-only (the mock image does not install shelfmark). The unit test cross-checks +this encoder against shelfmark's own ``bencode_decode`` / +``extract_info_hash_from_torrent`` so a divergence in either is caught. +""" + +from __future__ import annotations + +import hashlib + +DEFAULT_PIECE_LENGTH = 16384 # 16 KiB — fine for the tiny e2e payload + + +def bencode(value: object) -> bytes: + """Minimal bencode encoder (int / bytes / str / list / dict).""" + if isinstance(value, bool): # guard: bool is an int subclass + raise TypeError("bool is not bencodable") + if isinstance(value, int): + return b"i" + str(value).encode() + b"e" + if isinstance(value, bytes): + return str(len(value)).encode() + b":" + value + if isinstance(value, str): + return bencode(value.encode("utf-8")) + if isinstance(value, list): + return b"l" + b"".join(bencode(item) for item in value) + b"e" + if isinstance(value, dict): + out = b"d" + for key in sorted(value): # bencode dict keys must be sorted + key_bytes = key.encode("utf-8") if isinstance(key, str) else key + out += bencode(key_bytes) + bencode(value[key]) + return out + b"e" + raise TypeError(f"Cannot bencode value of type {type(value).__name__}") + + +def _pieces(data: bytes, piece_length: int) -> bytes: + return b"".join( + hashlib.sha1(data[i : i + piece_length]).digest() for i in range(0, len(data), piece_length) + ) + + +def build_info_dict(name: str, data: bytes, piece_length: int = DEFAULT_PIECE_LENGTH) -> dict: + return { + "name": name, + "piece length": piece_length, + "length": len(data), + "pieces": _pieces(data, piece_length), + } + + +def build_webseed_torrent( + name: str, + data: bytes, + webseed_url: str, + *, + piece_length: int = DEFAULT_PIECE_LENGTH, +) -> bytes: + """Build a tracker-less single-file .torrent whose only source is a webseed. + + Args: + name: file name inside the torrent (e.g. ``sample-book.epub``). + data: the exact file bytes the webseed URL must serve. + webseed_url: BEP-19 url-list entry — the direct HTTP URL for ``data``. + """ + info = build_info_dict(name, data, piece_length) + metainfo = { + "info": info, + # Single-entry webseed. For a single-file torrent the url-list entry is the + # direct file URL, so it must serve exactly ``data``. + "url-list": [webseed_url], + "comment": "shelfmark e2e webseed torrent", + "created by": "shelfmark-e2e", + } + return bencode(metainfo) + + +def info_hash(torrent_bytes: bytes) -> str: + """The btih (SHA1 of the bencoded ``info`` dict) as a hex string. + + Re-encodes via a tiny scan so we don't need a full decoder here. + """ + marker = b"4:infod" + start = torrent_bytes.find(marker) + if start < 0: + raise ValueError("no info dict found in torrent") + info_start = start + len(b"4:info") + # The info value begins at 'd'; find its matching 'e' by bencode-aware scan. + end = _scan_bencoded(torrent_bytes, info_start) + return hashlib.sha1(torrent_bytes[info_start:end]).hexdigest() + + +def _scan_bencoded(buf: bytes, pos: int) -> int: + """Return the index just past the bencoded value starting at ``pos``.""" + token = buf[pos : pos + 1] + if token == b"i": + return buf.index(b"e", pos) + 1 + if token in (b"l", b"d"): + pos += 1 + while buf[pos : pos + 1] != b"e": + if token == b"d": # dicts: key then value + pos = _scan_bencoded(buf, pos) + pos = _scan_bencoded(buf, pos) + return pos + 1 + # byte string: : + colon = buf.index(b":", pos) + length = int(buf[pos:colon]) + return colon + 1 + length diff --git a/tests/e2e/platform/mocks/mock_services.py b/tests/e2e/platform/mocks/mock_services.py new file mode 100644 index 0000000..de39b23 --- /dev/null +++ b/tests/e2e/platform/mocks/mock_services.py @@ -0,0 +1,577 @@ +"""Controllable mock services for the Shelfmark e2e Docker platform. + +A single Flask app that plays one of several *roles*, selected by the +``MOCK_ROLE`` environment variable. Running one image with different roles keeps +the platform image small and the behaviour in one auditable place. + +Roles +----- +``origin-aa`` Fake Anna's Archive: search results table, ``/md5/`` detail + pages, and a downloadable book file. The HTML mirrors the real + selectors the parser depends on (```` rows, last-cell + distant path, ``get.php?md5=..&key=..`` GET links) so parser + drift (#878/#879/#880) is caught here. Supports fault injection + via query flags to reproduce historical bugs deterministically. +``cloudflare`` Cloudflare-protected origin: returns a 403 "Just a moment..." + challenge page (with ``cf-mitigated: challenge``) until the + request carries a ``cf_clearance`` cookie, then serves the real + content. Exercises the app's CF detection + bypasser routing + (#284, #226, #202, #1030) without running real CF JS. +``flaresolverr`` Mock FlareSolverr implementing the ``/v1`` contract. It fetches + the requested URL *with* a clearance cookie and returns the + solved HTML + cookies, so ``_fetch_via_bypasser`` runs end to + end deterministically (no headless Chrome needed in CI). +``doh`` Minimal DNS-over-HTTPS (RFC 8484 + Google JSON) responder used + to verify the USE_DOH path resolves mock domains even when the + system resolver is poisoned (#1028). +``all`` Mounts every role at once (default; handy for local poking). + +Fault injection (origin-aa) rides INSIDE the search query as an +``E2EINJECT:`` token, because the app builds the AA URL itself and only +forwards the user query as ``q=``. The harness embeds it (see +PlatformClient.direct_search); the mock strips it before rendering: + ``no_files`` -> renders the literal "No files found." alongside a real row + (regression for the false-positive check). + ``empty`` -> renders an empty results page (true "No files found"). + ``layout_drift`` -> renders a structurally-changed page (cards, no ) so + a hardcoded-index parser yields zero rows — the app must + fail loudly, not silently (#878/#879/#880). + ``500`` -> returns HTTP 500 (mirror failover path). +""" + +from __future__ import annotations + +import base64 +import os +import re +import struct +from pathlib import Path + +from flask import Flask, Response, jsonify, make_response, request + +FIXTURES = Path(__file__).parent / "fixtures" +ROLE = os.environ.get("MOCK_ROLE", "all").strip().lower() +# Hostname the flaresolverr/cloudflare roles use to reach the AA origin from +# inside the compose network. +ORIGIN_INTERNAL_URL = os.environ.get("ORIGIN_INTERNAL_URL", "http://mock-aa") +CLEARANCE_COOKIE = "cf_clearance" +CLEARANCE_VALUE = "e2e-cleared-token" + +# Test book: *Moby-Dick* by Herman Melville (public domain), used both as the +# webseed-torrent payload (qBittorrent path) and the AA slow-download payload (real +# Chrome / Cloudflare path). PAYLOAD_NAME/URL must stay in sync between what mock-aa +# serves and what the torrent's url-list references. +PAYLOAD_NAME = "moby-dick.epub" +PAYLOAD_URL = f"{os.environ.get('PAYLOAD_PUBLIC_URL', 'http://mock-aa')}/payload/{PAYLOAD_NAME}" + +# Base for AA slow-download links on the detail page. When set to the Cloudflare +# gate (the `full` profile sets it to http://cf.mock.test), the *download* — not the +# search — is forced through the gate, so a real download triggers the internal +# Chrome bypasser to solve the challenge. Empty -> same-origin (no CF). +SLOW_DOWNLOAD_BASE = os.environ.get("SLOW_DOWNLOAD_BASE", "").rstrip("/") + +# Real public-domain opening of Moby-Dick (Chapter 1, "Loomings"). +_MOBY_DICK_TEXT = ( + "Call me Ishmael. Some years ago—never mind how long precisely—having little " + "or no money in my purse, and nothing particular to interest me on shore, I " + "thought I would sail about a little and see the watery part of the world. It " + "is a way I have of driving off the spleen and regulating the circulation. " + "Whenever I find myself growing grim about the mouth; whenever it is a damp, " + "drizzly November in my soul; whenever I find myself involuntarily pausing " + "before coffin warehouses, and bringing up the rear of every funeral I meet; " + "and especially whenever my hypos get such an upper hand of me, that it " + "requires a strong moral principle to prevent me from deliberately stepping " + "into the street, and methodically knocking people's hats off—then, I account " + "it high time to get to sea as soon as I can." +) + +app = Flask(__name__) + + +def _payload_bytes() -> bytes: + """Deterministic *Moby-Dick* EPUB used as the download payload. + + Determinism matters: the webseed torrent's piece hashes are computed from these + exact bytes, so any drift between what mock-aa serves and what the torrent + describes would make qBittorrent never complete. Fixed ZipInfo timestamps keep + the bytes byte-stable across runs. + """ + import io + import zipfile + + files = [ + ("mimetype", "application/epub+zip"), + ( + "META-INF/container.xml", + '' + '', + ), + ( + "OEBPS/content.opf", + '' + 'e2e-moby-dick' + "Moby-Dick; or, The Whale" + "Herman Melville" + "en" + '' + '', + ), + ( + "OEBPS/chapter1.xhtml", + '' + 'Loomings' + "

Chapter 1. Loomings.

" + # Repeat the opening so the EPUB clears shelfmark's 10 KB minimum-size + # check on the direct-download path (_MIN_VALID_FILE_SIZE). + + ("

" + _MOBY_DICK_TEXT + "

") * 24 + + "", + ), + ] + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf: + for name, content in files: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + zf.writestr(info, content) + return buf.getvalue() + + +def _read_fixture(name: str) -> str: + return (FIXTURES / name).read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- # +# Role: origin-aa (fake Anna's Archive) +# --------------------------------------------------------------------------- # +def _search_rows() -> str: + """One real result row in the exact shape the parser expects.""" + return _read_fixture("aa_search_results.html") + + +def register_origin_aa(flask_app: Flask) -> None: + @flask_app.route("/search") + def aa_search() -> Response: + query = request.args.get("q", "") + # Fault injection travels INSIDE the search query (the app builds the AA + # URL itself and won't forward arbitrary params), via a token the harness + # embeds: "E2EINJECT: ". + inject = "" + match = re.search(r"E2EINJECT:(\w+)", query) + if match: + inject = match.group(1) + query = re.sub(r"E2EINJECT:\w+\s*", "", query).strip() + if inject == "500": + return make_response("upstream error", 500) + if inject == "empty": + body = _read_fixture("aa_search_empty.html") + return make_response(body, 200) + if inject == "no_files": + # Real row present *and* the "No files found." string — the historical + # false-positive (#: 'No files found' check). The app must still + # surface the real row. + body = _search_rows().replace("", "
No files found.
") + return make_response(body, 200) + if inject == "layout_drift": + return make_response(_read_fixture("aa_search_layout_drift.html"), 200) + # Echo the query into the title so tests can assert routing worked. + body = _search_rows().replace("__QUERY__", query or "A Book Title") + return make_response(body, 200) + + @flask_app.route("/md5/") + def aa_detail(book_id: str) -> Response: + # __SLOW_BASE__ controls where the AA "slow partner server" links point. + # In the `full` profile it's the Cloudflare gate, so the *download* (not the + # search/detail) is what forces the internal Chrome bypasser to solve CF. + body = ( + _read_fixture("aa_detail.html") + .replace("__MD5__", book_id) + .replace("__SLOW_BASE__", SLOW_DOWNLOAD_BASE) + ) + return make_response(body, 200) + + @flask_app.route("/get.php") + def aa_getphp() -> Response: + # The actual file download link target (get.php?md5=..&key=..). + return _serve_book() + + @flask_app.route("/slow_download/") + def aa_slow(rest: str) -> Response: + # The AA "slow partner server" page: an HTML page whose "Download now" link + # is the final file URL. shelfmark's _extract_slow_download_url parses this. + # Reached (in the full profile) only after the internal Chrome bypasser + # solves the Cloudflare gate in front of it. + del rest + file_url = f"{os.environ.get('AA_FILE_BASE', 'http://mock-aa')}/file/{PAYLOAD_NAME}" + # The visible text must exceed the internal bypasser's "still loading" + # threshold (_LOADING_BODY_LENGTH_MAX = 50 chars of body.innerText) or it + # never considers the (cleared) page settled and loops until timeout. + html = ( + "Download" + "

Anna’s Archive — Slow Partner Server

" + "

Your download of Moby-Dick; or, The Whale by Herman Melville " + "is ready. Use the link below to download the file from this slow partner " + "server. The connection is slow but free, with no waitlist.

" + "
" + f'\U0001f4da Download now' + "
" + "

Thank you for supporting open access to knowledge.

" + "" + ) + return make_response(html, 200) + + @flask_app.route(f"/file/{PAYLOAD_NAME}") + def aa_file() -> Response: + # Final file URL extracted from the slow-download page. + return _serve_book() + + @flask_app.route("/dyn/api/fast_download.json") + def aa_fast() -> Response: + md5 = request.args.get("md5", "") + return jsonify({"download_url": f"{request.host_url.rstrip('/')}/get.php?md5={md5}&key=k"}) + + # --- webseed payload + torrent for the `full` real-client pipeline -------- + @flask_app.route(f"/payload/{PAYLOAD_NAME}") + def aa_payload() -> Response: + # Range support is required for transmission's GetRight webseed (it fetches + # pieces with `Range: bytes=...` and expects 206); libtorrent clients + # (qBittorrent/deluge) tolerate a plain 200, but transmission does not. + return _ranged_response(_payload_bytes(), "application/epub+zip") + + @flask_app.route("/payload.torrent") + def aa_torrent() -> Response: + torrent = _build_payload_torrent() + resp = make_response(torrent) + resp.headers["Content-Type"] = "application/x-bittorrent" + resp.headers["Content-Disposition"] = 'attachment; filename="sample-book.torrent"' + return resp + + +def _ranged_response(data: bytes, content_type: str) -> Response: + """Serve ``data`` honoring a single HTTP Range request (206 + Content-Range). + + Needed so transmission's webseed (which fetches via ``Range: bytes=...``) can + download piece by piece. A request without Range gets the full 200 body. + """ + total = len(data) + range_header = request.headers.get("Range", "") + if range_header.startswith("bytes="): + first = range_header[len("bytes=") :].split(",", 1)[0] + start_s, _, end_s = first.partition("-") + try: + start = int(start_s) if start_s else 0 + end = int(end_s) if end_s else total - 1 + except ValueError: + start, end = 0, total - 1 + end = min(end, total - 1) + start = max(0, min(start, end)) + chunk = data[start : end + 1] + resp = make_response(chunk, 206) + resp.headers["Content-Range"] = f"bytes {start}-{end}/{total}" + else: + resp = make_response(data) + resp.headers["Content-Type"] = content_type + resp.headers["Accept-Ranges"] = "bytes" + return resp + + +def _serve_book() -> Response: + resp = _ranged_response(_payload_bytes(), "application/epub+zip") + resp.headers["Content-Disposition"] = f'attachment; filename="{PAYLOAD_NAME}"' + return resp + + +def _build_payload_torrent() -> bytes: + """Webseed .torrent for the deterministic payload, sourced only from mock-aa. + + Imported lazily so the doh/cloudflare/flaresolverr roles don't need the + generator module on the path. + """ + import importlib.util + + gen_path = Path(__file__).parent / "make_webseed_torrent.py" + spec = importlib.util.spec_from_file_location("make_webseed_torrent", gen_path) + assert spec and spec.loader + gen = importlib.util.module_from_spec(spec) + spec.loader.exec_module(gen) + return gen.build_webseed_torrent(PAYLOAD_NAME, _payload_bytes(), PAYLOAD_URL) + + +# --------------------------------------------------------------------------- # +# Role: cloudflare (challenge until clearance cookie present) +# --------------------------------------------------------------------------- # +def register_cloudflare(flask_app: Flask) -> None: + @flask_app.route("/", defaults={"path": ""}) + @flask_app.route("/") + def cf_gate(path: str) -> Response: + if request.cookies.get(CLEARANCE_COOKIE) == CLEARANCE_VALUE: + # Cleared: proxy the request through to the real AA origin behaviour. + return _cleared_passthrough(path) + challenge = _read_fixture("cf_challenge.html") + resp = make_response(challenge, 403) + resp.headers["cf-mitigated"] = "challenge" + resp.headers["Server"] = "cloudflare" + return resp + + +def _cleared_passthrough(path: str) -> Response: + import requests as _rq + + target = f"{ORIGIN_INTERNAL_URL}/{path}" + upstream = _rq.get(target, params=request.args, timeout=10) + resp = make_response(upstream.content, upstream.status_code) + resp.headers["Content-Type"] = upstream.headers.get("Content-Type", "text/html") + return resp + + +# --------------------------------------------------------------------------- # +# Role: flaresolverr (mock external bypasser, /v1 contract) +# --------------------------------------------------------------------------- # +def register_flaresolverr(flask_app: Flask) -> None: + @flask_app.route("/v1", methods=["POST"]) + def v1() -> Response: + import requests as _rq + + payload = request.get_json(silent=True) or {} + url = payload.get("url", "") + if not url: + return jsonify({"status": "error", "message": "missing url"}), 400 + # "Solve" the challenge by fetching with the clearance cookie set. + upstream = _rq.get(url, cookies={CLEARANCE_COOKIE: CLEARANCE_VALUE}, timeout=15) + return jsonify( + { + "status": "ok", + "message": "Challenge solved!", + "solution": { + "url": url, + "status": upstream.status_code, + "response": upstream.text, + "cookies": [{"name": CLEARANCE_COOKIE, "value": CLEARANCE_VALUE, "domain": ""}], + "userAgent": "Mozilla/5.0 (e2e-flaresolverr)", + }, + } + ) + + +# --------------------------------------------------------------------------- # +# Role: prowlarr (minimal Prowlarr API for the `full` real-client pipeline) +# --------------------------------------------------------------------------- # +# Implements just the endpoints shelfmark's prowlarr client calls, returning one +# torrent release whose download is mock-aa's webseed .torrent. A real +# qBittorrent then completes the download over HTTP (no tracker/peer needed). +AA_INTERNAL_URL = os.environ.get("AA_INTERNAL_URL", "http://mock-aa") + + +def register_prowlarr(flask_app: Flask) -> None: + @flask_app.route("/api/v1/system/status") + def prowlarr_status() -> Response: + return jsonify({"appName": "Prowlarr", "version": "1.30.0.4000", "instanceName": "e2e"}) + + @flask_app.route("/api/v1/indexer") + def prowlarr_indexers() -> Response: + return jsonify( + [ + { + "id": 1, + "name": "Mock Torznab", + "enable": True, + "protocol": "torrent", + "implementation": "Torznab", + "implementationName": "Generic Torznab", + "definitionName": "mock-torznab", + "capabilities": { + "categories": [ + {"id": 7000, "name": "Books"}, + {"id": 7020, "name": "Books/EBook"}, + ] + }, + } + ] + ) + + @flask_app.route("/api/v1/indexer//newznab") + def prowlarr_torznab(indexer_id: int) -> Response: + del indexer_id + t = request.args.get("t", "search") + if t == "caps": + return Response(_torznab_caps(), mimetype="application/xml") + query = request.args.get("q", "") or "E2E Mock Book" + return Response(_torznab_search(query), mimetype="application/xml") + + +def _torznab_caps() -> str: + return ( + '' + '' + '' + '' + '' + '' + '' + ) + + +def _torznab_search(query: str) -> str: + torrent_url = f"{AA_INTERNAL_URL}/payload.torrent" + size = len(_payload_bytes()) + title = f"{query} - E2E Mock Book" + return ( + '' + '' + "" + "" + f"{title}" + "e2e-mock-release-1" + f"{torrent_url}" + f"{size}" + "Mon, 01 Jan 2024 00:00:00 +0000" + f'' + '' + '' + '' + '' + '' + "" + "" + ) + + +# --------------------------------------------------------------------------- # +# Role: doh (DNS over HTTPS responder) +# --------------------------------------------------------------------------- # +# Maps mock hostnames to the in-network IP the test wants them resolved to. +# Provided via DOH_MAP env: "host=ip,host2=ip2". +def _doh_map() -> dict[str, str]: + raw = os.environ.get("DOH_MAP", "") + out: dict[str, str] = {} + for pair in raw.split(","): + pair = pair.strip() + if "=" in pair: + host, ip = pair.split("=", 1) + out[host.strip().rstrip(".").lower()] = ip.strip() + return out + + +def _encode_a_answer(name: str, ip: str) -> bytes: + parts = ip.split(".") + return struct.pack("!HHHIH4B", 0xC00C, 1, 1, 60, 4, *(int(p) for p in parts)) + + +def register_doh(flask_app: Flask) -> None: + @flask_app.route("/dns-query", methods=["GET", "POST"]) + @flask_app.route("/resolve", methods=["GET", "POST"]) # Google-style JSON endpoint + def dns_query() -> Response: + mapping = _doh_map() + # Google/Cloudflare JSON form (?name=&type=A) + name = (request.args.get("name") or "").rstrip(".").lower() + if name: + ip = mapping.get(name) + answer = [{"name": name, "type": 1, "TTL": 60, "data": ip}] if ip else [] + return jsonify({"Status": 0 if ip else 3, "Answer": answer}) + # RFC 8484 wireformat (POST body or ?dns=) + if request.method == "POST": + wire = request.get_data() + else: + dns_b64 = request.args.get("dns", "") + wire = base64.urlsafe_b64decode(dns_b64 + "=" * (-len(dns_b64) % 4)) + return _wireformat_response(wire, mapping) + + def _wireformat_response(wire: bytes, mapping: dict[str, str]) -> Response: + # Minimal parser: echo header/question, append one A answer if known. + txid = wire[0:2] + qname, _ = _parse_qname(wire, 12) + question = wire[12:] + host = qname.rstrip(".").lower() + ip = mapping.get(host) + ancount = 1 if ip else 0 + header = txid + struct.pack("!HHHHH", 0x8180, 1, ancount, 0, 0) + body = question + (_encode_a_answer(host, ip) if ip else b"") + resp = make_response(header + body) + resp.headers["Content-Type"] = "application/dns-message" + return resp + + def _parse_qname(wire: bytes, offset: int) -> tuple[str, int]: + labels = [] + while True: + length = wire[offset] + offset += 1 + if length == 0: + break + labels.append(wire[offset : offset + length].decode("ascii", "ignore")) + offset += length + return ".".join(labels), offset + + +# --------------------------------------------------------------------------- # +# Health + role wiring +# --------------------------------------------------------------------------- # +@app.route("/healthz") +def healthz() -> Response: + return jsonify({"role": ROLE, "ok": True}) + + +_ROLES = { + "origin-aa": register_origin_aa, + "cloudflare": register_cloudflare, + "flaresolverr": register_flaresolverr, + "prowlarr": register_prowlarr, + "doh": register_doh, +} + +if ROLE == "all": + for _register in _ROLES.values(): + _register(app) +elif ROLE in _ROLES: + _ROLES[ROLE](app) +else: # pragma: no cover - misconfiguration guard + raise SystemExit(f"Unknown MOCK_ROLE={ROLE!r}; expected one of {[*sorted(_ROLES), 'all']}") + + +def _self_signed_cert() -> tuple[str, str]: + """Write a throwaway self-signed cert/key to /tmp and return their paths. + + Used only by the `doh` role's HTTPS server. The app reaches it with + CERTIFICATE_VALIDATION=disabled, so the cert's identity is irrelevant. + """ + import datetime + + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "e2e-doh")]) + now = datetime.datetime.now(datetime.UTC) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=3650)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("cloudflare-dns.com")]), False) + .sign(key, hashes.SHA256()) + ) + cert_path, key_path = "/tmp/doh.crt", "/tmp/doh.key" + Path(cert_path).write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + Path(key_path).write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return cert_path, key_path + + +if __name__ == "__main__": + if os.environ.get("DOH_TLS") == "1": + cert_path, key_path = _self_signed_cert() + app.run(host="0.0.0.0", port=443, ssl_context=(cert_path, key_path), threaded=True) + else: + app.run(host="0.0.0.0", port=int(os.environ.get("PORT", "80"))) diff --git a/tests/e2e/platform/mocks/requirements.txt b/tests/e2e/platform/mocks/requirements.txt new file mode 100644 index 0000000..e428dcc --- /dev/null +++ b/tests/e2e/platform/mocks/requirements.txt @@ -0,0 +1,4 @@ +flask>=3.0 +requests>=2.31 +# Only used by the `doh` role when DOH_TLS=1, to self-sign a cert at startup. +cryptography>=42.0 diff --git a/tests/e2e/platform/qbittorrent/qBittorrent.conf b/tests/e2e/platform/qbittorrent/qBittorrent.conf new file mode 100755 index 0000000..1e75a01 --- /dev/null +++ b/tests/e2e/platform/qbittorrent/qBittorrent.conf @@ -0,0 +1,22 @@ +[Application] +FileLogger\Enabled=true + +[BitTorrent] +Session\DefaultSavePath=/downloads +Session\TempPathEnabled=false +# Webseed (BEP-19) is how the e2e torrent completes with no tracker/peer. + +[LegalNotice] +Accepted=true + +[Preferences] +WebUI\Address=* +WebUI\Port=8080 +# Bypass WebUI auth for the e2e compose subnet so shelfmark connects without the +# linuxserver image's random temp password. +WebUI\LocalHostAuth=false +WebUI\AuthSubnetWhitelistEnabled=true +WebUI\AuthSubnetWhitelist=172.30.0.0/24 +WebUI\CSRFProtection=false +WebUI\HostHeaderValidation=false +Downloads\SavePath=/downloads/ diff --git a/tests/e2e/platform/run-e2e.sh b/tests/e2e/platform/run-e2e.sh new file mode 100755 index 0000000..b71f411 --- /dev/null +++ b/tests/e2e/platform/run-e2e.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Run the Shelfmark e2e platform for a single config profile. +# +# ./run-e2e.sh [env/.env] [extra pytest args...] +# +# Boots the stack defined by the profile env file, waits for health, runs the +# matching cluster tests (the suite skips tests not applicable to the profile), +# then tears down. Set KEEP_UP=1 to leave the stack running for debugging. +set -euo pipefail + +PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$PLATFORM_DIR" + +ENV_FILE="${1:-env/baseline.env}" +shift || true +PYTEST_ARGS=("$@") + +if [[ ! -f "$ENV_FILE" ]]; then + echo "error: env file not found: $ENV_FILE" >&2 + echo "available profiles:" >&2 + ls env/*.env >&2 + exit 2 +fi + +# shellcheck disable=SC1090 +set -a; source "$ENV_FILE"; set +a # export SM_*, COMPOSE_PROFILES, E2E_PROFILE +PROFILE="${E2E_PROFILE:-baseline}" +COMPOSE=(docker compose --env-file "$ENV_FILE" -f docker-compose.e2e.yml) + +STATE_DIR="$PLATFORM_DIR/.state" +LOG_FILE="$STATE_DIR/shelfmark.$PROFILE.log" +mkdir -p "$STATE_DIR/config" "$STATE_DIR/books" "$STATE_DIR/downloads" "$STATE_DIR/tmp" + +cleanup() { + if [[ "${KEEP_UP:-0}" != "1" ]]; then + echo "==> tearing down ($PROFILE)" + "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true + else + echo "==> KEEP_UP=1: leaving stack running ($PROFILE)" + fi +} +trap cleanup EXIT + +# E2E_NO_BUILD=1 reuses already-built images (see `make e2e-platform-build` / +# run-matrix.sh) so a matrix run builds the heavy shelfmark image only once. +if [[ "${E2E_NO_BUILD:-0}" == "1" ]]; then + echo "==> [$PROFILE] starting stack, reusing built images (profiles='${COMPOSE_PROFILES:-}')" + "${COMPOSE[@]}" up -d --no-build +else + echo "==> [$PROFILE] building + starting stack (profiles='${COMPOSE_PROFILES:-}')" + "${COMPOSE[@]}" up -d --build +fi + +echo "==> [$PROFILE] waiting for shelfmark health" +HEALTHY=0 +for _ in $(seq 1 60); do + if curl -fsS http://localhost:8084/api/health >/dev/null 2>&1; then HEALTHY=1; break; fi + sleep 2 +done + +# Capture boot diagnostics for the entrypoint/permission tests. +"${COMPOSE[@]}" logs shelfmark > "$LOG_FILE" 2>&1 || true +RESTARTS="$(docker inspect -f '{{.RestartCount}}' e2e-shelfmark 2>/dev/null || echo 0)" +echo "==> [$PROFILE] healthy=$HEALTHY restarts=$RESTARTS log=$LOG_FILE" + +if [[ "$HEALTHY" != "1" && "$PROFILE" != "tor" ]]; then + echo "error: shelfmark never became healthy under profile '$PROFILE'" >&2 + "${COMPOSE[@]}" logs --tail 40 shelfmark >&2 || true + exit 1 +fi + +# Hand context to the pytest suite. +export E2E_PROFILE="$PROFILE" +export E2E_BASE_URL="http://localhost:8084" +export E2E_BOOKS_DIR="$STATE_DIR/books" +export E2E_TMP_DIR="$STATE_DIR/tmp" +export E2E_SHELFMARK_LOG="$LOG_FILE" +export E2E_SHELFMARK_RESTARTS="$RESTARTS" + +echo "==> [$PROFILE] running suite" +set +e +( cd "$PLATFORM_DIR/../../.." && \ + uv run pytest tests/e2e/platform/suite -m platform -o addopts="--tb=short" "${PYTEST_ARGS[@]}" ) +RC=$? +set -e + +echo "==> [$PROFILE] pytest exit=$RC" +exit $RC diff --git a/tests/e2e/platform/run-matrix.sh b/tests/e2e/platform/run-matrix.sh new file mode 100755 index 0000000..0673379 --- /dev/null +++ b/tests/e2e/platform/run-matrix.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Run the full config matrix: every profile, in sequence, aggregating results. +# +# ./run-matrix.sh # all profiles +# ./run-matrix.sh baseline dns-manual # a subset (by profile name) +set -uo pipefail + +PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$PLATFORM_DIR" + +if [[ $# -gt 0 ]]; then + PROFILES=("$@") +else + PROFILES=(baseline bypasser-external bypasser-disabled dns-manual dns-blocked dns-doh \ + proxy-http proxy-socks tor client-transmission client-deluge) +fi + +# Build the (heavy) images once, then reuse them across every profile so the +# matrix doesn't rebuild the xvfb/chromium layer N times. Set NO_PREBUILD=1 to +# skip (e.g. to let each run rebuild from source). +if [[ "${NO_PREBUILD:-0}" != "1" ]]; then + echo "==> pre-building images once (reused by all profiles)" + ./build-images.sh + export E2E_NO_BUILD=1 +fi + +declare -A RESULT +FAILED=0 +for p in "${PROFILES[@]}"; do + echo "========================================================================" + echo " PROFILE: $p" + echo "========================================================================" + if ./run-e2e.sh "env/$p.env"; then + RESULT[$p]="PASS" + else + RESULT[$p]="FAIL" + FAILED=1 + fi +done + +echo "========================================================================" +echo " MATRIX SUMMARY" +echo "========================================================================" +for p in "${PROFILES[@]}"; do + printf " %-22s %s\n" "$p" "${RESULT[$p]:-SKIP}" +done +exit $FAILED diff --git a/tests/e2e/platform/suite/__init__.py b/tests/e2e/platform/suite/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/platform/suite/conftest.py b/tests/e2e/platform/suite/conftest.py new file mode 100644 index 0000000..e672fe3 --- /dev/null +++ b/tests/e2e/platform/suite/conftest.py @@ -0,0 +1,132 @@ +"""Harness for the Shelfmark e2e Docker platform. + +These tests run against a *live* Shelfmark booted by ``run-e2e.sh`` under a +particular config profile (env file). The active profile is read from +``E2E_PROFILE``; tests select which profiles they apply to with the +``@pytest.mark.profiles(...)`` marker. Unmarked tests are profile-agnostic +invariants and run under every profile — that is how the same cluster test +becomes the config matrix (search must succeed whether egress is direct, via a +proxy, via custom DNS, or through the bypasser). + +Run (handled by run-e2e.sh): + E2E_PROFILE=baseline uv run pytest tests/e2e/platform/suite -m platform +""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass, field + +import pytest +import requests + +BASE_URL = os.environ.get("E2E_BASE_URL", "http://localhost:8084") +ACTIVE_PROFILE = os.environ.get("E2E_PROFILE", "baseline") +DEFAULT_TIMEOUT = 15 +DOWNLOAD_TIMEOUT = int(os.environ.get("E2E_DOWNLOAD_TIMEOUT", "120")) +TERMINAL_OK = {"complete", "done", "available"} +TERMINAL_ERR = {"error", "cancelled"} + + +@dataclass +class PlatformClient: + base_url: str = BASE_URL + timeout: int = DEFAULT_TIMEOUT + session: requests.Session = field(default_factory=requests.Session) + + def get(self, path: str, **kw) -> requests.Response: + kw.setdefault("timeout", self.timeout) + return self.session.get(f"{self.base_url}{path}", **kw) + + def post(self, path: str, **kw) -> requests.Response: + kw.setdefault("timeout", self.timeout) + return self.session.post(f"{self.base_url}{path}", **kw) + + # --- domain helpers -------------------------------------------------- # + def wait_for_health(self, max_wait: int = 90) -> bool: + deadline = time.time() + max_wait + while time.time() < deadline: + try: + if self.get("/api/health").status_code == 200: + return True + except requests.RequestException: + pass + time.sleep(2) + return False + + def direct_search( + self, query: str, *, inject: str | None = None, **params + ) -> requests.Response: + """Source-native (hermetic) release search — no external metadata provider. + + Hits GET /api/releases?source=direct_download&query=... which drives + direct_download.search_books against the mock Anna's Archive. + + Fault injection rides *inside* the query text (the app builds the AA URL + itself and only forwards the query as ``q=``); the mock origin parses the + ``E2EINJECT:`` token. See mock_services.aa_search. + """ + effective_query = f"E2EINJECT:{inject} {query}" if inject else query + qp = {"source": "direct_download", "query": effective_query, **params} + return self.get("/api/releases", params=qp, timeout=60) + + def releases_from(self, resp: requests.Response) -> list[dict]: + if resp.status_code != 200: + return [] + data = resp.json() + if isinstance(data, dict): + rel = data.get("releases") + return rel if isinstance(rel, list) else [] + return data if isinstance(data, list) else [] + + def queue_download(self, release: dict) -> requests.Response: + return self.post("/api/releases/download", json=release, timeout=30) + + def wait_for_terminal(self, book_id: str, timeout: int = DOWNLOAD_TIMEOUT) -> tuple[str, dict]: + deadline = time.time() + timeout + last: dict = {} + while time.time() < deadline: + resp = self.get("/api/status") + if resp.status_code == 200 and isinstance(resp.json(), dict): + status = resp.json() + for state, entries in status.items(): + if isinstance(entries, dict) and book_id in entries: + last = entries[book_id] + if state in TERMINAL_OK or state in TERMINAL_ERR: + return state, last + time.sleep(2) + return "timeout", last + + +@pytest.fixture(scope="session") +def client() -> PlatformClient: + c = PlatformClient() + if not c.wait_for_health(): + pytest.fail(f"Shelfmark not healthy at {BASE_URL} (profile={ACTIVE_PROFILE})") + return c + + +@pytest.fixture(scope="session") +def active_profile() -> str: + return ACTIVE_PROFILE + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "platform: Shelfmark e2e docker platform test") + config.addinivalue_line("markers", "profiles(*names): only run under these E2E_PROFILE values") + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Skip tests whose declared profiles don't include the active profile. + + A test with no ``profiles`` marker is a profile-agnostic invariant and runs + everywhere (this is the matrix: invariants x profiles). + """ + for item in items: + item.add_marker(pytest.mark.platform) + marker = item.get_closest_marker("profiles") + if marker and ACTIVE_PROFILE not in marker.args: + item.add_marker( + pytest.mark.skip(reason=f"profile={ACTIVE_PROFILE!r} not in {marker.args}") + ) diff --git a/tests/e2e/platform/suite/test_cluster_bypasser.py b/tests/e2e/platform/suite/test_cluster_bypasser.py new file mode 100644 index 0000000..87cd218 --- /dev/null +++ b/tests/e2e/platform/suite/test_cluster_bypasser.py @@ -0,0 +1,86 @@ +"""Cluster 1 — Cloudflare bypasser wiring + clean-failure behavior. + +Reality discovered by running the stack: shelfmark's AA *search* and *detail* +fetches use ``html_get_page(allow_bypasser_fallback=False)``, so a search behind a +Cloudflare gate returns 503 **regardless** of the bypasser. The bypasser (internal +Chrome or external FlareSolverr) is a *download-time* mechanism +(``html_get_page(use_bypasser=True)``); it never runs for search. + +So these tests assert what is actually true and host-observable: + * the external bypasser is configured from env, and + * a CF-gated search fails *cleanly* (a 503 the client can act on, not a hang or + a crash) — both with the bypasser on (it isn't used for search) and off. + +Exercising shelfmark's *use* of the bypasser end-to-end (a real CF solve during a +download) needs the AA slow-download HTML flow mocked — see the README roadmap. +The bypass *mechanism* itself is verified to work: the mock FlareSolverr solves +the gate (manually confirmed; see README). Guards: #284 #226 #202 #1030 #410 #369. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import requests + + +def _boot_log() -> str: + path = os.environ.get("E2E_SHELFMARK_LOG") + if not path or not Path(path).exists(): + return "" + return Path(path).read_text(encoding="utf-8", errors="ignore") + + +def _cf_gated_search_has_no_releases(client) -> bool: + """A CF-gated AA search must not yield releases (the gate isn't bypassed for + search). + + NOTE (observed live): with ``USE_CF_BYPASS=false`` the search is *slow* to fail + — the app retries and can take ~60s, vs a fast 503 when the bypasser is enabled + (potential #1001 "hung on bypass protection"). We bound the wait and treat a + timeout the same as a clean failure: in both cases no releases were obtained, + which is the point of this negative control. + """ + try: + resp = client.get( + "/api/releases", + params={"source": "direct_download", "query": "Mistborn"}, + timeout=30, + ) + except requests.exceptions.Timeout: + return True # could not complete -> definitively no releases obtained + assert resp.status_code in (200, 404, 500, 503), ( + f"CF-gated search returned an unexpected status: {resp.status_code} {resp.text[:200]}" + ) + return not client.releases_from(resp) + + +@pytest.mark.profiles("bypasser-external") +def test_external_bypasser_is_configured(client) -> None: + """The external (FlareSolverr) bypasser path is selected via env.""" + assert client.get("/api/health").status_code == 200 + log = _boot_log() + if not log: + pytest.skip("E2E_SHELFMARK_LOG not available") + assert "USING_EXTERNAL_BYPASSER" in log and "EXT_BYPASSER_URL" in log, ( + "external bypasser config was not synced from env" + ) + + +@pytest.mark.profiles("bypasser-external") +def test_cf_gated_search_fails_cleanly_with_external_bypasser(client) -> None: + """Even with the external bypasser configured, a CF-gated *search* yields no + releases (the bypasser is download-time) — but it must fail cleanly.""" + assert _cf_gated_search_has_no_releases(client) + + +@pytest.mark.profiles("bypasser-disabled") +def test_cf_gated_search_fails_when_bypasser_off(client) -> None: + """Negative control: AA behind Cloudflare + bypasser OFF -> no releases, clean + failure. A regression that ignored the gate would wrongly return results.""" + assert _cf_gated_search_has_no_releases(client), ( + "results returned even though AA is Cloudflare-gated and the bypasser is " + "disabled — the challenge is being ignored (regression for #202/#410)" + ) diff --git a/tests/e2e/platform/suite/test_cluster_bypasser_tor.py b/tests/e2e/platform/suite/test_cluster_bypasser_tor.py new file mode 100644 index 0000000..8d49fee --- /dev/null +++ b/tests/e2e/platform/suite/test_cluster_bypasser_tor.py @@ -0,0 +1,32 @@ +"""Cluster 1/6 — Tor boot correctness. + +Tor has repeatedly boot-looped or pegged CPU on startup (#1021 loops on 1.3.0, +#940 USING_TOR loop, #801 gosu 100% CPU, #937 gunicorn missing). The hermetic, +fast assertion is: with USING_TOR=true the container reaches a healthy +/api/health and does NOT crash-loop. Real Tor egress (slow/flaky in CI) is left +to an opt-in 'tor-full' profile. +""" + +from __future__ import annotations + +import os + +import pytest + +pytestmark = pytest.mark.profiles("tor") + + +def test_app_becomes_healthy_under_tor(client) -> None: + """tor.sh + entrypoint must bring the app up, not boot-loop.""" + assert client.get("/api/health").status_code == 200 + + +def test_container_did_not_crash_loop(client) -> None: + """Restart count is captured by the runner into E2E_SHELFMARK_RESTARTS. + + A boot-loop shows up as repeated restarts; a healthy boot is 0. + """ + restarts = os.environ.get("E2E_SHELFMARK_RESTARTS") + if restarts is None: + pytest.skip("runner did not provide E2E_SHELFMARK_RESTARTS") + assert int(restarts) == 0, f"shelfmark restarted {restarts} times under Tor (boot-loop)" diff --git a/tests/e2e/platform/suite/test_cluster_clients.py b/tests/e2e/platform/suite/test_cluster_clients.py new file mode 100644 index 0000000..efb2558 --- /dev/null +++ b/tests/e2e/platform/suite/test_cluster_clients.py @@ -0,0 +1,87 @@ +"""Cluster 5 — real download clients (torrent), end to end. + +The `full` and `client-*` profiles each point shelfmark at a *real* torrent client +(qBittorrent / Transmission / Deluge / rTorrent) plus a mock Prowlarr that returns a +tracker-less BEP-19 webseed `.torrent` sourced from mock-aa. The client downloads +the payload over HTTP (no tracker/peer/seeder) and shelfmark's completion detection ++ post-process move lands Moby-Dick in `/books`. + +The test is **client-agnostic** — the active profile's env selects the client +(`PROWLARR_TORRENT_CLIENT` + that client's URL/creds) — so one test covers the whole +client matrix. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.profiles("full", "client-transmission", "client-deluge") + +BOOK = "Moby Dick" + + +def _books_dir() -> Path | None: + raw = os.environ.get("E2E_BOOKS_DIR") + return Path(raw) if raw else None + + +def _book_files(books: Path) -> set[str]: + return { + p.name for p in books.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"} + } + + +def _prowlarr_search(client, query: str): + return client.get( + "/api/releases", + params={ + "provider": "manual", + "book_id": "e2e-manual-1", + "source": "prowlarr", + "title": query, + "manual_query": query, + }, + timeout=60, + ) + + +def test_prowlarr_to_real_torrent_client_download(client, active_profile) -> None: + """Prowlarr release -> real torrent client (per profile) -> file in /books.""" + books = _books_dir() + if books is None or not books.exists(): + pytest.skip("E2E_BOOKS_DIR not visible to the test runner") + before = _book_files(books) + + resp = _prowlarr_search(client, BOOK) + assert resp.status_code == 200, f"[{active_profile}] prowlarr search failed: {resp.text[:300]}" + releases = client.releases_from(resp) + assert releases, f"[{active_profile}] mock prowlarr returned no releases" + + queued = client.queue_download(releases[0]) + assert queued.status_code in (200, 201, 202), ( + f"[{active_profile}] queue refused the release: {queued.status_code} {queued.text[:300]}" + ) + # The prowlarr source serializes download_url=None and resolves the real URL + # from its cache by source_id at download time. + book_id = releases[0].get("source_id") or releases[0].get("id") or releases[0].get("guid") + assert book_id, f"[{active_profile}] release missing a trackable id: {releases[0]!r}" + + state, info = client.wait_for_terminal(str(book_id)) + assert state in {"complete", "done", "available"}, ( + f"[{active_profile}] real torrent-client download did not complete: " + f"state={state} info={info!r}" + ) + + deadline = time.time() + 30 + new_files: set[str] = set() + while time.time() < deadline: + new_files = _book_files(books) - before + if new_files: + break + time.sleep(2) + assert new_files, f"[{active_profile}] client completed but no file landed in /books" + assert all(Path(n).suffix for n in new_files), f"file without extension: {new_files}" diff --git a/tests/e2e/platform/suite/test_cluster_download_permissions.py b/tests/e2e/platform/suite/test_cluster_download_permissions.py new file mode 100644 index 0000000..4e45f44 --- /dev/null +++ b/tests/e2e/platform/suite/test_cluster_download_permissions.py @@ -0,0 +1,62 @@ +"""Cluster 4 — download execution + file placement/permissions. + +Two halves, by what each profile can hermetically prove: + +* **baseline (no bypasser, AA-only):** AA's slow-download sources require a + Cloudflare bypass (``_CF_BYPASS_REQUIRED``), so a download here *cannot* succeed + and the app must say so cleanly — this is exactly the real-world #1028 shape + ("All download sources failed"). We assert that the failure is surfaced as a + terminal ``error`` with a message, not a hang/crash, and that nothing is left + orphaned in staging (#1040). +* **successful download + file move** (extension preserved #214, no orphaned + staging dir #1040) is proven for real in the ``full`` profile, where a real + qBittorrent completes a webseed torrent — see ``test_cluster_full_pipeline.py``. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.profiles("baseline") + + +def _staging_leftovers() -> list[Path]: + tmp_raw = os.environ.get("E2E_TMP_DIR") + if not tmp_raw or not Path(tmp_raw).exists(): + return [] + tmp = Path(tmp_raw) + return [p for p in tmp.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"}] + + +def test_no_bypass_download_fails_cleanly(client) -> None: + """Without a bypasser, AA is undownloadable — the app must report a clear + terminal error (the #1028 shape), not hang or crash.""" + resp = client.direct_search("Mistborn") + releases = client.releases_from(resp) + assert releases, "search should still return parsed releases even if undownloadable" + + queued = client.queue_download(releases[0]) + assert queued.status_code in (200, 201, 202), ( + f"queue refused the release: {queued.status_code} {queued.text[:300]}" + ) + book_id = releases[0].get("id") or releases[0].get("source_id") or releases[0].get("md5") + assert book_id, f"release missing an id to track: {releases[0]!r}" + + state, info = client.wait_for_terminal(str(book_id)) + assert state == "error", ( + f"expected a clean terminal error without a bypasser, got state={state} info={info!r}" + ) + message = str(info.get("status_message") or info.get("last_error_message") or "") + assert message.strip(), "download failed but surfaced no status message to the user" + + +def test_no_orphaned_staging_dir(client) -> None: + """#1040 guard: a failed/aborted download must not leave book payloads behind + in the staging/tmp area.""" + if not os.environ.get("E2E_TMP_DIR"): + pytest.skip("E2E_TMP_DIR not provided") + leftovers = _staging_leftovers() + assert not leftovers, f"book payload left orphaned in staging dir: {leftovers}" diff --git a/tests/e2e/platform/suite/test_cluster_full_pipeline.py b/tests/e2e/platform/suite/test_cluster_full_pipeline.py new file mode 100644 index 0000000..b54f3d6 --- /dev/null +++ b/tests/e2e/platform/suite/test_cluster_full_pipeline.py @@ -0,0 +1,146 @@ +"""The `full` profile — maximum-realism, heavy, nightly/manual only. + +Test book: *Moby-Dick* by Herman Melville (public domain). Validated live. + + * **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are + reachable (mock-aa), but the AA *slow-download* links point through the + Cloudflare gate (mock-cf). Downloading therefore forces the in-image headless + Chromium (seleniumbase CDP internal bypasser) to execute the challenge JS, + harvest ``cf_clearance``, and fetch the gated slow-download page. Moby-Dick + landing in ``/books`` is only possible if Chrome actually solved the gate — + that is the literal "spin a chrome browser" path. + * **DoH** enabled at boot (``USE_DOH=true``) without breaking startup. + +The real torrent-client download (the other half of the `full` profile) lives in +test_cluster_clients.py, which runs under `full` and the `client-*` profiles. +Only runs under the ``full`` profile booted by ``run-e2e.sh env/full.env``. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import time +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.profiles("full") + +BOOK = "Moby Dick" + + +def _boot_log() -> str: + path = os.environ.get("E2E_SHELFMARK_LOG") + if not path or not Path(path).exists(): + return "" + return Path(path).read_text(encoding="utf-8", errors="ignore") + + +def _books_dir() -> Path | None: + raw = os.environ.get("E2E_BOOKS_DIR") + return Path(raw) if raw else None + + +def _book_files(books: Path) -> set[str]: + return { + p.name for p in books.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"} + } + + +def _wait_for_new_book(books: Path, before: set[str], timeout: int = 40) -> set[str]: + deadline = time.time() + timeout + while time.time() < deadline: + new = _book_files(books) - before + if new: + return new + time.sleep(2) + return set() + + +def _track_id(release: dict) -> str: + return str(release.get("source_id") or release.get("id") or release.get("guid") or "") + + +# --------------------------------------------------------------------------- # +# 1. Real Chrome solves Cloudflare end-to-end (the "spin a chrome browser" path) +# --------------------------------------------------------------------------- # +def test_real_chrome_solves_cloudflare_end_to_end(client) -> None: + """Download an AA book whose slow-download is behind Cloudflare; the internal + headless Chrome must solve the gate for the file to arrive in /books.""" + books = _books_dir() + if books is None or not books.exists(): + pytest.skip("E2E_BOOKS_DIR not visible to the test runner") + before = _book_files(books) + + resp = client.direct_search(BOOK) + assert resp.status_code == 200, f"AA search failed: {resp.status_code} {resp.text[:300]}" + releases = client.releases_from(resp) + assert releases, "AA search returned no releases for Moby Dick" + + queued = client.queue_download(releases[0]) + assert queued.status_code in (200, 201, 202), ( + f"queue refused the AA release: {queued.status_code} {queued.text[:300]}" + ) + book_id = _track_id(releases[0]) + assert book_id, f"release missing a trackable id: {releases[0]!r}" + + state, info = client.wait_for_terminal(book_id) + assert state in {"complete", "done", "available"}, ( + f"AA download via the Chrome-solved Cloudflare gate did not complete: " + f"state={state} info={info!r}" + ) + + new_files = _wait_for_new_book(books, before) + assert new_files, ( + "download reported complete but no file landed in /books — the Cloudflare " + "gate in front of the slow-download was not solved by Chrome" + ) + assert all(Path(n).suffix for n in new_files), f"file written without extension: {new_files}" + + # Secondary, explicit signal that the internal bypasser (Chrome) was engaged. + _assert_bypasser_engaged() + + +def _assert_bypasser_engaged() -> None: + """Confirm shelfmark actually routed through the internal Chrome bypasser. + + Best-effort: reads the live shelfmark container logs. The file landing in /books + is already proof (the slow-download was CF-gated), but this pins the mechanism. + """ + if shutil.which("docker") is None: + return + result = subprocess.run( + ["docker", "logs", "e2e-shelfmark"], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + blob = (result.stdout + result.stderr).lower() + if not blob.strip(): + return + assert "bypass" in blob, ( + "no evidence the internal bypasser engaged during the download — the file " + "may have arrived via an unexpected (non-Chrome) path" + ) + + +# --------------------------------------------------------------------------- # +# 2. DoH +# --------------------------------------------------------------------------- # +def test_doh_enabled_and_app_healthy(client) -> None: + """DoH is enabled at boot and the app stays healthy (DoH init has historically + broken startup). ``/api/config`` doesn't expose ``USE_DOH``, so verify via the + boot log + health.""" + assert client.get("/api/health").status_code == 200 + log = _boot_log() + if not log: + pytest.skip("E2E_SHELFMARK_LOG not available to assert DoH") + assert "USE_DOH=true" in log or "'USE_DOH'" in log, "USE_DOH was not synced into config at boot" + + +# NOTE: the real torrent-client download (Prowlarr -> qBittorrent/transmission/ +# deluge/rtorrent -> /books) lives in test_cluster_clients.py, which runs under the +# `full` profile *and* the `client-*` profiles from one client-agnostic test. diff --git a/tests/e2e/platform/suite/test_cluster_health_entrypoint.py b/tests/e2e/platform/suite/test_cluster_health_entrypoint.py new file mode 100644 index 0000000..ad9b1f6 --- /dev/null +++ b/tests/e2e/platform/suite/test_cluster_health_entrypoint.py @@ -0,0 +1,49 @@ +"""Clusters 5/6/7 — container/entrypoint correctness, plus pointers. + +Cluster 6 (docker/entrypoint/PUID-PGID — 9 issues / 19 fix PRs, zero shell test +coverage today): the app must boot healthy under the configured PUID/PGID with no +permission errors, regardless of which config profile is active. This is a +profile-agnostic invariant, so it runs under every profile and catches +entrypoint/permission regressions (#801, #411, #434, #171) across the matrix. + +Clusters 5 (download clients) and 7 (audiobook/ABB) are exercised by dedicated +stacks/sources (docker-compose.test-clients.yml and an audiobookbay mock) — see +README. Pointers below keep them visible in the matrix without duplicating the +prowlarr e2e flow. +""" + +from __future__ import annotations + +import os + +import pytest + + +def test_health_endpoint_under_every_profile(client, active_profile) -> None: + """Entrypoint/permission boot must succeed under the active config.""" + resp = client.get("/api/health") + assert resp.status_code == 200, f"[{active_profile}] not healthy: {resp.text[:200]}" + + +def test_no_permission_errors_in_boot_logs() -> None: + """The runner captures shelfmark boot logs into E2E_SHELFMARK_LOG; assert no + permission/entrypoint failure markers (regression for #171/#447/#801).""" + log_path = os.environ.get("E2E_SHELFMARK_LOG") + if not log_path or not os.path.exists(log_path): + pytest.skip("E2E_SHELFMARK_LOG not provided by the runner") + with open(log_path, encoding="utf-8", errors="ignore") as fh: + text = fh.read().lower() + for marker in ("permission denied", "operation not permitted", "read-only file system"): + assert marker not in text, f"boot logs contain a permission failure: {marker!r}" + + +@pytest.mark.skip( + reason="cluster 5: covered by docker-compose.test-clients.yml + prowlarr e2e flow" +) +def test_download_clients_pointer() -> None: # pragma: no cover - documentation marker + ... + + +@pytest.mark.skip(reason="cluster 7: needs an audiobookbay mock role (tracked in README roadmap)") +def test_audiobook_pointer() -> None: # pragma: no cover - documentation marker + ... diff --git a/tests/e2e/platform/suite/test_cluster_search_aa.py b/tests/e2e/platform/suite/test_cluster_search_aa.py new file mode 100644 index 0000000..dad4702 --- /dev/null +++ b/tests/e2e/platform/suite/test_cluster_search_aa.py @@ -0,0 +1,70 @@ +"""Cluster 2/3 — search relevance and Anna's Archive HTML-parse robustness. + +The recurring root cause: AA changes its DOM and the hardcoded-index parser +silently returns zero rows, which users experience as "All download sources +failed" (#1028) or "book not found" (#198, #293). Evidence of brittleness: +#878/#879/#880 (hardcoded indices/selectors), plus the repeated +"Fix AA ... after they changed layout" PRs. + +These run under ``baseline`` (direct connection to the fake AA) so the assertions +isolate parsing from egress concerns. +""" + +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.profiles("baseline") + + +def test_search_returns_parsed_releases(client) -> None: + """Happy path: the parser turns the AA results table into releases.""" + resp = client.direct_search("Mistborn") + assert resp.status_code == 200, resp.text + releases = client.releases_from(resp) + assert releases, "expected at least one parsed release from the AA results table" + titles = " ".join(str(r.get("title", "")) for r in releases) + assert "Mistborn" in titles, f"query not reflected in parsed titles: {titles[:200]}" + + +def test_layout_drift_fails_loudly_not_silently(client) -> None: + """When AA's DOM changes so no row parses, the app must surface a clear + failure — NOT an empty 200 that reads as 'book does not exist'. + + Regression guard for #878/#879/#880 and the layout-change PRs. + """ + resp = client.direct_search("Mistborn", inject="layout_drift") + releases = client.releases_from(resp) + # Acceptable behaviours: an explicit error status, OR a 200 with an error + # field. NOT acceptable: 200 + empty releases with no signal. + if resp.status_code == 200: + body = ( + resp.json() + if resp.headers.get("content-type", "").startswith("application/json") + else {} + ) + has_error_signal = bool(body.get("error")) or bool(body.get("source_errors")) + assert not releases, "parser unexpectedly produced releases from drifted DOM" + assert has_error_signal, ( + "layout drift produced a silent empty 200 — the app must signal that " + "the source could not be parsed (regression for #878/#879/#880)" + ) + else: + assert resp.status_code >= 400, resp.status_code + + +def test_no_files_string_alongside_real_results(client) -> None: + """A real results table that also contains the literal 'No files found.' + must still yield releases (false-positive guard).""" + resp = client.direct_search("Mistborn", inject="no_files") + assert resp.status_code == 200, resp.text + assert client.releases_from(resp), ( + "'No files found.' substring caused a false-positive empty result" + ) + + +def test_genuinely_empty_results_handled_cleanly(client) -> None: + """A true 'No files found.' page yields zero releases without a 500.""" + resp = client.direct_search("Mistborn", inject="empty") + assert resp.status_code in (200, 404), resp.status_code + assert client.releases_from(resp) == [] diff --git a/tests/e2e/platform/suite/test_config_matrix.py b/tests/e2e/platform/suite/test_config_matrix.py new file mode 100644 index 0000000..d1292e9 --- /dev/null +++ b/tests/e2e/platform/suite/test_config_matrix.py @@ -0,0 +1,113 @@ +"""Config-matrix invariants. + +These tests carry NO ``profiles`` marker for the egress check, so they run under +every profile the runner boots. Reaching the (mock) book source must succeed +whether egress is direct, through an HTTP/SOCKS proxy, via custom DNS, or via the +Cloudflare bypasser. That cross-product *is* the config matrix. + +Covers the recurring "X setting silently breaks downloads" class: +proxy ignored (#956), DNS/ISP blocks (#1028, #108), bypasser config not adhered +(#410, #369, #267). +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +# Profiles where reaching the (non-CF-gated) source via search is expected to work +# end to end. Excluded: tor (boot-correctness only), and the bypasser profiles — +# their AA is behind a Cloudflare gate that search does NOT bypass (the bypasser is +# download-time only), so a *search* there fails by design (see test_cluster_bypasser). +EGRESS_PROFILES = ( + "baseline", + "dns-manual", + "dns-blocked", + "proxy-http", + "proxy-socks", +) + +# The proxy container whose logs prove egress actually traversed the proxy. +PROXY_CONTAINER = { + "proxy-http": "e2e-tinyproxy", + "proxy-socks": "e2e-microsocks", +} + + +def test_health_ok(client) -> None: + resp = client.get("/api/health") + assert resp.status_code == 200, resp.text + + +@pytest.mark.profiles(*EGRESS_PROFILES) +def test_source_reachable_under_active_profile(client, active_profile) -> None: + """The book source must be reachable regardless of egress configuration.""" + resp = client.direct_search("Mistborn") + assert resp.status_code == 200, ( + f"[{active_profile}] direct search failed: {resp.status_code} {resp.text[:300]}" + ) + releases = client.releases_from(resp) + assert releases, ( + f"[{active_profile}] expected releases from the mock source but got none — " + f"egress configuration is silently dropping the request" + ) + + +@pytest.mark.profiles("proxy-http") +def test_proxy_mode_synced_at_boot(client) -> None: + """The deployment PROXY_MODE env override is applied at boot. + + ``/api/config`` returns only a frontend-facing subset (not network/proxy + keys), so this is verified from the boot log where ENV→config sync is recorded. + """ + assert client.get("/api/health").status_code == 200 + path = os.environ.get("E2E_SHELFMARK_LOG") + if not path or not Path(path).exists(): + pytest.skip("E2E_SHELFMARK_LOG not available") + log = Path(path).read_text(encoding="utf-8", errors="ignore") + assert "PROXY_MODE" in log, "PROXY_MODE was not synced into network config at boot" + + +@pytest.mark.profiles("proxy-http", "proxy-socks") +def test_egress_actually_traverses_proxy(client, active_profile) -> None: + """#956 guard: a configured proxy must actually *carry* the app's egress. + + Reachability alone is not enough — the app and the mock AA share the e2e + network, so a regression that silently ignores the proxy config would still + reach AA directly and pass ``test_source_reachable_under_active_profile``. + Here we drive a search and then inspect the proxy container's logs: if the + proxy never saw the traffic, the proxy was bypassed (regression for #956). + """ + if shutil.which("docker") is None: + pytest.skip("docker CLI not available to the test host") + container = PROXY_CONTAINER[active_profile] + + # Generate egress that *must* go through the proxy. + resp = client.direct_search("Mistborn") + assert resp.status_code == 200 and client.releases_from(resp), ( + f"[{active_profile}] search failed under proxy: {resp.status_code} {resp.text[:200]}" + ) + + logs = subprocess.run( + ["docker", "logs", container], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + blob = (logs.stdout + logs.stderr).lower() + assert blob.strip(), ( + f"[{active_profile}] proxy container {container!r} produced no logs after a " + f"search — egress did not traverse the proxy (regression for #956 proxy-ignored)" + ) + # Strongest signal (HTTP proxy logs the destination host/request explicitly). + if active_profile == "proxy-http": + markers = ("mock-aa", "aa.mock.test", "connect", "request", "get ") + assert any(m in blob for m in markers), ( + f"tinyproxy logs show no AA request — proxy may be passing traffic without " + f"the app routing through it. logs tail: ...{blob[-300:]!r}" + )