e2e testing 2 (#1083)

2026-09-24 21:00:31 +01:00 · 2026-06-23 16:58:24 -04:00
parent 2f70ed36e4
commit 1c19326bd0
33 changed files with 2774 additions and 0 deletions
@@ -0,0 +1,103 @@
+"""Cluster 7 (audiobook/ABB) parse-contract guards.
+
+ABB forces ``https://`` for search and detail fetches, so it can't be exercised
+hermetically in the HTTP e2e docker platform. Its recurring bugs are instead in
+*parsing*: magnet/info-hash extraction ("Fix ABB magnet parsing", and the
+qbittorrent hash-length issue #386) and DOM/layout drift. These contract tests
+feed golden HTML through the real scraper — the same fail-on-drift philosophy as
+the AA layout-drift guard — and run in normal CI.
+
+They deliberately cover cases the existing ``test_scraper.py`` does not: info-hash
+*normalization* (whitespace/case), the in-page magnet *fallback*, and a layout
+drift that must degrade to an empty result rather than crash.
+"""
+
+from __future__ import annotations
+
+import re
+from unittest.mock import patch
+
+from shelfmark.release_sources.audiobookbay import scraper
+
+# Detail page where the Info Hash is lowercase and split by whitespace/newlines —
+# the exact shape that produced malformed magnets / wrong hash lengths (#386).
+DETAIL_HTML_MESSY_HASH = """
+<html><body><table>
+  <tr><td>Info Hash</td><td>abc123def456789012345678
+  901234567890abcd</td></tr>
+  <tr><td>Tracker 1</td><td>udp://tracker.openbittorrent.com:80</td></tr>
+</table></body></html>
+"""
+
+# Info Hash cell is junk, but a full magnet link is posted elsewhere on the page.
+DETAIL_HTML_MAGNET_FALLBACK = """
+<html><body>
+  <table><tr><td>Info Hash</td><td>n/a</td></tr></table>
+  <p>Mirror: magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=x</p>
+</body></html>
+"""
+
+# DOM drift: results are present but the .post / .postTitle structure changed.
+SEARCH_HTML_LAYOUT_DRIFT = """
+<html><body>
+  <article class="result-card">
+    <header><a href="/abss/drifted/">Drifted Audiobook - Author</a></header>
+    <span class="lang">English</span>
+  </article>
+</body></html>
+"""
+
+
+def _patch_detail(html: str):
+    return patch(
+        "shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
+        return_value=html,
+    )
+
+
+def test_info_hash_is_normalized_to_canonical_btih() -> None:
+    """Whitespace/newlines are stripped and the hash upper-cased to a valid
+    40-char btih (regression for #386 / 'Fix ABB magnet parsing')."""
+    with _patch_detail(DETAIL_HTML_MESSY_HASH):
+        magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/x/", "audiobookbay.lu")
+    assert magnet is not None, "messy-but-valid info hash should still yield a magnet"
+    btih = re.search(r"xt=urn:btih:([0-9A-Fa-f]+)", magnet)
+    assert btih is not None, magnet
+    assert btih.group(1) == "ABC123DEF456789012345678901234567890ABCD"
+    assert len(btih.group(1)) == 40
+    assert "tr=" in magnet  # tracker carried through
+
+
+def test_magnet_fallback_when_info_hash_cell_is_junk() -> None:
+    """When the Info Hash cell is invalid, the scraper recovers the hash from an
+    in-page magnet link rather than failing."""
+    with _patch_detail(DETAIL_HTML_MAGNET_FALLBACK):
+        magnet = scraper.extract_magnet_link("https://audiobookbay.lu/abss/y/", "audiobookbay.lu")
+    assert magnet is not None
+    assert "btih:1111111111111111111111111111111111111111" in magnet
+
+
+def test_missing_info_hash_returns_none_not_crash() -> None:
+    """No hash anywhere -> None (clean failure), never an exception."""
+    with _patch_detail("<html><body><p>nothing here</p></body></html>"):
+        assert (
+            scraper.extract_magnet_link("https://audiobookbay.lu/abss/z/", "audiobookbay.lu")
+            is None
+        )
+
+
+def test_search_layout_drift_degrades_to_empty() -> None:
+    """A changed results DOM yields zero parsed results without raising — the
+    ABB analogue of the AA layout-drift guard."""
+    with (
+        patch(
+            "shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
+            return_value=(SEARCH_HTML_LAYOUT_DRIFT, "https://audiobookbay.lu/?s=test"),
+        ),
+        patch(
+            "shelfmark.release_sources.audiobookbay.scraper.config.get",
+            return_value=0.0,
+        ),
+    ):
+        results = scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
+    assert results == [], f"drifted DOM should parse to no results, got {results!r}"
@@ -0,0 +1,79 @@
+"""DoH resolver integration against the e2e platform's mock DoH responder.
+
+The config-cluster analysis flagged DNS/DoH as a recurring break surface (#1028,
+#108). A fully hermetic DoH-over-the-network profile isn't feasible in the HTTP
+docker platform (DoH provider URLs are HTTPS + IP-pinned), so we exercise the
+*real* ``DoHResolver`` client against the platform's mock ``doh`` role here, over
+plain HTTP on localhost. This runs in normal CI (not just the nightly docker
+matrix) and guards the DoH JSON-parsing path the app relies on.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import threading
+from pathlib import Path
+from wsgiref.simple_server import WSGIServer, make_server
+
+import pytest
+
+MOCK_PATH = Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "mock_services.py"
+
+
+def _load_mock_doh_app(doh_map: str):
+    """Import the platform mock_services module wired for the ``doh`` role.
+
+    The module wires its routes at import time from ``MOCK_ROLE``/``DOH_MAP``, so
+    those must be set before loading it.
+    """
+    os.environ["MOCK_ROLE"] = "doh"
+    os.environ["DOH_MAP"] = doh_map
+    spec = importlib.util.spec_from_file_location("mock_doh_services", MOCK_PATH)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.app
+
+
+@pytest.fixture(scope="module")
+def doh_url():
+    if not MOCK_PATH.exists():
+        pytest.skip(f"platform mock not found at {MOCK_PATH}")
+    app = _load_mock_doh_app("aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11")
+    server: WSGIServer = make_server("127.0.0.1", 0, app)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        # Google JSON DoH style uses the /resolve endpoint.
+        yield f"http://127.0.0.1:{server.server_port}/resolve"
+    finally:
+        server.shutdown()
+
+
+def _resolver(doh_url: str):
+    from shelfmark.download.network import DoHResolver
+
+    # hostname/ip args are the DoH server's own identity (used only for recursion
+    # avoidance); the localhost values here are irrelevant to the lookups under test.
+    return DoHResolver(doh_url, "127.0.0.1", "127.0.0.1")
+
+
+def test_doh_resolves_mapped_host(doh_url) -> None:
+    """The real DoH client parses the mock's JSON answer into an A record."""
+    assert _resolver(doh_url).resolve("aa.mock.test", "A") == ["172.30.0.10"]
+
+
+def test_doh_nxdomain_returns_empty_not_error(doh_url) -> None:
+    """An unmapped name yields an empty list (Status 3), not an exception —
+    the path that, when mishandled, surfaced as silent download failures."""
+    assert _resolver(doh_url).resolve("unmapped.invalid", "A") == []
+
+
+def test_doh_resolver_caches_within_ttl(doh_url) -> None:
+    """A second lookup is served from cache (the resolver's documented behaviour)."""
+    resolver = _resolver(doh_url)
+    first = resolver.resolve("cf.mock.test", "A")
+    assert first == ["172.30.0.11"]
+    assert ("cf.mock.test", "A") in resolver._cache
+    assert resolver.resolve("cf.mock.test", "A") == first
@@ -0,0 +1,87 @@
+"""Validate the e2e platform's webseed .torrent generator.
+
+The ``full`` e2e profile relies on a tracker-less webseed torrent so a real
+qBittorrent can complete a real download from the mock origin over HTTP. If the
+generator emits malformed bencode or mismatched piece hashes, qBittorrent would
+silently never complete — so we cross-check the generator against shelfmark's own
+``bencode_decode`` / ``extract_info_hash_from_torrent`` here, in normal CI.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+from shelfmark.download.clients.torrent_utils import (
+    bencode_decode,
+    extract_info_hash_from_torrent,
+)
+
+GEN_PATH = (
+    Path(__file__).resolve().parents[1] / "e2e" / "platform" / "mocks" / "make_webseed_torrent.py"
+)
+
+
+def _load_generator():
+    if not GEN_PATH.exists():
+        pytest.skip(f"generator not found at {GEN_PATH}")
+    spec = importlib.util.spec_from_file_location("make_webseed_torrent", GEN_PATH)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+PAYLOAD = b"E2E webseed payload \x00\x01\x02 " * 2000  # ~50 KiB -> multiple pieces
+NAME = "sample-book.epub"
+WEBSEED = "http://mock-aa/payload/sample-book.epub"
+
+
+def test_generated_torrent_decodes_with_shelfmark_bencode() -> None:
+    gen = _load_generator()
+    raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=16384)
+
+    decoded, _ = bencode_decode(raw)
+    assert isinstance(decoded, dict)
+    info = decoded[b"info"]
+    assert info[b"name"] == NAME.encode()
+    assert info[b"length"] == len(PAYLOAD)
+    # url-list (webseed) must point at the file the mock serves.
+    assert decoded[b"url-list"] == [WEBSEED.encode()]
+    # No tracker — the whole point is HTTP-only completion.
+    assert b"announce" not in decoded
+
+
+def test_piece_hashes_match_payload_bytes() -> None:
+    gen = _load_generator()
+    piece_len = 16384
+    raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED, piece_length=piece_len)
+    decoded, _ = bencode_decode(raw)
+    pieces = decoded[b"info"][b"pieces"]
+
+    expected = b"".join(
+        hashlib.sha1(PAYLOAD[i : i + piece_len]).digest() for i in range(0, len(PAYLOAD), piece_len)
+    )
+    assert pieces == expected, "piece hashes do not match payload — qbit would never complete"
+    assert len(pieces) % 20 == 0
+
+
+def test_info_hash_matches_shelfmark_extractor() -> None:
+    """Our infohash helper must agree with shelfmark's torrent parser."""
+    gen = _load_generator()
+    raw = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
+
+    ours = gen.info_hash(raw)
+    theirs = extract_info_hash_from_torrent(raw)
+    assert theirs is not None
+    assert ours.lower() == theirs.lower(), (ours, theirs)
+
+
+def test_generator_is_deterministic() -> None:
+    gen = _load_generator()
+    a = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
+    b = gen.build_webseed_torrent(NAME, PAYLOAD, WEBSEED)
+    assert a == b, "torrent generation must be byte-deterministic for stable infohash"
@@ -0,0 +1,7 @@
+# Runtime state created by run-e2e.sh: per-profile shelfmark logs, the app's
+# /config (settings, users.db, secrets), downloaded books, and staging dirs.
+.state/
+
+# Python bytecode from the mock services + suite.
+__pycache__/
+*.pyc
@@ -0,0 +1,231 @@
+# Shelfmark E2E Docker Testing Platform
+
+A hermetic, container-based end-to-end platform that boots the real Shelfmark app
+against **controllable** dependencies — a fake Anna's Archive, a Cloudflare gate, a
+mock FlareSolverr bypasser, a mock Prowlarr + a real qBittorrent, custom DNS
+servers, HTTP/SOCKS5 proxies, and a Tor profile — and runs a cluster test suite
+under each **config profile**.
+
+It exists to make the recurring bug clusters from the issue/PR analysis impossible
+to reintroduce silently. The biggest one — Tor/Cloudflare/bypasser (37 issues /
+67 fix PRs) — had almost no automated coverage; this platform changes that.
+
+```
+  pytest suite (host :8084)
+        │
+        ▼
+  shelfmark (under test) ── egress depends on the active profile:
+        ├─ direct ──────────────► mock-aa (.10)                  fake Anna's Archive
+        ├─ Cloudflare gate ─────► mock-cf (.11) ─► mock-aa       [full: real Chrome solves it]
+        ├─ FlareSolverr ────────► mock-cf (.11) ─► mock-flaresolverr (.12)  [bypasser-external]
+        ├─ custom DNS ──────────► coredns (.20) / coredns-blocked (.22)     [dns-manual/blocked]
+        ├─ HTTP / SOCKS proxy ──► tinyproxy (.30) / microsocks (.31)        [proxy-http/socks]
+        ├─ Tor (transparent) ───► in-image tor.sh                           [tor]
+        └─ Prowlarr → client ───► mock-prowlarr (.40) ─► qBittorrent        [full: real download]
+
+  (all on one e2e docker network, 172.30.0.0/24, static IPs for DNS determinism)
+```
+
+## Quick start
+
+```bash
+# one profile
+make e2e-platform                       # baseline
+make e2e-platform-profile PROFILE=bypasser-external
+make e2e-platform-profile PROFILE=client-deluge
+make e2e-platform-full                  # heavy: real Chrome + DoH + real qBittorrent
+
+# the whole matrix
+make e2e-platform-matrix
+
+# build the heavy image once, then reuse it (matrix does this automatically)
+make e2e-platform-build
+E2E_NO_BUILD=1 tests/e2e/platform/run-e2e.sh env/dns-doh.env
+
+# debug: leave the stack up after the run
+KEEP_UP=1 tests/e2e/platform/run-e2e.sh env/dns-blocked.env
+```
+
+Requirements: Docker + Compose v2, and `uv` (for the pytest runner). The runner
+builds the Shelfmark image from the repo `Dockerfile`, boots the profile's stack,
+waits for `/api/health`, runs the suite, and tears down. `run-matrix.sh` builds the
+image **once** and reuses it across profiles (`E2E_NO_BUILD=1`) so the slow
+xvfb/chromium layer isn't rebuilt per profile.
+
+## How profiles work
+
+Each profile is an env file in `env/`. It sets:
+- `COMPOSE_PROFILES` — which optional services start (compose `profiles:`).
+- `SM_*` — the app's config, injected as container env. Shelfmark treats
+  deployment ENV as authoritative (`config.get`: "Deployment-level ENV values
+  always win"), so a profile fully determines the app's DNS/proxy/bypasser/source
+  configuration with no runtime mutation.
+- `E2E_PROFILE` — handed to pytest so the suite selects applicable tests.
+
+Tests declare applicability with `@pytest.mark.profiles(...)`. **A test with no
+marker is a profile-agnostic invariant and runs under every profile** — that is
+how one cluster test ("source must be reachable") becomes the config matrix.
+
+## The matrix (cluster × profile)
+
+Status column: ✅ = run live on Docker and passing. Every profile below was run
+end-to-end (`docker compose up` + suite + teardown) and passes.
+
+| Profile | Egress / what it proves | Clusters | Regression targets | Status |
+|---|---|---|---|---|
+| `baseline` | Direct to fake AA; search/parse + #1028 clean-failure | 2,3,4 | #198 #293 #214 #1040 #1028 | ✅ 9 passed |
+| `bypasser-external` | External bypasser wired; CF-gated search fails cleanly | 1 | #284 #202 #410 #369 | ✅ 5 passed |
+| `bypasser-disabled` | CF-gated AA + bypasser OFF → no results (control) | 1 | #202 #410 | ✅ 4 passed |
+| `dns-manual` | AA only resolvable via custom DNS (coredns) | config: DNS | #108 | ✅ 4 passed |
+| `dns-blocked` | System DNS NXDOMAINs AA; custom DNS resolves it | config: DNS | **#1028** | ✅ 4 passed |
+| `dns-doh` | System DNS blocks AA; **DoH over real HTTPS** resolves it | config: DoH | **#1028** #108 | ✅ 3 passed |
+| `proxy-http` | All egress via tinyproxy, **proven by proxy logs** | config: proxy | **#956** | ✅ 6 passed |
+| `proxy-socks` | All egress via SOCKS5 (microsocks), traversal-checked | config: proxy | #956 | ✅ 5 passed |
+| `tor` | `USING_TOR=true` boots clean (restarts=0) | 1/6 Tor boot | #1021 #940 #801 | ✅ 5 passed |
+| `client-transmission` | Prowlarr → **real Transmission** webseed download → /books | 5 clients | #1022 #634 | ✅ 4 passed |
+| `client-deluge` | Prowlarr → **real Deluge** webseed download → /books | 5 clients | #530 | ✅ 4 passed |
+| `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed |
+| *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ |
+
+> **The bypasser is download-time, not search-time.** Running the stack revealed
+> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a
+> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser
+> (internal Chrome or external FlareSolverr) only runs during a file *download*
+> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean*
+> CF-gated-search failure, while the **`full` profile exercises the real end-to-end
+> CF solve**: AA search/detail are reachable, but the AA slow-download link points
+> at the gate, so downloading Moby-Dick forces the in-image headless Chromium to
+> detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file —
+> verified live (`Challenge detected: cloudflare` → `Bypass successful` → Moby-Dick
+> in `/books`).
+>
+> **`bypasser-external` must set `SM_USING_EXTERNAL_BYPASSER=true`** — shelfmark does
+> **not** derive it from `EXT_BYPASSER_URL`; without it the app silently uses the
+> in-image Chrome bypasser instead of FlareSolverr.
+
+Coverage of the 7 clusters from the analysis:
+
+1. **Bypasser/Tor/Cloudflare** → `bypasser-external`, `bypasser-disabled`, `tor`.
+2. **Search/metadata** → `baseline` (`test_cluster_search_aa.py`, hermetic via the
+   `direct_download` source so no external metadata provider is needed).
+3. **AA parsing/mirrors** → `baseline` parse guards incl. the **layout-drift
+   fail-loud** test (#878/#879/#880).
+4. **Permissions/file-move** → `baseline` (`test_cluster_download_permissions.py`).
+5. **Torrent/usenet clients** → a mock Prowlarr + webseed torrent drives **three
+   real torrent clients** end to end (`full`=qBittorrent, `client-transmission`,
+   `client-deluge`) — completion detection + file move into `/books`. One
+   client-agnostic test (`test_cluster_clients.py`) covers all three.
+6. **Docker/entrypoint/PUID-PGID** → profile-agnostic health + boot-log checks,
+   run under every profile.
+7. **Audiobook/ABB** → parse-contract guards in
+   `tests/audiobookbay/test_scraper_contract.py` (info-hash normalization #386,
+   magnet fallback, layout drift). These run in **normal CI**, not the docker
+   matrix, because ABB hardcodes `https://` for its fetches (see Roadmap).
+
+### Proxy traversal (not just reachability)
+Because the app and the mock AA share the e2e network, a regression that ignores
+the proxy config would still reach AA directly. `test_egress_actually_traverses_proxy`
+drives a search and then inspects the proxy container's logs, so the proxy
+profiles prove the egress *went through* the proxy — a real guard for #956.
+
+### DoH — two layers
+- **Offline** (`tests/download/test_doh_resolver_mock.py`, normal CI): the real
+  `DoHResolver` is driven against the mock `doh` role over localhost HTTP, covering
+  JSON-answer parsing, NXDOMAIN → empty, and caching.
+- **In-stack** (`dns-doh` profile): the mock `doh` role serves the DNS JSON API over
+  **real HTTPS** (self-signed). The system resolver (coredns-blocked) NXDOMAINs
+  `aa.mock.test`, so the host can *only* be resolved via DoH; compose `extra_hosts`
+  redirects the `cloudflare-dns.com` provider to the in-stack mock and
+  `CERTIFICATE_VALIDATION=disabled` accepts the self-signed cert. The search reaching
+  AA proves the app's DoH path resolved the name end to end — **no app code change**.
+
+### The `full` profile — real Chrome + real client (`make e2e-platform-full`)
+The "everything real" heavy profile (test book: **Moby-Dick**), run nightly / on
+demand (excluded from the PR matrix). It spins up, with **no** mock bypasser, and
+**passes live** (6 passed):
+
+- **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are
+  reachable (`mock-aa`), but the AA *slow-download* link points at the Cloudflare
+  gate (`mock-cf`), whose challenge page runs JS that issues `cf_clearance` and
+  reloads. Downloading Moby-Dick forces the in-image headless Chromium (seleniumbase
+  CDP, in the `shelfmark` image via `xvfb`+`chromium`) to load the gate, detect the
+  challenge (`Challenge detected: cloudflare`), solve it (`_bypass_method_cdp_solve`),
+  and fetch the cleared "Download now" page → the file lands in `/books`. That
+  outcome is *only* reachable if Chrome solved the gate — the literal "spin a Chrome
+  browser" path and the strongest guard for the #1 cluster. Two subtleties this
+  surfaced, now handled by the mock: the cleared page must exceed the bypasser's
+  `_LOADING_BODY_LENGTH_MAX` (50 chars of innerText) or it loops as "still loading",
+  and the AA detail page must satisfy the brittle `original_nodes[-6]` parse (#880).
+- **DoH** on at boot.
+- **Real qBittorrent download.** A mock Prowlarr (`/api/v1/system/status`,
+  `/api/v1/indexer`, torznab search) returns one release whose `.torrent` is a
+  **tracker-less BEP-19 webseed** pointing at `mock-aa`'s HTTP payload. A real
+  qBittorrent completes the download over HTTP (no tracker/peer/seeder), and
+  shelfmark's completion detection + file move lands the book in `/books`. The
+  webseed torrent is generated by `mocks/make_webseed_torrent.py` (cross-checked
+  against shelfmark's own bencode/infohash parser in
+  `tests/download/test_webseed_torrent_generator.py`), and the whole
+  prowlarr→qBittorrent path is configured declaratively via env (`env/full.env`).
+
+## Components
+
+| Path | Purpose |
+|---|---|
+| `mocks/mock_services.py` | One Flask app, five roles (`origin-aa`, `cloudflare`, `flaresolverr`, `prowlarr`, `doh`) selected by `MOCK_ROLE`. `origin-aa` also serves the webseed payload + `.torrent`. |
+| `mocks/make_webseed_torrent.py` | Stdlib bencode + BEP-19 webseed `.torrent` generator for the `full` real-client download. |
+| `qbittorrent/qBittorrent.conf` | Real qBittorrent config (auth bypassed for the e2e subnet) for the `full` profile. |
+| `env/full.env` | The heavy `full` profile: real Chrome bypasser + DoH + real qBittorrent. |
+| `mocks/fixtures/*.html` | AA search/detail HTML in the **exact** shape the parser expects, plus drift/empty/no-files variants. |
+| `docker-compose.e2e.yml` | The stack; optional services gated by compose profiles, static IPs for DNS determinism. |
+| `dns/Corefile*`, `dns/mock.test.db` | coredns zones — working + ISP-block (NXDOMAIN). |
+| `env/*.env` | The config profiles (matrix rows). |
+| `suite/` | The pytest harness + cluster tests. |
+| `run-e2e.sh` / `run-matrix.sh` | Boot one profile / loop the matrix. |
+| `build-images.sh` | Build the heavy image once (`make e2e-platform-build`); reused via `E2E_NO_BUILD=1`. |
+
+### Fault injection
+The mock AA reproduces historical bugs deterministically. Injection rides inside
+the search query as `E2EINJECT:<name>` (the app builds the AA URL itself and only
+forwards the user query as `q=`). Names: `no_files`, `empty`, `layout_drift`,
+`500`. The harness embeds them via `PlatformClient.direct_search(..., inject=...)`.
+
+## Gating PRs (block merge on e2e failure)
+
+The `.github/workflows/e2e-platform.yml` workflow runs on every PR. On a PR that
+touches relevant code (`shelfmark/**`, `Dockerfile`, `entrypoint.sh`, `tor.sh`,
+`tests/e2e/platform/**`) it runs the fast PR subset **and** the heavy `full`
+profile (real Chrome solving Cloudflare + DoH + real qBittorrent), then a single
+**`e2e required`** job aggregates them: it fails if any e2e job failed, and passes
+(so it never hangs) when the e2e jobs are skipped on an unrelated PR.
+
+The workflow producing a failing check is **not enough on its own** — GitHub only
+*blocks merge* on checks listed in branch protection. A repo **admin** must, once:
+
+- **UI:** Settings → Branches → branch protection rule for `main` →
+  *Require status checks to pass before merging* → add **`e2e required`**.
+- **or `gh` (admin token):**
+  ```bash
+  gh api -X PUT repos/calibrain/shelfmark/branches/main/protection \
+    -H "Accept: application/vnd.github+json" --input - <<'JSON'
+  { "required_status_checks": { "strict": true, "contexts": ["e2e required"] },
+    "enforce_admins": true, "required_pull_request_reviews": null, "restrictions": null }
+  JSON
+  ```
+
+After that, any failure in the e2e platform tests (including the `full` profile)
+blocks the PR from merging. Requiring just the one `e2e required` context covers
+the whole dynamic matrix, so the list never needs updating as profiles change.
+
+## Known limitations / follow-ups
+
+- **rTorrent.** Not in the matrix: its rakshasa-libtorrent has **no GetRight/webseed
+  support**, so the hermetic webseed torrent (which qBittorrent/Transmission/Deluge
+  all complete) leaves rTorrent stuck at 0%. Supporting it needs a real tracker +
+  seeder (peer download) — a follow-up that the webseed design intentionally avoids.
+- **Usenet clients (SABnzbd/NZBGet).** Not yet covered — completing a usenet download
+  hermetically needs a mock NNTP server serving the yEnc-encoded payload plus an NZB,
+  which is a separate (larger) build than the torrent webseed path.
+- **Audiobook (cluster 7) in-stack.** ABB hardcodes `https://`, so it's covered
+  offline (`tests/audiobookbay/test_scraper_contract.py`); an in-stack
+  `audiobookbay` role needs the same self-signed-HTTPS plumbing the `dns-doh` profile
+  now uses for DoH.
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Build every buildable image in the e2e stack once (the heavy `shelfmark` image
+# plus the mock-* role images), so run-matrix.sh / run-e2e.sh with E2E_NO_BUILD=1
+# can reuse them instead of rebuilding the xvfb/chromium layer per profile.
+set -euo pipefail
+
+PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$PLATFORM_DIR"
+
+# Activate every profile that owns a buildable service so they all get built.
+# (Download clients, coredns, proxies are pre-built images — nothing to build.)
+export COMPOSE_PROFILES="bypasser-external,full,dns-doh"
+echo "==> building shelfmark + mock images (one cold build of the chromium layer)"
+docker compose -f docker-compose.e2e.yml build
+echo "==> done. Reuse with: E2E_NO_BUILD=1 ./run-e2e.sh env/<profile>.env"
@@ -0,0 +1,13 @@
+# coredns: authoritative for mock.test, forwards everything else.
+mock.test:53 {
+    file /zones/mock.test.db
+    log
+    errors
+}
+
+. :53 {
+    forward . 1.1.1.1 8.8.8.8
+    cache 30
+    log
+    errors
+}
@@ -0,0 +1,18 @@
+# coredns: ISP-DNS-block simulation. Resolves nothing under mock.test
+# (returns NXDOMAIN), so an app that relies on *system* DNS for book sources
+# fails — while an app that uses its own custom DNS resolver still works.
+# Regression harness for #1028 (internal bypasser used system DNS in subprocess).
+mock.test:53 {
+    template IN ANY mock.test {
+        rcode NXDOMAIN
+    }
+    log
+    errors
+}
+
+. :53 {
+    # Everything else still resolves, so only the book-source host is "blocked".
+    forward . 1.1.1.1 8.8.8.8
+    log
+    errors
+}
@@ -0,0 +1,13 @@
+$ORIGIN mock.test.
+$TTL 60
+@   IN  SOA ns.mock.test. admin.mock.test. (
+        1          ; serial
+        7200       ; refresh
+        3600       ; retry
+        1209600    ; expire
+        3600 )     ; minimum
+@   IN  NS  ns.mock.test.
+ns  IN  A   172.30.0.20
+aa  IN  A   172.30.0.10
+cf  IN  A   172.30.0.11
+doh IN  A   172.30.0.21
@@ -0,0 +1,294 @@
+# Shelfmark e2e Docker testing platform
+# ---------------------------------------------------------------------------
+# A hermetic stack: the app under test plus *controllable* dependencies
+# (fake Anna's Archive, a Cloudflare gate, a mock FlareSolverr bypasser, a DoH
+# responder, a DNS server, HTTP + SOCKS5 proxies, and a Tor profile).
+#
+# Config profiles are env-driven: pick a profile with an env file under env/
+# (it sets COMPOSE_PROFILES + the SM_* config the app boots with). The runner
+# brings the stack up per profile, runs the matching cluster tests, tears down.
+#
+#   ./run-e2e.sh env/baseline.env
+#   ./run-e2e.sh env/bypasser-external.env
+#   ./run-e2e.sh env/dns-blocked.env
+#
+# See README.md for the full cluster x profile matrix.
+
+x-mock-build: &mock-build
+  build:
+    context: ./mocks
+    dockerfile: Dockerfile
+
+networks:
+  e2e:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.30.0.0/24
+
+services:
+  # ----- App under test --------------------------------------------------- #
+  shelfmark:
+    build:
+      context: ../../..
+      dockerfile: Dockerfile
+      target: ${SM_BUILD_TARGET:-shelfmark}
+    container_name: e2e-shelfmark
+    cap_add:
+      - NET_ADMIN   # required by tor.sh iptables when USING_TOR=true
+      - NET_RAW
+    environment:
+      TZ: UTC
+      DEBUG: "true"
+      ONBOARDING: "false"          # skip wizard; ephemeral storage
+      DISABLE_LOCAL_AUTH: "${SM_DISABLE_LOCAL_AUTH:-true}"
+      PUID: "${SM_PUID:-1000}"
+      PGID: "${SM_PGID:-1000}"
+      # --- source / mirror config (cluster 2/3) ---
+      DIRECT_DOWNLOAD_ENABLED: "${SM_DIRECT_DOWNLOAD_ENABLED:-true}"
+      AA_ADDITIONAL_URLS: "${SM_AA_URL:-http://mock-aa}"
+      # --- bypasser config (cluster 1) ---
+      USE_CF_BYPASS: "${SM_USE_CF_BYPASS:-false}"
+      # Selects the external (FlareSolverr) bypasser; when false the in-image
+      # Chrome (internal) bypasser is used. NOT derived from EXT_BYPASSER_URL.
+      USING_EXTERNAL_BYPASSER: "${SM_USING_EXTERNAL_BYPASSER:-false}"
+      EXT_BYPASSER_URL: "${SM_EXT_BYPASSER_URL:-}"
+      # --- prowlarr indexer + torrent client (cluster 5, `full` profile) ---
+      PROWLARR_ENABLED: "${SM_PROWLARR_ENABLED:-false}"
+      PROWLARR_URL: "${SM_PROWLARR_URL:-}"
+      PROWLARR_API_KEY: "${SM_PROWLARR_API_KEY:-}"
+      PROWLARR_TORRENT_CLIENT: "${SM_PROWLARR_TORRENT_CLIENT:-}"
+      QBITTORRENT_URL: "${SM_QBITTORRENT_URL:-}"
+      QBITTORRENT_USERNAME: "${SM_QBITTORRENT_USERNAME:-}"
+      QBITTORRENT_PASSWORD: "${SM_QBITTORRENT_PASSWORD:-}"
+      QBITTORRENT_CATEGORY: "${SM_QBITTORRENT_CATEGORY:-}"
+      # transmission / deluge / rtorrent (client-* profiles)
+      TRANSMISSION_URL: "${SM_TRANSMISSION_URL:-}"
+      TRANSMISSION_USERNAME: "${SM_TRANSMISSION_USERNAME:-}"
+      TRANSMISSION_PASSWORD: "${SM_TRANSMISSION_PASSWORD:-}"
+      DELUGE_HOST: "${SM_DELUGE_HOST:-}"
+      DELUGE_PORT: "${SM_DELUGE_PORT:-}"
+      DELUGE_PASSWORD: "${SM_DELUGE_PASSWORD:-}"
+      RTORRENT_URL: "${SM_RTORRENT_URL:-}"
+      # --- DNS / DoH (config cluster) ---
+      CUSTOM_DNS: "${SM_CUSTOM_DNS:-}"
+      CUSTOM_DNS_MANUAL: "${SM_CUSTOM_DNS_MANUAL:-}"
+      USE_DOH: "${SM_USE_DOH:-false}"
+      # Disable TLS verification so the in-stack DoH-over-HTTPS mock (self-signed)
+      # is accepted in the dns-doh profile. Default keeps verification ON.
+      CERTIFICATE_VALIDATION: "${SM_CERTIFICATE_VALIDATION:-enabled}"
+      # --- proxy (config cluster) ---
+      PROXY_MODE: "${SM_PROXY_MODE:-none}"
+      HTTP_PROXY_URL: "${SM_HTTP_PROXY:-}"
+      HTTP_PROXY: "${SM_HTTP_PROXY:-}"
+      SOCKS5_PROXY: "${SM_SOCKS5_PROXY:-}"
+      NO_PROXY: "${SM_NO_PROXY:-}"
+      # --- tor (cluster 1/6) ---
+      USING_TOR: "${SM_USING_TOR:-false}"
+    ports:
+      - "8084:8084"
+    volumes:
+      - ./.state/config:/config
+      - ./.state/books:/books
+      - ./.state/downloads:/downloads
+      - ./.state/tmp:/tmp/shelfmark
+    networks:
+      - e2e
+    dns:
+      # When a DNS profile is active, point the container's system resolver at
+      # our controllable server; otherwise Docker's embedded DNS (127.0.0.11).
+      - ${SM_SYSTEM_DNS:-127.0.0.11}
+    extra_hosts:
+      # Redirect the DoH provider hostname to the in-stack mock-doh (dns-doh
+      # profile). Harmless elsewhere — only the dns-doh profile enables DoH against
+      # the cloudflare provider, and /etc/hosts is consulted before the resolver.
+      - "cloudflare-dns.com:172.30.0.21"
+    restart: "no"
+
+  # ----- Fake Anna's Archive origin (always on) --------------------------- #
+  mock-aa:
+    <<: *mock-build
+    container_name: e2e-mock-aa
+    environment:
+      MOCK_ROLE: origin-aa
+      # When set (the `full` profile sets it to the CF gate), AA slow-download
+      # links point through Cloudflare so a real download forces the internal
+      # Chrome bypasser to solve the challenge. Empty -> same-origin (no CF).
+      SLOW_DOWNLOAD_BASE: "${SM_SLOW_DOWNLOAD_BASE:-}"
+      AA_FILE_BASE: "http://mock-aa"
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.10
+        aliases:
+          - aa.mock.test
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request;urllib.request.urlopen('http://localhost/healthz')"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+
+  # ----- Cloudflare gate (profiles: bypasser-external, full) -------------- #
+  # `full` puts AA's slow-download behind this gate so the real Chrome bypasser
+  # must solve it; `bypasser-external` puts AA search behind it for the
+  # FlareSolverr negative/positive controls.
+  mock-cf:
+    <<: *mock-build
+    container_name: e2e-mock-cf
+    profiles: ["bypasser-external", "full"]
+    environment:
+      MOCK_ROLE: cloudflare
+      ORIGIN_INTERNAL_URL: http://mock-aa
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.11
+        aliases:
+          - cf.mock.test
+
+  # ----- Mock FlareSolverr external bypasser (profile: bypasser-external) -- #
+  mock-flaresolverr:
+    <<: *mock-build
+    container_name: e2e-mock-flaresolverr
+    profiles: ["bypasser-external"]
+    environment:
+      MOCK_ROLE: flaresolverr
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.12
+
+  # NOTE: the mock `doh` role lives in mock_services.py and is exercised by
+  # tests/download/test_doh_resolver_mock.py (real DoHResolver over localhost HTTP).
+  # An in-stack DoH service is intentionally absent — see README "Known limitations".
+
+  # ----- DNS server (profile: dns-manual) --------------------------------- #
+  coredns:
+    image: coredns/coredns:1.11.1
+    container_name: e2e-coredns
+    profiles: ["dns-manual"]
+    command: ["-conf", "/Corefile"]
+    volumes:
+      - ./dns/Corefile:/Corefile:ro
+      - ./dns/mock.test.db:/zones/mock.test.db:ro
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.20
+
+  # ----- DNS server that NXDOMAINs the AA host (profile: dns-blocked) ------ #
+  # Simulates ISP DNS blocking (#1028); the app must fall back (DoH / direct).
+  coredns-blocked:
+    image: coredns/coredns:1.11.1
+    container_name: e2e-coredns-blocked
+    profiles: ["dns-blocked"]
+    command: ["-conf", "/Corefile"]
+    volumes:
+      - ./dns/Corefile.blocked:/Corefile:ro
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.22
+
+  # ----- HTTP proxy (profile: proxy-http) --------------------------------- #
+  tinyproxy:
+    image: monokal/tinyproxy:latest
+    container_name: e2e-tinyproxy
+    profiles: ["proxy-http"]
+    command: ANY
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.30
+
+  # ----- SOCKS5 proxy (profile: proxy-socks) ------------------------------ #
+  microsocks:
+    image: vimagick/microsocks:latest
+    container_name: e2e-microsocks
+    profiles: ["proxy-socks"]
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.31
+
+  # ----- Mock Prowlarr indexer (full + client-* profiles) ----------------- #
+  # Minimal Prowlarr API returning one torrent release whose .torrent is a
+  # webseed pointing at mock-aa. Drives the real torrent-client download in the
+  # `full` (qBittorrent) and `client-*` (transmission/deluge/rtorrent) profiles.
+  mock-prowlarr:
+    <<: *mock-build
+    container_name: e2e-mock-prowlarr
+    profiles: ["full", "client-transmission", "client-deluge"]
+    environment:
+      MOCK_ROLE: prowlarr
+      AA_INTERNAL_URL: http://mock-aa
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.40
+        aliases:
+          - prowlarr.mock.test
+
+  # ----- Real qBittorrent download client (profile: full) ----------------- #
+  # Auth is bypassed for the e2e subnet (qBittorrent.conf) so shelfmark connects
+  # without juggling the image's random temp password.
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    container_name: e2e-qbittorrent
+    profiles: ["full"]
+    environment:
+      PUID: "1000"
+      PGID: "1000"
+      TZ: UTC
+      WEBUI_PORT: "8080"
+    volumes:
+      - ./qbittorrent/qBittorrent.conf:/config/qBittorrent/qBittorrent.conf
+      # Shared with shelfmark so completed files are visible for the move step.
+      - ./.state/downloads:/downloads
+    networks:
+      - e2e
+
+  # ----- Real Transmission client (profile: client-transmission) ---------- #
+  # Same webseed torrent + mock Prowlarr as `full`, different real client.
+  transmission:
+    image: lscr.io/linuxserver/transmission:latest
+    container_name: e2e-transmission
+    profiles: ["client-transmission"]
+    environment:
+      PUID: "1000"
+      PGID: "1000"
+      TZ: UTC
+      USER: admin
+      PASS: admin
+    volumes:
+      - ./.state/downloads:/downloads
+    networks:
+      - e2e
+
+  # ----- Real Deluge client (profile: client-deluge) ---------------------- #
+  # shelfmark talks to deluge-web (default WebUI password "deluge"), which
+  # connects to the bundled daemon — no daemon auth-file juggling.
+  deluge:
+    image: lscr.io/linuxserver/deluge:latest
+    container_name: e2e-deluge
+    profiles: ["client-deluge"]
+    environment:
+      PUID: "1000"
+      PGID: "1000"
+      TZ: UTC
+    volumes:
+      - ./.state/downloads:/downloads
+    networks:
+      - e2e
+
+  # NOTE: rTorrent is intentionally absent — its rakshasa-libtorrent has no
+  # GetRight/webseed support, so the hermetic webseed torrent can't drive it.
+  # Supporting it would need a real tracker + seeder (see README limitations).
+
+  # ----- DoH-over-HTTPS responder (profile: dns-doh) ---------------------- #
+  # In-stack DoH: serves the DNS JSON API over HTTPS with a self-signed cert.
+  # shelfmark reaches it because the profile (a) maps the DoH provider hostname
+  # to this container via extra_hosts and (b) sets CERTIFICATE_VALIDATION=disabled.
+  mock-doh:
+    <<: *mock-build
+    container_name: e2e-mock-doh
+    profiles: ["dns-doh"]
+    environment:
+      MOCK_ROLE: doh
+      DOH_TLS: "1"
+      DOH_MAP: "aa.mock.test=172.30.0.10,cf.mock.test=172.30.0.11"
+    networks:
+      e2e:
+        ipv4_address: 172.30.0.21
@@ -0,0 +1,25 @@
+# Mock services image for the Shelfmark e2e platform.
+# One image, many roles (selected via MOCK_ROLE): origin-aa, cloudflare,
+# flaresolverr, prowlarr, doh. See mock_services.py for the role contracts.
+FROM python:3.14-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY mock_services.py .
+# Needed by the origin-aa role to build the `full` profile's webseed .torrent.
+COPY make_webseed_torrent.py .
+COPY fixtures ./fixtures
+
+ENV PORT=80
+ENV MOCK_ROLE=all
+
+# gunicorn keeps the mock responsive under concurrent e2e load.
+RUN pip install --no-cache-dir gunicorn
+EXPOSE 80
+EXPOSE 443
+# DOH_TLS=1 (the `doh` role) self-signs a cert and serves HTTPS on 443; every
+# other role serves plain HTTP on $PORT via gunicorn.
+CMD ["sh", "-c", "if [ \"$DOH_TLS\" = 1 ]; then python mock_services.py; else gunicorn -w 2 -b 0.0.0.0:${PORT} mock_services:app; fi"]
@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+<head><title>Anna's Archive (mock) - __MD5__</title></head>
+<body>
+<main>
+  <div>
+    <div><img src="/img/cover-__MD5__.jpg"></div>
+    <div class="main-inner">
+<div><div class="md-meta"><div><div><span>Language</span><span>English [en]</span></div><div><span>Year</span><span>1851</span></div></div></div><div>🔍 Moby Dick</div><div>epub · 1.2 MB · fiction</div><div><a href="__SLOW_BASE__/slow_download/__MD5__/0/0">Slow Partner Server #1</a> — no waitlist, but the download is slow</div><div><a href="__SLOW_BASE__/slow_download/__MD5__/0/1">Slow Partner Server #2</a> — waitlist, but faster</div><div><a href="/get.php?md5=__MD5__&key=e2ekey">GET</a></div></div>
+    </div>
+    <div class="js-md5-top-box-description">Moby-Dick; or, The Whale by Herman Melville.</div>
+  </div>
+</main>
+</body>
+</html>
@@ -0,0 +1,9 @@
+<!doctype html>
+<html lang="en">
+<head><title>Anna's Archive (mock)</title></head>
+<body>
+<main>
+  <div>No files found.</div>
+</main>
+</body>
+</html>
@@ -0,0 +1,27 @@
+<!doctype html>
+<html lang="en">
+<head><title>Anna's Archive (mock - layout drift)</title></head>
+<body>
+<main>
+  <!--
+    Simulates Anna's Archive changing its DOM (the recurring root cause behind
+    #878/#879/#880 and the repeated "Fix AA ... after they changed layout" PRs).
+    The result rows now use <div class="result"> cards instead of a <table> of
+    <tr>/<td>. A parser with hardcoded table/cell indices yields ZERO rows here.
+    The matching e2e test asserts the app FAILS LOUDLY (SearchUnavailableError /
+    explicit "no results" surfaced to the client) rather than silently returning
+    an empty list that users read as "book doesn't exist".
+  -->
+  <div class="results">
+    <div class="result">
+      <a href="/md5/cccccccccccccccccccccccccccccccc">
+        <h3>Drifted Title</h3>
+        <span class="author">Brandon Sanderson</span>
+        <span class="ext">epub</span>
+        <span class="size">1.2 MB</span>
+      </a>
+    </div>
+  </div>
+</main>
+</body>
+</html>
@@ -0,0 +1,41 @@
+<!doctype html>
+<html lang="en">
+<head><title>Anna's Archive (mock)</title></head>
+<body>
+<main>
+  <!-- NO_FILES_MARKER -->
+  <table>
+    <tbody>
+      <tr>
+        <td><a href="/md5/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"><img src="/img/cover1.jpg"></a></td>
+        <td><span>__QUERY__</span></td>
+        <td><span>Herman Melville</span></td>
+        <td><span>Harper &amp; Brothers</span></td>
+        <td><span>2024</span></td>
+        <td><span>-</span></td>
+        <td><span>-</span></td>
+        <td><span>English [en]</span></td>
+        <td><span>fiction</span></td>
+        <td><span>epub</span></td>
+        <td><span>1.2 MB</span></td>
+        <td><span>lgli/N:\fiction\en\E2E\__QUERY__.epub</span></td>
+      </tr>
+      <tr>
+        <td><a href="/md5/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"><img src="/img/cover2.jpg"></a></td>
+        <td><span>__QUERY__ (Annotated Edition)</span></td>
+        <td><span>Herman Melville</span></td>
+        <td><span>Harper &amp; Brothers</span></td>
+        <td><span>2023</span></td>
+        <td><span>-</span></td>
+        <td><span>-</span></td>
+        <td><span>English [en]</span></td>
+        <td><span>fiction</span></td>
+        <td><span>pdf</span></td>
+        <td><span>8.4 MB</span></td>
+        <td><span>lgli/N:\fiction\en\E2E\__QUERY__.pdf</span></td>
+      </tr>
+    </tbody>
+  </table>
+</main>
+</body>
+</html>
@@ -0,0 +1,39 @@
+<!doctype html>
+<html lang="en-US">
+<head>
+  <title>Just a moment...</title>
+  <!-- No-JS fallback: a browser with scripting disabled just keeps waiting,
+       exactly like a real managed challenge. The internal bypasser drives a real
+       Chrome, so the script branch below is what actually clears it. -->
+  <meta http-equiv="refresh" content="8">
+</head>
+<body>
+  <div class="main-wrapper" role="main">
+    <div class="main-content">
+      <noscript>
+        <div id="challenge-error-title">Enable JavaScript and cookies to continue</div>
+      </noscript>
+      <div id="cf-please-wait">
+        <p>Checking if the site connection is secure</p>
+        <p>Verifying you are human. This may take a few seconds.</p>
+      </div>
+    </div>
+  </div>
+  <script>
+    // Mimics a managed Cloudflare challenge: after a short "verification"
+    // delay a real browser is issued cf_clearance, then the page reloads and
+    // the (now cookie-bearing) request is passed through to the real origin.
+    // A regression that ignores the challenge / bypasser never reaches here.
+    window._cf_chl_opt = { cType: 'managed' };
+    (function () {
+      function solve() {
+        document.cookie = 'cf_clearance=e2e-cleared-token; path=/; SameSite=Lax';
+        // Reload so the cleared cookie is sent on the next request.
+        window.location.reload();
+      }
+      // Small delay so the bypasser observes a genuine challenge first.
+      setTimeout(solve, 1200);
+    })();
+  </script>
+</body>
+</html>
@@ -0,0 +1,113 @@
+"""Generate a single-file BitTorrent metainfo (.torrent) with a BEP-19 webseed.
+
+Used by the e2e platform's ``full`` profile so a *real* torrent client
+(qBittorrent) can complete a *real* download hermetically: the torrent carries no
+tracker and a single ``url-list`` webseed pointing at the mock origin's HTTP file
+endpoint, so libtorrent fetches the payload over HTTP — no tracker, peer, or
+seeder container required.
+
+Stdlib-only (the mock image does not install shelfmark). The unit test cross-checks
+this encoder against shelfmark's own ``bencode_decode`` /
+``extract_info_hash_from_torrent`` so a divergence in either is caught.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+DEFAULT_PIECE_LENGTH = 16384  # 16 KiB — fine for the tiny e2e payload
+
+
+def bencode(value: object) -> bytes:
+    """Minimal bencode encoder (int / bytes / str / list / dict)."""
+    if isinstance(value, bool):  # guard: bool is an int subclass
+        raise TypeError("bool is not bencodable")
+    if isinstance(value, int):
+        return b"i" + str(value).encode() + b"e"
+    if isinstance(value, bytes):
+        return str(len(value)).encode() + b":" + value
+    if isinstance(value, str):
+        return bencode(value.encode("utf-8"))
+    if isinstance(value, list):
+        return b"l" + b"".join(bencode(item) for item in value) + b"e"
+    if isinstance(value, dict):
+        out = b"d"
+        for key in sorted(value):  # bencode dict keys must be sorted
+            key_bytes = key.encode("utf-8") if isinstance(key, str) else key
+            out += bencode(key_bytes) + bencode(value[key])
+        return out + b"e"
+    raise TypeError(f"Cannot bencode value of type {type(value).__name__}")
+
+
+def _pieces(data: bytes, piece_length: int) -> bytes:
+    return b"".join(
+        hashlib.sha1(data[i : i + piece_length]).digest() for i in range(0, len(data), piece_length)
+    )
+
+
+def build_info_dict(name: str, data: bytes, piece_length: int = DEFAULT_PIECE_LENGTH) -> dict:
+    return {
+        "name": name,
+        "piece length": piece_length,
+        "length": len(data),
+        "pieces": _pieces(data, piece_length),
+    }
+
+
+def build_webseed_torrent(
+    name: str,
+    data: bytes,
+    webseed_url: str,
+    *,
+    piece_length: int = DEFAULT_PIECE_LENGTH,
+) -> bytes:
+    """Build a tracker-less single-file .torrent whose only source is a webseed.
+
+    Args:
+        name: file name inside the torrent (e.g. ``sample-book.epub``).
+        data: the exact file bytes the webseed URL must serve.
+        webseed_url: BEP-19 url-list entry — the direct HTTP URL for ``data``.
+    """
+    info = build_info_dict(name, data, piece_length)
+    metainfo = {
+        "info": info,
+        # Single-entry webseed. For a single-file torrent the url-list entry is the
+        # direct file URL, so it must serve exactly ``data``.
+        "url-list": [webseed_url],
+        "comment": "shelfmark e2e webseed torrent",
+        "created by": "shelfmark-e2e",
+    }
+    return bencode(metainfo)
+
+
+def info_hash(torrent_bytes: bytes) -> str:
+    """The btih (SHA1 of the bencoded ``info`` dict) as a hex string.
+
+    Re-encodes via a tiny scan so we don't need a full decoder here.
+    """
+    marker = b"4:infod"
+    start = torrent_bytes.find(marker)
+    if start < 0:
+        raise ValueError("no info dict found in torrent")
+    info_start = start + len(b"4:info")
+    # The info value begins at 'd'; find its matching 'e' by bencode-aware scan.
+    end = _scan_bencoded(torrent_bytes, info_start)
+    return hashlib.sha1(torrent_bytes[info_start:end]).hexdigest()
+
+
+def _scan_bencoded(buf: bytes, pos: int) -> int:
+    """Return the index just past the bencoded value starting at ``pos``."""
+    token = buf[pos : pos + 1]
+    if token == b"i":
+        return buf.index(b"e", pos) + 1
+    if token in (b"l", b"d"):
+        pos += 1
+        while buf[pos : pos + 1] != b"e":
+            if token == b"d":  # dicts: key then value
+                pos = _scan_bencoded(buf, pos)
+            pos = _scan_bencoded(buf, pos)
+        return pos + 1
+    # byte string: <len>:<bytes>
+    colon = buf.index(b":", pos)
+    length = int(buf[pos:colon])
+    return colon + 1 + length
@@ -0,0 +1,577 @@
+"""Controllable mock services for the Shelfmark e2e Docker platform.
+
+A single Flask app that plays one of several *roles*, selected by the
+``MOCK_ROLE`` environment variable. Running one image with different roles keeps
+the platform image small and the behaviour in one auditable place.
+
+Roles
+-----
+``origin-aa``     Fake Anna's Archive: search results table, ``/md5/<id>`` detail
+                  pages, and a downloadable book file. The HTML mirrors the real
+                  selectors the parser depends on (``<tr>`` rows, last-cell
+                  distant path, ``get.php?md5=..&key=..`` GET links) so parser
+                  drift (#878/#879/#880) is caught here. Supports fault injection
+                  via query flags to reproduce historical bugs deterministically.
+``cloudflare``    Cloudflare-protected origin: returns a 403 "Just a moment..."
+                  challenge page (with ``cf-mitigated: challenge``) until the
+                  request carries a ``cf_clearance`` cookie, then serves the real
+                  content. Exercises the app's CF detection + bypasser routing
+                  (#284, #226, #202, #1030) without running real CF JS.
+``flaresolverr``  Mock FlareSolverr implementing the ``/v1`` contract. It fetches
+                  the requested URL *with* a clearance cookie and returns the
+                  solved HTML + cookies, so ``_fetch_via_bypasser`` runs end to
+                  end deterministically (no headless Chrome needed in CI).
+``doh``           Minimal DNS-over-HTTPS (RFC 8484 + Google JSON) responder used
+                  to verify the USE_DOH path resolves mock domains even when the
+                  system resolver is poisoned (#1028).
+``all``           Mounts every role at once (default; handy for local poking).
+
+Fault injection (origin-aa) rides INSIDE the search query as an
+``E2EINJECT:<name>`` token, because the app builds the AA URL itself and only
+forwards the user query as ``q=``. The harness embeds it (see
+PlatformClient.direct_search); the mock strips it before rendering:
+  ``no_files``      -> renders the literal "No files found." alongside a real row
+                       (regression for the false-positive check).
+  ``empty``         -> renders an empty results page (true "No files found").
+  ``layout_drift``  -> renders a structurally-changed page (cards, no <table>) so
+                       a hardcoded-index parser yields zero rows — the app must
+                       fail loudly, not silently (#878/#879/#880).
+  ``500``           -> returns HTTP 500 (mirror failover path).
+"""
+
+from __future__ import annotations
+
+import base64
+import os
+import re
+import struct
+from pathlib import Path
+
+from flask import Flask, Response, jsonify, make_response, request
+
+FIXTURES = Path(__file__).parent / "fixtures"
+ROLE = os.environ.get("MOCK_ROLE", "all").strip().lower()
+# Hostname the flaresolverr/cloudflare roles use to reach the AA origin from
+# inside the compose network.
+ORIGIN_INTERNAL_URL = os.environ.get("ORIGIN_INTERNAL_URL", "http://mock-aa")
+CLEARANCE_COOKIE = "cf_clearance"
+CLEARANCE_VALUE = "e2e-cleared-token"
+
+# Test book: *Moby-Dick* by Herman Melville (public domain), used both as the
+# webseed-torrent payload (qBittorrent path) and the AA slow-download payload (real
+# Chrome / Cloudflare path). PAYLOAD_NAME/URL must stay in sync between what mock-aa
+# serves and what the torrent's url-list references.
+PAYLOAD_NAME = "moby-dick.epub"
+PAYLOAD_URL = f"{os.environ.get('PAYLOAD_PUBLIC_URL', 'http://mock-aa')}/payload/{PAYLOAD_NAME}"
+
+# Base for AA slow-download links on the detail page. When set to the Cloudflare
+# gate (the `full` profile sets it to http://cf.mock.test), the *download* — not the
+# search — is forced through the gate, so a real download triggers the internal
+# Chrome bypasser to solve the challenge. Empty -> same-origin (no CF).
+SLOW_DOWNLOAD_BASE = os.environ.get("SLOW_DOWNLOAD_BASE", "").rstrip("/")
+
+# Real public-domain opening of Moby-Dick (Chapter 1, "Loomings").
+_MOBY_DICK_TEXT = (
+    "Call me Ishmael. Some years ago—never mind how long precisely—having little "
+    "or no money in my purse, and nothing particular to interest me on shore, I "
+    "thought I would sail about a little and see the watery part of the world. It "
+    "is a way I have of driving off the spleen and regulating the circulation. "
+    "Whenever I find myself growing grim about the mouth; whenever it is a damp, "
+    "drizzly November in my soul; whenever I find myself involuntarily pausing "
+    "before coffin warehouses, and bringing up the rear of every funeral I meet; "
+    "and especially whenever my hypos get such an upper hand of me, that it "
+    "requires a strong moral principle to prevent me from deliberately stepping "
+    "into the street, and methodically knocking people's hats off—then, I account "
+    "it high time to get to sea as soon as I can."
+)
+
+app = Flask(__name__)
+
+
+def _payload_bytes() -> bytes:
+    """Deterministic *Moby-Dick* EPUB used as the download payload.
+
+    Determinism matters: the webseed torrent's piece hashes are computed from these
+    exact bytes, so any drift between what mock-aa serves and what the torrent
+    describes would make qBittorrent never complete. Fixed ZipInfo timestamps keep
+    the bytes byte-stable across runs.
+    """
+    import io
+    import zipfile
+
+    files = [
+        ("mimetype", "application/epub+zip"),
+        (
+            "META-INF/container.xml",
+            '<?xml version="1.0"?><container version="1.0" '
+            'xmlns="urn:oasis:names:tc:opendocument:xmlns:container">'
+            '<rootfiles><rootfile full-path="OEBPS/content.opf" '
+            'media-type="application/oebps-package+xml"/></rootfiles></container>',
+        ),
+        (
+            "OEBPS/content.opf",
+            '<?xml version="1.0"?><package xmlns="http://www.idpf.org/2007/opf" '
+            'version="3.0" unique-identifier="id"><metadata '
+            'xmlns:dc="http://purl.org/dc/elements/1.1/">'
+            '<dc:identifier id="id">e2e-moby-dick</dc:identifier>'
+            "<dc:title>Moby-Dick; or, The Whale</dc:title>"
+            "<dc:creator>Herman Melville</dc:creator>"
+            "<dc:language>en</dc:language></metadata>"
+            '<manifest><item id="c1" href="chapter1.xhtml" '
+            'media-type="application/xhtml+xml"/></manifest>'
+            '<spine><itemref idref="c1"/></spine></package>',
+        ),
+        (
+            "OEBPS/chapter1.xhtml",
+            '<?xml version="1.0" encoding="utf-8"?>'
+            '<html xmlns="http://www.w3.org/1999/xhtml"><head><title>Loomings</title>'
+            "</head><body><h1>Chapter 1. Loomings.</h1>"
+            # Repeat the opening so the EPUB clears shelfmark's 10 KB minimum-size
+            # check on the direct-download path (_MIN_VALID_FILE_SIZE).
+            + ("<p>" + _MOBY_DICK_TEXT + "</p>") * 24
+            + "</body></html>",
+        ),
+    ]
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf:
+        for name, content in files:
+            info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
+            zf.writestr(info, content)
+    return buf.getvalue()
+
+
+def _read_fixture(name: str) -> str:
+    return (FIXTURES / name).read_text(encoding="utf-8")
+
+
+# --------------------------------------------------------------------------- #
+# Role: origin-aa  (fake Anna's Archive)
+# --------------------------------------------------------------------------- #
+def _search_rows() -> str:
+    """One real result row in the exact shape the parser expects."""
+    return _read_fixture("aa_search_results.html")
+
+
+def register_origin_aa(flask_app: Flask) -> None:
+    @flask_app.route("/search")
+    def aa_search() -> Response:
+        query = request.args.get("q", "")
+        # Fault injection travels INSIDE the search query (the app builds the AA
+        # URL itself and won't forward arbitrary params), via a token the harness
+        # embeds: "E2EINJECT:<name> <real query>".
+        inject = ""
+        match = re.search(r"E2EINJECT:(\w+)", query)
+        if match:
+            inject = match.group(1)
+            query = re.sub(r"E2EINJECT:\w+\s*", "", query).strip()
+        if inject == "500":
+            return make_response("upstream error", 500)
+        if inject == "empty":
+            body = _read_fixture("aa_search_empty.html")
+            return make_response(body, 200)
+        if inject == "no_files":
+            # Real row present *and* the "No files found." string — the historical
+            # false-positive (#: 'No files found' check). The app must still
+            # surface the real row.
+            body = _search_rows().replace("<!-- NO_FILES_MARKER -->", "<div>No files found.</div>")
+            return make_response(body, 200)
+        if inject == "layout_drift":
+            return make_response(_read_fixture("aa_search_layout_drift.html"), 200)
+        # Echo the query into the title so tests can assert routing worked.
+        body = _search_rows().replace("__QUERY__", query or "A Book Title")
+        return make_response(body, 200)
+
+    @flask_app.route("/md5/<book_id>")
+    def aa_detail(book_id: str) -> Response:
+        # __SLOW_BASE__ controls where the AA "slow partner server" links point.
+        # In the `full` profile it's the Cloudflare gate, so the *download* (not the
+        # search/detail) is what forces the internal Chrome bypasser to solve CF.
+        body = (
+            _read_fixture("aa_detail.html")
+            .replace("__MD5__", book_id)
+            .replace("__SLOW_BASE__", SLOW_DOWNLOAD_BASE)
+        )
+        return make_response(body, 200)
+
+    @flask_app.route("/get.php")
+    def aa_getphp() -> Response:
+        # The actual file download link target (get.php?md5=..&key=..).
+        return _serve_book()
+
+    @flask_app.route("/slow_download/<path:rest>")
+    def aa_slow(rest: str) -> Response:
+        # The AA "slow partner server" page: an HTML page whose "Download now" link
+        # is the final file URL. shelfmark's _extract_slow_download_url parses this.
+        # Reached (in the full profile) only after the internal Chrome bypasser
+        # solves the Cloudflare gate in front of it.
+        del rest
+        file_url = f"{os.environ.get('AA_FILE_BASE', 'http://mock-aa')}/file/{PAYLOAD_NAME}"
+        # The visible text must exceed the internal bypasser's "still loading"
+        # threshold (_LOADING_BODY_LENGTH_MAX = 50 chars of body.innerText) or it
+        # never considers the (cleared) page settled and loops until timeout.
+        html = (
+            "<!doctype html><html><head><title>Download</title></head><body>"
+            "<h1>Anna&rsquo;s Archive &mdash; Slow Partner Server</h1>"
+            "<p>Your download of <em>Moby-Dick; or, The Whale</em> by Herman Melville "
+            "is ready. Use the link below to download the file from this slow partner "
+            "server. The connection is slow but free, with no waitlist.</p>"
+            "<div class='top-row'>"
+            f'<a href="{file_url}" download>\U0001f4da Download now</a>'
+            "</div>"
+            "<p>Thank you for supporting open access to knowledge.</p>"
+            "</body></html>"
+        )
+        return make_response(html, 200)
+
+    @flask_app.route(f"/file/{PAYLOAD_NAME}")
+    def aa_file() -> Response:
+        # Final file URL extracted from the slow-download page.
+        return _serve_book()
+
+    @flask_app.route("/dyn/api/fast_download.json")
+    def aa_fast() -> Response:
+        md5 = request.args.get("md5", "")
+        return jsonify({"download_url": f"{request.host_url.rstrip('/')}/get.php?md5={md5}&key=k"})
+
+    # --- webseed payload + torrent for the `full` real-client pipeline --------
+    @flask_app.route(f"/payload/{PAYLOAD_NAME}")
+    def aa_payload() -> Response:
+        # Range support is required for transmission's GetRight webseed (it fetches
+        # pieces with `Range: bytes=...` and expects 206); libtorrent clients
+        # (qBittorrent/deluge) tolerate a plain 200, but transmission does not.
+        return _ranged_response(_payload_bytes(), "application/epub+zip")
+
+    @flask_app.route("/payload.torrent")
+    def aa_torrent() -> Response:
+        torrent = _build_payload_torrent()
+        resp = make_response(torrent)
+        resp.headers["Content-Type"] = "application/x-bittorrent"
+        resp.headers["Content-Disposition"] = 'attachment; filename="sample-book.torrent"'
+        return resp
+
+
+def _ranged_response(data: bytes, content_type: str) -> Response:
+    """Serve ``data`` honoring a single HTTP Range request (206 + Content-Range).
+
+    Needed so transmission's webseed (which fetches via ``Range: bytes=...``) can
+    download piece by piece. A request without Range gets the full 200 body.
+    """
+    total = len(data)
+    range_header = request.headers.get("Range", "")
+    if range_header.startswith("bytes="):
+        first = range_header[len("bytes=") :].split(",", 1)[0]
+        start_s, _, end_s = first.partition("-")
+        try:
+            start = int(start_s) if start_s else 0
+            end = int(end_s) if end_s else total - 1
+        except ValueError:
+            start, end = 0, total - 1
+        end = min(end, total - 1)
+        start = max(0, min(start, end))
+        chunk = data[start : end + 1]
+        resp = make_response(chunk, 206)
+        resp.headers["Content-Range"] = f"bytes {start}-{end}/{total}"
+    else:
+        resp = make_response(data)
+    resp.headers["Content-Type"] = content_type
+    resp.headers["Accept-Ranges"] = "bytes"
+    return resp
+
+
+def _serve_book() -> Response:
+    resp = _ranged_response(_payload_bytes(), "application/epub+zip")
+    resp.headers["Content-Disposition"] = f'attachment; filename="{PAYLOAD_NAME}"'
+    return resp
+
+
+def _build_payload_torrent() -> bytes:
+    """Webseed .torrent for the deterministic payload, sourced only from mock-aa.
+
+    Imported lazily so the doh/cloudflare/flaresolverr roles don't need the
+    generator module on the path.
+    """
+    import importlib.util
+
+    gen_path = Path(__file__).parent / "make_webseed_torrent.py"
+    spec = importlib.util.spec_from_file_location("make_webseed_torrent", gen_path)
+    assert spec and spec.loader
+    gen = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(gen)
+    return gen.build_webseed_torrent(PAYLOAD_NAME, _payload_bytes(), PAYLOAD_URL)
+
+
+# --------------------------------------------------------------------------- #
+# Role: cloudflare  (challenge until clearance cookie present)
+# --------------------------------------------------------------------------- #
+def register_cloudflare(flask_app: Flask) -> None:
+    @flask_app.route("/", defaults={"path": ""})
+    @flask_app.route("/<path:path>")
+    def cf_gate(path: str) -> Response:
+        if request.cookies.get(CLEARANCE_COOKIE) == CLEARANCE_VALUE:
+            # Cleared: proxy the request through to the real AA origin behaviour.
+            return _cleared_passthrough(path)
+        challenge = _read_fixture("cf_challenge.html")
+        resp = make_response(challenge, 403)
+        resp.headers["cf-mitigated"] = "challenge"
+        resp.headers["Server"] = "cloudflare"
+        return resp
+
+
+def _cleared_passthrough(path: str) -> Response:
+    import requests as _rq
+
+    target = f"{ORIGIN_INTERNAL_URL}/{path}"
+    upstream = _rq.get(target, params=request.args, timeout=10)
+    resp = make_response(upstream.content, upstream.status_code)
+    resp.headers["Content-Type"] = upstream.headers.get("Content-Type", "text/html")
+    return resp
+
+
+# --------------------------------------------------------------------------- #
+# Role: flaresolverr  (mock external bypasser, /v1 contract)
+# --------------------------------------------------------------------------- #
+def register_flaresolverr(flask_app: Flask) -> None:
+    @flask_app.route("/v1", methods=["POST"])
+    def v1() -> Response:
+        import requests as _rq
+
+        payload = request.get_json(silent=True) or {}
+        url = payload.get("url", "")
+        if not url:
+            return jsonify({"status": "error", "message": "missing url"}), 400
+        # "Solve" the challenge by fetching with the clearance cookie set.
+        upstream = _rq.get(url, cookies={CLEARANCE_COOKIE: CLEARANCE_VALUE}, timeout=15)
+        return jsonify(
+            {
+                "status": "ok",
+                "message": "Challenge solved!",
+                "solution": {
+                    "url": url,
+                    "status": upstream.status_code,
+                    "response": upstream.text,
+                    "cookies": [{"name": CLEARANCE_COOKIE, "value": CLEARANCE_VALUE, "domain": ""}],
+                    "userAgent": "Mozilla/5.0 (e2e-flaresolverr)",
+                },
+            }
+        )
+
+
+# --------------------------------------------------------------------------- #
+# Role: prowlarr  (minimal Prowlarr API for the `full` real-client pipeline)
+# --------------------------------------------------------------------------- #
+# Implements just the endpoints shelfmark's prowlarr client calls, returning one
+# torrent release whose download is mock-aa's webseed .torrent. A real
+# qBittorrent then completes the download over HTTP (no tracker/peer needed).
+AA_INTERNAL_URL = os.environ.get("AA_INTERNAL_URL", "http://mock-aa")
+
+
+def register_prowlarr(flask_app: Flask) -> None:
+    @flask_app.route("/api/v1/system/status")
+    def prowlarr_status() -> Response:
+        return jsonify({"appName": "Prowlarr", "version": "1.30.0.4000", "instanceName": "e2e"})
+
+    @flask_app.route("/api/v1/indexer")
+    def prowlarr_indexers() -> Response:
+        return jsonify(
+            [
+                {
+                    "id": 1,
+                    "name": "Mock Torznab",
+                    "enable": True,
+                    "protocol": "torrent",
+                    "implementation": "Torznab",
+                    "implementationName": "Generic Torznab",
+                    "definitionName": "mock-torznab",
+                    "capabilities": {
+                        "categories": [
+                            {"id": 7000, "name": "Books"},
+                            {"id": 7020, "name": "Books/EBook"},
+                        ]
+                    },
+                }
+            ]
+        )
+
+    @flask_app.route("/api/v1/indexer/<int:indexer_id>/newznab")
+    def prowlarr_torznab(indexer_id: int) -> Response:
+        del indexer_id
+        t = request.args.get("t", "search")
+        if t == "caps":
+            return Response(_torznab_caps(), mimetype="application/xml")
+        query = request.args.get("q", "") or "E2E Mock Book"
+        return Response(_torznab_search(query), mimetype="application/xml")
+
+
+def _torznab_caps() -> str:
+    return (
+        '<?xml version="1.0" encoding="UTF-8"?>'
+        '<caps><server title="Mock Torznab"/>'
+        '<limits max="100" default="50"/>'
+        '<searching><search available="yes" supportedParams="q"/>'
+        '<book-search available="yes" supportedParams="q,author,title"/></searching>'
+        '<categories><category id="7000" name="Books">'
+        '<subcat id="7020" name="EBook"/></category></categories></caps>'
+    )
+
+
+def _torznab_search(query: str) -> str:
+    torrent_url = f"{AA_INTERNAL_URL}/payload.torrent"
+    size = len(_payload_bytes())
+    title = f"{query} - E2E Mock Book"
+    return (
+        '<?xml version="1.0" encoding="UTF-8"?>'
+        '<rss version="2.0" xmlns:torznab="http://torznab.com/schemas/2015/feed">'
+        "<channel>"
+        "<item>"
+        f"<title>{title}</title>"
+        "<guid>e2e-mock-release-1</guid>"
+        f"<link>{torrent_url}</link>"
+        f"<size>{size}</size>"
+        "<pubDate>Mon, 01 Jan 2024 00:00:00 +0000</pubDate>"
+        f'<enclosure url="{torrent_url}" length="{size}" type="application/x-bittorrent"/>'
+        '<torznab:attr name="category" value="7020"/>'
+        '<torznab:attr name="seeders" value="10"/>'
+        '<torznab:attr name="peers" value="11"/>'
+        '<torznab:attr name="downloadvolumefactor" value="0"/>'
+        '<torznab:attr name="uploadvolumefactor" value="1"/>'
+        "</item>"
+        "</channel></rss>"
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Role: doh  (DNS over HTTPS responder)
+# --------------------------------------------------------------------------- #
+# Maps mock hostnames to the in-network IP the test wants them resolved to.
+# Provided via DOH_MAP env: "host=ip,host2=ip2".
+def _doh_map() -> dict[str, str]:
+    raw = os.environ.get("DOH_MAP", "")
+    out: dict[str, str] = {}
+    for pair in raw.split(","):
+        pair = pair.strip()
+        if "=" in pair:
+            host, ip = pair.split("=", 1)
+            out[host.strip().rstrip(".").lower()] = ip.strip()
+    return out
+
+
+def _encode_a_answer(name: str, ip: str) -> bytes:
+    parts = ip.split(".")
+    return struct.pack("!HHHIH4B", 0xC00C, 1, 1, 60, 4, *(int(p) for p in parts))
+
+
+def register_doh(flask_app: Flask) -> None:
+    @flask_app.route("/dns-query", methods=["GET", "POST"])
+    @flask_app.route("/resolve", methods=["GET", "POST"])  # Google-style JSON endpoint
+    def dns_query() -> Response:
+        mapping = _doh_map()
+        # Google/Cloudflare JSON form (?name=&type=A)
+        name = (request.args.get("name") or "").rstrip(".").lower()
+        if name:
+            ip = mapping.get(name)
+            answer = [{"name": name, "type": 1, "TTL": 60, "data": ip}] if ip else []
+            return jsonify({"Status": 0 if ip else 3, "Answer": answer})
+        # RFC 8484 wireformat (POST body or ?dns=)
+        if request.method == "POST":
+            wire = request.get_data()
+        else:
+            dns_b64 = request.args.get("dns", "")
+            wire = base64.urlsafe_b64decode(dns_b64 + "=" * (-len(dns_b64) % 4))
+        return _wireformat_response(wire, mapping)
+
+    def _wireformat_response(wire: bytes, mapping: dict[str, str]) -> Response:
+        # Minimal parser: echo header/question, append one A answer if known.
+        txid = wire[0:2]
+        qname, _ = _parse_qname(wire, 12)
+        question = wire[12:]
+        host = qname.rstrip(".").lower()
+        ip = mapping.get(host)
+        ancount = 1 if ip else 0
+        header = txid + struct.pack("!HHHHH", 0x8180, 1, ancount, 0, 0)
+        body = question + (_encode_a_answer(host, ip) if ip else b"")
+        resp = make_response(header + body)
+        resp.headers["Content-Type"] = "application/dns-message"
+        return resp
+
+    def _parse_qname(wire: bytes, offset: int) -> tuple[str, int]:
+        labels = []
+        while True:
+            length = wire[offset]
+            offset += 1
+            if length == 0:
+                break
+            labels.append(wire[offset : offset + length].decode("ascii", "ignore"))
+            offset += length
+        return ".".join(labels), offset
+
+
+# --------------------------------------------------------------------------- #
+# Health + role wiring
+# --------------------------------------------------------------------------- #
+@app.route("/healthz")
+def healthz() -> Response:
+    return jsonify({"role": ROLE, "ok": True})
+
+
+_ROLES = {
+    "origin-aa": register_origin_aa,
+    "cloudflare": register_cloudflare,
+    "flaresolverr": register_flaresolverr,
+    "prowlarr": register_prowlarr,
+    "doh": register_doh,
+}
+
+if ROLE == "all":
+    for _register in _ROLES.values():
+        _register(app)
+elif ROLE in _ROLES:
+    _ROLES[ROLE](app)
+else:  # pragma: no cover - misconfiguration guard
+    raise SystemExit(f"Unknown MOCK_ROLE={ROLE!r}; expected one of {[*sorted(_ROLES), 'all']}")
+
+
+def _self_signed_cert() -> tuple[str, str]:
+    """Write a throwaway self-signed cert/key to /tmp and return their paths.
+
+    Used only by the `doh` role's HTTPS server. The app reaches it with
+    CERTIFICATE_VALIDATION=disabled, so the cert's identity is irrelevant.
+    """
+    import datetime
+
+    from cryptography import x509
+    from cryptography.hazmat.primitives import hashes, serialization
+    from cryptography.hazmat.primitives.asymmetric import rsa
+    from cryptography.x509.oid import NameOID
+
+    key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "e2e-doh")])
+    now = datetime.datetime.now(datetime.UTC)
+    cert = (
+        x509.CertificateBuilder()
+        .subject_name(name)
+        .issuer_name(name)
+        .public_key(key.public_key())
+        .serial_number(x509.random_serial_number())
+        .not_valid_before(now - datetime.timedelta(days=1))
+        .not_valid_after(now + datetime.timedelta(days=3650))
+        .add_extension(x509.SubjectAlternativeName([x509.DNSName("cloudflare-dns.com")]), False)
+        .sign(key, hashes.SHA256())
+    )
+    cert_path, key_path = "/tmp/doh.crt", "/tmp/doh.key"
+    Path(cert_path).write_bytes(cert.public_bytes(serialization.Encoding.PEM))
+    Path(key_path).write_bytes(
+        key.private_bytes(
+            serialization.Encoding.PEM,
+            serialization.PrivateFormat.TraditionalOpenSSL,
+            serialization.NoEncryption(),
+        )
+    )
+    return cert_path, key_path
+
+
+if __name__ == "__main__":
+    if os.environ.get("DOH_TLS") == "1":
+        cert_path, key_path = _self_signed_cert()
+        app.run(host="0.0.0.0", port=443, ssl_context=(cert_path, key_path), threaded=True)
+    else:
+        app.run(host="0.0.0.0", port=int(os.environ.get("PORT", "80")))
@@ -0,0 +1,4 @@
+flask>=3.0
+requests>=2.31
+# Only used by the `doh` role when DOH_TLS=1, to self-sign a cert at startup.
+cryptography>=42.0
@@ -0,0 +1,22 @@
+[Application]
+FileLogger\Enabled=true
+
+[BitTorrent]
+Session\DefaultSavePath=/downloads
+Session\TempPathEnabled=false
+# Webseed (BEP-19) is how the e2e torrent completes with no tracker/peer.
+
+[LegalNotice]
+Accepted=true
+
+[Preferences]
+WebUI\Address=*
+WebUI\Port=8080
+# Bypass WebUI auth for the e2e compose subnet so shelfmark connects without the
+# linuxserver image's random temp password.
+WebUI\LocalHostAuth=false
+WebUI\AuthSubnetWhitelistEnabled=true
+WebUI\AuthSubnetWhitelist=172.30.0.0/24
+WebUI\CSRFProtection=false
+WebUI\HostHeaderValidation=false
+Downloads\SavePath=/downloads/
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Run the Shelfmark e2e platform for a single config profile.
+#
+#   ./run-e2e.sh [env/<profile>.env] [extra pytest args...]
+#
+# Boots the stack defined by the profile env file, waits for health, runs the
+# matching cluster tests (the suite skips tests not applicable to the profile),
+# then tears down. Set KEEP_UP=1 to leave the stack running for debugging.
+set -euo pipefail
+
+PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$PLATFORM_DIR"
+
+ENV_FILE="${1:-env/baseline.env}"
+shift || true
+PYTEST_ARGS=("$@")
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "error: env file not found: $ENV_FILE" >&2
+  echo "available profiles:" >&2
+  ls env/*.env >&2
+  exit 2
+fi
+
+# shellcheck disable=SC1090
+set -a; source "$ENV_FILE"; set +a   # export SM_*, COMPOSE_PROFILES, E2E_PROFILE
+PROFILE="${E2E_PROFILE:-baseline}"
+COMPOSE=(docker compose --env-file "$ENV_FILE" -f docker-compose.e2e.yml)
+
+STATE_DIR="$PLATFORM_DIR/.state"
+LOG_FILE="$STATE_DIR/shelfmark.$PROFILE.log"
+mkdir -p "$STATE_DIR/config" "$STATE_DIR/books" "$STATE_DIR/downloads" "$STATE_DIR/tmp"
+
+cleanup() {
+  if [[ "${KEEP_UP:-0}" != "1" ]]; then
+    echo "==> tearing down ($PROFILE)"
+    "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+  else
+    echo "==> KEEP_UP=1: leaving stack running ($PROFILE)"
+  fi
+}
+trap cleanup EXIT
+
+# E2E_NO_BUILD=1 reuses already-built images (see `make e2e-platform-build` /
+# run-matrix.sh) so a matrix run builds the heavy shelfmark image only once.
+if [[ "${E2E_NO_BUILD:-0}" == "1" ]]; then
+  echo "==> [$PROFILE] starting stack, reusing built images (profiles='${COMPOSE_PROFILES:-<none>}')"
+  "${COMPOSE[@]}" up -d --no-build
+else
+  echo "==> [$PROFILE] building + starting stack (profiles='${COMPOSE_PROFILES:-<none>}')"
+  "${COMPOSE[@]}" up -d --build
+fi
+
+echo "==> [$PROFILE] waiting for shelfmark health"
+HEALTHY=0
+for _ in $(seq 1 60); do
+  if curl -fsS http://localhost:8084/api/health >/dev/null 2>&1; then HEALTHY=1; break; fi
+  sleep 2
+done
+
+# Capture boot diagnostics for the entrypoint/permission tests.
+"${COMPOSE[@]}" logs shelfmark > "$LOG_FILE" 2>&1 || true
+RESTARTS="$(docker inspect -f '{{.RestartCount}}' e2e-shelfmark 2>/dev/null || echo 0)"
+echo "==> [$PROFILE] healthy=$HEALTHY restarts=$RESTARTS log=$LOG_FILE"
+
+if [[ "$HEALTHY" != "1" && "$PROFILE" != "tor" ]]; then
+  echo "error: shelfmark never became healthy under profile '$PROFILE'" >&2
+  "${COMPOSE[@]}" logs --tail 40 shelfmark >&2 || true
+  exit 1
+fi
+
+# Hand context to the pytest suite.
+export E2E_PROFILE="$PROFILE"
+export E2E_BASE_URL="http://localhost:8084"
+export E2E_BOOKS_DIR="$STATE_DIR/books"
+export E2E_TMP_DIR="$STATE_DIR/tmp"
+export E2E_SHELFMARK_LOG="$LOG_FILE"
+export E2E_SHELFMARK_RESTARTS="$RESTARTS"
+
+echo "==> [$PROFILE] running suite"
+set +e
+( cd "$PLATFORM_DIR/../../.." && \
+  uv run pytest tests/e2e/platform/suite -m platform -o addopts="--tb=short" "${PYTEST_ARGS[@]}" )
+RC=$?
+set -e
+
+echo "==> [$PROFILE] pytest exit=$RC"
+exit $RC
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Run the full config matrix: every profile, in sequence, aggregating results.
+#
+#   ./run-matrix.sh                 # all profiles
+#   ./run-matrix.sh baseline dns-manual   # a subset (by profile name)
+set -uo pipefail
+
+PLATFORM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$PLATFORM_DIR"
+
+if [[ $# -gt 0 ]]; then
+  PROFILES=("$@")
+else
+  PROFILES=(baseline bypasser-external bypasser-disabled dns-manual dns-blocked dns-doh \
+            proxy-http proxy-socks tor client-transmission client-deluge)
+fi
+
+# Build the (heavy) images once, then reuse them across every profile so the
+# matrix doesn't rebuild the xvfb/chromium layer N times. Set NO_PREBUILD=1 to
+# skip (e.g. to let each run rebuild from source).
+if [[ "${NO_PREBUILD:-0}" != "1" ]]; then
+  echo "==> pre-building images once (reused by all profiles)"
+  ./build-images.sh
+  export E2E_NO_BUILD=1
+fi
+
+declare -A RESULT
+FAILED=0
+for p in "${PROFILES[@]}"; do
+  echo "========================================================================"
+  echo "  PROFILE: $p"
+  echo "========================================================================"
+  if ./run-e2e.sh "env/$p.env"; then
+    RESULT[$p]="PASS"
+  else
+    RESULT[$p]="FAIL"
+    FAILED=1
+  fi
+done
+
+echo "========================================================================"
+echo "  MATRIX SUMMARY"
+echo "========================================================================"
+for p in "${PROFILES[@]}"; do
+  printf "  %-22s %s\n" "$p" "${RESULT[$p]:-SKIP}"
+done
+exit $FAILED
@@ -0,0 +1,132 @@
+"""Harness for the Shelfmark e2e Docker platform.
+
+These tests run against a *live* Shelfmark booted by ``run-e2e.sh`` under a
+particular config profile (env file). The active profile is read from
+``E2E_PROFILE``; tests select which profiles they apply to with the
+``@pytest.mark.profiles(...)`` marker. Unmarked tests are profile-agnostic
+invariants and run under every profile — that is how the same cluster test
+becomes the config matrix (search must succeed whether egress is direct, via a
+proxy, via custom DNS, or through the bypasser).
+
+Run (handled by run-e2e.sh):
+    E2E_PROFILE=baseline uv run pytest tests/e2e/platform/suite -m platform
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass, field
+
+import pytest
+import requests
+
+BASE_URL = os.environ.get("E2E_BASE_URL", "http://localhost:8084")
+ACTIVE_PROFILE = os.environ.get("E2E_PROFILE", "baseline")
+DEFAULT_TIMEOUT = 15
+DOWNLOAD_TIMEOUT = int(os.environ.get("E2E_DOWNLOAD_TIMEOUT", "120"))
+TERMINAL_OK = {"complete", "done", "available"}
+TERMINAL_ERR = {"error", "cancelled"}
+
+
+@dataclass
+class PlatformClient:
+    base_url: str = BASE_URL
+    timeout: int = DEFAULT_TIMEOUT
+    session: requests.Session = field(default_factory=requests.Session)
+
+    def get(self, path: str, **kw) -> requests.Response:
+        kw.setdefault("timeout", self.timeout)
+        return self.session.get(f"{self.base_url}{path}", **kw)
+
+    def post(self, path: str, **kw) -> requests.Response:
+        kw.setdefault("timeout", self.timeout)
+        return self.session.post(f"{self.base_url}{path}", **kw)
+
+    # --- domain helpers -------------------------------------------------- #
+    def wait_for_health(self, max_wait: int = 90) -> bool:
+        deadline = time.time() + max_wait
+        while time.time() < deadline:
+            try:
+                if self.get("/api/health").status_code == 200:
+                    return True
+            except requests.RequestException:
+                pass
+            time.sleep(2)
+        return False
+
+    def direct_search(
+        self, query: str, *, inject: str | None = None, **params
+    ) -> requests.Response:
+        """Source-native (hermetic) release search — no external metadata provider.
+
+        Hits GET /api/releases?source=direct_download&query=... which drives
+        direct_download.search_books against the mock Anna's Archive.
+
+        Fault injection rides *inside* the query text (the app builds the AA URL
+        itself and only forwards the query as ``q=``); the mock origin parses the
+        ``E2EINJECT:<name>`` token. See mock_services.aa_search.
+        """
+        effective_query = f"E2EINJECT:{inject} {query}" if inject else query
+        qp = {"source": "direct_download", "query": effective_query, **params}
+        return self.get("/api/releases", params=qp, timeout=60)
+
+    def releases_from(self, resp: requests.Response) -> list[dict]:
+        if resp.status_code != 200:
+            return []
+        data = resp.json()
+        if isinstance(data, dict):
+            rel = data.get("releases")
+            return rel if isinstance(rel, list) else []
+        return data if isinstance(data, list) else []
+
+    def queue_download(self, release: dict) -> requests.Response:
+        return self.post("/api/releases/download", json=release, timeout=30)
+
+    def wait_for_terminal(self, book_id: str, timeout: int = DOWNLOAD_TIMEOUT) -> tuple[str, dict]:
+        deadline = time.time() + timeout
+        last: dict = {}
+        while time.time() < deadline:
+            resp = self.get("/api/status")
+            if resp.status_code == 200 and isinstance(resp.json(), dict):
+                status = resp.json()
+                for state, entries in status.items():
+                    if isinstance(entries, dict) and book_id in entries:
+                        last = entries[book_id]
+                        if state in TERMINAL_OK or state in TERMINAL_ERR:
+                            return state, last
+            time.sleep(2)
+        return "timeout", last
+
+
+@pytest.fixture(scope="session")
+def client() -> PlatformClient:
+    c = PlatformClient()
+    if not c.wait_for_health():
+        pytest.fail(f"Shelfmark not healthy at {BASE_URL} (profile={ACTIVE_PROFILE})")
+    return c
+
+
+@pytest.fixture(scope="session")
+def active_profile() -> str:
+    return ACTIVE_PROFILE
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line("markers", "platform: Shelfmark e2e docker platform test")
+    config.addinivalue_line("markers", "profiles(*names): only run under these E2E_PROFILE values")
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    """Skip tests whose declared profiles don't include the active profile.
+
+    A test with no ``profiles`` marker is a profile-agnostic invariant and runs
+    everywhere (this is the matrix: invariants x profiles).
+    """
+    for item in items:
+        item.add_marker(pytest.mark.platform)
+        marker = item.get_closest_marker("profiles")
+        if marker and ACTIVE_PROFILE not in marker.args:
+            item.add_marker(
+                pytest.mark.skip(reason=f"profile={ACTIVE_PROFILE!r} not in {marker.args}")
+            )
@@ -0,0 +1,86 @@
+"""Cluster 1 — Cloudflare bypasser wiring + clean-failure behavior.
+
+Reality discovered by running the stack: shelfmark's AA *search* and *detail*
+fetches use ``html_get_page(allow_bypasser_fallback=False)``, so a search behind a
+Cloudflare gate returns 503 **regardless** of the bypasser. The bypasser (internal
+Chrome or external FlareSolverr) is a *download-time* mechanism
+(``html_get_page(use_bypasser=True)``); it never runs for search.
+
+So these tests assert what is actually true and host-observable:
+  * the external bypasser is configured from env, and
+  * a CF-gated search fails *cleanly* (a 503 the client can act on, not a hang or
+    a crash) — both with the bypasser on (it isn't used for search) and off.
+
+Exercising shelfmark's *use* of the bypasser end-to-end (a real CF solve during a
+download) needs the AA slow-download HTML flow mocked — see the README roadmap.
+The bypass *mechanism* itself is verified to work: the mock FlareSolverr solves
+the gate (manually confirmed; see README). Guards: #284 #226 #202 #1030 #410 #369.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import pytest
+import requests
+
+
+def _boot_log() -> str:
+    path = os.environ.get("E2E_SHELFMARK_LOG")
+    if not path or not Path(path).exists():
+        return ""
+    return Path(path).read_text(encoding="utf-8", errors="ignore")
+
+
+def _cf_gated_search_has_no_releases(client) -> bool:
+    """A CF-gated AA search must not yield releases (the gate isn't bypassed for
+    search).
+
+    NOTE (observed live): with ``USE_CF_BYPASS=false`` the search is *slow* to fail
+    — the app retries and can take ~60s, vs a fast 503 when the bypasser is enabled
+    (potential #1001 "hung on bypass protection"). We bound the wait and treat a
+    timeout the same as a clean failure: in both cases no releases were obtained,
+    which is the point of this negative control.
+    """
+    try:
+        resp = client.get(
+            "/api/releases",
+            params={"source": "direct_download", "query": "Mistborn"},
+            timeout=30,
+        )
+    except requests.exceptions.Timeout:
+        return True  # could not complete -> definitively no releases obtained
+    assert resp.status_code in (200, 404, 500, 503), (
+        f"CF-gated search returned an unexpected status: {resp.status_code} {resp.text[:200]}"
+    )
+    return not client.releases_from(resp)
+
+
+@pytest.mark.profiles("bypasser-external")
+def test_external_bypasser_is_configured(client) -> None:
+    """The external (FlareSolverr) bypasser path is selected via env."""
+    assert client.get("/api/health").status_code == 200
+    log = _boot_log()
+    if not log:
+        pytest.skip("E2E_SHELFMARK_LOG not available")
+    assert "USING_EXTERNAL_BYPASSER" in log and "EXT_BYPASSER_URL" in log, (
+        "external bypasser config was not synced from env"
+    )
+
+
+@pytest.mark.profiles("bypasser-external")
+def test_cf_gated_search_fails_cleanly_with_external_bypasser(client) -> None:
+    """Even with the external bypasser configured, a CF-gated *search* yields no
+    releases (the bypasser is download-time) — but it must fail cleanly."""
+    assert _cf_gated_search_has_no_releases(client)
+
+
+@pytest.mark.profiles("bypasser-disabled")
+def test_cf_gated_search_fails_when_bypasser_off(client) -> None:
+    """Negative control: AA behind Cloudflare + bypasser OFF -> no releases, clean
+    failure. A regression that ignored the gate would wrongly return results."""
+    assert _cf_gated_search_has_no_releases(client), (
+        "results returned even though AA is Cloudflare-gated and the bypasser is "
+        "disabled — the challenge is being ignored (regression for #202/#410)"
+    )
@@ -0,0 +1,32 @@
+"""Cluster 1/6 — Tor boot correctness.
+
+Tor has repeatedly boot-looped or pegged CPU on startup (#1021 loops on 1.3.0,
+#940 USING_TOR loop, #801 gosu 100% CPU, #937 gunicorn missing). The hermetic,
+fast assertion is: with USING_TOR=true the container reaches a healthy
+/api/health and does NOT crash-loop. Real Tor egress (slow/flaky in CI) is left
+to an opt-in 'tor-full' profile.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+pytestmark = pytest.mark.profiles("tor")
+
+
+def test_app_becomes_healthy_under_tor(client) -> None:
+    """tor.sh + entrypoint must bring the app up, not boot-loop."""
+    assert client.get("/api/health").status_code == 200
+
+
+def test_container_did_not_crash_loop(client) -> None:
+    """Restart count is captured by the runner into E2E_SHELFMARK_RESTARTS.
+
+    A boot-loop shows up as repeated restarts; a healthy boot is 0.
+    """
+    restarts = os.environ.get("E2E_SHELFMARK_RESTARTS")
+    if restarts is None:
+        pytest.skip("runner did not provide E2E_SHELFMARK_RESTARTS")
+    assert int(restarts) == 0, f"shelfmark restarted {restarts} times under Tor (boot-loop)"
@@ -0,0 +1,87 @@
+"""Cluster 5 — real download clients (torrent), end to end.
+
+The `full` and `client-*` profiles each point shelfmark at a *real* torrent client
+(qBittorrent / Transmission / Deluge / rTorrent) plus a mock Prowlarr that returns a
+tracker-less BEP-19 webseed `.torrent` sourced from mock-aa. The client downloads
+the payload over HTTP (no tracker/peer/seeder) and shelfmark's completion detection
+ post-process move lands Moby-Dick in `/books`.
+
+The test is **client-agnostic** — the active profile's env selects the client
+(`PROWLARR_TORRENT_CLIENT` + that client's URL/creds) — so one test covers the whole
+client matrix.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.profiles("full", "client-transmission", "client-deluge")
+
+BOOK = "Moby Dick"
+
+
+def _books_dir() -> Path | None:
+    raw = os.environ.get("E2E_BOOKS_DIR")
+    return Path(raw) if raw else None
+
+
+def _book_files(books: Path) -> set[str]:
+    return {
+        p.name for p in books.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"}
+    }
+
+
+def _prowlarr_search(client, query: str):
+    return client.get(
+        "/api/releases",
+        params={
+            "provider": "manual",
+            "book_id": "e2e-manual-1",
+            "source": "prowlarr",
+            "title": query,
+            "manual_query": query,
+        },
+        timeout=60,
+    )
+
+
+def test_prowlarr_to_real_torrent_client_download(client, active_profile) -> None:
+    """Prowlarr release -> real torrent client (per profile) -> file in /books."""
+    books = _books_dir()
+    if books is None or not books.exists():
+        pytest.skip("E2E_BOOKS_DIR not visible to the test runner")
+    before = _book_files(books)
+
+    resp = _prowlarr_search(client, BOOK)
+    assert resp.status_code == 200, f"[{active_profile}] prowlarr search failed: {resp.text[:300]}"
+    releases = client.releases_from(resp)
+    assert releases, f"[{active_profile}] mock prowlarr returned no releases"
+
+    queued = client.queue_download(releases[0])
+    assert queued.status_code in (200, 201, 202), (
+        f"[{active_profile}] queue refused the release: {queued.status_code} {queued.text[:300]}"
+    )
+    # The prowlarr source serializes download_url=None and resolves the real URL
+    # from its cache by source_id at download time.
+    book_id = releases[0].get("source_id") or releases[0].get("id") or releases[0].get("guid")
+    assert book_id, f"[{active_profile}] release missing a trackable id: {releases[0]!r}"
+
+    state, info = client.wait_for_terminal(str(book_id))
+    assert state in {"complete", "done", "available"}, (
+        f"[{active_profile}] real torrent-client download did not complete: "
+        f"state={state} info={info!r}"
+    )
+
+    deadline = time.time() + 30
+    new_files: set[str] = set()
+    while time.time() < deadline:
+        new_files = _book_files(books) - before
+        if new_files:
+            break
+        time.sleep(2)
+    assert new_files, f"[{active_profile}] client completed but no file landed in /books"
+    assert all(Path(n).suffix for n in new_files), f"file without extension: {new_files}"
@@ -0,0 +1,62 @@
+"""Cluster 4 — download execution + file placement/permissions.
+
+Two halves, by what each profile can hermetically prove:
+
+* **baseline (no bypasser, AA-only):** AA's slow-download sources require a
+  Cloudflare bypass (``_CF_BYPASS_REQUIRED``), so a download here *cannot* succeed
+  and the app must say so cleanly — this is exactly the real-world #1028 shape
+  ("All download sources failed"). We assert that the failure is surfaced as a
+  terminal ``error`` with a message, not a hang/crash, and that nothing is left
+  orphaned in staging (#1040).
+* **successful download + file move** (extension preserved #214, no orphaned
+  staging dir #1040) is proven for real in the ``full`` profile, where a real
+  qBittorrent completes a webseed torrent — see ``test_cluster_full_pipeline.py``.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.profiles("baseline")
+
+
+def _staging_leftovers() -> list[Path]:
+    tmp_raw = os.environ.get("E2E_TMP_DIR")
+    if not tmp_raw or not Path(tmp_raw).exists():
+        return []
+    tmp = Path(tmp_raw)
+    return [p for p in tmp.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"}]
+
+
+def test_no_bypass_download_fails_cleanly(client) -> None:
+    """Without a bypasser, AA is undownloadable — the app must report a clear
+    terminal error (the #1028 shape), not hang or crash."""
+    resp = client.direct_search("Mistborn")
+    releases = client.releases_from(resp)
+    assert releases, "search should still return parsed releases even if undownloadable"
+
+    queued = client.queue_download(releases[0])
+    assert queued.status_code in (200, 201, 202), (
+        f"queue refused the release: {queued.status_code} {queued.text[:300]}"
+    )
+    book_id = releases[0].get("id") or releases[0].get("source_id") or releases[0].get("md5")
+    assert book_id, f"release missing an id to track: {releases[0]!r}"
+
+    state, info = client.wait_for_terminal(str(book_id))
+    assert state == "error", (
+        f"expected a clean terminal error without a bypasser, got state={state} info={info!r}"
+    )
+    message = str(info.get("status_message") or info.get("last_error_message") or "")
+    assert message.strip(), "download failed but surfaced no status message to the user"
+
+
+def test_no_orphaned_staging_dir(client) -> None:
+    """#1040 guard: a failed/aborted download must not leave book payloads behind
+    in the staging/tmp area."""
+    if not os.environ.get("E2E_TMP_DIR"):
+        pytest.skip("E2E_TMP_DIR not provided")
+    leftovers = _staging_leftovers()
+    assert not leftovers, f"book payload left orphaned in staging dir: {leftovers}"
@@ -0,0 +1,146 @@
+"""The `full` profile — maximum-realism, heavy, nightly/manual only.
+
+Test book: *Moby-Dick* by Herman Melville (public domain). Validated live.
+
+  * **Real Chrome solves Cloudflare, end to end (VERIFIED).** AA search/detail are
+    reachable (mock-aa), but the AA *slow-download* links point through the
+    Cloudflare gate (mock-cf). Downloading therefore forces the in-image headless
+    Chromium (seleniumbase CDP internal bypasser) to execute the challenge JS,
+    harvest ``cf_clearance``, and fetch the gated slow-download page. Moby-Dick
+    landing in ``/books`` is only possible if Chrome actually solved the gate —
+    that is the literal "spin a chrome browser" path.
+  * **DoH** enabled at boot (``USE_DOH=true``) without breaking startup.
+
+The real torrent-client download (the other half of the `full` profile) lives in
+test_cluster_clients.py, which runs under `full` and the `client-*` profiles.
+Only runs under the ``full`` profile booted by ``run-e2e.sh env/full.env``.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.profiles("full")
+
+BOOK = "Moby Dick"
+
+
+def _boot_log() -> str:
+    path = os.environ.get("E2E_SHELFMARK_LOG")
+    if not path or not Path(path).exists():
+        return ""
+    return Path(path).read_text(encoding="utf-8", errors="ignore")
+
+
+def _books_dir() -> Path | None:
+    raw = os.environ.get("E2E_BOOKS_DIR")
+    return Path(raw) if raw else None
+
+
+def _book_files(books: Path) -> set[str]:
+    return {
+        p.name for p in books.rglob("*") if p.is_file() and p.suffix.lower() in {".epub", ".pdf"}
+    }
+
+
+def _wait_for_new_book(books: Path, before: set[str], timeout: int = 40) -> set[str]:
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        new = _book_files(books) - before
+        if new:
+            return new
+        time.sleep(2)
+    return set()
+
+
+def _track_id(release: dict) -> str:
+    return str(release.get("source_id") or release.get("id") or release.get("guid") or "")
+
+
+# --------------------------------------------------------------------------- #
+# 1. Real Chrome solves Cloudflare end-to-end (the "spin a chrome browser" path)
+# --------------------------------------------------------------------------- #
+def test_real_chrome_solves_cloudflare_end_to_end(client) -> None:
+    """Download an AA book whose slow-download is behind Cloudflare; the internal
+    headless Chrome must solve the gate for the file to arrive in /books."""
+    books = _books_dir()
+    if books is None or not books.exists():
+        pytest.skip("E2E_BOOKS_DIR not visible to the test runner")
+    before = _book_files(books)
+
+    resp = client.direct_search(BOOK)
+    assert resp.status_code == 200, f"AA search failed: {resp.status_code} {resp.text[:300]}"
+    releases = client.releases_from(resp)
+    assert releases, "AA search returned no releases for Moby Dick"
+
+    queued = client.queue_download(releases[0])
+    assert queued.status_code in (200, 201, 202), (
+        f"queue refused the AA release: {queued.status_code} {queued.text[:300]}"
+    )
+    book_id = _track_id(releases[0])
+    assert book_id, f"release missing a trackable id: {releases[0]!r}"
+
+    state, info = client.wait_for_terminal(book_id)
+    assert state in {"complete", "done", "available"}, (
+        f"AA download via the Chrome-solved Cloudflare gate did not complete: "
+        f"state={state} info={info!r}"
+    )
+
+    new_files = _wait_for_new_book(books, before)
+    assert new_files, (
+        "download reported complete but no file landed in /books — the Cloudflare "
+        "gate in front of the slow-download was not solved by Chrome"
+    )
+    assert all(Path(n).suffix for n in new_files), f"file written without extension: {new_files}"
+
+    # Secondary, explicit signal that the internal bypasser (Chrome) was engaged.
+    _assert_bypasser_engaged()
+
+
+def _assert_bypasser_engaged() -> None:
+    """Confirm shelfmark actually routed through the internal Chrome bypasser.
+
+    Best-effort: reads the live shelfmark container logs. The file landing in /books
+    is already proof (the slow-download was CF-gated), but this pins the mechanism.
+    """
+    if shutil.which("docker") is None:
+        return
+    result = subprocess.run(
+        ["docker", "logs", "e2e-shelfmark"],
+        capture_output=True,
+        text=True,
+        timeout=20,
+        check=False,
+    )
+    blob = (result.stdout + result.stderr).lower()
+    if not blob.strip():
+        return
+    assert "bypass" in blob, (
+        "no evidence the internal bypasser engaged during the download — the file "
+        "may have arrived via an unexpected (non-Chrome) path"
+    )
+
+
+# --------------------------------------------------------------------------- #
+# 2. DoH
+# --------------------------------------------------------------------------- #
+def test_doh_enabled_and_app_healthy(client) -> None:
+    """DoH is enabled at boot and the app stays healthy (DoH init has historically
+    broken startup). ``/api/config`` doesn't expose ``USE_DOH``, so verify via the
+    boot log + health."""
+    assert client.get("/api/health").status_code == 200
+    log = _boot_log()
+    if not log:
+        pytest.skip("E2E_SHELFMARK_LOG not available to assert DoH")
+    assert "USE_DOH=true" in log or "'USE_DOH'" in log, "USE_DOH was not synced into config at boot"
+
+
+# NOTE: the real torrent-client download (Prowlarr -> qBittorrent/transmission/
+# deluge/rtorrent -> /books) lives in test_cluster_clients.py, which runs under the
+# `full` profile *and* the `client-*` profiles from one client-agnostic test.
@@ -0,0 +1,49 @@
+"""Clusters 5/6/7 — container/entrypoint correctness, plus pointers.
+
+Cluster 6 (docker/entrypoint/PUID-PGID — 9 issues / 19 fix PRs, zero shell test
+coverage today): the app must boot healthy under the configured PUID/PGID with no
+permission errors, regardless of which config profile is active. This is a
+profile-agnostic invariant, so it runs under every profile and catches
+entrypoint/permission regressions (#801, #411, #434, #171) across the matrix.
+
+Clusters 5 (download clients) and 7 (audiobook/ABB) are exercised by dedicated
+stacks/sources (docker-compose.test-clients.yml and an audiobookbay mock) — see
+README. Pointers below keep them visible in the matrix without duplicating the
+prowlarr e2e flow.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+
+def test_health_endpoint_under_every_profile(client, active_profile) -> None:
+    """Entrypoint/permission boot must succeed under the active config."""
+    resp = client.get("/api/health")
+    assert resp.status_code == 200, f"[{active_profile}] not healthy: {resp.text[:200]}"
+
+
+def test_no_permission_errors_in_boot_logs() -> None:
+    """The runner captures shelfmark boot logs into E2E_SHELFMARK_LOG; assert no
+    permission/entrypoint failure markers (regression for #171/#447/#801)."""
+    log_path = os.environ.get("E2E_SHELFMARK_LOG")
+    if not log_path or not os.path.exists(log_path):
+        pytest.skip("E2E_SHELFMARK_LOG not provided by the runner")
+    with open(log_path, encoding="utf-8", errors="ignore") as fh:
+        text = fh.read().lower()
+    for marker in ("permission denied", "operation not permitted", "read-only file system"):
+        assert marker not in text, f"boot logs contain a permission failure: {marker!r}"
+
+
+@pytest.mark.skip(
+    reason="cluster 5: covered by docker-compose.test-clients.yml + prowlarr e2e flow"
+)
+def test_download_clients_pointer() -> None:  # pragma: no cover - documentation marker
+    ...
+
+
+@pytest.mark.skip(reason="cluster 7: needs an audiobookbay mock role (tracked in README roadmap)")
+def test_audiobook_pointer() -> None:  # pragma: no cover - documentation marker
+    ...
@@ -0,0 +1,70 @@
+"""Cluster 2/3 — search relevance and Anna's Archive HTML-parse robustness.
+
+The recurring root cause: AA changes its DOM and the hardcoded-index parser
+silently returns zero rows, which users experience as "All download sources
+failed" (#1028) or "book not found" (#198, #293). Evidence of brittleness:
+#878/#879/#880 (hardcoded indices/selectors), plus the repeated
+"Fix AA ... after they changed layout" PRs.
+
+These run under ``baseline`` (direct connection to the fake AA) so the assertions
+isolate parsing from egress concerns.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytestmark = pytest.mark.profiles("baseline")
+
+
+def test_search_returns_parsed_releases(client) -> None:
+    """Happy path: the parser turns the AA results table into releases."""
+    resp = client.direct_search("Mistborn")
+    assert resp.status_code == 200, resp.text
+    releases = client.releases_from(resp)
+    assert releases, "expected at least one parsed release from the AA results table"
+    titles = " ".join(str(r.get("title", "")) for r in releases)
+    assert "Mistborn" in titles, f"query not reflected in parsed titles: {titles[:200]}"
+
+
+def test_layout_drift_fails_loudly_not_silently(client) -> None:
+    """When AA's DOM changes so no row parses, the app must surface a clear
+    failure — NOT an empty 200 that reads as 'book does not exist'.
+
+    Regression guard for #878/#879/#880 and the layout-change PRs.
+    """
+    resp = client.direct_search("Mistborn", inject="layout_drift")
+    releases = client.releases_from(resp)
+    # Acceptable behaviours: an explicit error status, OR a 200 with an error
+    # field. NOT acceptable: 200 + empty releases with no signal.
+    if resp.status_code == 200:
+        body = (
+            resp.json()
+            if resp.headers.get("content-type", "").startswith("application/json")
+            else {}
+        )
+        has_error_signal = bool(body.get("error")) or bool(body.get("source_errors"))
+        assert not releases, "parser unexpectedly produced releases from drifted DOM"
+        assert has_error_signal, (
+            "layout drift produced a silent empty 200 — the app must signal that "
+            "the source could not be parsed (regression for #878/#879/#880)"
+        )
+    else:
+        assert resp.status_code >= 400, resp.status_code
+
+
+def test_no_files_string_alongside_real_results(client) -> None:
+    """A real results table that also contains the literal 'No files found.'
+    must still yield releases (false-positive guard)."""
+    resp = client.direct_search("Mistborn", inject="no_files")
+    assert resp.status_code == 200, resp.text
+    assert client.releases_from(resp), (
+        "'No files found.' substring caused a false-positive empty result"
+    )
+
+
+def test_genuinely_empty_results_handled_cleanly(client) -> None:
+    """A true 'No files found.' page yields zero releases without a 500."""
+    resp = client.direct_search("Mistborn", inject="empty")
+    assert resp.status_code in (200, 404), resp.status_code
+    assert client.releases_from(resp) == []
@@ -0,0 +1,113 @@
+"""Config-matrix invariants.
+
+These tests carry NO ``profiles`` marker for the egress check, so they run under
+every profile the runner boots. Reaching the (mock) book source must succeed
+whether egress is direct, through an HTTP/SOCKS proxy, via custom DNS, or via the
+Cloudflare bypasser. That cross-product *is* the config matrix.
+
+Covers the recurring "X setting silently breaks downloads" class:
+proxy ignored (#956), DNS/ISP blocks (#1028, #108), bypasser config not adhered
+(#410, #369, #267).
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+# Profiles where reaching the (non-CF-gated) source via search is expected to work
+# end to end. Excluded: tor (boot-correctness only), and the bypasser profiles —
+# their AA is behind a Cloudflare gate that search does NOT bypass (the bypasser is
+# download-time only), so a *search* there fails by design (see test_cluster_bypasser).
+EGRESS_PROFILES = (
+    "baseline",
+    "dns-manual",
+    "dns-blocked",
+    "proxy-http",
+    "proxy-socks",
+)
+
+# The proxy container whose logs prove egress actually traversed the proxy.
+PROXY_CONTAINER = {
+    "proxy-http": "e2e-tinyproxy",
+    "proxy-socks": "e2e-microsocks",
+}
+
+
+def test_health_ok(client) -> None:
+    resp = client.get("/api/health")
+    assert resp.status_code == 200, resp.text
+
+
+@pytest.mark.profiles(*EGRESS_PROFILES)
+def test_source_reachable_under_active_profile(client, active_profile) -> None:
+    """The book source must be reachable regardless of egress configuration."""
+    resp = client.direct_search("Mistborn")
+    assert resp.status_code == 200, (
+        f"[{active_profile}] direct search failed: {resp.status_code} {resp.text[:300]}"
+    )
+    releases = client.releases_from(resp)
+    assert releases, (
+        f"[{active_profile}] expected releases from the mock source but got none — "
+        f"egress configuration is silently dropping the request"
+    )
+
+
+@pytest.mark.profiles("proxy-http")
+def test_proxy_mode_synced_at_boot(client) -> None:
+    """The deployment PROXY_MODE env override is applied at boot.
+
+    ``/api/config`` returns only a frontend-facing subset (not network/proxy
+    keys), so this is verified from the boot log where ENV→config sync is recorded.
+    """
+    assert client.get("/api/health").status_code == 200
+    path = os.environ.get("E2E_SHELFMARK_LOG")
+    if not path or not Path(path).exists():
+        pytest.skip("E2E_SHELFMARK_LOG not available")
+    log = Path(path).read_text(encoding="utf-8", errors="ignore")
+    assert "PROXY_MODE" in log, "PROXY_MODE was not synced into network config at boot"
+
+
+@pytest.mark.profiles("proxy-http", "proxy-socks")
+def test_egress_actually_traverses_proxy(client, active_profile) -> None:
+    """#956 guard: a configured proxy must actually *carry* the app's egress.
+
+    Reachability alone is not enough — the app and the mock AA share the e2e
+    network, so a regression that silently ignores the proxy config would still
+    reach AA directly and pass ``test_source_reachable_under_active_profile``.
+    Here we drive a search and then inspect the proxy container's logs: if the
+    proxy never saw the traffic, the proxy was bypassed (regression for #956).
+    """
+    if shutil.which("docker") is None:
+        pytest.skip("docker CLI not available to the test host")
+    container = PROXY_CONTAINER[active_profile]
+
+    # Generate egress that *must* go through the proxy.
+    resp = client.direct_search("Mistborn")
+    assert resp.status_code == 200 and client.releases_from(resp), (
+        f"[{active_profile}] search failed under proxy: {resp.status_code} {resp.text[:200]}"
+    )
+
+    logs = subprocess.run(
+        ["docker", "logs", container],
+        capture_output=True,
+        text=True,
+        timeout=15,
+        check=False,
+    )
+    blob = (logs.stdout + logs.stderr).lower()
+    assert blob.strip(), (
+        f"[{active_profile}] proxy container {container!r} produced no logs after a "
+        f"search — egress did not traverse the proxy (regression for #956 proxy-ignored)"
+    )
+    # Strongest signal (HTTP proxy logs the destination host/request explicitly).
+    if active_profile == "proxy-http":
+        markers = ("mock-aa", "aa.mock.test", "connect", "request", "get ")
+        assert any(m in blob for m in markers), (
+            f"tinyproxy logs show no AA request — proxy may be passing traffic without "
+            f"the app routing through it. logs tail: ...{blob[-300:]!r}"
+        )