posts, the book page carries one POST form per format, and
+the form's response is an interstitial with a meta refresh to a signed file URL.
+"""
+
+from io import BytesIO
+from threading import Event
+from types import SimpleNamespace
+
+import pytest
+from bs4 import BeautifulSoup
+
+from shelfmark.core import mirrors
+from shelfmark.core.models import SearchFilters
+from shelfmark.release_sources import BrowseRecord
+from shelfmark.release_sources.direct_download import oceanofpdf as op
+from shelfmark.release_sources.direct_download import registry
+from shelfmark.release_sources.direct_download.common import DirectDownloadUnavailableError
+
+BASE = "https://ocean.example"
+DUNE_URL = f"{BASE}/authors/frank-herbert/pdf-epub-dune-download-85663993149/"
+SONG_URL = f"{BASE}/authors/anissa-m-bouziane/pdf-epub-dune-song-download/"
+ART_URL = f"{BASE}/authors/alejandro-jodorowsky/pdf-dune-art-book-download/"
+FETCH_URL = f"{BASE}/Fetching_Resource.php"
+FILE_URL = "https://files.example/OceanofPDF.com/Dune_1_-_Frank_Herbert.epub?md5=abc&expires=1"
+
+
+def _article(url: str, title: str, label: str, summary: str, cover: str = "") -> str:
+ return f"""
+
+
+
+ """
+
+
+def _page(body: str) -> str:
+ return f"""OceanofPDF
+ Skip to main content
+ {body}"""
+
+
+SEARCH_PAGE = _page(
+ _article(
+ DUNE_URL,
+ "Dune",
+ "[PDF] [EPUB] Dune Download",
+ "[PDF] [EPUB] Dune Download by Frank Herbert. Download Dune by Frank Herbert in PDF "
+ "EPUB format complete free.",
+ cover="https://media.example/dune.jpg",
+ )
+ + _article(
+ SONG_URL,
+ "Dune Song",
+ "[PDF] [EPUB] Dune Song Download",
+ "[PDF] [EPUB] Dune Song Download by Anissa M. Bouziane. Download Dune Song by Anissa "
+ "M. Bouziane in PDF EPUB format complete free.",
+ )
+ + _article(
+ ART_URL,
+ "Dune Art Book",
+ "[PDF] Dune Art Book Download",
+ "[PDF] Dune Art Book Download by Alejandro Jodorowsky. Download Dune Art Book in PDF "
+ "format complete free.",
+ )
+)
+
+EMPTY_SEARCH_PAGE = _page("Nothing Found
")
+
+BOOK_PAGE = _page(
+ """
+
+
- Full Book Name: Dune
- Author Name: Frank Herbert
+ - PDF File Size: 2.1 MB
- EPUB File Size: 952 KB
+
+
+
+
Don't worry! Your download will still start.
"
+)
+
+CHALLENGE_PAGE = (
+ "Just a moment..."
+ ''
+ ""
+)
+
+
+@pytest.fixture
+def configured(monkeypatch):
+ """Configure an OceanofPDF mirror and keep every request off the network.
+
+ Returns the settings the provider reads, so a test can flip the toggle or the bypass.
+ """
+ settings = {
+ "USE_CF_BYPASS": True,
+ "SOURCE_PRIORITY": [{"id": "zlib", "enabled": True}, {"id": "oceanofpdf", "enabled": True}],
+ }
+ monkeypatch.setattr(
+ op, "config", SimpleNamespace(get=lambda key, default=None: settings.get(key, default))
+ )
+ monkeypatch.setattr(mirrors, "_get_config", lambda: {"OCEANOFPDF_MIRROR_URLS": [BASE]})
+ monkeypatch.setattr(op.network, "AAMirrorSelector", lambda: SimpleNamespace(last_failure=None))
+ monkeypatch.setattr(op.network, "get_proxies", lambda _url: {})
+ monkeypatch.setattr(op.network, "get_ssl_verify", lambda _url: True)
+ monkeypatch.setattr(op.downloader, "get_cf_cookies_for_domain", lambda _host: {})
+ monkeypatch.setattr(op.downloader, "get_cf_user_agent_for_domain", lambda _host: None)
+ return settings
+
+
+def _plan(**overrides):
+ values = {
+ "manual_query": None,
+ "source_filters": None,
+ "languages": None,
+ "author": "Frank Herbert",
+ "grouped_title_variants": [SimpleNamespace(title="Dune")],
+ }
+ values.update(overrides)
+ return SimpleNamespace(**values)
+
+
+def _serve_pages(monkeypatch, pages: dict[str, str]) -> list[str]:
+ """Answer html_get_page from ``pages`` by URL, recording each URL requested."""
+ requested: list[str] = []
+
+ def fake_html_get_page(url, *_args, **_kwargs):
+ requested.append(url)
+ return pages.get(url, "")
+
+ monkeypatch.setattr(op.downloader, "html_get_page", fake_html_get_page)
+ return requested
+
+
+# --- Search parsing ---
+
+
+def test_search_results_split_into_one_record_per_format():
+ records = op.parse_search_results(SEARCH_PAGE, SearchFilters(format=["pdf", "epub"]))
+
+ assert [(r.title, r.format) for r in records] == [
+ ("Dune", "pdf"),
+ ("Dune", "epub"),
+ ("Dune Song", "pdf"),
+ ("Dune Song", "epub"),
+ ("Dune Art Book", "pdf"),
+ ]
+ dune_pdf, dune_epub = records[0], records[1]
+ assert dune_pdf.id.startswith("oceanofpdf:")
+ assert dune_pdf.id != dune_epub.id
+ assert dune_pdf.source_url == DUNE_URL
+ assert dune_pdf.author == "Frank Herbert"
+ assert dune_pdf.preview == "https://media.example/dune.jpg"
+ assert dune_pdf.source == "direct_download"
+
+
+def test_record_ids_are_stable_across_searches():
+ first = op.parse_search_results(SEARCH_PAGE, SearchFilters())
+ second = op.parse_search_results(SEARCH_PAGE, SearchFilters())
+
+ assert [r.id for r in first] == [r.id for r in second]
+
+
+def test_author_with_initials_is_not_cut_at_the_first_full_stop():
+ records = op.parse_search_results(SEARCH_PAGE, SearchFilters())
+
+ assert {r.author for r in records if r.title == "Dune Song"} == {"Anissa M. Bouziane"}
+
+
+def test_unfiltered_search_keeps_only_the_configured_supported_formats(monkeypatch):
+ monkeypatch.setattr(
+ "shelfmark.release_sources.direct_download.common.get_supported_formats",
+ lambda: ["epub", "mobi"],
+ )
+
+ records = op.parse_search_results(SEARCH_PAGE, SearchFilters())
+
+ assert {r.format for r in records} == {"epub"}
+
+
+def test_format_filter_drops_other_formats():
+ records = op.parse_search_results(SEARCH_PAGE, SearchFilters(format=["epub"]))
+
+ assert {r.format for r in records} == {"epub"}
+ assert "Dune Art Book" not in {r.title for r in records}
+
+
+def test_results_without_a_language_survive_a_language_filter():
+ records = op.parse_search_results(SEARCH_PAGE, SearchFilters(lang=["en"]))
+
+ assert records
+
+
+def test_empty_search_page_parses_to_no_records():
+ assert op.parse_search_results(EMPTY_SEARCH_PAGE, SearchFilters()) == []
+
+
+# --- Book page and interstitial parsing ---
+
+
+def test_download_form_is_picked_by_format():
+ soup = BeautifulSoup(BOOK_PAGE, "html.parser")
+
+ action, fields = op.pick_download_form(soup, "epub")
+
+ assert action == FETCH_URL
+ assert fields == {"id": "srv3", "filename": "Dune_1_-_Frank_Herbert.epub"}
+
+
+def test_download_form_ignores_the_search_form_and_defaults_to_the_first():
+ soup = BeautifulSoup(BOOK_PAGE, "html.parser")
+
+ _action, fields = op.pick_download_form(soup, None)
+
+ assert fields["filename"].endswith(".pdf")
+
+
+def test_download_form_is_none_for_a_format_the_page_lacks():
+ soup = BeautifulSoup(BOOK_PAGE, "html.parser")
+
+ assert op.pick_download_form(soup, "mobi") is None
+
+
+def test_file_size_is_read_for_the_requested_format():
+ soup = BeautifulSoup(BOOK_PAGE, "html.parser")
+
+ assert op.file_size_from_page(soup, "epub") == "952 KB"
+ assert op.file_size_from_page(soup, "pdf") == "2.1 MB"
+ assert op.file_size_from_page(soup, "mobi") is None
+
+
+def test_meta_refresh_gives_the_file_url_and_delay():
+ html = f''
+
+ assert op.parse_file_redirect(html, FETCH_URL) == (FILE_URL, 5)
+
+
+def test_relative_refresh_target_is_resolved_against_the_page():
+ html = ''
+
+ assert op.parse_file_redirect(html, FETCH_URL) == (f"{BASE}/files/book.epub", 3)
+
+
+def test_plain_file_link_is_used_when_there_is_no_meta_refresh():
+ html = 'Download'
+
+ assert op.parse_file_redirect(html, FETCH_URL) == ("https://files.example/x/book.epub?md5=1", 0)
+
+
+def test_interstitial_without_a_file_link_resolves_to_none():
+ assert op.parse_file_redirect("Please donate
", FETCH_URL) is None
+
+
+# --- Provider search ---
+
+
+def test_search_queries_title_and_author_first(configured, monkeypatch):
+ requested = _serve_pages(monkeypatch, {f"{BASE}/?s=Dune+Frank+Herbert": SEARCH_PAGE})
+
+ records = op.OceanofPDFProvider().search(SimpleNamespace(), _plan())
+
+ assert requested == [f"{BASE}/?s=Dune+Frank+Herbert"]
+ assert records[0].title == "Dune"
+
+
+def test_search_falls_back_to_the_bare_title(configured, monkeypatch):
+ requested = _serve_pages(
+ monkeypatch,
+ {f"{BASE}/?s=Dune+Frank+Herbert": EMPTY_SEARCH_PAGE, f"{BASE}/?s=Dune": SEARCH_PAGE},
+ )
+
+ records = op.OceanofPDFProvider().search(SimpleNamespace(), _plan())
+
+ assert requested == [f"{BASE}/?s=Dune+Frank+Herbert", f"{BASE}/?s=Dune"]
+ assert records
+
+
+def test_manual_search_sends_the_query_as_typed(configured, monkeypatch):
+ requested = _serve_pages(monkeypatch, {f"{BASE}/?s=dune+song": SEARCH_PAGE})
+
+ op.OceanofPDFProvider().search(
+ SimpleNamespace(), _plan(manual_query="dune song", source_filters=SearchFilters())
+ )
+
+ assert requested == [f"{BASE}/?s=dune+song"]
+
+
+def test_unreachable_site_is_reported_not_treated_as_no_results(configured, monkeypatch):
+ _serve_pages(monkeypatch, {})
+
+ with pytest.raises(DirectDownloadUnavailableError, match="Unable to reach OceanofPDF"):
+ op.OceanofPDFProvider().search(SimpleNamespace(), _plan())
+
+
+def test_uncleared_challenge_is_reported(configured, monkeypatch):
+ _serve_pages(monkeypatch, {f"{BASE}/?s=Dune+Frank+Herbert": CHALLENGE_PAGE})
+
+ with pytest.raises(DirectDownloadUnavailableError, match="Cloudflare challenge"):
+ op.OceanofPDFProvider().search(SimpleNamespace(), _plan())
+
+
+def test_real_page_carrying_challenge_markers_is_not_misread(configured, monkeypatch):
+ page = SEARCH_PAGE.replace("
+