fix(audiobookbay): decode base64-encoded post markup (#1656)

AudiobookBay now serves its search results base64-encoded inside hidden
.post divs and decodes them in the browser. The scraper looked for
.postTitle directly, so every search came back with zero results and no
error.

_decode_post base64-decodes each .post payload and reparses it, and
hands the element back unchanged when it is not base64 or has no
postTitle, so a mirror still serving plain markup keeps working. A page
that has .post elements but parses none of them now logs a warning, so
the next markup change shows up instead of looking like an empty result.

Ported from DrNgo/shelfmark-fork@7fb5f27 by @DrNgo, with the commit
co-authored to him. The tests cover both markups and the warning. I have
not run this build against the live site yet, which is why it is a
draft: from the box I tried, a search gets redirected to the homepage
with or without the fix, so that proved nothing. I will mark it ready
once it has run on my instance.


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

<!-- coderabbit-ai-generated:v1 -->

## Summary by CodeRabbit

* **Bug Fixes**
* AudiobookBay search can now display results when listing details are
wrapped in encoded markup. These results include the same available
information as listings in plain markup, such as title, format, file
size, date, and cover image. This improves consistency when searching
across different AudiobookBay page formats.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Co-authored-by: Michael Ngo <michaeln56@gmail.com>

shelfmark/release_sources/audiobookbay/scraper.py

@@ -1,8 +1,10 @@
 """Web scraping functions for AudiobookBay."""
 
+import base64
 import re
 import time
 from threading import Lock
+from typing import TYPE_CHECKING
 from urllib.parse import quote, quote_plus
 
 import requests
@@ -14,6 +16,9 @@ from shelfmark.download import http as downloader
 from shelfmark.download.postprocess.packs import PackFile
 from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation
 
+if TYPE_CHECKING:
+    from bs4.element import Tag
+
 logger = setup_logger(__name__)
 
 # Default trackers if none found on page
@@ -119,6 +124,26 @@ def _encode_search_query(query: str, *, exact_phrase: bool) -> str:
     return quote_plus(search_query)
 
 
+def _decode_post(post: Tag) -> Tag | BeautifulSoup:
+    """Decode ABB's base64-encoded post markup when present.
+
+    ABB serves post content base64-encoded inside a hidden ``.post`` div and
+    decodes it client-side with JavaScript; the decoded payload is the classic
+    ``.postTitle``/``.postInfo``/``.postContent`` markup. Plain-markup posts
+    (older mirrors) are returned unchanged, so never assume base64.
+    """
+    raw = "".join(post.get_text(strip=True).split())
+    if not raw or len(raw) < 32:
+        return post
+    try:
+        html = base64.b64decode(raw, validate=True).decode("utf-8", "replace")
+    except ValueError:
+        return post
+    if "postTitle" not in html:
+        return post
+    return BeautifulSoup(html, "html.parser")
+
+
 def _normalize_result_url(url: str, hostname: str) -> str:
     """Normalize ABB result URLs to absolute HTTPS URLs."""
     normalized_url = (url or "").strip()
@@ -249,12 +274,13 @@ def search_audiobookbay(
             # Parse HTML
             soup = BeautifulSoup(page_html, "html.parser")
 
-            # Extract book entries
-            posts = soup.select(".post")
+            # Extract book entries (decoding ABB's base64-wrapped markup when present)
+            posts = [_decode_post(p) for p in soup.select(".post")]
             if not posts:
                 # No more results
                 break
 
+            parsed_before = len(results)
             for post in posts:
                 try:
                     # Extract title
@@ -351,6 +377,16 @@ def search_audiobookbay(
                     logger.debug("Skipping post due to error: %s", e)
                     continue
 
+            # A page with .post elements that all fail to parse usually means ABB
+            # changed its markup; without this the source silently returns [].
+            if len(results) == parsed_before:
+                logger.warning(
+                    "ABB page %s: found %d .post elements but parsed 0 results, "
+                    "site markup may have changed",
+                    page,
+                    len(posts),
+                )
+
             # Rate limiting delay between pages
             if page < max_pages and rate_limit_delay > 0:
                 time.sleep(rate_limit_delay)

tests/audiobookbay/test_scraper.py

@@ -2,6 +2,7 @@
 Tests for AudiobookBay scraper functions.
 """
 
+import base64
 from unittest.mock import patch
 
 from shelfmark.release_sources.audiobookbay import scraper
@@ -51,6 +52,41 @@ EMPTY_SEARCH_HTML = """
 </html>
 """
 
+# ABB serves post markup base64-encoded inside hidden .post divs, decoded
+# client-side by JavaScript. The decoded payload is the classic markup.
+_POST_INNER_HTML = (
+    '<div class="postTitle"><h2><a href="/abss/encoded-book-title/" rel="bookmark">'
+    "Encoded Book Title - Encoded Author</a></h2></div>"
+    '<div class="postInfo">Category: Crime&nbsp; <br>Language: English'
+    '<span style="margin-left:100px;">Keywords: Test&nbsp;</span><br></div>'
+    '<div class="postContent">'
+    '<div class="center">'
+    '<p class="center"><a href="https://audiobookbay.lu/abss/encoded-book-title/">'
+    '<img src="https://example.com/encoded-cover.jpg" alt="Cover" width="250"></a></p>'
+    "</div>"
+    '<p style="text-align:center;">Posted: 12 Jul 2026<br>'
+    'Format: <span style="color:#a00;">M4B</span> / '
+    'Bitrate: <span style="color:#a00;">64 Kbps</span><br>'
+    'File Size: <span style="color:#00f;">1.55</span> GBs</p>'
+    "</div>"
+)
+
+_BASE64_PAYLOAD = base64.b64encode(_POST_INNER_HTML.encode("utf-8")).decode("ascii")
+
+PLAIN_POST_SEARCH_HTML = f'<html><body><div class="post">{_POST_INNER_HTML}</div></body></html>'
+
+BASE64_SEARCH_HTML = f'<html><body><div class="post re-ab" style="display:none;">{_BASE64_PAYLOAD}</div></body></html>'
+
+# .post elements whose content is neither valid base64 nor classic markup.
+UNPARSEABLE_SEARCH_HTML = """
+<html>
+<body>
+<div class="post re-ab" style="display:none;">this is not base64 and has no postTitle markup!</div>
+<div class="post re-ab" style="display:none;">another opaque blob without any known structure</div>
+</body>
+</html>
+"""
+
 # Mock HTML for detail page with info hash and trackers
 SAMPLE_DETAIL_HTML = """
 <html>
@@ -381,6 +417,56 @@ class TestSearchAudiobookbay:
         )
 
 
+class TestSearchAudiobookbayBase64Posts:
+    """Tests for ABB's base64-encoded post markup."""
+
+    @staticmethod
+    def _search(html):
+        with (
+            patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page") as get,
+            patch("shelfmark.release_sources.audiobookbay.scraper.config.get") as cfg,
+        ):
+            cfg.return_value = 0.0
+            get.return_value = (html, "https://audiobookbay.lu/?s=test&cat=undefined%2Cundefined")
+            return scraper.search_audiobookbay("test", max_pages=1, hostname="audiobookbay.lu")
+
+    def test_base64_page_parses_same_posts_as_plain_page(self):
+        plain = self._search(PLAIN_POST_SEARCH_HTML)
+        encoded = self._search(BASE64_SEARCH_HTML)
+
+        assert len(encoded) == 1
+        assert encoded == plain
+        assert encoded[0]["title"] == "Encoded Book Title - Encoded Author"
+        assert encoded[0]["link"] == "https://audiobookbay.lu/abss/encoded-book-title/"
+        assert encoded[0]["format"] == "M4B"
+        assert encoded[0]["bitrate"] == "64 Kbps"
+        assert encoded[0]["size"] == "1.55 GB"
+        assert encoded[0]["posted_date"] == "12 Jul 2026"
+        assert encoded[0]["cover"] == "https://example.com/encoded-cover.jpg"
+
+    def test_warns_when_posts_found_but_none_parsed(self):
+        # setup_logger builds parentless loggers (no propagation), so caplog
+        # can't see them; spy on the module logger instead.
+        with patch.object(scraper.logger, "warning") as mock_warning:
+            results = self._search(UNPARSEABLE_SEARCH_HTML)
+
+        assert results == []
+        warning_texts = [
+            (call.args[0] % tuple(call.args[1:])) if len(call.args) > 1 else call.args[0]
+            for call in mock_warning.call_args_list
+        ]
+        assert any("2 .post elements" in t and "parsed 0" in t for t in warning_texts), (
+            warning_texts
+        )
+
+    def test_no_warning_when_posts_parse(self):
+        with patch.object(scraper.logger, "warning") as mock_warning:
+            results = self._search(SAMPLE_SEARCH_HTML)
+
+        assert results
+        mock_warning.assert_not_called()
+
+
 class TestExtractMagnetLink:
     """Tests for the extract_magnet_link function."""