fix(audiobookbay): search the ASCII punctuation ABB actually stores (#1242)

WordPress texturizes punctuation on output only, so a post stored as
"The
Stranger's Wife" renders as "The Stranger’s Wife". ABB's search matches
the
stored value and ANDs its terms, so one typographic character in the
query
empties the entire result set rather than merely ranking worse. Book
metadata
and mobile keyboards both hand us those characters.

Map curly quotes, dashes and ellipses to ASCII before a query goes out,
and on
both sides of the relevance comparison, since scraped titles carry the
rendered
forms. Release titles are still stored and displayed exactly as ABB
renders
them; only matching normalizes.

Also percent-encode the search query properly. The hand-rolled encoder
only
escaped double quotes and spaces, so a bare "&" started a new query
parameter
and silently truncated the search: "detective dan riley books 1 & 2
weatherley"
reached ABB as "detective dan riley books 1" and returned six
confident-looking
results without the requested book among them. "%" and "+" were mangled
too.

shelfmark/release_sources/audiobookbay/scraper.py

@@ -2,7 +2,7 @@
 
 import re
 import time
-from urllib.parse import quote
+from urllib.parse import quote, quote_plus
 
 import requests
 from bs4 import BeautifulSoup
@@ -10,6 +10,7 @@ from bs4 import BeautifulSoup
 from shelfmark.core.config import config
 from shelfmark.core.logger import setup_logger
 from shelfmark.download import http as downloader
+from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation
 
 logger = setup_logger(__name__)
 
@@ -98,8 +99,10 @@ def _encode_search_query(query: str, *, exact_phrase: bool) -> str:
         and not (search_query.startswith('"') and search_query.endswith('"'))
     ):
         search_query = f'"{search_query}"'
-    # Keep ABB-friendly encoding style (spaces as '+') while percent-encoding quotes.
-    return search_query.replace('"', "%22").replace(" ", "+")
+    # Keep ABB's space-as-'+' style, but percent-encode everything else: a bare
+    # '&' would otherwise start a new query parameter, '%' would open an invalid
+    # escape, and a literal '+' would arrive as a space.
+    return quote_plus(search_query)
 
 
 def _normalize_result_url(url: str, hostname: str) -> str:
@@ -153,6 +156,9 @@ def search_audiobookbay(
 
     """
     results = []
+    # ABB matches the stored, untexturized title, so a curly apostrophe reaching
+    # the search returns nothing at all rather than merely ranking worse.
+    query = normalize_search_punctuation(query)
     rate_limit_delay = _coerce_non_negative_float(config.get("ABB_RATE_LIMIT_DELAY", 1.0), 1.0)
     session = requests.Session()
 

shelfmark/release_sources/audiobookbay/source.py

@@ -23,7 +23,11 @@ from shelfmark.release_sources import (
     register_source,
 )
 from shelfmark.release_sources.audiobookbay import scraper
-from shelfmark.release_sources.audiobookbay.utils import normalize_hostname, parse_size
+from shelfmark.release_sources.audiobookbay.utils import (
+    normalize_hostname,
+    normalize_search_punctuation,
+    parse_size,
+)
 
 logger = setup_logger(__name__)
 MIN_RELEVANCE_QUERY_WORD_LENGTH = 2
@@ -227,10 +231,12 @@ class AudiobookBaySource(ReleaseSource):
                         deduped_queries[index + 1].lower(),
                     )
 
-            # Extract query words for relevance checking
+            # Extract query words for relevance checking. Both sides of the
+            # comparison are punctuation-normalized: scraped titles carry the
+            # typographic forms WordPress renders, queries carry the ASCII ones.
             query_words = {
                 word.lower()
-                for word in query_lower.split()
+                for word in normalize_search_punctuation(query_lower).split()
                 if len(word) > MIN_RELEVANCE_QUERY_WORD_LENGTH
             }
 
@@ -239,7 +245,7 @@ class AudiobookBaySource(ReleaseSource):
                 try:
                     raw_title = result["title"]
                     title, author = _split_title_and_author(raw_title)
-                    title_for_filter = raw_title.lower()
+                    title_for_filter = normalize_search_punctuation(raw_title).lower()
 
                     # Basic relevance check: ensure title contains at least one query word
                     # This filters out homepage "Latest" feed items that may leak through

shelfmark/release_sources/audiobookbay/utils.py

@@ -2,6 +2,63 @@
 
 import re
 
+# WordPress texturizes punctuation on output only: a post stored as "The
+# Stranger's Wife" is rendered as "The Stranger’s Wife". ABB's search matches the
+# stored value, so a query carrying the typographic form matches nothing -- and
+# because ABB ANDs its search terms, one such term empties the entire result set.
+# Book metadata and phone keyboards both hand us the typographic forms, so map
+# them back before they reach a search or a title comparison.
+_ASCII_PUNCTUATION = str.maketrans(
+    {
+        # Single quotes
+        "‘": "'",  # left single quotation mark
+        "’": "'",  # right single quotation mark
+        "‚": "'",  # single low-9 quotation mark
+        "‛": "'",  # single high-reversed-9 quotation mark
+        "′": "'",  # prime
+        "´": "'",  # acute accent
+        "`": "'",  # grave accent
+        # Double quotes
+        "“": '"',  # left double quotation mark
+        "”": '"',  # right double quotation mark
+        "„": '"',  # double low-9 quotation mark
+        "‟": '"',  # double high-reversed-9 quotation mark
+        "″": '"',  # double prime
+        # Dashes
+        "‐": "-",  # hyphen
+        "‑": "-",  # non-breaking hyphen
+        "‒": "-",  # figure dash
+        "–": "-",  # en dash
+        "—": "-",  # em dash
+        "―": "-",  # horizontal bar
+        "−": "-",  # minus sign
+        "﹘": "-",  # small em dash
+        "﹣": "-",  # small hyphen-minus
+        "－": "-",  # fullwidth hyphen-minus
+        # Ellipsis
+        "…": "...",  # horizontal ellipsis
+    }
+)
+
+
+def normalize_search_punctuation(text: str) -> str:
+    """Replace typographic punctuation with the ASCII forms ABB stores.
+
+    Each character is mapped individually rather than collapsing runs, so an
+    ASCII "--" is left alone: only characters ABB cannot have stored are
+    rewritten.
+
+    Args:
+        text: A search query, or a scraped title being compared against one.
+
+    Returns:
+        The text with curly quotes, dashes and ellipses mapped to ASCII.
+
+    """
+    if not text:
+        return text
+    return text.translate(_ASCII_PUNCTUATION)
+
 
 def normalize_hostname(raw: str | None) -> str:
     """Normalize a user-supplied hostname for URL construction.

tests/audiobookbay/test_scraper.py

@@ -317,6 +317,44 @@ class TestSearchAudiobookbay:
         assert "s=%22test+query%22" in requested_url
         assert "cat=undefined%2Cundefined" in requested_url
 
+    @patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
+    @patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
+    def test_search_audiobookbay_normalizes_curly_apostrophe(self, mock_config_get, mock_html_get):
+        """Test curly apostrophes are searched as the ASCII form ABB stores."""
+        mock_config_get.return_value = 0.0
+        mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
+
+        scraper.search_audiobookbay(
+            "the stranger’s wife",
+            max_pages=1,
+            hostname="audiobookbay.lu",
+        )
+
+        requested_url = mock_html_get.call_args.args[0]
+        assert "s=the+stranger%27s+wife" in requested_url
+        assert "%E2%80%99" not in requested_url
+
+    @patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
+    @patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
+    def test_search_audiobookbay_percent_encodes_reserved_characters(
+        self, mock_config_get, mock_html_get
+    ):
+        """Test reserved characters cannot break out of the search parameter."""
+        mock_config_get.return_value = 0.0
+        mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
+
+        scraper.search_audiobookbay(
+            "sense & sensibility 100% c++",
+            max_pages=1,
+            hostname="audiobookbay.lu",
+        )
+
+        requested_url = mock_html_get.call_args.args[0]
+        assert "s=sense+%26+sensibility+100%25+c%2B%2B" in requested_url
+        # The only surviving '&' introduces the legacy category parameter.
+        assert requested_url.count("&") == 1
+        assert requested_url.endswith("&cat=undefined%2Cundefined")
+
     @patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
     @patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
     def test_search_audiobookbay_always_uses_legacy_category_query(

tests/audiobookbay/test_source.py

@@ -306,6 +306,42 @@ class TestAudiobookBaySource:
         assert len(results) == 1
         assert results[0].title == "Test Book by Test Author"
 
+    @patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
+    def test_search_relevance_filtering_spans_typographic_punctuation(self, mock_search):
+        """Test an ASCII query still matches the typographic title ABB renders."""
+        mock_search.return_value = [
+            {
+                "title": "The Stranger’s Wife — Anna‑Lou Weatherley",
+                "link": "https://audiobookbay.lu/abss/the-strangers-wife/",
+                "format": "M4B",
+                "size": "259 MB",
+                "language": "English",
+            },
+        ]
+
+        source = AudiobookBaySource()
+        book = BookMetadata(
+            provider="test",
+            provider_id="123",
+            title="Stranger's",
+            authors=["Anna-Lou Weatherley"],
+        )
+        # Every query word carries punctuation, so the result survives only when
+        # both sides of the comparison are normalized.
+        plan = ReleaseSearchPlan(
+            languages=["en"],
+            isbn_candidates=[],
+            author="",
+            title_variants=[ReleaseSearchVariant(title="Stranger's", author="")],
+            grouped_title_variants=[],
+        )
+
+        results = source.search(book, plan, content_type="audiobook")
+
+        assert len(results) == 1
+        # The release keeps the title as ABB rendered it; only matching normalizes.
+        assert results[0].title == "The Stranger’s Wife — Anna‑Lou Weatherley"
+
     @patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
     def test_search_result_mapping(self, mock_search):
         """Test conversion of scraper results to Release objects."""

tests/audiobookbay/test_utils.py

@@ -2,7 +2,52 @@
 Tests for AudiobookBay utility functions.
 """
 
-from shelfmark.release_sources.audiobookbay.utils import parse_size
+from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation, parse_size
+
+
+class TestNormalizeSearchPunctuation:
+    """Tests for the normalize_search_punctuation function."""
+
+    def test_curly_apostrophe_becomes_ascii(self):
+        """ABB matches the stored ASCII apostrophe, not the rendered curly one."""
+        assert normalize_search_punctuation("The Stranger’s Wife") == "The Stranger's Wife"
+
+    def test_all_single_quote_variants(self):
+        """Every single-quote lookalike collapses to the ASCII apostrophe."""
+        for variant in ("‘", "’", "‚", "‛", "′", "´", "`"):
+            assert normalize_search_punctuation(f"don{variant}t") == "don't"
+
+    def test_all_double_quote_variants(self):
+        """Every double-quote lookalike collapses to the ASCII double quote."""
+        for variant in ("“", "”", "„", "‟", "″"):
+            assert normalize_search_punctuation(f"{variant}quoted{variant}") == '"quoted"'
+
+    def test_all_dash_variants(self):
+        """Every dash lookalike collapses to the ASCII hyphen."""
+        for variant in ("‐", "‑", "‒", "–", "—", "―", "−", "﹘", "﹣", "－"):
+            assert normalize_search_punctuation(f"anna{variant}lou") == "anna-lou"
+
+    def test_ellipsis_expands_to_three_dots(self):
+        """WordPress renders '...' as a single ellipsis character."""
+        assert normalize_search_punctuation("And Then…") == "And Then..."
+
+    def test_ascii_query_is_unchanged(self):
+        """A query that is already ASCII passes through untouched."""
+        assert normalize_search_punctuation("The Stranger's Wife") == "The Stranger's Wife"
+
+    def test_ascii_dash_runs_are_not_collapsed(self):
+        """Only characters ABB cannot have stored are rewritten."""
+        assert normalize_search_punctuation("Book -- Subtitle") == "Book -- Subtitle"
+
+    def test_other_punctuation_is_preserved(self):
+        """Colons and commas carry search signal and are left alone."""
+        assert normalize_search_punctuation("Weatherley: Book 3, Part 1") == (
+            "Weatherley: Book 3, Part 1"
+        )
+
+    def test_empty_query(self):
+        """An empty query is returned as-is."""
+        assert normalize_search_punctuation("") == ""
 
 
 class TestParseSize: