diff --git a/shelfmark/release_sources/audiobookbay/scraper.py b/shelfmark/release_sources/audiobookbay/scraper.py index 1faeb4a4..217fe7d3 100644 --- a/shelfmark/release_sources/audiobookbay/scraper.py +++ b/shelfmark/release_sources/audiobookbay/scraper.py @@ -2,7 +2,7 @@ import re import time -from urllib.parse import quote +from urllib.parse import quote, quote_plus import requests from bs4 import BeautifulSoup @@ -10,6 +10,7 @@ from bs4 import BeautifulSoup from shelfmark.core.config import config from shelfmark.core.logger import setup_logger from shelfmark.download import http as downloader +from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation logger = setup_logger(__name__) @@ -98,8 +99,10 @@ def _encode_search_query(query: str, *, exact_phrase: bool) -> str: and not (search_query.startswith('"') and search_query.endswith('"')) ): search_query = f'"{search_query}"' - # Keep ABB-friendly encoding style (spaces as '+') while percent-encoding quotes. - return search_query.replace('"', "%22").replace(" ", "+") + # Keep ABB's space-as-'+' style, but percent-encode everything else: a bare + # '&' would otherwise start a new query parameter, '%' would open an invalid + # escape, and a literal '+' would arrive as a space. + return quote_plus(search_query) def _normalize_result_url(url: str, hostname: str) -> str: @@ -153,6 +156,9 @@ def search_audiobookbay( """ results = [] + # ABB matches the stored, untexturized title, so a curly apostrophe reaching + # the search returns nothing at all rather than merely ranking worse. + query = normalize_search_punctuation(query) rate_limit_delay = _coerce_non_negative_float(config.get("ABB_RATE_LIMIT_DELAY", 1.0), 1.0) session = requests.Session() diff --git a/shelfmark/release_sources/audiobookbay/source.py b/shelfmark/release_sources/audiobookbay/source.py index c672c34a..0c48dc4c 100644 --- a/shelfmark/release_sources/audiobookbay/source.py +++ b/shelfmark/release_sources/audiobookbay/source.py @@ -23,7 +23,11 @@ from shelfmark.release_sources import ( register_source, ) from shelfmark.release_sources.audiobookbay import scraper -from shelfmark.release_sources.audiobookbay.utils import normalize_hostname, parse_size +from shelfmark.release_sources.audiobookbay.utils import ( + normalize_hostname, + normalize_search_punctuation, + parse_size, +) logger = setup_logger(__name__) MIN_RELEVANCE_QUERY_WORD_LENGTH = 2 @@ -227,10 +231,12 @@ class AudiobookBaySource(ReleaseSource): deduped_queries[index + 1].lower(), ) - # Extract query words for relevance checking + # Extract query words for relevance checking. Both sides of the + # comparison are punctuation-normalized: scraped titles carry the + # typographic forms WordPress renders, queries carry the ASCII ones. query_words = { word.lower() - for word in query_lower.split() + for word in normalize_search_punctuation(query_lower).split() if len(word) > MIN_RELEVANCE_QUERY_WORD_LENGTH } @@ -239,7 +245,7 @@ class AudiobookBaySource(ReleaseSource): try: raw_title = result["title"] title, author = _split_title_and_author(raw_title) - title_for_filter = raw_title.lower() + title_for_filter = normalize_search_punctuation(raw_title).lower() # Basic relevance check: ensure title contains at least one query word # This filters out homepage "Latest" feed items that may leak through diff --git a/shelfmark/release_sources/audiobookbay/utils.py b/shelfmark/release_sources/audiobookbay/utils.py index 58719fc4..ea49f5ce 100644 --- a/shelfmark/release_sources/audiobookbay/utils.py +++ b/shelfmark/release_sources/audiobookbay/utils.py @@ -2,6 +2,63 @@ import re +# WordPress texturizes punctuation on output only: a post stored as "The +# Stranger's Wife" is rendered as "The Stranger’s Wife". ABB's search matches the +# stored value, so a query carrying the typographic form matches nothing -- and +# because ABB ANDs its search terms, one such term empties the entire result set. +# Book metadata and phone keyboards both hand us the typographic forms, so map +# them back before they reach a search or a title comparison. +_ASCII_PUNCTUATION = str.maketrans( + { + # Single quotes + "‘": "'", # left single quotation mark + "’": "'", # right single quotation mark + "‚": "'", # single low-9 quotation mark + "‛": "'", # single high-reversed-9 quotation mark + "′": "'", # prime + "´": "'", # acute accent + "`": "'", # grave accent + # Double quotes + "“": '"', # left double quotation mark + "”": '"', # right double quotation mark + "„": '"', # double low-9 quotation mark + "‟": '"', # double high-reversed-9 quotation mark + "″": '"', # double prime + # Dashes + "‐": "-", # hyphen + "‑": "-", # non-breaking hyphen + "‒": "-", # figure dash + "–": "-", # en dash + "—": "-", # em dash + "―": "-", # horizontal bar + "−": "-", # minus sign + "﹘": "-", # small em dash + "﹣": "-", # small hyphen-minus + "－": "-", # fullwidth hyphen-minus + # Ellipsis + "…": "...", # horizontal ellipsis + } +) + + +def normalize_search_punctuation(text: str) -> str: + """Replace typographic punctuation with the ASCII forms ABB stores. + + Each character is mapped individually rather than collapsing runs, so an + ASCII "--" is left alone: only characters ABB cannot have stored are + rewritten. + + Args: + text: A search query, or a scraped title being compared against one. + + Returns: + The text with curly quotes, dashes and ellipses mapped to ASCII. + + """ + if not text: + return text + return text.translate(_ASCII_PUNCTUATION) + def normalize_hostname(raw: str | None) -> str: """Normalize a user-supplied hostname for URL construction. diff --git a/tests/audiobookbay/test_scraper.py b/tests/audiobookbay/test_scraper.py index 074b1bc1..5d111e8e 100644 --- a/tests/audiobookbay/test_scraper.py +++ b/tests/audiobookbay/test_scraper.py @@ -317,6 +317,44 @@ class TestSearchAudiobookbay: assert "s=%22test+query%22" in requested_url assert "cat=undefined%2Cundefined" in requested_url + @patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page") + @patch("shelfmark.release_sources.audiobookbay.scraper.config.get") + def test_search_audiobookbay_normalizes_curly_apostrophe(self, mock_config_get, mock_html_get): + """Test curly apostrophes are searched as the ASCII form ABB stores.""" + mock_config_get.return_value = 0.0 + mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x") + + scraper.search_audiobookbay( + "the stranger’s wife", + max_pages=1, + hostname="audiobookbay.lu", + ) + + requested_url = mock_html_get.call_args.args[0] + assert "s=the+stranger%27s+wife" in requested_url + assert "%E2%80%99" not in requested_url + + @patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page") + @patch("shelfmark.release_sources.audiobookbay.scraper.config.get") + def test_search_audiobookbay_percent_encodes_reserved_characters( + self, mock_config_get, mock_html_get + ): + """Test reserved characters cannot break out of the search parameter.""" + mock_config_get.return_value = 0.0 + mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x") + + scraper.search_audiobookbay( + "sense & sensibility 100% c++", + max_pages=1, + hostname="audiobookbay.lu", + ) + + requested_url = mock_html_get.call_args.args[0] + assert "s=sense+%26+sensibility+100%25+c%2B%2B" in requested_url + # The only surviving '&' introduces the legacy category parameter. + assert requested_url.count("&") == 1 + assert requested_url.endswith("&cat=undefined%2Cundefined") + @patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page") @patch("shelfmark.release_sources.audiobookbay.scraper.config.get") def test_search_audiobookbay_always_uses_legacy_category_query( diff --git a/tests/audiobookbay/test_source.py b/tests/audiobookbay/test_source.py index 3b227f68..81cb9326 100644 --- a/tests/audiobookbay/test_source.py +++ b/tests/audiobookbay/test_source.py @@ -306,6 +306,42 @@ class TestAudiobookBaySource: assert len(results) == 1 assert results[0].title == "Test Book by Test Author" + @patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay") + def test_search_relevance_filtering_spans_typographic_punctuation(self, mock_search): + """Test an ASCII query still matches the typographic title ABB renders.""" + mock_search.return_value = [ + { + "title": "The Stranger’s Wife — Anna‑Lou Weatherley", + "link": "https://audiobookbay.lu/abss/the-strangers-wife/", + "format": "M4B", + "size": "259 MB", + "language": "English", + }, + ] + + source = AudiobookBaySource() + book = BookMetadata( + provider="test", + provider_id="123", + title="Stranger's", + authors=["Anna-Lou Weatherley"], + ) + # Every query word carries punctuation, so the result survives only when + # both sides of the comparison are normalized. + plan = ReleaseSearchPlan( + languages=["en"], + isbn_candidates=[], + author="", + title_variants=[ReleaseSearchVariant(title="Stranger's", author="")], + grouped_title_variants=[], + ) + + results = source.search(book, plan, content_type="audiobook") + + assert len(results) == 1 + # The release keeps the title as ABB rendered it; only matching normalizes. + assert results[0].title == "The Stranger’s Wife — Anna‑Lou Weatherley" + @patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay") def test_search_result_mapping(self, mock_search): """Test conversion of scraper results to Release objects.""" diff --git a/tests/audiobookbay/test_utils.py b/tests/audiobookbay/test_utils.py index 38a72aba..d4be76d7 100644 --- a/tests/audiobookbay/test_utils.py +++ b/tests/audiobookbay/test_utils.py @@ -2,7 +2,52 @@ Tests for AudiobookBay utility functions. """ -from shelfmark.release_sources.audiobookbay.utils import parse_size +from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation, parse_size + + +class TestNormalizeSearchPunctuation: + """Tests for the normalize_search_punctuation function.""" + + def test_curly_apostrophe_becomes_ascii(self): + """ABB matches the stored ASCII apostrophe, not the rendered curly one.""" + assert normalize_search_punctuation("The Stranger’s Wife") == "The Stranger's Wife" + + def test_all_single_quote_variants(self): + """Every single-quote lookalike collapses to the ASCII apostrophe.""" + for variant in ("‘", "’", "‚", "‛", "′", "´", "`"): + assert normalize_search_punctuation(f"don{variant}t") == "don't" + + def test_all_double_quote_variants(self): + """Every double-quote lookalike collapses to the ASCII double quote.""" + for variant in ("“", "”", "„", "‟", "″"): + assert normalize_search_punctuation(f"{variant}quoted{variant}") == '"quoted"' + + def test_all_dash_variants(self): + """Every dash lookalike collapses to the ASCII hyphen.""" + for variant in ("‐", "‑", "‒", "–", "—", "―", "−", "﹘", "﹣", "－"): + assert normalize_search_punctuation(f"anna{variant}lou") == "anna-lou" + + def test_ellipsis_expands_to_three_dots(self): + """WordPress renders '...' as a single ellipsis character.""" + assert normalize_search_punctuation("And Then…") == "And Then..." + + def test_ascii_query_is_unchanged(self): + """A query that is already ASCII passes through untouched.""" + assert normalize_search_punctuation("The Stranger's Wife") == "The Stranger's Wife" + + def test_ascii_dash_runs_are_not_collapsed(self): + """Only characters ABB cannot have stored are rewritten.""" + assert normalize_search_punctuation("Book -- Subtitle") == "Book -- Subtitle" + + def test_other_punctuation_is_preserved(self): + """Colons and commas carry search signal and are left alone.""" + assert normalize_search_punctuation("Weatherley: Book 3, Part 1") == ( + "Weatherley: Book 3, Part 1" + ) + + def test_empty_query(self): + """An empty query is returned as-is.""" + assert normalize_search_punctuation("") == "" class TestParseSize: