mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 22:05:20 +01:00
FlyinPancakeandCopilot Autofix powered by AI
3a9cff9816
First of all, I don't know if you even want to merge a scraper-based metadata provider. I made this just for my use-case. If you'd rather not, I completely understand it. An alternative would be adopting [Audiobookshelf's Metadata Provider API](https://audiobookshelf.org/docs/documentation/community/community-providers) which I contributed to it for exactly the reason to not have scrapers. ## What Adds [Moly.hu](https://moly.hu) — the Hungarian community book catalog — as a metadata provider, following the existing provider plugin architecture (`@register_provider` + settings tab with enable checkbox and Test Connection button, disabled by default). ## Why None of the current providers cover Hungarian editions well: Hardcover and Open Library rarely index them, and Google Books coverage is spotty. Moly.hu is the de-facto catalog for Hungarian books (local editions *and* Hungarian translations of foreign works). With this provider, Universal mode works end-to-end for Hungarian titles: moly search → localized title/author feed the release search → indexers that carry Hungarian content can actually match. Related pain points: #595 (books missing from metadata providers), #1035 (interest in niche sources). ## How - HTML scraping with BeautifulSoup (already a dependency), no API key needed - Scraping approach (search URL, page structure, language-tag mapping) adapted from the long-lived Calibre `Moly_hu` plugin (GPL v3, credited in the module docstring), with fallback selector chains inherited from it - Sliding-window rate limit (30 req/min) to stay polite to a small community site - Standard `@cacheable` decorators; fetch failures return `None` so they are not cached (same behavior as the Google Books provider) - Search results carry cover thumbnails, rating and series info as display fields; `get_book` parses title (zero-width chars stripped, nested series link excluded), authors, ISBN-13/10, publisher, publish year, description (spoiler-warning prefix stripped), tags/genres, cover, and language (from moly's language tags, defaulting to `hu`) - ISBN search resolves through moly's site search ## Testing - `tests/metadata/test_moly_parse.py`: offline tests with fixture HTML mirroring live moly.hu markup — search parsing/dedup, pagination guard, failure-not-cached behavior, book-page parsing, ISBN resolution, ISBN validation helper - `uv run pytest tests/metadata` green (45 passed), `ruff check` / `ruff format` clean - Verified live against moly.hu (search, get_book, ISBN lookup) and running in Docker alongside Hardcover --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
203 lines
7.1 KiB
Python
203 lines
7.1 KiB
Python
import requests
|
||
|
||
from shelfmark.core.cache import get_metadata_cache
|
||
from shelfmark.metadata_providers import MetadataSearchOptions, SearchType
|
||
from shelfmark.metadata_providers.moly import MolyProvider, _valid_isbn
|
||
|
||
SEARCH_HTML = """
|
||
<html><body>
|
||
<div id="content">
|
||
<div class="search_area">
|
||
<div class="book_with_shop">
|
||
<a href="/konyvek/mocsidzuki-mai-a-telihold-kavezo">
|
||
<img alt="Mocsidzuki Mai: A Telihold kávézó" class="tooltip"
|
||
src="https://assets.moly.hu/system/covers/normal/covers_864003.jpg" />
|
||
</a>
|
||
</div>
|
||
<p>
|
||
<a class="book_selector" data-id="1"
|
||
href="/konyvek/mocsidzuki-mai-a-telihold-kavezo">Mocsidzuki Mai:
|
||
<strong class="highlight">A Telihold</strong> kávézó</a>
|
||
<span class="like_count">82%</span>
|
||
<a rel="modal" class="action" href="/sorozatok/a-telihold-kavezo">(A Telihold kávézó 1.)</a>
|
||
</p>
|
||
<p>
|
||
<a class="book_selector" data-id="2"
|
||
href="/konyvek/mocsidzuki-mai-a-telihold-kavezo">Mocsidzuki Mai: A Telihold kávézó</a>
|
||
</p>
|
||
<p>
|
||
<a class="book_selector" data-id="3"
|
||
href="/konyvek/mocsidzuki-mai-az-igazi-kivansag">Mocsidzuki Mai: Az igazi kívánság</a>
|
||
</p>
|
||
<p>
|
||
<a class="book_selector" data-id="4"
|
||
href="/konyvek/lisa-jewell-a-fold-nyelte-el">Lis<strong class="highlight">a</strong> Jew<strong class="highlight">el</strong>l: <strong class="highlight">A föld nyelte el</strong></a>
|
||
</p>
|
||
<p><a class="book_selector" data-id="5" href="/konyvek/broken">Not a book row</a></p>
|
||
</div>
|
||
</div>
|
||
</body></html>
|
||
"""
|
||
|
||
BOOK_HTML = """
|
||
<html><body>
|
||
<div id="content">
|
||
<div class="head_title">
|
||
<h1><span class="item">A Telihold kávézó
|
||
<a href="/sorozatok/a-telihold-kavezo">(A Telihold kávézó 1.)</a></span></h1>
|
||
<div class="rating"><span class="like_count">82%</span></div>
|
||
</div>
|
||
<div class="authors"><a href="/alkotok/mocsidzuki-mai">Mocsidzuki Mai</a></div>
|
||
<div class="coverbox">
|
||
<a rel="light" class="zoom" href="/system/covers/big/covers_864003.jpg?1712583436"></a>
|
||
</div>
|
||
<div id="full_description">
|
||
<p>Vigyázat! Cselekményleírást tartalmaz.</p>
|
||
<p>A japánok úgy tartják, hogy ha gondoskodsz egy macskáról,
|
||
az egy napon meghálálja.</p>
|
||
</div>
|
||
<div class="items">
|
||
<div class="edition">
|
||
<div><a href="/kiadok/athenaeum">Athenaeum</a>, Budapest, <abbr>2024</abbr></div>
|
||
<div>180 oldal · ISBN: 9789635434435 · Fordította: Nagy Anita</div>
|
||
</div>
|
||
</div>
|
||
<div id="book_tags">
|
||
<a class="tag" href="/cimkek/asztrologia">asztrológia</a>
|
||
<a class="tag" href="/cimkek/japan">japán</a>
|
||
</div>
|
||
</div>
|
||
</body></html>
|
||
"""
|
||
|
||
|
||
class _MolyResponse:
|
||
def __init__(self, text):
|
||
self.text = text
|
||
|
||
def raise_for_status(self):
|
||
return None
|
||
|
||
|
||
class _MolySession:
|
||
"""Serves canned HTML per URL substring."""
|
||
|
||
def __init__(self, pages):
|
||
self.pages = pages
|
||
self.calls = 0
|
||
self.headers = {}
|
||
|
||
def get(self, url, **kwargs):
|
||
self.calls += 1
|
||
for fragment, html in self.pages.items():
|
||
if fragment in url:
|
||
return _MolyResponse(html)
|
||
raise requests.HTTPError(f"unexpected URL: {url}")
|
||
|
||
|
||
class _FlakyMolySession(_MolySession):
|
||
def get(self, url, **kwargs):
|
||
if self.calls == 0:
|
||
self.calls += 1
|
||
raise requests.Timeout
|
||
return super().get(url, **kwargs)
|
||
|
||
|
||
def _provider(session):
|
||
provider = MolyProvider()
|
||
provider.session = session
|
||
return provider
|
||
|
||
|
||
class TestMolySearch:
|
||
def test_search_parses_results(self):
|
||
get_metadata_cache().clear()
|
||
provider = _provider(_MolySession({"/kereses": SEARCH_HTML}))
|
||
|
||
books = provider.search(MetadataSearchOptions(query="telihold kávézó"))
|
||
|
||
assert len(books) == 3 # duplicate slug deduped, colon-less row skipped
|
||
|
||
# Mid-word <strong class="highlight"> wrapping must not split words
|
||
highlighted = books[2]
|
||
assert highlighted.title == "A föld nyelte el"
|
||
assert highlighted.authors == ["Lisa Jewell"]
|
||
book = books[0]
|
||
assert book.provider == "moly"
|
||
assert book.provider_id == "mocsidzuki-mai-a-telihold-kavezo"
|
||
assert book.title == "A Telihold kávézó"
|
||
assert book.authors == ["Mocsidzuki Mai"]
|
||
assert book.cover_url == "https://assets.moly.hu/system/covers/normal/covers_864003.jpg"
|
||
assert book.source_url == "https://moly.hu/konyvek/mocsidzuki-mai-a-telihold-kavezo"
|
||
labels = {f.label: f.value for f in book.display_fields}
|
||
assert labels["Rating"] == "82%"
|
||
assert labels["Series"] == "A Telihold kávézó 1."
|
||
|
||
def test_search_second_page_is_empty(self):
|
||
provider = _provider(_MolySession({"/kereses": SEARCH_HTML}))
|
||
|
||
assert provider.search(MetadataSearchOptions(query="telihold", page=2)) == []
|
||
|
||
def test_search_does_not_cache_request_failures(self):
|
||
get_metadata_cache().clear()
|
||
session = _FlakyMolySession({"/kereses": SEARCH_HTML})
|
||
provider = _provider(session)
|
||
options = MetadataSearchOptions(query="telihold kávézó")
|
||
|
||
assert provider.search(options) == []
|
||
|
||
books = provider.search(options)
|
||
|
||
assert session.calls == 2
|
||
assert [book.provider_id for book in books] == [
|
||
"mocsidzuki-mai-a-telihold-kavezo",
|
||
"mocsidzuki-mai-az-igazi-kivansag",
|
||
"lisa-jewell-a-fold-nyelte-el",
|
||
]
|
||
|
||
|
||
class TestMolyGetBook:
|
||
def test_get_book_parses_details(self):
|
||
get_metadata_cache().clear()
|
||
provider = _provider(_MolySession({"/konyvek/": BOOK_HTML}))
|
||
|
||
book = provider.get_book("mocsidzuki-mai-a-telihold-kavezo")
|
||
|
||
assert book is not None
|
||
# Zero-width space stripped, nested series link excluded from title
|
||
assert book.title == "A Telihold kávézó"
|
||
assert book.authors == ["Mocsidzuki Mai"]
|
||
assert book.isbn_13 == "9789635434435"
|
||
assert book.publisher == "Athenaeum"
|
||
assert book.publish_year == 2024
|
||
assert book.language == "hu"
|
||
assert book.cover_url == "https://moly.hu/system/covers/big/covers_864003.jpg?1712583436"
|
||
assert "asztrológia" in book.genres
|
||
# Spoiler warning stripped from the description
|
||
assert book.description is not None
|
||
assert not book.description.startswith("Vigyázat!")
|
||
assert "japánok" in book.description
|
||
|
||
def test_isbn_search_resolves_first_result(self):
|
||
get_metadata_cache().clear()
|
||
provider = _provider(_MolySession({"/kereses": SEARCH_HTML, "/konyvek/": BOOK_HTML}))
|
||
|
||
book = provider.search(
|
||
MetadataSearchOptions(query="9789635434435", search_type=SearchType.ISBN)
|
||
)[0]
|
||
|
||
assert book.provider_id == "mocsidzuki-mai-a-telihold-kavezo"
|
||
assert book.isbn_13 == "9789635434435"
|
||
|
||
|
||
class TestValidIsbn:
|
||
def test_isbn_13(self):
|
||
assert _valid_isbn("978-963-543-443-5") == "9789635434435"
|
||
|
||
def test_isbn_10_with_check_x(self):
|
||
assert _valid_isbn("963-543-443-X") == "963543443X"
|
||
|
||
def test_rejects_page_counts_and_years(self):
|
||
assert _valid_isbn("2024") is None
|
||
assert _valid_isbn("1234567") is None
|