From 86fff322976699dc6c27c23fdf8f413dfd00be76 Mon Sep 17 00:00:00 2001 From: splitsec2 <35583321+splitsec2@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:08:00 -0600 Subject: [PATCH] feat(hardcover): owned books, simpler queries, dead sources and failed releases (#1535) A request whose book turns out to be in the library is closed with the note `[auto] Already in the library.` instead of sitting in Needs Review. A request an admin handled by hand while a pass was running is skipped. A series label in front of a Calibre title no longer hides an owned book, and a Calibre title marked as another edition (full cast, dramatized, GraphicAudio) is not the book. When no source matches, up to three simpler forms of the title are tried on each source under the same strict match. Optionally (off by default), an unmatched audiobook request is looked up on Audible US and UK through Audiobookshelf's metadata search and the request says what the store answered. It never declines anything. A Soulseek peer whose recent transfers all failed, and an AudiobookBay post whose comments say it has no seeders, are tried after the other matches and never excluded. A failed or stalled release is kept out for AUTO_DOWNLOAD_RETRY_FAILED_RELEASE_DAYS and then tried again, and a torrent that died for one request is not offered to another. A request stuck on queued or downloading is settled from the download history. Optionally (off by default) a failed download is retried with another release within minutes. --- docs/hardcover-sync.md | 23 + docs/library-check.md | 2 + shelfmark/config/hardcover_sync_settings.py | 51 + shelfmark/core/auto_download.py | 743 ++++++++++-- shelfmark/core/existence_check.py | 286 +++++ shelfmark/core/existence_match.py | 273 +++++ shelfmark/core/hardcover_scheduler.py | 55 +- .../core/library_providers/audiobookshelf.py | 32 + shelfmark/core/library_providers/calibre.py | 20 +- shelfmark/core/requests_service.py | 12 + shelfmark/release_sources/__init__.py | 8 + .../release_sources/audiobookbay/handler.py | 12 + .../release_sources/audiobookbay/scraper.py | 20 + tests/audiobookbay/test_scraper.py | 29 + .../fixtures/audible_search/uk_hobbit.json | 283 +++++ .../audible_search/uk_young_zaphod.json | 1 + .../audible_search/us_advocates_dilemma.json | 34 + .../fixtures/audible_search/us_hobbit.json | 302 +++++ .../fixtures/audible_search/us_the_mummy.json | 39 + .../audible_search/us_young_zaphod.json | 1 + tests/core/test_auto_download.py | 1078 ++++++++++++++++- tests/core/test_existence_check.py | 432 +++++++ tests/core/test_hardcover_scheduler.py | 76 ++ .../test_library_provider_audiobookshelf.py | 82 ++ tests/core/test_library_provider_calibre.py | 141 ++- 25 files changed, 3948 insertions(+), 87 deletions(-) create mode 100644 shelfmark/core/existence_check.py create mode 100644 shelfmark/core/existence_match.py create mode 100644 tests/core/fixtures/audible_search/uk_hobbit.json create mode 100644 tests/core/fixtures/audible_search/uk_young_zaphod.json create mode 100644 tests/core/fixtures/audible_search/us_advocates_dilemma.json create mode 100644 tests/core/fixtures/audible_search/us_hobbit.json create mode 100644 tests/core/fixtures/audible_search/us_the_mummy.json create mode 100644 tests/core/fixtures/audible_search/us_young_zaphod.json create mode 100644 tests/core/test_existence_check.py diff --git a/docs/hardcover-sync.md b/docs/hardcover-sync.md index 4e5111c3..6ad84beb 100644 --- a/docs/hardcover-sync.md +++ b/docs/hardcover-sync.md @@ -33,11 +33,30 @@ A release is a strict match only when all of the following hold: AudiobookBay requires every word of a query, and its posts for one volume of a multi-part book often spell the part differently from the request ("Part Two" for "Part II"), so a query carrying the part finds only the packs that list it. When no row a query returns carries the requested title, the source tries the next query, ending with the title minus its trailing *, Part N* or *Volume N*. The strict match above still decides which part was found. +When no source yields a strict match for the request's own search title, up to three simpler titles are tried in turn, each across the sources in priority order and all within the request's search budget: the title without a subtitle after `:` or ` - `, the title without an alternate title (*The Hobbit, or There and Back Again* becomes *The Hobbit*), and the title with its numbers written as digits (*Part II* becomes *Part 2*). The first one that yields a strict match wins, and the log names it. A simpler title is matched by exactly the rules above, so a release has to name the shorter title, the right author, and no sequel, edition, derivative or pack. The subtitle is only dropped when the book's metadata says it is one (a series name is known, or the book's subtitle is what follows the separator), and never when what is left is only the series or author name: *Star Wars: Thrawn* is not searched as *Star Wars*. + Among matching releases from the winning source, format ranks first (`m4b` over `mp3` for audiobooks; `epub` over `azw3` over `mobi` for ebooks), then the copy other people chose — more downloads (direct-download sources report them), then more seeders — then larger size. If no source produces a confident match the request stays **pending** for manual review; nothing is guessed. +A request whose book turns out to be in your library already is closed, not left pending: it is marked fulfilled with the admin note `[auto] Already in the library.`, so it stops being looked up every pass. Before this it sat in Needs Review indefinitely. **Clear Completed** clears these too. + +Hardcover allows a short burst of lookups and then answers 429. Each pass paces its book lookups and backs off when told to, following `Retry-After`. A lookup that still fails counts as an error for that request, not as "book not found", and the request is tried again next pass. + A download that fails, or is interrupted (a restart while it was queued, say), does not stay that way forever. Once the request has sat there for **Retry failed downloads after (days)** (7 by default), it goes back to pending and the next pass searches again. Each pass reopens at most five, oldest first, so a backlog is spread out instead of hitting your sources at once. An interrupted download may pick the same release again, since a torrent already in your client is joined instead of added twice. A download that itself failed, or that Shelfmark's stall timer cancelled, has its release remembered and skipped next time, and the request is given up on after three failed retries. Requests you or an admin cancelled are never retried, and neither are rejected ones: both are decisions. Set the value to 0 to turn retries off. A request that no source has a match for is not searched every cycle for ever. After three passes in a row with no match it is searched once a day, then every two days, and no further apart than **Longest wait between searches for an unmatched request (days)** (3 by default, 0 searches every unmatched request every cycle). A match, a search that failed (which says nothing about the book) or **Sync now** starts it over. +**Trying another release sooner.** Set **Try another release after a failed download (minutes)** (off by default) and a download that fails, or that is given up on (stalled, or no seeders when **Give Up With No Seeders** is on), does not wait for the days above. Between sync cycles Shelfmark looks for such requests, reopens them after that many minutes and searches again, for those requests only. The release that failed is skipped, and a request is still given up on after three failed attempts. + +**Sources known to be dead are tried last.** When choosing among matching releases, a Soulseek peer whose transfers failed in the last three days and who delivered nothing in that time comes after the others. AudiobookBay shows no seed counts, so the comments on its five best posts are read, and a post where comments say it has no seeders comes after the others. Nothing is excluded: if a dead-looking release is the only match, it is still tried. + +### Audiobooks no store sells + +Some requested books were never recorded, so no source will ever match them and the request waits in Needs Review for good. With **Mark audiobooks no store sells** on, an audiobook request that has gone **Look the book up after (unmatched passes)** consecutive passes (3 by default) without a strict match is looked up on Audible's US and UK stores, through the metadata search of the Audiobookshelf server set up for the library check (its URL and token are needed; the library check itself can stay off). + +- **On neither store:** the request stays pending and its failure reason reads `Not found on Audible (US/UK) after searches, `. It is not declined: Audible search misses real recordings (novellas sold only in a collection, books retitled before release, older or fan recordings), so the call is left to you. The store is asked again a week later. +- **On a store:** the request stays pending and its failure reason reads `Exists on Audible (), not on our sources yet`. The store is asked again a week later, not every pass. +- **Lookup failed:** nothing changes and the next pass asks again. A failed lookup never marks a request. Audiobookshelf answers a failed Audible search with an empty list, which looks like "no such book", so before marking a request as not found Shelfmark also looks up a book both stores are known to sell (*The Hobbit*). If that comes back missing too, the answer is treated as a failure. + +A store result counts as the book when its title matches the request's once both are normalised (accents, punctuation, "Part II"/"Part Two"/"Part 2", a subtitle after `:` or `, or`, bracketed notes like "(Unabridged)" and a leading series label are all ignored, and a title that begins the other with at least two words also counts) and the request author's surname is in its author field. The matching leans towards "found": a wrong "found" only leaves the request pending. The pass counter is reset when a source matches the book and when the request is rejected, so a reopened request counts afresh. Store lookups come out of the request's search time; when too little is left, the lookup waits for the next pass. Ebook requests are not checked. ### Library check @@ -61,6 +80,10 @@ The check **fails closed** here: if an enabled library cannot be read and has no | `AUTO_DOWNLOAD_MIN_SEEDERS` | Minimum seeders (torrents) | 1 | Torrent releases below this are ignored | | `AUTO_DOWNLOAD_RETRY_DAYS` | Retry failed downloads after (days) | 7 | A failed or interrupted download goes back to pending after this long. 0 turns retries off | | `AUTO_DOWNLOAD_BACKOFF_MAX_DAYS` | Longest wait between searches for an unmatched request (days) | 3 | 0 searches every unmatched request every cycle | +| `AUTO_DOWNLOAD_FAILURE_RETRY_MINUTES` | Try another release after a failed download (minutes) | 0 (off) | Reopens failed or given-up downloads and picks another release this long after, between sync cycles | +| `AUTO_DOWNLOAD_RETRY_FAILED_RELEASE_DAYS` | Try a failed release again after (days) | 7 | A release that failed or stalled is kept out of the search for this long, then may be picked again; a request is still given up on after three failed attempts | +| `AUTO_DOWNLOAD_EXISTENCE_CHECK_ENABLED` | Mark audiobooks no store sells | Off | See [Audiobooks no store sells](#audiobooks-no-store-sells). Needs the Audiobookshelf URL and token | +| `AUTO_DOWNLOAD_EXISTENCE_CHECK_AFTER_PASSES` | Look the book up after (unmatched passes) | 3 | Consecutive passes without a strict match before the stores are asked | One action button sits at the bottom of the tab: diff --git a/docs/library-check.md b/docs/library-check.md index 8413c9ac..197cdd09 100644 --- a/docs/library-check.md +++ b/docs/library-check.md @@ -52,6 +52,8 @@ In order of confidence: ## Behaviour worth knowing +Sequels and other near-miss titles fail the title rule; the same title by a different author fails the surname rule. A series label in front of a library title ("Alex Cross 02: Kiss the Girls", "HHG 5: Mostly Harmless") is read as the series, not as part of the title, when the entry is in a series, so renaming a shelf to show reading order doesn't make its books look unowned. A shelf title whose bracketed words name another edition ("Full Cast Edition", "Dramatized Adaptation") is not the book either: owning the adaptation is not owning the recording asked for. Printing words ("Illustrated Edition", "Unabridged") change nothing, and in a Calibre title a numbered parenthetical ("(Alex Cross Series #11)") is read as the series, not as an edition. + - **It fails open for badges.** If a library cannot be read, Shelfmark logs a warning, reuses the last successful read if it has one, and otherwise treats the book as not owned. A broken path degrades the badge, it never blocks a search. diff --git a/shelfmark/config/hardcover_sync_settings.py b/shelfmark/config/hardcover_sync_settings.py index 15707765..3a3647e7 100644 --- a/shelfmark/config/hardcover_sync_settings.py +++ b/shelfmark/config/hardcover_sync_settings.py @@ -199,6 +199,33 @@ def hardcover_sync_settings() -> list[SettingsField]: min_value=0, max_value=365, ), + NumberField( + key="AUTO_DOWNLOAD_FAILURE_RETRY_MINUTES", + label="Try another release after a failed download (minutes)", + description=( + "A download that failed, or that was given up on (stalled, no seeders), goes " + "back to pending this many minutes later and another release is chosen, " + "without waiting for the retry days above or the next sync cycle. The release " + "that failed is not picked again, and a request is given up on after three " + "failed attempts. 0 turns this off." + ), + default=0, + min_value=0, + max_value=1440, + ), + NumberField( + key="AUTO_DOWNLOAD_RETRY_FAILED_RELEASE_DAYS", + label="Try a failed release again after (days)", + description=( + "A release that failed or stalled is kept out of the search for this many " + "days, then it may be picked again. A torrent that stalled can be revived " + "by new seeds, so a failure should not shut a book out for good. A request " + "is still given up on after three failed attempts." + ), + default=7, + min_value=1, + max_value=365, + ), NumberField( key="AUTO_DOWNLOAD_BACKOFF_MAX_DAYS", label="Longest wait between searches for an unmatched request (days)", @@ -212,6 +239,30 @@ def hardcover_sync_settings() -> list[SettingsField]: min_value=0, max_value=30, ), + CheckboxField( + key="AUTO_DOWNLOAD_EXISTENCE_CHECK_ENABLED", + label="Mark audiobooks no store sells", + description=( + "An audiobook request that stays unmatched is looked up on Audible (US and " + "UK) through your Audiobookshelf server. Either way it stays pending and its " + "reason says what the store answered (with the ASIN when listed), for you to " + "decline or keep; it is checked again after a week. A lookup that " + "fails decides nothing. Needs the Audiobookshelf URL and token below." + ), + default=False, + ), + NumberField( + key="AUTO_DOWNLOAD_EXISTENCE_CHECK_AFTER_PASSES", + label="Look the book up after (unmatched passes)", + description=( + "How many consecutive passes with no strict match a request goes through " + "before the store is asked. A pass that matched, or a reopen, starts the " + "count over." + ), + default=3, + min_value=1, + max_value=100, + ), NumberField( key="AUTO_DOWNLOAD_AUDIOBOOK_MIN_SIZE_PERCENT", label="Smallest acceptable audiobook copy (%)", diff --git a/shelfmark/core/auto_download.py b/shelfmark/core/auto_download.py index fb8f1585..10ce369d 100644 --- a/shelfmark/core/auto_download.py +++ b/shelfmark/core/auto_download.py @@ -15,12 +15,14 @@ Design goals: from __future__ import annotations +import json import math import re import sqlite3 import statistics +import time import unicodedata -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any @@ -39,6 +41,9 @@ from shelfmark.core.text_match import ( author_surname, extra_work_tokens, is_bundle_title, + normalize_numerals, + significant_tokens, + strip_alternate_title, title_tokens_match, ) from shelfmark.core.text_match import ( @@ -59,6 +64,10 @@ if TYPE_CHECKING: logger = setup_logger(__name__) +# Admin note on a pending request auto-download closed because the library already has it. +# "[auto]" marks an admin action no person took; the note is all a request records about why. +IN_LIBRARY_NOTE = "[auto] Already in the library." + # Fraction of significant book-title tokens that must appear in the release title. TITLE_MATCH_THRESHOLD = DEFAULT_TITLE_MATCH_THRESHOLD @@ -636,6 +645,196 @@ def _demote_undersized(ranked: list[Release], content_type: str) -> list[Release return [r for r in ranked if r not in thin] + thin +# A Soulseek peer whose recent downloads all failed is tried after the others, not before. +DEAD_PEER_DAYS = 3 +# What a Soulseek task records when the peer is the problem: a transfer that errored or timed +# out, or a peer that did not answer slskd's queue request (it answers 500, and the download is +# tried twice before giving up). +_PEER_FAILURE_MESSAGES = ("peer transfer failed", "failed to add to slskd") +# AudiobookBay shows no seed counts, so its best few candidates have their comments read for +# "0 seeders" and the like. A page costs a fetch (cached), so only this many are looked at. +MAX_COMPLAINT_CHECKS = 5 + + +def recent_failed_peers( + db_path: str | None, *, now: datetime | None = None, days: int = DEAD_PEER_DAYS +) -> frozenset[str]: + """Soulseek peers that failed a transfer in the last ``days`` and delivered nothing in it.""" + if not db_path: + return frozenset() + since = ((now or datetime.now(UTC)) - timedelta(days=days)).isoformat(timespec="seconds") + try: + conn = sqlite3.connect(db_path) + try: + rows = conn.execute( + "SELECT final_status, status_message, retry_payload FROM download_history " + "WHERE source = 'slskd' AND queued_at >= ?", + (since,), + ).fetchall() + finally: + conn.close() + except sqlite3.Error as exc: + logger.debug("auto-download: could not read recent Soulseek failures: %s", exc) + return frozenset() + failed: set[str] = set() + delivered: set[str] = set() + for final_status, message, payload in rows: + try: + context = (json.loads(payload) if payload else {}).get("retry_source_context") or {} + except ValueError, AttributeError: + continue + peer = str(context.get("username") or "").strip() if isinstance(context, dict) else "" + if not peer: + continue + if final_status == "complete": + delivered.add(peer) + elif final_status == "error" and str(message or "").lower().startswith( + _PEER_FAILURE_MESSAGES + ): + failed.add(peer) + return frozenset(failed - delivered) + + +# A release that failed is not offered again for this long, then it is tried once more: a torrent +# that stalled can be revived by new seeds, and a failure may simply have been a bad hour. +FAILED_RELEASE_RETRY_DAYS_SETTING = "AUTO_DOWNLOAD_RETRY_FAILED_RELEASE_DAYS" +DEFAULT_FAILED_RELEASE_RETRY_DAYS = 7 +# A torrent release that stalled or had no seeders is not offered to ANY request in that time. +# Soulseek failures are about a peer, not a release, and are handled by the dead-peer rule. +_DEAD_RELEASE_MESSAGES = ("download stalled", "no seeders") +# Rows that say nothing about the release: the process stopped, or a later download replaced it. +_NOT_THE_RELEASES_FAULT = ("interrupted", "superseded") + + +def failed_release_retry_days() -> int: + """Days a failed release is kept out before it may be picked again (at least 1).""" + return max( + coerce_int( + app_config.get(FAILED_RELEASE_RETRY_DAYS_SETTING, DEFAULT_FAILED_RELEASE_RETRY_DAYS), + DEFAULT_FAILED_RELEASE_RETRY_DAYS, + ), + 1, + ) + + +def recent_failed_releases( + db_path: str | None, *, now: datetime | None = None, days: int | None = None +) -> frozenset[tuple[str, str]]: + """Releases (source, source_id) whose latest download failed within the last ``days``. + + The history row of a release is the one place its failure time is recorded, so a failure + older than that no longer counts against the release, whatever a request remembers. A release + that completed in the window is not failed. + """ + if not db_path: + return frozenset() + window = failed_release_retry_days() if days is None else days + since = ((now or datetime.now(UTC)) - timedelta(days=window)).isoformat(timespec="seconds") + try: + conn = sqlite3.connect(db_path) + try: + rows = conn.execute( + "SELECT task_id, source, final_status, status_message FROM download_history " + "WHERE COALESCE(terminal_at, queued_at) >= ?", + (since,), + ).fetchall() + finally: + conn.close() + except sqlite3.Error as exc: + logger.debug("auto-download: could not read recent failed releases: %s", exc) + return frozenset() + failed: set[tuple[str, str]] = set() + delivered: set[tuple[str, str]] = set() + for task_id, source, final_status, message in rows: + key = (str(source), str(task_id)) + if final_status == "complete": + delivered.add(key) + elif final_status in ("cancelled", "error") and not str(message or "").lower().startswith( + _NOT_THE_RELEASES_FAULT + ): + failed.add(key) + return frozenset(failed - delivered) + + +def recent_dead_releases( + db_path: str | None, *, now: datetime | None = None, days: int | None = None +) -> frozenset[tuple[str, str]]: + """Releases (source, source_id) that stalled or had no seeders in the last ``days``. + + Another request's failure counts: a request remembers only the releases that failed for it, + so a torrent that died for one book was picked again for the next. A release that also + completed in the window is not dead. + """ + if not db_path: + return frozenset() + window = failed_release_retry_days() if days is None else days + since = ((now or datetime.now(UTC)) - timedelta(days=window)).isoformat(timespec="seconds") + try: + conn = sqlite3.connect(db_path) + try: + rows = conn.execute( + "SELECT task_id, source, final_status, status_message FROM download_history " + "WHERE source != 'slskd' AND COALESCE(terminal_at, queued_at) >= ?", + (since,), + ).fetchall() + finally: + conn.close() + except sqlite3.Error as exc: + logger.debug("auto-download: could not read recent dead releases: %s", exc) + return frozenset() + dead: set[tuple[str, str]] = set() + delivered: set[tuple[str, str]] = set() + for task_id, source, final_status, message in rows: + key = (str(source), str(task_id)) + if final_status == "complete": + delivered.add(key) + elif final_status in ("cancelled", "error") and str(message or "").lower().startswith( + _DEAD_RELEASE_MESSAGES + ): + dead.add(key) + return frozenset(dead - delivered) + + +def _dead_source_reason( + release: Release, dead_peers: frozenset[str], complaint_checks: list[int] +) -> str | None: + extra = release.extra if isinstance(release.extra, dict) else {} + if release.source == "slskd": + peer = str(extra.get("username") or "").strip() + return f"peer {peer} failed recently" if peer and peer in dead_peers else None + if release.source == "audiobookbay" and complaint_checks[0] < MAX_COMPLAINT_CHECKS: + if search_deadline.expired(): + return None + complaint_checks[0] += 1 + from shelfmark.release_sources import get_handler + + try: + complaints = get_handler(release.source).seed_complaints( + {"source": release.source, "download_url": release.download_url} + ) + except Exception as exc: # noqa: BLE001 - a page that cannot be read just is not judged + logger.debug("auto-download: no seed check for %s: %s", release.title, exc) + return None + if complaints: + return f"{complaints} comment(s) say it has no seeders" + return None + + +def _demote_dead_sources(ranked: list[Release], dead_peers: frozenset[str]) -> list[Release]: + """Move releases from sources known to be dead behind the rest, keeping order within each.""" + live: list[Release] = [] + dead: list[Release] = [] + complaint_checks = [0] + for release in ranked: + reason = _dead_source_reason(release, dead_peers, complaint_checks) + if reason is None: + live.append(release) + continue + logger.info("auto-download: trying %s last, %s", release.title, reason) + dead.append(release) + return live + dead + + def pick_best_release(releases: list[Release], content_type: str = "audiobook") -> Release | None: if not releases: return None @@ -797,7 +996,11 @@ def _last_delivery_change(row: dict[str, Any]) -> datetime | None: def _latest_history(db_path: str, request_ids: list[int]) -> dict[int, tuple[str, str | None]]: - """The newest (status, message) in download_history per request id; none means absent.""" + """The newest (status, message) in download_history per request id; none means absent. + + Newest by when it was queued: a reused release keeps its old row id, so id order is not + chronological. + """ latest: dict[int, tuple[str, str | None]] = {} if not request_ids: return latest @@ -808,7 +1011,7 @@ def _latest_history(db_path: str, request_ids: list[int]) -> dict[int, tuple[str marks = ",".join("?" * len(chunk)) rows = conn.execute( "SELECT request_id, final_status, status_message FROM download_history " # noqa: S608 - placeholders only - f"WHERE request_id IN ({marks}) ORDER BY id", + f"WHERE request_id IN ({marks}) ORDER BY queued_at, id", chunk, ) for request_id, final_status, status_message in rows: @@ -818,6 +1021,15 @@ def _latest_history(db_path: str, request_ids: list[int]) -> dict[int, tuple[str return latest +# What a download records when its client could not be reached. The release is not to blame. +_CLIENT_UNREACHABLE = ( + "failed to connect", + "cannot connect", + "connection refused", + "connection aborted", +) + + def _failure_kind(latest: tuple[str, str | None] | None) -> str | None: """Why a request whose delivery never completed never arrived, from its latest history row. @@ -832,7 +1044,9 @@ def _failure_kind(latest: tuple[str, str | None] | None) -> str | None: if final_status == "active": return _INTERRUPTED if final_status == "error": - if (message or "").strip().lower() == _INTERRUPTED_MESSAGE: + lowered = (message or "").strip().lower() + if lowered == _INTERRUPTED_MESSAGE or any(m in lowered for m in _CLIENT_UNREACHABLE): + # Nothing was wrong with the release: the process stopped, or its client was down. return _INTERRUPTED return _FAILED if final_status == "cancelled" and (message or "").strip().lower().startswith( @@ -861,6 +1075,89 @@ def _failed_release_keys(book_data: object) -> set[tuple[str, str]]: return keys +# Delivery states that say a task is alive. Only the status poll moves a request off them, and a +# finished task leaves the queue an hour after it ends, so with nobody watching they can stay +# for good. +_LIVE_DELIVERY_STATES = frozenset({QueueStatus.QUEUED.value, QueueStatus.DOWNLOADING.value}) +_SETTLED_STATUSES = frozenset( + {QueueStatus.COMPLETE.value, QueueStatus.ERROR.value, QueueStatus.CANCELLED.value} +) + + +def _parse_moment(value: object) -> datetime | None: + try: + moment = datetime.fromisoformat(str(value)) + except ValueError: + return None + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +def settle_delivery_states_from_history(user_db: UserDB, *, db_path: str | None) -> int: + """Record how a download ended for requests that still say queued or downloading. + + The status poll does this while someone watches. Here the history decides: the newest row of + the request, if it ended after the request last changed, is the outcome, and the request is + dated from when it ended so the retry cooldown counts from then. A row still + "active", or one that ended before the request was fulfilled again, says nothing about the + current attempt and is left alone. Returns how many requests were updated. + """ + if not db_path: + return 0 + candidates = [ + row + for row in user_db.list_requests(status=RequestStatus.FULFILLED) + if str(row.get("delivery_state") or "none").lower() in _LIVE_DELIVERY_STATES + ] + if not candidates: + return 0 + ids = [int(row["id"]) for row in candidates] + latest: dict[int, tuple[str, object]] = {} + try: + conn = sqlite3.connect(db_path) + try: + for start in range(0, len(ids), 500): + chunk = ids[start : start + 500] + marks = ",".join("?" * len(chunk)) + for request_id, final_status, terminal_at in conn.execute( + "SELECT request_id, final_status, terminal_at FROM download_history " # noqa: S608 - placeholders only + f"WHERE request_id IN ({marks}) ORDER BY queued_at, id", + chunk, + ): + latest[int(request_id)] = (str(final_status), terminal_at) + finally: + conn.close() + except sqlite3.Error as exc: + logger.warning("auto-download: could not settle delivery states from history: %s", exc) + return 0 + settled = 0 + for row in candidates: + request_id = int(row["id"]) + entry = latest.get(request_id) + if entry is None or entry[0] not in _SETTLED_STATUSES: + continue + ended = _parse_moment(entry[1]) + changed = _last_delivery_change(row) + if ended is None or (changed is not None and ended <= changed): + continue + try: + user_db.update_request( + request_id, + delivery_state=entry[0], + delivery_updated_at=ended.isoformat(timespec="seconds"), + ) + except sqlite3.Error as exc: + logger.warning("auto-download: could not settle request %s: %s", request_id, exc) + continue + logger.info( + "auto-download: request %s said %s but its download ended (%s); recorded", + request_id, + row.get("delivery_state"), + entry[0], + ) + settled += 1 + return settled + + def reopen_stale_failures( user_db: UserDB, *, @@ -869,9 +1166,17 @@ def reopen_stale_failures( provider_filter: str | None = "hardcover", max_reopen: int = MAX_RETRIES_PER_PASS, now: datetime | None = None, + failure_cooldown: timedelta | None = None, + only_kinds: frozenset[str] | None = None, + reopened_ids: list[int] | None = None, ) -> int: """Return synced requests whose download never arrived to pending once the cooldown passes. + ``failure_cooldown`` shortens the wait for a download that itself failed or that the stall + timer ended (a dead release says nothing about when the book will turn up, so another + release can be tried soon); interrupted downloads keep the ``cooldown_days`` wait. + ``only_kinds`` limits which of those are reopened, and ``reopened_ids`` collects the ids. + Nothing else retries them: the sync treats a request in any status as handled, and the auto-download pass only acts on pending ones. Reopening goes through ``UserDB.reopen_failed_request`` (upstream's own path for a failed request), which clears @@ -884,9 +1189,14 @@ def reopen_stale_failures( and rejected ones are decisions and are never touched. Returns how many were reopened; ``cooldown_days`` of 0 or less turns retries off, and without readable history nothing is retried, since guessing would be worse. """ - if cooldown_days <= 0: + if cooldown_days <= 0 and failure_cooldown is None: return 0 - cutoff = (now or datetime.now(UTC)) - timedelta(days=cooldown_days) + settle_delivery_states_from_history(user_db, db_path=db_path) + moment = now or datetime.now(UTC) + cutoff = moment - timedelta(days=cooldown_days) if cooldown_days > 0 else None + failure_cutoff = moment - failure_cooldown if failure_cooldown is not None else None + # A row can only be reopened once it is past the shorter of the two waits. + earliest = max(c for c in (cutoff, failure_cutoff) if c is not None) stale: list[tuple[datetime, dict[str, Any], str]] = [] for row in user_db.list_requests(status=RequestStatus.FULFILLED): @@ -899,7 +1209,7 @@ def reopen_stale_failures( ): continue changed = _last_delivery_change(row) - if changed is None or changed > cutoff: + if changed is None or changed > earliest: continue stale.append((changed, row, state)) if not stale or not db_path: @@ -920,6 +1230,13 @@ def reopen_stale_failures( kind = _failure_kind(history.get(request_id)) if kind is None or (state == _CANCELLED and kind != _STALLED): continue + if only_kinds is not None and kind not in only_kinds: + continue + threshold = ( + failure_cutoff if kind in (_FAILED, _STALLED) and failure_cutoff is not None else cutoff + ) + if threshold is None or _changed > threshold: + continue try: if kind in (_FAILED, _STALLED): book_data = dict(row["book_data"]) @@ -933,13 +1250,20 @@ def reopen_stale_failures( ) not in _failed_release_keys(book_data): book_data["failed_releases"] = [*tried, key] user_db.update_request(request_id, book_data=book_data) - reason = f"Retrying after a {cooldown_days}-day cooldown ({kind}; delivery was {state})" + wait = ( + f"{int(failure_cooldown.total_seconds() // 60)}-minute" + if failure_cooldown is not None and threshold is failure_cutoff + else f"{cooldown_days}-day" + ) + reason = f"Retrying after a {wait} cooldown ({kind}; delivery was {state})" result = user_db.reopen_failed_request(request_id, failure_reason=reason) except sqlite3.Error as exc: logger.warning("auto-download: could not reopen request %s: %s", request_id, exc) continue if result is not None: reopened += 1 + if reopened_ids is not None: + reopened_ids.append(request_id) return reopened @@ -991,6 +1315,87 @@ def _configured_source_priority(content_type: str) -> list[str]: return [src["name"] for src in available if src.get("enabled")] +def _acted_on_since_listed(user_db: UserDB, request_id: int) -> bool: + """True only when the request is read back and is positively no longer waiting. + + A pass lists its pending requests once and can run for minutes, so an admin may have + fulfilled, rejected or cancelled one since. Anything doubtful (the read fails, the row + is missing, an unexpected value) answers False, so the request is searched exactly as + it was before this check existed. + """ + try: + current = user_db.get_request(request_id) + except Exception: # noqa: BLE001 - a failed read must never stop a download + return False + if not isinstance(current, dict): + return False + status = str(current.get("status") or "").lower() + delivery = str(current.get("delivery_state") or "none").lower() + closed = {RequestStatus.FULFILLED, RequestStatus.REJECTED, RequestStatus.CANCELLED} + return status in closed or (status == RequestStatus.PENDING and delivery not in {"none", ""}) + + +# Simpler queries tried, in order, when a request's own query finds no strict match. +MAX_QUERY_VARIANTS = 3 +# "Find Your Why: A Practical Guide..." and "Dune - Deluxe Edition" both name the book +# before the separator. +_SUBTITLE_SPLIT = re.compile(r"^(?P
.+?)(?:\s*:\s+|\s+-\s+)\S") + + +def _without_subtitle(title: str, book: BookMetadata) -> str: + """The title before a ':' or ' - ' subtitle; the title itself when that would not do. + + A prefix that is only series or author words ("Star Wars: Thrawn", "Alex Cross: + Kiss the Girls") names a whole series rather than this book, so it is not taken; + nor is one followed by only series words (the "Thrawn" series' "Star Wars: + Thrawn"), where the prefix is a franchise. + Nor is it when the metadata cannot say so: without a series name, or a subtitle + the part after the separator holds, "Star Wars: Thrawn" would be searched and + matched as "Star Wars". + """ + match = _SUBTITLE_SPLIT.match(title) + if not match: + return title + main = match.group("main").strip() + words = set(significant_tokens(main)) + known = set(significant_tokens(" ".join([book.series_name or "", *(book.authors or [])]))) + rest = set(significant_tokens(title[len(main) :])) + series = set(significant_tokens(book.series_name)) + if not words or words <= known or (rest and rest <= series): + return title + subtitle = set(significant_tokens(book.subtitle)) + confirmed = bool(subtitle) and subtitle <= rest + if not (book.series_name or confirmed): + return title + return main + + +def query_variants(book: BookMetadata) -> list[tuple[str, str]]: + """Simpler titles to search when the book's own search title finds no strict match. + + Returns ``(label, title)`` pairs, at most ``MAX_QUERY_VARIANTS``, each different + from the title already searched and from the ones before it: the title without a + subtitle, without an alternate title (", or There and Back Again"), and with its + numbers written as digits. Each is searched and matched as the book's search title, + so every strict-match rule still applies to it. + """ + searched = book.search_title or book.title or "" + candidates = [ + ("without subtitle", _without_subtitle(searched, book)), + ("without alternate title", strip_alternate_title(searched)), + ("numerals as digits", normalize_numerals(searched)), + ] + seen = {" ".join(searched.lower().split())} + variants: list[tuple[str, str]] = [] + for label, title in candidates: + key = " ".join(title.lower().split()) + if not key or key in seen or not significant_tokens(title): + continue + seen.add(key) + variants.append((label, title.strip())) + return variants[:MAX_QUERY_VARIANTS] + + def auto_download_request( user_db: UserDB, request_row: dict[str, Any], @@ -1001,6 +1406,9 @@ def auto_download_request( queue_release: Callable[..., tuple[bool, str | None]], admin_user_id: int, providers: dict[str, Any] | None = None, + dead_peers: frozenset[str] = frozenset(), + dead_releases: frozenset[tuple[str, str]] = frozenset(), + recent_failures: frozenset[tuple[str, str]] | None = None, ) -> AutoDownloadOutcome: """Search, strict-match, and queue a single pending request. @@ -1015,6 +1423,9 @@ def auto_download_request( ) request_id = int(request_row["id"]) + if _acted_on_since_listed(user_db, request_id): + logger.info("auto-download: request %s was handled since this pass listed it", request_id) + return AutoDownloadOutcome(request_id, "skipped", "no longer pending") book_data = request_row.get("book_data") or {} if not isinstance(book_data, dict): return AutoDownloadOutcome(request_id, "skipped", "no book_data") @@ -1050,89 +1461,146 @@ def auto_download_request( ) return AutoDownloadOutcome(request_id, "error", f"library check unavailable: {exc}") if in_library: + # Close it, or it sits in Needs Review and is looked up again every pass. + # Same end state as an admin approving it with no release. + try: + user_db.update_request( + request_id, + expected_current_status=RequestStatus.PENDING, + status=RequestStatus.FULFILLED, + delivery_state=QueueStatus.COMPLETE, + delivery_updated_at=datetime.now(UTC).isoformat(timespec="seconds"), + admin_note=IN_LIBRARY_NOTE, + reviewed_by=admin_user_id, + reviewed_at=datetime.now(UTC).isoformat(timespec="seconds"), + ) + except ValueError as exc: + # Changed under us (an admin acted on it mid-pass); leave it as it now is. + logger.info( + "auto-download: request %s already in library, not closed: %s", request_id, exc + ) + return AutoDownloadOutcome(request_id, "in_library", "already in library") logger.info( - "auto-download: request %s (%s) already in library; skipping", + "auto-download: request %s (%s) already in library; closed", request_id, book.title, ) - return AutoDownloadOutcome(request_id, "in_library", "already in library") + return AutoDownloadOutcome(request_id, "in_library", "already in library; closed") audiobook_formats = _audiobook_formats() ebook_formats = _ebook_formats() languages = _wanted_languages(coerce_int(request_row.get("user_id"), 0) or None) - skipped_pack = False - # Releases that already failed for this request, so a retry does not pick the same one. - failed_releases = _failed_release_keys(book_data) + # Releases that already failed for this request, so a retry does not pick the same one. Only + # those whose failure is recent: an older one is given another chance (None keeps all). + remembered = _failed_release_keys(book_data) + if recent_failures is not None: + remembered &= recent_failures + failed_releases = remembered | dead_releases source_errors: list[str] = [] + skipped_pack = False - # Walk sources in priority order; take the first source with a strict match. - for source_name in sources: - # The requester's id lets the search plan apply their default languages. - _source, releases, search_error = search_source_releases( - source_name, - book, - expand_search=True, - content_type=content_type, - user_id=coerce_int(request_row.get("user_id"), 0) or None, - ) - if search_error: - source_errors.append(f"{source_name}: {search_error}") - match_kwargs: dict[str, Any] = { - "content_type": content_type, - "min_seeders": min_seeders, - "audiobook_formats": audiobook_formats, - "ebook_formats": ebook_formats, - "languages": languages, - } - releases = [r for r in releases if (r.source, r.source_id) not in failed_releases] - if not any(strict_match(release, book, **match_kwargs) for release in releases): - _fill_missing_authors(releases, book, match_kwargs) - candidates = [ - release for release in releases if strict_match(release, book, **match_kwargs) - ] - chosen = None - release_data: dict[str, Any] = {} - ranked = sorted(candidates, key=lambda r: _release_sort_key(r, content_type), reverse=True) - thin = _undersized_copies(ranked, content_type) - ranked = _demote_undersized(ranked, content_type) - for index, candidate in enumerate(ranked): - if index >= MAX_PACK_CHECKS: - break # the best few were all packs; don't queue an uninspected tail - data = build_release_data(candidate, book, content_type) - is_pack, single_file = _inspect_candidate(data) - if is_pack: - logger.info("auto-download: skipping multi-book pack %s", candidate.title) - skipped_pack = True - continue - if single_file and candidate not in thin: - # One m4b/m4a beats a set of chapter files: no merge step afterwards. - chosen, release_data = candidate, data - break - if chosen is None: - chosen, release_data = candidate, data # best-ranked fallback - if content_type != "audiobook" or _audiobook_format(candidate) == "m4b": - break # already as good as it gets; don't inspect the rest - if chosen is None: - continue - - try: - fulfil_request( - user_db, - request_id=request_id, - admin_user_id=admin_user_id, - queue_release=queue_release, - release_data=release_data, + def _walk(searched_book: BookMetadata) -> AutoDownloadOutcome | None: + """Search the sources in priority order; the first with a strict match wins.""" + nonlocal skipped_pack + for source_name in sources: + # The requester's id lets the search plan apply their default languages. + _source, releases, search_error = search_source_releases( + source_name, + searched_book, + expand_search=True, + content_type=content_type, + user_id=coerce_int(request_row.get("user_id"), 0) or None, ) - except RequestServiceError as exc: - logger.warning("auto-download: fulfil failed for request %s: %s", request_id, exc) - return AutoDownloadOutcome(request_id, "error", str(exc), source=source_name) + if search_error: + source_errors.append(f"{source_name}: {search_error}") + match_kwargs: dict[str, Any] = { + "content_type": content_type, + "min_seeders": min_seeders, + "audiobook_formats": audiobook_formats, + "ebook_formats": ebook_formats, + "languages": languages, + } + releases = [r for r in releases if (r.source, r.source_id) not in failed_releases] + if not any( + strict_match(release, searched_book, **match_kwargs) for release in releases + ): + _fill_missing_authors(releases, searched_book, match_kwargs) + candidates = [ + release + for release in releases + if strict_match(release, searched_book, **match_kwargs) + ] + chosen = None + release_data: dict[str, Any] = {} + ranked = sorted( + candidates, key=lambda r: _release_sort_key(r, content_type), reverse=True + ) + thin = _undersized_copies(ranked, content_type) + ranked = _demote_undersized(ranked, content_type) + ranked = _demote_dead_sources(ranked, dead_peers) + for index, candidate in enumerate(ranked): + if index >= MAX_PACK_CHECKS: + break # the best few were all packs; don't queue an uninspected tail + data = build_release_data(candidate, searched_book, content_type) + is_pack, single_file = _inspect_candidate(data) + if is_pack: + logger.info("auto-download: skipping multi-book pack %s", candidate.title) + skipped_pack = True + continue + if single_file and candidate not in thin: + # One m4b/m4a beats a set of chapter files: no merge step afterwards. + chosen, release_data = candidate, data + break + if chosen is None: + chosen, release_data = candidate, data # best-ranked fallback + if content_type != "audiobook" or _audiobook_format(candidate) == "m4b": + break # already as good as it gets; don't inspect the rest + if chosen is None: + continue + + try: + fulfil_request( + user_db, + request_id=request_id, + admin_user_id=admin_user_id, + queue_release=queue_release, + release_data=release_data, + ) + except RequestServiceError as exc: + logger.warning("auto-download: fulfil failed for request %s: %s", request_id, exc) + return AutoDownloadOutcome(request_id, "error", str(exc), source=source_name) + logger.info( + "auto-download: queued request %s from %s (%s)", + request_id, + source_name, + chosen.title, + ) + return AutoDownloadOutcome(request_id, "queued", chosen.title, source=source_name) + return None + + outcome = _walk(book) + if outcome is not None: + return outcome + # Nothing matched the request's own query. Try a few simpler ones, stopping at the + # first that yields a strict match; the match rules are the same for each. + for label, title in query_variants(book): logger.info( - "auto-download: queued request %s from %s (%s)", + "auto-download: request %s (%s): no strict match, trying %s: %r", request_id, - source_name, - chosen.title, + book.title, + label, + title, ) - return AutoDownloadOutcome(request_id, "queued", chosen.title, source=source_name) + outcome = _walk(replace(book, search_title=title)) + if outcome is not None: + if outcome.status == "queued": + logger.info( + "auto-download: request %s matched on the query %s: %r", + request_id, + label, + title, + ) + return outcome if source_errors: # A source that failed has not said the book is missing. @@ -1147,6 +1615,61 @@ def auto_download_request( return AutoDownloadOutcome(request_id, "no_match", "no strict match across sources") +FAILURE_RETRY_MINUTES_SETTING = "AUTO_DOWNLOAD_FAILURE_RETRY_MINUTES" + + +def retry_failed_downloads( + user_db: UserDB, + *, + queue_release: Callable[..., tuple[bool, str | None]], + provider_filter: str | None = "hardcover", + admin_user_id: int | None = None, + db_path: str | None = None, +) -> dict[str, int] | None: + """Pick another release for requests whose download just failed or stalled. + + Runs between full passes. Only the requests it reopens are searched, so it costs nothing + when no download has failed. The release that failed is noted on the request first, and a + request is given up on after ``MAX_FAILURE_RETRIES`` attempts. None when switched off. + """ + minutes = coerce_int(app_config.get(FAILURE_RETRY_MINUTES_SETTING, 0), 0) + if minutes <= 0 or not bool(app_config.get("AUTO_DOWNLOAD_ENABLED", False)): + return None + reopened_ids: list[int] = [] + reopen_stale_failures( + user_db, + cooldown_days=0, + db_path=db_path, + provider_filter=provider_filter, + failure_cooldown=timedelta(minutes=minutes), + only_kinds=frozenset({_FAILED, _STALLED}), + reopened_ids=reopened_ids, + ) + # A failed download the status poll already reopened is pending with no release; those + # are searched here as well, so they are not left for the next full pass. + pending_after_failure = { + int(row["id"]) + for row in user_db.list_requests(status="pending") + if row.get("last_failure_reason") + and str(row.get("delivery_state") or "none").lower() in {"none", ""} + and isinstance(row.get("book_data"), dict) + and row["book_data"].get("failed_releases") + and len(row["book_data"]["failed_releases"]) < MAX_FAILURE_RETRIES + } + ids = frozenset(reopened_ids) | pending_after_failure + if not ids: + return None + logger.info("auto-download: trying another release for %d request(s)", len(ids)) + return auto_download_pending( + user_db, + queue_release=queue_release, + provider_filter=provider_filter, + admin_user_id=admin_user_id, + db_path=db_path, + only_request_ids=ids, + ) + + def auto_download_pending( user_db: UserDB, *, @@ -1155,9 +1678,13 @@ def auto_download_pending( admin_user_id: int | None = None, db_path: str | None = None, ignore_backoff: bool = False, + only_request_ids: frozenset[int] | None = None, ) -> dict[str, int]: """Run the auto-download pass over all eligible pending requests. + ``only_request_ids`` limits the pass to those requests (the quick retry after a failed + download), and skips the 7-day reopen. + ``ignore_backoff`` (a manual "Sync now") searches every request, however many passes in a row found nothing for it. @@ -1181,8 +1708,12 @@ def auto_download_pending( retry_days = coerce_int( app_config.get(RETRY_DAYS_SETTING, DEFAULT_RETRY_DAYS), DEFAULT_RETRY_DAYS ) - reopened = reopen_stale_failures( - user_db, cooldown_days=retry_days, db_path=db_path, provider_filter=provider_filter + reopened = ( + 0 + if only_request_ids is not None + else reopen_stale_failures( + user_db, cooldown_days=retry_days, db_path=db_path, provider_filter=provider_filter + ) ) if reopened: logger.info( @@ -1203,7 +1734,13 @@ def auto_download_pending( else coerce_int(app_config.get(BACKOFF_SETTING, DEFAULT_BACKOFF_MAX_DAYS), 0) ) + dead_peers = recent_failed_peers(db_path) + dead_releases = recent_dead_releases(db_path) + # Without a history to read there is no way to tell how old a failure is, so all count. + recent_failures = recent_failed_releases(db_path) if db_path else None for row in pending: + if only_request_ids is not None and int(row["id"]) not in only_request_ids: + continue book_data = row.get("book_data") or {} if provider_filter and ( not isinstance(book_data, dict) or book_data.get("provider") != provider_filter @@ -1235,6 +1772,7 @@ def auto_download_pending( summary["skipped"] += 1 continue + started = time.monotonic() try: outcome = auto_download_request( user_db, @@ -1245,6 +1783,9 @@ def auto_download_pending( queue_release=queue_release, admin_user_id=admin_user_id, providers=providers, + dead_peers=dead_peers, + dead_releases=dead_releases, + recent_failures=recent_failures, ) except Exception: # One bad request must not take the rest of the pass with it. @@ -1252,7 +1793,51 @@ def auto_download_pending( summary["error"] += 1 continue summary[outcome.status] = summary.get(outcome.status, 0) + 1 + _existence_check( + user_db, + row, + outcome, + admin_user_id=admin_user_id, + content_type=content_type, + # What is left of this request's search budget; the store lookups spend it. + budget_seconds=max( + 0.0, search_deadline.budget_seconds() - (time.monotonic() - started) + ), + summary=summary, + ) _record_search_outcome(user_db, int(row["id"]), outcome.status, datetime.now(UTC)) logger.info("auto-download pass complete: %s", summary) return summary + + +def _existence_check( + user_db: UserDB, + row: dict[str, Any], + outcome: AutoDownloadOutcome, + *, + admin_user_id: int, + content_type: str, + budget_seconds: float, + summary: dict[str, int], +) -> None: + """Hand a request's outcome to the audiobook existence check (see ``existence_check``). + + Its failure must not take the pass with it, so it is logged and the pass moves on. + """ + from shelfmark.core import existence_check + + try: + verdict = existence_check.after_request( + user_db, + row, + outcome, + admin_user_id=admin_user_id, + content_type=content_type, + budget_seconds=budget_seconds, + ) + except Exception: + logger.exception("auto-download: existence check for request %s failed", row.get("id")) + return + if verdict == "not_listed": + summary["not_listed"] = summary.get("not_listed", 0) + 1 diff --git a/shelfmark/core/existence_check.py b/shelfmark/core/existence_check.py new file mode 100644 index 00000000..10a87f68 --- /dev/null +++ b/shelfmark/core/existence_check.py @@ -0,0 +1,286 @@ +"""Mark audiobook requests for books that no store sells as an audiobook. + +A pending audiobook request that no source ever matches sits in Needs Review for good, +and an admin ends up looking it up by hand to learn that there is no recording to find. +This does that lookup: once a request has gone a configured number of consecutive +auto-download passes without a strict match, Audible's US and UK catalogues are asked, +through the Audiobookshelf server Shelfmark already talks to, whether the book exists as +an audiobook at all. + +- Not listed on either store: the request stays pending and its failure reason says so, + for an admin to decline or keep. Audible search misses real books (collection-only + novellas, retitled releases), so the check never declines on its own. +- Listed: the request stays pending and its failure reason says so, with the ASIN, so + the admin sees why it waits. Either way the store is not asked again for a week. +- The store or the server failed, or the request's search budget was spent: nothing is + decided. An error is not "no audiobook". Audiobookshelf answers a failed Audible search + with an empty list rather than an error, so before a "not listed" counts, a book both + stores are known to sell (``CANARY``) is looked up too; if that is not found either, + the answer is "unknown". + +The pass counter lives in the request's ``book_data`` beside the retry bookkeeping. A +pass that matched the book clears it, and so does any rejection (``requests_service`` +drops it), so a request an admin reopens starts counting afresh. Only audiobook requests are checked: ebooks have no store +that could vouch for their absence. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any + +import requests + +from shelfmark.core.config import config as app_config +from shelfmark.core.existence_match import StoreMatch, find_match, search_title, us_spelling +from shelfmark.core.library_providers import audiobookshelf +from shelfmark.core.logger import setup_logger +from shelfmark.core.request_helpers import coerce_int +from shelfmark.core.request_validation import RequestStatus + +if TYPE_CHECKING: + from shelfmark.core.auto_download import AutoDownloadOutcome + from shelfmark.core.user_db import UserDB + +logger = setup_logger(__name__) + +ENABLED_SETTING = "AUTO_DOWNLOAD_EXISTENCE_CHECK_ENABLED" +AFTER_PASSES_SETTING = "AUTO_DOWNLOAD_EXISTENCE_CHECK_AFTER_PASSES" +DEFAULT_AFTER_PASSES = 3 + +# A book the store listed is asked about again after this long, not every pass. +RECHECK_DAYS = 7 + +# Audiobookshelf's provider ids for the two Audible stores asked, in order. +STORES: tuple[tuple[str, str], ...] = (("audible", "US"), ("audible.uk", "UK")) +US_STORE = "audible" + +# Where the bookkeeping sits in a request's book_data, beside ``failed_releases``. +BOOK_DATA_KEY = "existence_check" + +# One store lookup may take this long, and one is not started with less than this of the +# request's search budget left: the pass must not stall on a store that stopped answering. +_LOOKUP_TIMEOUT = 15.0 +_MIN_LOOKUP_BUDGET = 5.0 + +# A book both stores sell, asked about before a "not listed" is believed: Audiobookshelf +# answers a failed Audible search with an empty list, which reads the same as "no such book". +CANARY = ("The Hobbit", "J.R.R. Tolkien") + +# Outcomes of a pass that mean the request found its book, so the count starts over. +_MATCHED = frozenset({"queued", "in_library"}) + +NOT_LISTED_REASON = "Not found on Audible (US/UK) after {passes} searches, {date}" +LISTED_REASON = "Exists on Audible ({asin}), not on our sources yet" + + +@dataclass(frozen=True) +class ExistenceResult: + """What the stores said: ``found``, ``not_found`` or ``unknown`` (nothing decided).""" + + status: str + match: StoreMatch | None = None + store: str | None = None + detail: str = "" + + +def enabled() -> bool: + return bool(app_config.get(ENABLED_SETTING, False)) + + +def passes_before_check() -> int: + """How many consecutive no-match passes a request goes through before the lookup.""" + raw = coerce_int(app_config.get(AFTER_PASSES_SETTING, DEFAULT_AFTER_PASSES), 0) + return raw if raw >= 1 else DEFAULT_AFTER_PASSES + + +def _search( + title: str, author: str, provider: str, store: str, deadline: float | None +) -> list[dict[str, Any]] | ExistenceResult: + """One store search, or the ``unknown`` result that stands in for it.""" + timeout = _LOOKUP_TIMEOUT + if deadline is not None: + left = deadline - time.monotonic() + if left < _MIN_LOOKUP_BUDGET: + return ExistenceResult( + "unknown", store=store, detail=f"no search time left for the {store} store" + ) + timeout = min(timeout, left) + try: + return audiobookshelf.search_books(title, author, provider, timeout=timeout) + except (requests.RequestException, ValueError) as exc: + return ExistenceResult("unknown", store=store, detail=f"{store} lookup failed: {exc}") + + +def look_up( + title: str | None, author: str | None, *, budget_seconds: float | None = None +) -> ExistenceResult: + """Ask Audible US, then UK, whether ``title`` by ``author`` exists as an audiobook. + + The lookups share ``budget_seconds`` when one is given; a store that cannot be asked + within it is not asked, and the answer is ``unknown``. Any server or store error is + ``unknown`` too, and so is a "not listed" that the stores also give for ``CANARY``: + an error never says the book is missing. + """ + query = search_title(title, author) + if not query: + return ExistenceResult("unknown", detail="the request has no title to search") + deadline = None if budget_seconds is None else time.monotonic() + budget_seconds + for provider, store in STORES: + store_query = us_spelling(query) if provider == US_STORE else query + results = _search(store_query, author or "", provider, store, deadline) + if isinstance(results, ExistenceResult): + return results + match = find_match(title, author, results) + if match is not None: + return ExistenceResult("found", match=match, store=store) + canary_title, canary_author = CANARY + for provider, store in STORES: + results = _search(canary_title, canary_author, provider, store, deadline) + if isinstance(results, ExistenceResult): + return results + if find_match(canary_title, canary_author, results) is None: + return ExistenceResult( + "unknown", + store=store, + detail=f"the {store} store did not list {canary_title} either; taken as a failure", + ) + return ExistenceResult("not_found") + + +def without_state(book_data: dict[str, Any]) -> dict[str, Any] | None: + """``book_data`` without this check's bookkeeping, or None when it has none.""" + if BOOK_DATA_KEY not in book_data: + return None + return _with_state(book_data, None) + + +def _state(book_data: dict[str, Any]) -> dict[str, Any]: + raw = book_data.get(BOOK_DATA_KEY) + return dict(raw) if isinstance(raw, dict) else {} + + +def _with_state(book_data: dict[str, Any], state: dict[str, Any] | None) -> dict[str, Any]: + cleared = {key: value for key, value in book_data.items() if key != BOOK_DATA_KEY} + if state: + cleared[BOOK_DATA_KEY] = state + return cleared + + +def _checked_recently(state: dict[str, Any], now: datetime) -> bool: + """True when the store said "found" less than ``RECHECK_DAYS`` ago.""" + if state.get("result") not in ("found", "not_found"): + return False + try: + checked_at = datetime.fromisoformat(str(state.get("checked_at") or "")) + except ValueError: + return False + if checked_at.tzinfo is None: + checked_at = checked_at.replace(tzinfo=UTC) + return now - checked_at < timedelta(days=RECHECK_DAYS) + + +def after_request( + user_db: UserDB, + row: dict[str, Any], + outcome: AutoDownloadOutcome, + *, + admin_user_id: int, + content_type: str, + budget_seconds: float | None = None, + now: datetime | None = None, +) -> str | None: + """Count the pass for one request and, when it is time, ask the stores. + + Called by the auto-download pass after each request with that request's outcome. + Returns what happened: ``counted`` (not yet time), ``waiting`` (the store listed it + recently), ``listed`` (the store lists it; the request stays pending and says so), + ``not_listed`` (not on either store; stays pending and says so), ``unknown`` (lookup failed, nothing changed) or None when + the request was not an audiobook no-match, or the check is off. + """ + if content_type != "audiobook" or not enabled(): + return None + request_id = int(row["id"]) + # The pass may have written to the request (a failed release, a status change), so + # work from what is stored now, not from the row the pass started with. + current = user_db.get_request(request_id) + if current is None: + return None + book_data = current.get("book_data") + if not isinstance(book_data, dict): + return None + state = _state(book_data) + + if outcome.status != "no_match": + if outcome.status in _MATCHED and state: + # The book was found; a retry after a failed download starts counting afresh. + user_db.update_request(request_id, book_data=_with_state(book_data, None)) + return None + if current.get("status") != RequestStatus.PENDING: + return None + + moment = now or datetime.now(UTC) + passes = coerce_int(state.get("no_match_passes"), 0) + 1 + state["no_match_passes"] = passes + # Written before the lookup, so a pass still counts if the lookup never finishes. + user_db.update_request( + request_id, + expected_current_status=RequestStatus.PENDING, + book_data=_with_state(book_data, state), + ) + if passes < passes_before_check(): + return "counted" + if _checked_recently(state, moment): + return "waiting" + + title = str(book_data.get("title") or "") + author = str(book_data.get("author") or "") + result = look_up(title, author, budget_seconds=budget_seconds) + if result.status == "unknown": + logger.warning( + "existence check: request %s (%s) not decided: %s", request_id, title, result.detail + ) + return "unknown" + + if result.status == "found" and result.match is not None: + state.update( + result="found", + checked_at=moment.isoformat(timespec="seconds"), + asin=result.match.asin, + store=result.store, + ) + user_db.update_request( + request_id, + expected_current_status=RequestStatus.PENDING, + book_data=_with_state(book_data, state), + last_failure_reason=LISTED_REASON.format(asin=result.match.asin), + ) + logger.info( + "existence check: request %s (%s) is on Audible %s as %s; still pending", + request_id, + title, + result.store, + result.match.asin, + ) + return "listed" + + # Not on either store. Audible search misses too much to decline on that alone (books + # sold only in a collection, retitled releases, fan recordings), so the request stays + # pending for an admin to decide, with the reason saying what the store answered. + state.update(result="not_found", checked_at=moment.isoformat(timespec="seconds")) + user_db.update_request( + request_id, + expected_current_status=RequestStatus.PENDING, + book_data=_with_state(book_data, state), + last_failure_reason=NOT_LISTED_REASON.format(passes=passes, date=moment.date().isoformat()), + ) + logger.info( + "existence check: request %s (%s) not on Audible US or UK after %d searches; " + "left pending for review", + request_id, + title, + passes, + ) + return "not_listed" diff --git a/shelfmark/core/existence_match.py b/shelfmark/core/existence_match.py new file mode 100644 index 00000000..95d8c9af --- /dev/null +++ b/shelfmark/core/existence_match.py @@ -0,0 +1,273 @@ +"""Does an Audible catalogue result name the audiobook we want? + +Text work only, for the results Audiobookshelf's ``/api/search/books`` returns from its +Audible providers (``audible`` is the US store, ``audible.uk`` the UK one). A result names +the wanted book when its title, normalised the same way as the wanted title, is the same +sequence of words (or one begins the other, see ``titles_match``), and one of the wanted +author's surnames appears in its author field. + +Normalising a title: diacritics are folded; bracketed notes such as "(Unabridged)" go; a +leading series label ("Robert Langdon, Book 4 - ") goes; the subtitle after ":", ", or", +" aka " or a spaced dash is dropped; "&" reads as "and"; a possessive "'s" is dropped; and +a roman, spelled-out or digit numeral all become the digit ("Part II", "Part Two" and +"Part 2" are one title). Stopwords are left out of the comparison. + +The matching leans towards "found" on purpose. A wrong "found" leaves a request pending, +where it already was; a wrong "not found" declines a book that exists. + +Standard library only, on purpose: the stress test against a live library runs this file +as-is next to the Audiobookshelf server, so what is tested is what ships. +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass + +_STOPWORDS = frozenset({"a", "an", "the", "of", "and", "or", "to", "in", "on", "for", "with", "by"}) + +_NUMERALS = { + "zero": "0", + "one": "1", + "two": "2", + "three": "3", + "four": "4", + "five": "5", + "six": "6", + "seven": "7", + "eight": "8", + "nine": "9", + "ten": "10", + "eleven": "11", + "twelve": "12", + "thirteen": "13", + "fourteen": "14", + "fifteen": "15", + "sixteen": "16", + "seventeen": "17", + "eighteen": "18", + "nineteen": "19", + "twenty": "20", + "i": "1", + "ii": "2", + "iii": "3", + "iv": "4", + "v": "5", + "vi": "6", + "vii": "7", + "viii": "8", + "ix": "9", + "x": "10", + "xi": "11", + "xii": "12", + "xiii": "13", + "xiv": "14", + "xv": "15", + "xvi": "16", + "xvii": "17", + "xviii": "18", + "xix": "19", + "xx": "20", +} + +# British spellings the US store does not find ("The Armour of Light" is "The Armor of +# Light" there). Titles compare in the US spelling, and the US store is asked in it. +_US_SPELLING = { + "armour": "armor", + "behaviour": "behavior", + "colour": "color", + "colours": "colors", + "endeavour": "endeavor", + "favour": "favor", + "harbour": "harbor", + "honour": "honor", + "humour": "humor", + "labour": "labor", + "neighbour": "neighbor", + "neighbours": "neighbors", + "rumour": "rumor", + "rumours": "rumors", + "saviour": "savior", + "splendour": "splendor", + "valour": "valor", +} +_US_SPELLING_WORD = re.compile(r"\b(" + "|".join(_US_SPELLING) + r")\b", re.IGNORECASE) + +_BRACKETED = re.compile(r"\([^)]*\)|\[[^\]]*\]|\{[^}]*\}") +# A dash with space on at least one side splits off a part: "Born a Crime- Stories from +# ...", "Robert Langdon, Book 4 - Inferno". Hyphenated words ("Sword-Sworn") survive. +_DASH = re.compile(r"\s+[-–—]+\s*|\s*[-–—]+\s+") +# ": subtitle", ", or Alternative Title", " aka Other Title": the first ends the title. +_SUBTITLE = re.compile(r"\s*(?::|,\s+or\b|\s+aka\b).*$", re.IGNORECASE) +# A leading part that ends in a series position: "Mayfair Witches 01", "Robert Langdon, +# Book 4", "Discworld #3". A plain number does not count ("Fahrenheit 451 - A Novel"). +_NUMBERED_PART = re.compile( + r"(?:\b(?:book|vol(?:ume)?|part|no)\.?\s*\d+(?:\.\d+)?|\b0\d+|#\d+(?:\.\d+)?)$", re.IGNORECASE +) +# A series label marked by a bracketed number in front of the title: "HGTTG [01] The +# Hitchhiker's Guide to the Galaxy". +_BRACKETED_LABEL = re.compile(r"^[^\[\]()]{0,40}?[\[(]\s*#?\d+(?:\.\d+)?\s*[\])]\s*(?=\S)") +# A zero-padded track number in front of a title ("04 The Visitor"). +_TRACK_NUMBER = re.compile(r"^0\d\s+(?=\S)") +_POSSESSIVE = re.compile(r"[’']s\b") +_SPLIT = re.compile(r"[^a-z0-9]+") +# Separators between the names in an author field: "Anne Rice, Christopher Rice", +# "Kristin Hannah/Polly Stone", "Lee Child & Andrew Child", "Douglas Adams and ...". +_AUTHOR_SPLIT = re.compile(r"\s*(?:[,;/&]|\band\b)\s*", re.IGNORECASE) + +# A prefix match ("Start with Why" for "Start with Why 15th Anniversary Edition") needs +# this many words in the shorter title, so a one-word title never matches a longer one. +_MIN_PREFIX_WORDS = 2 + + +def _fold(text: str) -> str: + """ASCII-folded, lower-cased text: diacritics and typographic quotes removed.""" + decomposed = unicodedata.normalize("NFKD", text) + stripped = "".join(ch for ch in decomposed if not unicodedata.combining(ch)) + return stripped.replace("’", "'").replace("‘", "'").lower() + + +def _words(text: str) -> list[str]: + folded = _fold(text).replace("&", " and ") + folded = _POSSESSIVE.sub("", folded).replace("'", "") + return [_US_SPELLING.get(word, word) for word in _SPLIT.split(folded) if word] + + +def us_spelling(text: str) -> str: + """``text`` with the British spellings in ``_US_SPELLING`` turned American.""" + + def _swap(match: re.Match[str]) -> str: + word = match.group(0) + us = _US_SPELLING[word.lower()] + return us.capitalize() if word[0].isupper() else us + + return _US_SPELLING_WORD.sub(_swap, text) + + +def search_title( + title: str | None, author: str | None = None, *, keep_subtitle: bool = False +) -> str: + """The title alone, as it is sent to the store. + + Bracketed notes ("(Unabridged)", "[Series 02]") go. Of the parts a spaced dash + separates, leading ones that are a numbered series label or the author's name go too, + and the first that is left is the title; the rest is subtitle. A subtitle after ":", + ", or" and " aka " goes (unless ``keep_subtitle``), and so does a trailing ellipsis. + """ + original = (title or "").strip() + text = _BRACKETED_LABEL.sub("", original) + text = _BRACKETED.sub(" - ", text) + parts = [part.strip(" ,;") for part in _DASH.split(text)] + parts = [part for part in parts if part] + author_words = _words(author or "") + while len(parts) > 1 and ( + _NUMBERED_PART.search(parts[0]) or (author_words and _words(parts[0]) == author_words) + ): + parts.pop(0) + if keep_subtitle: + text = " ".join(parts) + else: + text = parts[0] if parts else "" + text = _SUBTITLE.sub("", text) + text = _TRACK_NUMBER.sub("", text) + text = re.sub(r"\s+", " ", text).strip(" ,;") + text = re.sub(r"\s*(?:\.{3}|…)$", "", text) + return text or original + + +def title_words( + title: str | None, author: str | None = None, *, keep_subtitle: bool = False +) -> tuple[str, ...]: + """The comparable words of a title, in order.""" + words = _words(search_title(title, author, keep_subtitle=keep_subtitle)) + return tuple(_NUMERALS.get(word, word) for word in words if word not in _STOPWORDS) + + +def _same_title(wanted: tuple[str, ...], candidate: tuple[str, ...]) -> bool: + if not wanted or not candidate: + return False + if wanted == candidate: + return True + shorter, longer = sorted((wanted, candidate), key=len) + return len(shorter) >= _MIN_PREFIX_WORDS and longer[: len(shorter)] == shorter + + +def titles_match(wanted: str | None, candidate: str | None, author: str | None = None) -> bool: + """True when the two titles name the same book once normalised. + + Same words, or one title's words begin the other's when the shorter has at least two + ("Merry Christmas" and "Merry Christmas, Alex Cross"). Each title is compared both + without and with its subtitle, so "The Mummy, or Ramses the Damned" matches "The Mummy + or Ramses the Damned". A store title with a series in front of a colon ("Ramses the + Damned: The Reign of Osiris") is also tried without it. + """ + wanted_forms = { + title_words(wanted, author), + title_words(wanted, author, keep_subtitle=True), + } + candidate_title = candidate or "" + candidate_forms = { + title_words(candidate_title), + title_words(candidate_title, keep_subtitle=True), + } + if ":" in candidate_title: + candidate_forms.add(title_words(candidate_title.split(":", 1)[1])) + return any(_same_title(w, c) for w in wanted_forms for c in candidate_forms) + + +def author_surnames(author: str | None) -> set[str]: + """The surnames in an author field, one per name. + + "Lee Child" gives {"child"}; "R.C. Bray, Andy Weir" gives {"bray", "weir"}; "Child, + Lee" gives {"child", "lee"}, which is harmless as the match is on any of them. + """ + surnames = set() + for name in _AUTHOR_SPLIT.split(author or ""): + words = [w for w in _SPLIT.split(_fold(name)) if len(w) >= 2 and w not in _STOPWORDS] + if words: + surnames.add(words[-1]) + return surnames + + +def author_matches(wanted_author: str | None, candidate_author: str | None) -> bool: + """True when one of the wanted author's surnames is in the result's author field. + + A wanted author with no surname (empty, or initials only) matches on title alone. + """ + surnames = author_surnames(wanted_author) + if not surnames: + return True + return bool(surnames & set(_SPLIT.split(_fold(candidate_author or "")))) + + +@dataclass(frozen=True) +class StoreMatch: + """The catalogue entry that names the wanted book.""" + + asin: str + title: str + author: str + + +def find_match( + wanted_title: str | None, wanted_author: str | None, results: list[dict] +) -> StoreMatch | None: + """The first store result that is the wanted book, or None. + + ``results`` is what ``/api/search/books`` returns: dicts with ``title``, ``author`` and + ``asin``. Results without an ASIN are not catalogue entries and are skipped. + """ + for result in results: + if not isinstance(result, dict): + continue + asin = str(result.get("asin") or "").strip().upper() + title = str(result.get("title") or "") + author = str(result.get("author") or "") + if not asin: + continue + if titles_match(wanted_title, title, wanted_author) and author_matches( + wanted_author, author + ): + return StoreMatch(asin=asin, title=title, author=author) + return None diff --git a/shelfmark/core/hardcover_scheduler.py b/shelfmark/core/hardcover_scheduler.py index 943f1b6c..d258d28a 100644 --- a/shelfmark/core/hardcover_scheduler.py +++ b/shelfmark/core/hardcover_scheduler.py @@ -109,6 +109,59 @@ def trigger_async(*, force: bool = True) -> bool: return True +def _quick_retry_interval_seconds() -> float: + """How often failed downloads are checked between cycles, 0 when switched off.""" + raw = app_config.get("AUTO_DOWNLOAD_FAILURE_RETRY_MINUTES", 0) + try: + minutes = float(raw) if isinstance(raw, (int, float, str)) else 0.0 + except ValueError: + return 0.0 + return max(minutes * 60.0, _MIN_INTERVAL_SECONDS) if minutes > 0 else 0.0 + + +def run_quick_retry() -> dict[str, int] | None: + """Pick another release for downloads that failed since the last look. + + Skipped (None) while a cycle or a manual sync runs, and when no download failed. + """ + if not _run_lock.acquire(blocking=False): + return None + try: + with _ctx_lock: + user_db = _ctx.get("user_db") + queue_release = _ctx.get("queue_release") + db_path = _ctx.get("db_path") + if user_db is None or queue_release is None: + return None + from shelfmark.core.auto_download import retry_failed_downloads + + return retry_failed_downloads(user_db, queue_release=queue_release, db_path=db_path) + finally: + _run_lock.release() + + +def _wait_for_next_cycle() -> None: + """Sleep until the next cycle, looking at failed downloads on the way when that is on.""" + deadline = time.monotonic() + _interval_seconds() + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return + step = _quick_retry_interval_seconds() + if step <= 0: + time.sleep(remaining) + return + time.sleep(min(step, remaining)) + if time.monotonic() >= deadline: + return + try: + result = run_quick_retry() + if result is not None: + logger.info("Hardcover scheduler: quick retry result %s", result) + except Exception: + logger.exception("Hardcover scheduler: quick retry failed") + + def _scheduler_loop() -> None: logger.info("Hardcover scheduler started") time.sleep(_WARMUP_SECONDS) @@ -120,7 +173,7 @@ def _scheduler_loop() -> None: logger.info("Hardcover scheduler: cycle result %s", result) except Exception: logger.exception("Hardcover scheduler: cycle failed") - time.sleep(_interval_seconds()) + _wait_for_next_cycle() def start( diff --git a/shelfmark/core/library_providers/audiobookshelf.py b/shelfmark/core/library_providers/audiobookshelf.py index cad67527..46427cb2 100644 --- a/shelfmark/core/library_providers/audiobookshelf.py +++ b/shelfmark/core/library_providers/audiobookshelf.py @@ -130,6 +130,38 @@ def _fetch_library_entries(url: str, token: str, lib_ids: list[str]) -> list[Lib return entries +def search_books( + title: str, + author: str, + provider: str, + *, + timeout: float = _REQUEST_TIMEOUT, + overrides: Mapping[str, Any] | None = None, +) -> list[dict[str, Any]]: + """Ask Audiobookshelf's metadata search (``GET /api/search/books``) about a book. + + ``provider`` names one of the server's metadata providers: ``audible`` is the US + store, ``audible.uk`` the UK one. The result is the server's list of matches, best + first, each with at least ``title``, ``author`` and (for Audible) ``asin``. Raises + ValueError without a URL and token, and the requests error when the server or the + store fails; the caller decides what an error means. + """ + _enabled, url, token, _lib_ids = _config(overrides) + if not url or not token: + raise ValueError("Set the Audiobookshelf URL and token first.") + resp = _session(url, token).get( + f"{url}/api/search/books", + params={"title": title, "author": author, "provider": provider}, + timeout=timeout, + ) + resp.raise_for_status() + payload = resp.json() + if not isinstance(payload, list): + msg = "Audiobookshelf answered the book search with something other than a list" + raise ValueError(msg) # noqa: TRY004 - a bad answer from the server, not a bad argument + return [item for item in payload if isinstance(item, dict)] + + class AudiobookshelfLibrary: """Audiobook ownership via the Audiobookshelf REST API.""" diff --git a/shelfmark/core/library_providers/calibre.py b/shelfmark/core/library_providers/calibre.py index 80ecc017..cdc3a483 100644 --- a/shelfmark/core/library_providers/calibre.py +++ b/shelfmark/core/library_providers/calibre.py @@ -29,8 +29,12 @@ if TYPE_CHECKING: logger = setup_logger(__name__) -# "(Alex Cross Series #11)", "[Illustrated Edition]" and friends. -_PARENTHETICAL = re.compile(r"[(\[][^)\]]*[)\]]") +# "(Alex Cross Series #11)", "(Book 2)": a bracketed segment with a number in it. +_NUMBERED_PARENTHETICAL = re.compile(r"[(\[][^)\]]*\d[^)\]]*[)\]]") +# "Label NN: Title", a series label and number in front of the work's own title. The +# label often abbreviates the series ("HHG 5: Mostly Harmless" for The Hitchhiker's +# Guide to the Galaxy), so its words are in neither the series nor the author name. +_SERIES_LABEL = re.compile(r"^\s*(?P