From d0e008adde718389440950bd011e157a7f4e166f Mon Sep 17 00:00:00 2001 From: CaliBrain Date: Thu, 13 Aug 2026 13:51:40 -0400 Subject: [PATCH] Stop dropping audiobook releases that are not m4b or mp3 (#1199) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An IRC audiobook search returned nothing while OpenBooks, reading the same @search answer from the same channel, listed results. Three separate defects were discarding them. The audiobook format list was maintained by hand in four places and had drifted. The settings UI offered only m4b/mp3/m4a/zip/rar, and that list is the only one a user's config can be built from, so flac, opus, ogg, aac, wav and wma were unreachable everywhere — even though the IRC parser recognized them, the IRC sorter ranked them (dead code that could never fire), archive extraction knew them and Prowlarr searched for them. A FLAC audiobook was invisible in search and, if it arrived anyway, rejected after download as "format not supported". AUDIOBOOK_FORMATS and ARCHIVE_FORMATS now live once in core.utils and every layer derives from them, which also restored the missing .opus in the post-download scan's trackable extensions. Widening the default alone would not have reached anyone already affected: initialize_default_configs() writes field defaults only when a tab has no config file yet, so an existing install keeps its persisted m4b/mp3 list forever. migrate_audiobook_formats rewrites a list that still matches the old default exactly and leaves every other value alone — re-enabling formats someone had deliberately turned off would be worse than leaving them narrow. The IRC parser filtered by file extension alone. Multi-file audiobooks ship as a .rar or .zip of MP3s, which matched neither SUPPORTED_FORMATS nor SUPPORTED_AUDIOBOOK_FORMATS, so they fell out of the ebook bucket and the audiobook bucket both. Results are now classified before the format filter is applied: an audio extension means audiobook, an ebook extension means ebook, and for a container — where the extension says nothing about the contents — the release name decides. An ebook archive stays out of audiobook results. RESULT_LINE_REGEX matched \w+ after any dot, so a line carrying no file extension parsed as format "5mb" out of "::INFO:: 620.5MB", taking the title and the size down with it and guaranteeing every downstream filter dropped it. Any decimal size did this. The extension is now matched against the known formats, so such a line falls through to the simple pattern and comes back as "unknown", which the rest of the parser already handles. ALL_RECOGNIZED_FORMATS became an ordered tuple in the process: it was a set, so which extension won for a line naming two of them depended on set iteration order and could vary between restarts. Refs #1129 --- docs/environment-variables.md | 4 +- shelfmark/config/migrations.py | 54 ++++++++++- shelfmark/config/settings.py | 24 +++-- shelfmark/core/utils.py | 15 +++ shelfmark/download/archive.py | 3 +- shelfmark/download/postprocess/scan.py | 5 +- shelfmark/main.py | 24 ++--- shelfmark/release_sources/irc/parser.py | 87 +++++++++++++----- shelfmark/release_sources/prowlarr/source.py | 3 +- .../config/test_audiobook_format_migration.py | 91 +++++++++++++++++++ .../core/test_audiobook_format_consistency.py | 38 ++++++++ tests/irc/test_parser.py | 91 +++++++++++++++++++ 12 files changed, 388 insertions(+), 51 deletions(-) create mode 100644 tests/config/test_audiobook_format_migration.py create mode 100644 tests/core/test_audiobook_format_consistency.py diff --git a/docs/environment-variables.md b/docs/environment-variables.md index e3b281e..205b052 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -247,7 +247,7 @@ Seconds since the last WireGuard handshake before the healthcheck bounces the tu | `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). | string | _none_ | | `AUDIOBOOK_LIBRARY_URL` | Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text. | string | _none_ | | `SUPPORTED_FORMATS` | Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found. | string (comma-separated) | `epub,mobi,azw3,fb2,djvu,cbz,cbr` | -| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3` | +| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar` | | `BOOK_LANGUAGE` | Default language filter for searches. | string (comma-separated) | `en` |
@@ -296,7 +296,7 @@ Book formats to include in search results. ZIP/RAR archives are extracted automa Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. - **Type:** string (comma-separated) -- **Default:** `m4b,mp3` +- **Default:** `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar` #### `BOOK_LANGUAGE` diff --git a/shelfmark/config/migrations.py b/shelfmark/config/migrations.py index 566825d..b2f7008 100644 --- a/shelfmark/config/migrations.py +++ b/shelfmark/config/migrations.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from os import PathLike _DEPRECATED_SETTINGS_RESTRICTION_KEYS = ( @@ -16,6 +16,13 @@ _DEPRECATED_SETTINGS_RESTRICTION_KEYS = ( "RESTRICT_SETTINGS_TO_ADMIN", ) +# The audiobook format list shipped as the default until the format sets were unified. +# It only covered m4b/mp3, so FLAC/OPUS/OGG/M4A releases were dropped from search results +# and rejected after download - and the wider default alone would never reach existing +# installs, because initialize_default_configs() only writes defaults when the config +# file does not exist yet. +_LEGACY_AUDIOBOOK_FORMATS_DEFAULT = ("m4b", "mp3") + class MigrationLogger(Protocol): """Logger surface used by config migration helpers.""" @@ -57,6 +64,51 @@ def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None: return None +def migrate_audiobook_formats( + *, + load_general_config: Callable[[], dict[str, Any]], + save_general_config: Callable[[dict[str, Any]], None], + widened_formats: Sequence[str], + logger: MigrationLogger, +) -> None: + """Widen an untouched audiobook format list to the current, fuller default. + + Only a list that still matches the old default exactly is rewritten. Any other value + means someone chose it deliberately, and a migration that "helpfully" re-enabled + formats a user had turned off would be worse than leaving them on the narrow list. + """ + try: + config = load_general_config() + + if "SUPPORTED_AUDIOBOOK_FORMATS" not in config: + # Nothing persisted, so the field default already applies. + logger.debug("No persisted audiobook formats - the current default applies") + return + + current = config.get("SUPPORTED_AUDIOBOOK_FORMATS") + if not isinstance(current, list): + return + + normalized = {str(fmt).strip().lower() for fmt in current if str(fmt).strip()} + if normalized != set(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT): + logger.debug( + "Audiobook formats were customized (%s) - left unchanged", sorted(normalized) + ) + return + + save_general_config({"SUPPORTED_AUDIOBOOK_FORMATS": list(widened_formats)}) + logger.info( + "Widened audiobook formats from the legacy default %s to %s", + list(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT), + list(widened_formats), + ) + + except FileNotFoundError: + logger.debug("No existing general config file found - nothing to migrate") + except Exception: + logger.exception("Failed to migrate audiobook formats") + + def migrate_security_settings( *, load_security_config: Callable[[], dict[str, Any]], diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index b9701c4..202668a 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -14,6 +14,7 @@ from shelfmark.config.download_settings_handlers import ( check_books_destination, ) from shelfmark.config.email_settings import check_email_connection +from shelfmark.config.migrations import migrate_audiobook_formats from shelfmark.core.languages import supported_book_languages from shelfmark.core.logger import setup_logger from shelfmark.core.settings_registry import ( @@ -35,6 +36,7 @@ from shelfmark.core.settings_registry import ( register_on_save, register_settings, ) +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60 _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600 @@ -134,6 +136,20 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]: logger = setup_logger(__name__) + + +def migrate_audiobook_format_settings() -> None: + """Bring installs created before the audiobook format sets were unified up to date.""" + from shelfmark.core.settings_registry import load_config_file, save_config_file + + migrate_audiobook_formats( + load_general_config=lambda: load_config_file("general"), + save_general_config=lambda values: save_config_file("general", values), + widened_formats=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS], + logger=logger, + ) + + _SMTP_PORT_MAX = 65535 _EMAIL_ATTACHMENT_LIMIT_MB_MAX = 600 @@ -215,11 +231,7 @@ _FORMAT_OPTIONS = [ ] _AUDIOBOOK_FORMAT_OPTIONS = [ - {"value": "m4b", "label": "M4B"}, - {"value": "mp3", "label": "MP3"}, - {"value": "m4a", "label": "M4A"}, - {"value": "zip", "label": "ZIP"}, - {"value": "rar", "label": "RAR"}, + {"value": fmt, "label": fmt.upper()} for fmt in (*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS) ] _DOWNLOAD_TO_BROWSER_CONTENT_TYPE_OPTIONS = [ @@ -416,7 +428,7 @@ def general_settings() -> list[SettingsField]: label="Supported Audiobook Formats", description="Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.", options=_AUDIOBOOK_FORMAT_OPTIONS, - default=["m4b", "mp3"], + default=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS], ), MultiSelectField( key="BOOK_LANGUAGE", diff --git a/shelfmark/core/utils.py b/shelfmark/core/utils.py index a012bf4..3dce751 100644 --- a/shelfmark/core/utils.py +++ b/shelfmark/core/utils.py @@ -115,6 +115,21 @@ def is_audiobook(content_type: str | None) -> bool: return bool(content_type and "audiobook" in content_type.lower()) +# Every audio format an audiobook can legitimately arrive in, and the single source of +# truth for that list. The settings UI, release-source parsing, archive extraction and +# post-download scanning all derive from it, so a format added here becomes selectable, +# searchable AND downloadable at once. These used to be four hand-maintained copies that +# had drifted apart: the settings UI only offered m4b/mp3/m4a, which meant a FLAC +# audiobook could never be enabled, was silently dropped from every search result, and +# was rejected after download as "format not supported". +AUDIOBOOK_FORMATS = ("m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus") + +# Multi-file audiobooks are almost always distributed as an archive. These are containers +# rather than formats: they are what a *release* looks like, and the formats above are +# what comes out of one after extraction. +ARCHIVE_FORMATS = ("zip", "rar") + + CONTENT_TYPES = [ "book (fiction)", "book (non-fiction)", diff --git a/shelfmark/download/archive.py b/shelfmark/download/archive.py index 73a242d..1dfb288 100644 --- a/shelfmark/download/archive.py +++ b/shelfmark/download/archive.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, cast from shelfmark.core.logger import setup_logger +from shelfmark.core.utils import AUDIOBOOK_FORMATS from shelfmark.core.utils import is_audiobook as check_audiobook from shelfmark.download.fs import atomic_move from shelfmark.download.postprocess.policy import ( @@ -98,7 +99,7 @@ ALL_EBOOK_EXTENSIONS = { } # All known audio extensions (superset of what user might enable for audiobooks) -ALL_AUDIO_EXTENSIONS = {".m4b", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".wma", ".wav", ".opus"} +ALL_AUDIO_EXTENSIONS = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} def _filter_files( diff --git a/shelfmark/download/postprocess/scan.py b/shelfmark/download/postprocess/scan.py index 5b732fd..e1523e9 100644 --- a/shelfmark/download/postprocess/scan.py +++ b/shelfmark/download/postprocess/scan.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING from shelfmark.core.logger import setup_logger +from shelfmark.core.utils import AUDIOBOOK_FORMATS from shelfmark.core.utils import is_audiobook as check_audiobook from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive from shelfmark.download.fs import run_blocking_io @@ -142,7 +143,7 @@ def scan_directory_tree( is_audiobook = check_audiobook(content_type) if is_audiobook: - trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"} + trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} else: trackable_exts = { ".pdf", @@ -367,7 +368,7 @@ def collect_staged_files( is_audiobook = check_audiobook(task.content_type) if is_audiobook: - trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"} + trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} else: trackable_exts = { ".pdf", diff --git a/shelfmark/main.py b/shelfmark/main.py index 0cb94d1..78157dd 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -38,7 +38,10 @@ from shelfmark.config.env import ( string_to_bool, ) from shelfmark.config.security import _migrate_security_settings -from shelfmark.config.settings import _SUPPORTED_BOOK_LANGUAGE +from shelfmark.config.settings import ( + _SUPPORTED_BOOK_LANGUAGE, + migrate_audiobook_format_settings, +) from shelfmark.core.activity_view_state_service import ActivityViewStateService from shelfmark.core.auth_modes import ( get_auth_check_admin_status, @@ -80,7 +83,7 @@ from shelfmark.core.requests_service import ( sync_delivery_states_from_queue_status, ) from shelfmark.core.user_db import UserDB -from shelfmark.core.utils import normalize_base_path +from shelfmark.core.utils import AUDIOBOOK_FORMATS, normalize_base_path from shelfmark.download import orchestrator as backend from shelfmark.release_sources import ( BrowseRecord, @@ -168,6 +171,9 @@ except ImportError as e: # Migrate legacy security settings if needed _migrate_security_settings() +# Widen audiobook formats for installs that still carry the old m4b/mp3-only default +migrate_audiobook_format_settings() + # Initialize user database and register multi-user routes # If CONFIG_DIR doesn't exist or is read-only, multi-user features will be disabled _user_db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db") @@ -319,19 +325,7 @@ def get_auth_mode() -> str: _AUDIOBOOK_CATEGORY_RANGE = (3030, 3049) -_AUDIOBOOK_FORMAT_HINTS = frozenset( - { - "m4b", - "mp3", - "m4a", - "flac", - "ogg", - "wma", - "aac", - "wav", - "opus", - } -) +_AUDIOBOOK_FORMAT_HINTS = frozenset(AUDIOBOOK_FORMATS) def _contains_audiobook_format_hint(value: Any) -> bool: diff --git a/shelfmark/release_sources/irc/parser.py b/shelfmark/release_sources/irc/parser.py index 53eb034..d8ff791 100644 --- a/shelfmark/release_sources/irc/parser.py +++ b/shelfmark/release_sources/irc/parser.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from shelfmark.core.config import config from shelfmark.core.logger import setup_logger +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS from shelfmark.core.utils import is_audiobook as check_audiobook if TYPE_CHECKING: @@ -18,11 +19,8 @@ if TYPE_CHECKING: logger = setup_logger(__name__) -# All recognized formats for parsing IRC result lines. -# This comprehensive list is used to identify file extensions in results. -# User-configured formats are used separately for filtering. -ALL_RECOGNIZED_FORMATS = { - # Ebook formats +# Ebook formats recognized in IRC result lines. +EBOOK_FORMATS = ( "epub", "mobi", "azw3", @@ -41,19 +39,17 @@ ALL_RECOGNIZED_FORMATS = { "cbz", "cdr", "jpg", - "rar", - "zip", - # Audiobook formats - "m4b", - "mp3", - "m4a", - "flac", - "ogg", - "wma", - "aac", - "wav", - "opus", -} +) + +# All recognized formats for parsing IRC result lines. +# This comprehensive list is used to identify file extensions in results. +# User-configured formats are used separately for filtering. +# Ordered longest-first so that scanning a line matches "azw3" before "azw" and "docx" +# before "doc". It used to be a set, which made the winning format for a line naming more +# than one extension depend on set iteration order, and therefore vary between restarts. +ALL_RECOGNIZED_FORMATS = tuple( + sorted({*EBOOK_FORMATS, *ARCHIVE_FORMATS, *AUDIOBOOK_FORMATS}, key=len, reverse=True) +) def _normalize_config_formats(raw_formats: object) -> set[str]: @@ -84,13 +80,22 @@ def _get_supported_formats(content_type: str | None = None) -> set[str]: # Regex to parse result lines # Format: !Server Author - Title.format ::INFO:: size +# +# The extension is matched against the known formats rather than a bare \w+. A bare \w+ +# happily matched the decimal point in the size, so a line with no file extension parsed +# as format="5mb" out of "::INFO:: 620.5MB" - taking the title and size down with it, and +# leaving the result to be discarded by every format filter downstream. Restricting the +# alternation makes such a line fall through to SIMPLE_RESULT_REGEX and come back as +# "unknown", which is what the rest of the parser already expects. +_FORMAT_ALTERNATION = "|".join(re.escape(fmt) for fmt in ALL_RECOGNIZED_FORMATS) RESULT_LINE_REGEX = re.compile( r"^!(\S+)\s+" # !ServerName r"(.+?)\s+-\s+" # Author Name - - r"(.+?)\.(\w+)" # Title.format + rf"(.+?)\.({_FORMAT_ALTERNATION})\b" # Title.format r"(?:\s+::INFO::\s*(.+?))?" # Optional ::INFO:: metadata r"(?:\s+::HASH::\s*(\S+))?" # Optional ::HASH:: - r"\s*$" + r"\s*$", + re.IGNORECASE, ) # Simpler fallback pattern @@ -187,18 +192,54 @@ def parse_result_line(line: str) -> SearchResult | None: return None +# Words that mark an archive as holding an audiobook rather than an ebook. Multi-file +# audiobooks ship as .rar/.zip, so for those the extension says nothing about the content +# and the release name is the only evidence there is. +_AUDIOBOOK_MARKER_REGEX = re.compile( + r"\b(?:audio ?books?|unabridged|abridged|narrat(?:ed|or)|audible|\d+ ?kbps|" + + "|".join(re.escape(fmt) for fmt in AUDIOBOOK_FORMATS) + + r")\b", + re.IGNORECASE, +) + +_AUDIOBOOK_FORMAT_SET = frozenset(AUDIOBOOK_FORMATS) +_EBOOK_FORMAT_SET = frozenset(EBOOK_FORMATS) + + +def detect_content_type(result: SearchResult) -> str: + """Classify a parsed result as an audiobook or an ebook. + + Extension alone is not enough. It settles the plain cases, but the common audiobook + release is a .rar or .zip of MP3s, which is indistinguishable by extension from an + ebook archive - so for containers (and for lines with no usable extension) the + release name decides. + """ + if result.format in _AUDIOBOOK_FORMAT_SET: + return "audiobook" + if result.format in _EBOOK_FORMAT_SET: + return "ebook" + return "audiobook" if _AUDIOBOOK_MARKER_REGEX.search(result.full_line) else "ebook" + + def parse_results_file(content: str, content_type: str | None = None) -> list[SearchResult]: """Parse a search results file into SearchResult objects.""" results = [] supported = _get_supported_formats(content_type) + requested = "audiobook" if check_audiobook(content_type) else "ebook" for line in content.splitlines(): result = parse_result_line(line) - if result and (result.format in supported or result.format == "unknown"): - # Filter to user's configured formats + if not result: + continue + # Classify first, then apply the user's format filter within that bucket. Doing it + # the other way round is what lost audiobooks entirely: an audiobook .rar matched + # neither the ebook nor the audiobook format list, so it fell out of both. + if detect_content_type(result) != requested: + continue + if result.format in supported or result.format == "unknown": results.append(result) - logger.info("Parsed %s results from search file", len(results)) + logger.info("Parsed %s %s results from search file", len(results), requested) return results diff --git a/shelfmark/release_sources/prowlarr/source.py b/shelfmark/release_sources/prowlarr/source.py index 0205d35..527e9d9 100644 --- a/shelfmark/release_sources/prowlarr/source.py +++ b/shelfmark/release_sources/prowlarr/source.py @@ -16,6 +16,7 @@ from shelfmark.core.languages import normalize_language from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import normalize_optional_text from shelfmark.core.search_plan import ReleaseSearchVariant +from shelfmark.core.utils import AUDIOBOOK_FORMATS as CORE_AUDIOBOOK_FORMATS from shelfmark.core.utils import normalize_http_url from shelfmark.release_sources import ( ColumnAlign, @@ -222,7 +223,7 @@ EBOOK_FORMATS = [ ] # Common audiobook formats -AUDIOBOOK_FORMATS = ["m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus"] +AUDIOBOOK_FORMATS = list(CORE_AUDIOBOOK_FORMATS) # Combined list for format detection (audiobook formats first for priority) ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS diff --git a/tests/config/test_audiobook_format_migration.py b/tests/config/test_audiobook_format_migration.py new file mode 100644 index 0000000..816b5cf --- /dev/null +++ b/tests/config/test_audiobook_format_migration.py @@ -0,0 +1,91 @@ +"""Tests for widening the audiobook format list on existing installs. + +`initialize_default_configs()` only writes field defaults when a tab has no config file +yet, so widening the default alone would have reached fresh installs only - exactly not +the installs already carrying the narrow m4b/mp3 list that loses FLAC/OPUS releases. +""" + +import logging + +import pytest + +from shelfmark.config.migrations import migrate_audiobook_formats +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS + +WIDENED = [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS] + + +@pytest.fixture +def migrate(): + """Run the migration over an in-memory config, returning the resulting config.""" + + def run(config: dict | None) -> dict: + stored = {} if config is None else dict(config) + saved: dict = {} + + def save(values: dict) -> None: + saved.update(values) + stored.update(values) + + migrate_audiobook_formats( + load_general_config=lambda: stored, + save_general_config=save, + widened_formats=WIDENED, + logger=logging.getLogger("test"), + ) + return stored + + return run + + +def test_legacy_default_is_widened(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"]}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED + + +def test_legacy_default_is_widened_regardless_of_order_or_case(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["MP3", " m4b "]}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED + + +def test_other_settings_are_preserved(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"], "SUPPORTED_FORMATS": ["epub"]}) + + assert result["SUPPORTED_FORMATS"] == ["epub"] + + +@pytest.mark.parametrize( + "customized", + [ + ["m4b"], # deliberately narrowed - re-enabling formats would override the choice + ["mp3", "flac"], + ["m4b", "mp3", "zip"], + [], + ], +) +def test_customized_lists_are_left_alone(migrate, customized): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": customized}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == customized + + +def test_absent_key_is_left_alone(migrate): + """Nothing persisted means the field default already applies - don't write one.""" + result = migrate({"SUPPORTED_FORMATS": ["epub"]}) + + assert "SUPPORTED_AUDIOBOOK_FORMATS" not in result + + +def test_unexpected_type_is_left_alone(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": "m4b,mp3"}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == "m4b,mp3" + + +def test_migration_is_idempotent(migrate): + once = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"]}) + twice = migrate(once) + + assert twice["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED diff --git a/tests/core/test_audiobook_format_consistency.py b/tests/core/test_audiobook_format_consistency.py new file mode 100644 index 0000000..a4bfcac --- /dev/null +++ b/tests/core/test_audiobook_format_consistency.py @@ -0,0 +1,38 @@ +"""The audiobook format list must stay in agreement across every layer that gates on it. + +These lists were maintained by hand in four places and drifted: the settings UI offered +only m4b/mp3/m4a, so FLAC/OPUS/OGG could never be enabled even though the parsers +recognized them, the sorter ranked them, and archive extraction knew them. The result was +a FLAC audiobook that was invisible in search and rejected after download. They now all +derive from `shelfmark.core.utils.AUDIOBOOK_FORMATS`; this test fails if one drifts again. +""" + +from shelfmark.config.settings import _AUDIOBOOK_FORMAT_OPTIONS +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS +from shelfmark.download.archive import ALL_AUDIO_EXTENSIONS +from shelfmark.release_sources.irc import parser +from shelfmark.release_sources.prowlarr.source import AUDIOBOOK_FORMATS as PROWLARR_FORMATS + + +def test_archive_extraction_knows_every_audiobook_format(): + assert ALL_AUDIO_EXTENSIONS == {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} + + +def test_prowlarr_knows_every_audiobook_format(): + assert PROWLARR_FORMATS == list(AUDIOBOOK_FORMATS) + + +def test_irc_parser_knows_every_audiobook_format(): + assert set(AUDIOBOOK_FORMATS) <= set(parser.ALL_RECOGNIZED_FORMATS) + + +def test_every_audiobook_format_is_selectable_in_settings(): + """The settings list is the only one a user's config can be built from.""" + selectable = {option["value"] for option in _AUDIOBOOK_FORMAT_OPTIONS} + + assert selectable == {*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS} + + +def test_audiobook_and_ebook_formats_do_not_overlap(): + """Overlap would make content-type classification by extension ambiguous.""" + assert not set(AUDIOBOOK_FORMATS) & set(parser.EBOOK_FORMATS) diff --git a/tests/irc/test_parser.py b/tests/irc/test_parser.py index 32841e5..ade17f3 100644 --- a/tests/irc/test_parser.py +++ b/tests/irc/test_parser.py @@ -1,5 +1,96 @@ +import pytest + +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS from shelfmark.release_sources.irc import parser +# What a stock install actually filters with, so these tests fail if the defaults regress. +_DEFAULT_CONFIG = { + "SUPPORTED_FORMATS": ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"], + "SUPPORTED_AUDIOBOOK_FORMATS": [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS], +} + + +@pytest.fixture +def default_formats(monkeypatch): + """Filter with the shipped default format lists.""" + monkeypatch.setattr( + parser.config, "get", lambda key, default=None: _DEFAULT_CONFIG.get(key, default) + ) + + +def test_audiobook_archives_are_found_with_default_settings(default_formats): + """Regression for #1129: audiobooks ship as .rar/.zip and were dropped by both buckets. + + A single "@search" answers with one file holding every format. These are the lines an + audiobook actually occupies in it - the extension is a container, and the release name + is the only thing saying what is inside. + """ + content = "\n".join( + [ + "!Oatmeal Andy Weir - Project Hail Mary (Audiobook) [MP3 64kbps].rar ::INFO:: 620.5MB", + "!DV8 Andy Weir - Project Hail Mary - Audiobook.zip ::INFO:: 700MB", + "!Horla Andy Weir - Project Hail Mary [Unabridged].m4b ::INFO:: 850.1MB", + ] + ) + + results = parser.parse_results_file(content, content_type="audiobook") + + assert [result.format for result in results] == ["rar", "zip", "m4b"] + + +def test_ebook_archive_does_not_leak_into_audiobook_results(default_formats): + """An ebook .rar must not be offered as an audiobook just because it is an archive.""" + content = "!bald Andy Weir - Project Hail Mary (retail).rar ::INFO:: 2.1MB" + + assert parser.parse_results_file(content, content_type="audiobook") == [] + + +def test_flac_audiobook_is_reachable_with_default_settings(default_formats): + """FLAC was recognized by the parser and ranked by the sorter, but never selectable.""" + content = "!Ook Andy Weir - Project Hail Mary.flac ::INFO:: 1.1GB" + + results = parser.parse_results_file(content, content_type="audiobook") + + assert [result.format for result in results] == ["flac"] + + +def test_decimal_size_is_not_mistaken_for_a_file_extension(): + """A line with no extension used to parse as format="5mb" out of "::INFO:: 620.5MB".""" + line = "!Ook Andy Weir - Project Hail Mary (2021) Audiobook ::INFO:: 620.5MB" + + result = parser.parse_result_line(line) + + assert result.format == "unknown" + assert result.title == "Project Hail Mary (2021) Audiobook" + assert result.size == "620.5MB" + + +@pytest.mark.parametrize( + ("line", "expected"), + [ + ("!s A - T.epub ::INFO:: 1MB", "ebook"), + ("!s A - T.mp3 ::INFO:: 1MB", "audiobook"), + ("!s A - T.flac ::INFO:: 1MB", "audiobook"), + # Archives carry no format information, so the name has to decide. + ("!s A - T (Audiobook).rar ::INFO:: 1MB", "audiobook"), + ("!s A - T [Unabridged].zip ::INFO:: 1MB", "audiobook"), + ("!s A - T (Narrated by Someone).rar ::INFO:: 1MB", "audiobook"), + ("!s A - T [64kbps].zip ::INFO:: 1MB", "audiobook"), + ("!s A - T (retail).rar ::INFO:: 1MB", "ebook"), + ("!s A - T.zip ::INFO:: 1MB", "ebook"), + ], +) +def test_detect_content_type(line, expected): + assert parser.detect_content_type(parser.parse_result_line(line)) == expected + + +def test_recognized_formats_order_is_deterministic(): + """This was a set, so which format won for a multi-extension line varied per restart.""" + assert parser.ALL_RECOGNIZED_FORMATS == tuple(parser.ALL_RECOGNIZED_FORMATS) + # Longest-first, so ".azw3" cannot be truncated to "azw" (nor ".docx" to "doc"). + lengths = [len(fmt) for fmt in parser.ALL_RECOGNIZED_FORMATS] + assert lengths == sorted(lengths, reverse=True) + def test_parse_results_file_uses_audiobook_format_settings(monkeypatch): values = {