diff --git a/docs/environment-variables.md b/docs/environment-variables.md index e3b281e..205b052 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -247,7 +247,7 @@ Seconds since the last WireGuard handshake before the healthcheck bounces the tu | `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). | string | _none_ | | `AUDIOBOOK_LIBRARY_URL` | Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text. | string | _none_ | | `SUPPORTED_FORMATS` | Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found. | string (comma-separated) | `epub,mobi,azw3,fb2,djvu,cbz,cbr` | -| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3` | +| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar` | | `BOOK_LANGUAGE` | Default language filter for searches. | string (comma-separated) | `en` |
@@ -296,7 +296,7 @@ Book formats to include in search results. ZIP/RAR archives are extracted automa Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. - **Type:** string (comma-separated) -- **Default:** `m4b,mp3` +- **Default:** `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar` #### `BOOK_LANGUAGE` diff --git a/shelfmark/config/migrations.py b/shelfmark/config/migrations.py index 566825d..b2f7008 100644 --- a/shelfmark/config/migrations.py +++ b/shelfmark/config/migrations.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from os import PathLike _DEPRECATED_SETTINGS_RESTRICTION_KEYS = ( @@ -16,6 +16,13 @@ _DEPRECATED_SETTINGS_RESTRICTION_KEYS = ( "RESTRICT_SETTINGS_TO_ADMIN", ) +# The audiobook format list shipped as the default until the format sets were unified. +# It only covered m4b/mp3, so FLAC/OPUS/OGG/M4A releases were dropped from search results +# and rejected after download - and the wider default alone would never reach existing +# installs, because initialize_default_configs() only writes defaults when the config +# file does not exist yet. +_LEGACY_AUDIOBOOK_FORMATS_DEFAULT = ("m4b", "mp3") + class MigrationLogger(Protocol): """Logger surface used by config migration helpers.""" @@ -57,6 +64,51 @@ def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None: return None +def migrate_audiobook_formats( + *, + load_general_config: Callable[[], dict[str, Any]], + save_general_config: Callable[[dict[str, Any]], None], + widened_formats: Sequence[str], + logger: MigrationLogger, +) -> None: + """Widen an untouched audiobook format list to the current, fuller default. + + Only a list that still matches the old default exactly is rewritten. Any other value + means someone chose it deliberately, and a migration that "helpfully" re-enabled + formats a user had turned off would be worse than leaving them on the narrow list. + """ + try: + config = load_general_config() + + if "SUPPORTED_AUDIOBOOK_FORMATS" not in config: + # Nothing persisted, so the field default already applies. + logger.debug("No persisted audiobook formats - the current default applies") + return + + current = config.get("SUPPORTED_AUDIOBOOK_FORMATS") + if not isinstance(current, list): + return + + normalized = {str(fmt).strip().lower() for fmt in current if str(fmt).strip()} + if normalized != set(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT): + logger.debug( + "Audiobook formats were customized (%s) - left unchanged", sorted(normalized) + ) + return + + save_general_config({"SUPPORTED_AUDIOBOOK_FORMATS": list(widened_formats)}) + logger.info( + "Widened audiobook formats from the legacy default %s to %s", + list(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT), + list(widened_formats), + ) + + except FileNotFoundError: + logger.debug("No existing general config file found - nothing to migrate") + except Exception: + logger.exception("Failed to migrate audiobook formats") + + def migrate_security_settings( *, load_security_config: Callable[[], dict[str, Any]], diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index b9701c4..202668a 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -14,6 +14,7 @@ from shelfmark.config.download_settings_handlers import ( check_books_destination, ) from shelfmark.config.email_settings import check_email_connection +from shelfmark.config.migrations import migrate_audiobook_formats from shelfmark.core.languages import supported_book_languages from shelfmark.core.logger import setup_logger from shelfmark.core.settings_registry import ( @@ -35,6 +36,7 @@ from shelfmark.core.settings_registry import ( register_on_save, register_settings, ) +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60 _DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600 @@ -134,6 +136,20 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]: logger = setup_logger(__name__) + + +def migrate_audiobook_format_settings() -> None: + """Bring installs created before the audiobook format sets were unified up to date.""" + from shelfmark.core.settings_registry import load_config_file, save_config_file + + migrate_audiobook_formats( + load_general_config=lambda: load_config_file("general"), + save_general_config=lambda values: save_config_file("general", values), + widened_formats=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS], + logger=logger, + ) + + _SMTP_PORT_MAX = 65535 _EMAIL_ATTACHMENT_LIMIT_MB_MAX = 600 @@ -215,11 +231,7 @@ _FORMAT_OPTIONS = [ ] _AUDIOBOOK_FORMAT_OPTIONS = [ - {"value": "m4b", "label": "M4B"}, - {"value": "mp3", "label": "MP3"}, - {"value": "m4a", "label": "M4A"}, - {"value": "zip", "label": "ZIP"}, - {"value": "rar", "label": "RAR"}, + {"value": fmt, "label": fmt.upper()} for fmt in (*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS) ] _DOWNLOAD_TO_BROWSER_CONTENT_TYPE_OPTIONS = [ @@ -416,7 +428,7 @@ def general_settings() -> list[SettingsField]: label="Supported Audiobook Formats", description="Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.", options=_AUDIOBOOK_FORMAT_OPTIONS, - default=["m4b", "mp3"], + default=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS], ), MultiSelectField( key="BOOK_LANGUAGE", diff --git a/shelfmark/core/utils.py b/shelfmark/core/utils.py index a012bf4..3dce751 100644 --- a/shelfmark/core/utils.py +++ b/shelfmark/core/utils.py @@ -115,6 +115,21 @@ def is_audiobook(content_type: str | None) -> bool: return bool(content_type and "audiobook" in content_type.lower()) +# Every audio format an audiobook can legitimately arrive in, and the single source of +# truth for that list. The settings UI, release-source parsing, archive extraction and +# post-download scanning all derive from it, so a format added here becomes selectable, +# searchable AND downloadable at once. These used to be four hand-maintained copies that +# had drifted apart: the settings UI only offered m4b/mp3/m4a, which meant a FLAC +# audiobook could never be enabled, was silently dropped from every search result, and +# was rejected after download as "format not supported". +AUDIOBOOK_FORMATS = ("m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus") + +# Multi-file audiobooks are almost always distributed as an archive. These are containers +# rather than formats: they are what a *release* looks like, and the formats above are +# what comes out of one after extraction. +ARCHIVE_FORMATS = ("zip", "rar") + + CONTENT_TYPES = [ "book (fiction)", "book (non-fiction)", diff --git a/shelfmark/download/archive.py b/shelfmark/download/archive.py index 73a242d..1dfb288 100644 --- a/shelfmark/download/archive.py +++ b/shelfmark/download/archive.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, cast from shelfmark.core.logger import setup_logger +from shelfmark.core.utils import AUDIOBOOK_FORMATS from shelfmark.core.utils import is_audiobook as check_audiobook from shelfmark.download.fs import atomic_move from shelfmark.download.postprocess.policy import ( @@ -98,7 +99,7 @@ ALL_EBOOK_EXTENSIONS = { } # All known audio extensions (superset of what user might enable for audiobooks) -ALL_AUDIO_EXTENSIONS = {".m4b", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".wma", ".wav", ".opus"} +ALL_AUDIO_EXTENSIONS = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} def _filter_files( diff --git a/shelfmark/download/postprocess/scan.py b/shelfmark/download/postprocess/scan.py index 5b732fd..e1523e9 100644 --- a/shelfmark/download/postprocess/scan.py +++ b/shelfmark/download/postprocess/scan.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING from shelfmark.core.logger import setup_logger +from shelfmark.core.utils import AUDIOBOOK_FORMATS from shelfmark.core.utils import is_audiobook as check_audiobook from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive from shelfmark.download.fs import run_blocking_io @@ -142,7 +143,7 @@ def scan_directory_tree( is_audiobook = check_audiobook(content_type) if is_audiobook: - trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"} + trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} else: trackable_exts = { ".pdf", @@ -367,7 +368,7 @@ def collect_staged_files( is_audiobook = check_audiobook(task.content_type) if is_audiobook: - trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"} + trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} else: trackable_exts = { ".pdf", diff --git a/shelfmark/main.py b/shelfmark/main.py index 0cb94d1..78157dd 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -38,7 +38,10 @@ from shelfmark.config.env import ( string_to_bool, ) from shelfmark.config.security import _migrate_security_settings -from shelfmark.config.settings import _SUPPORTED_BOOK_LANGUAGE +from shelfmark.config.settings import ( + _SUPPORTED_BOOK_LANGUAGE, + migrate_audiobook_format_settings, +) from shelfmark.core.activity_view_state_service import ActivityViewStateService from shelfmark.core.auth_modes import ( get_auth_check_admin_status, @@ -80,7 +83,7 @@ from shelfmark.core.requests_service import ( sync_delivery_states_from_queue_status, ) from shelfmark.core.user_db import UserDB -from shelfmark.core.utils import normalize_base_path +from shelfmark.core.utils import AUDIOBOOK_FORMATS, normalize_base_path from shelfmark.download import orchestrator as backend from shelfmark.release_sources import ( BrowseRecord, @@ -168,6 +171,9 @@ except ImportError as e: # Migrate legacy security settings if needed _migrate_security_settings() +# Widen audiobook formats for installs that still carry the old m4b/mp3-only default +migrate_audiobook_format_settings() + # Initialize user database and register multi-user routes # If CONFIG_DIR doesn't exist or is read-only, multi-user features will be disabled _user_db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db") @@ -319,19 +325,7 @@ def get_auth_mode() -> str: _AUDIOBOOK_CATEGORY_RANGE = (3030, 3049) -_AUDIOBOOK_FORMAT_HINTS = frozenset( - { - "m4b", - "mp3", - "m4a", - "flac", - "ogg", - "wma", - "aac", - "wav", - "opus", - } -) +_AUDIOBOOK_FORMAT_HINTS = frozenset(AUDIOBOOK_FORMATS) def _contains_audiobook_format_hint(value: Any) -> bool: diff --git a/shelfmark/release_sources/irc/parser.py b/shelfmark/release_sources/irc/parser.py index 53eb034..d8ff791 100644 --- a/shelfmark/release_sources/irc/parser.py +++ b/shelfmark/release_sources/irc/parser.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from shelfmark.core.config import config from shelfmark.core.logger import setup_logger +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS from shelfmark.core.utils import is_audiobook as check_audiobook if TYPE_CHECKING: @@ -18,11 +19,8 @@ if TYPE_CHECKING: logger = setup_logger(__name__) -# All recognized formats for parsing IRC result lines. -# This comprehensive list is used to identify file extensions in results. -# User-configured formats are used separately for filtering. -ALL_RECOGNIZED_FORMATS = { - # Ebook formats +# Ebook formats recognized in IRC result lines. +EBOOK_FORMATS = ( "epub", "mobi", "azw3", @@ -41,19 +39,17 @@ ALL_RECOGNIZED_FORMATS = { "cbz", "cdr", "jpg", - "rar", - "zip", - # Audiobook formats - "m4b", - "mp3", - "m4a", - "flac", - "ogg", - "wma", - "aac", - "wav", - "opus", -} +) + +# All recognized formats for parsing IRC result lines. +# This comprehensive list is used to identify file extensions in results. +# User-configured formats are used separately for filtering. +# Ordered longest-first so that scanning a line matches "azw3" before "azw" and "docx" +# before "doc". It used to be a set, which made the winning format for a line naming more +# than one extension depend on set iteration order, and therefore vary between restarts. +ALL_RECOGNIZED_FORMATS = tuple( + sorted({*EBOOK_FORMATS, *ARCHIVE_FORMATS, *AUDIOBOOK_FORMATS}, key=len, reverse=True) +) def _normalize_config_formats(raw_formats: object) -> set[str]: @@ -84,13 +80,22 @@ def _get_supported_formats(content_type: str | None = None) -> set[str]: # Regex to parse result lines # Format: !Server Author - Title.format ::INFO:: size +# +# The extension is matched against the known formats rather than a bare \w+. A bare \w+ +# happily matched the decimal point in the size, so a line with no file extension parsed +# as format="5mb" out of "::INFO:: 620.5MB" - taking the title and size down with it, and +# leaving the result to be discarded by every format filter downstream. Restricting the +# alternation makes such a line fall through to SIMPLE_RESULT_REGEX and come back as +# "unknown", which is what the rest of the parser already expects. +_FORMAT_ALTERNATION = "|".join(re.escape(fmt) for fmt in ALL_RECOGNIZED_FORMATS) RESULT_LINE_REGEX = re.compile( r"^!(\S+)\s+" # !ServerName r"(.+?)\s+-\s+" # Author Name - - r"(.+?)\.(\w+)" # Title.format + rf"(.+?)\.({_FORMAT_ALTERNATION})\b" # Title.format r"(?:\s+::INFO::\s*(.+?))?" # Optional ::INFO:: metadata r"(?:\s+::HASH::\s*(\S+))?" # Optional ::HASH:: - r"\s*$" + r"\s*$", + re.IGNORECASE, ) # Simpler fallback pattern @@ -187,18 +192,54 @@ def parse_result_line(line: str) -> SearchResult | None: return None +# Words that mark an archive as holding an audiobook rather than an ebook. Multi-file +# audiobooks ship as .rar/.zip, so for those the extension says nothing about the content +# and the release name is the only evidence there is. +_AUDIOBOOK_MARKER_REGEX = re.compile( + r"\b(?:audio ?books?|unabridged|abridged|narrat(?:ed|or)|audible|\d+ ?kbps|" + + "|".join(re.escape(fmt) for fmt in AUDIOBOOK_FORMATS) + + r")\b", + re.IGNORECASE, +) + +_AUDIOBOOK_FORMAT_SET = frozenset(AUDIOBOOK_FORMATS) +_EBOOK_FORMAT_SET = frozenset(EBOOK_FORMATS) + + +def detect_content_type(result: SearchResult) -> str: + """Classify a parsed result as an audiobook or an ebook. + + Extension alone is not enough. It settles the plain cases, but the common audiobook + release is a .rar or .zip of MP3s, which is indistinguishable by extension from an + ebook archive - so for containers (and for lines with no usable extension) the + release name decides. + """ + if result.format in _AUDIOBOOK_FORMAT_SET: + return "audiobook" + if result.format in _EBOOK_FORMAT_SET: + return "ebook" + return "audiobook" if _AUDIOBOOK_MARKER_REGEX.search(result.full_line) else "ebook" + + def parse_results_file(content: str, content_type: str | None = None) -> list[SearchResult]: """Parse a search results file into SearchResult objects.""" results = [] supported = _get_supported_formats(content_type) + requested = "audiobook" if check_audiobook(content_type) else "ebook" for line in content.splitlines(): result = parse_result_line(line) - if result and (result.format in supported or result.format == "unknown"): - # Filter to user's configured formats + if not result: + continue + # Classify first, then apply the user's format filter within that bucket. Doing it + # the other way round is what lost audiobooks entirely: an audiobook .rar matched + # neither the ebook nor the audiobook format list, so it fell out of both. + if detect_content_type(result) != requested: + continue + if result.format in supported or result.format == "unknown": results.append(result) - logger.info("Parsed %s results from search file", len(results)) + logger.info("Parsed %s %s results from search file", len(results), requested) return results diff --git a/shelfmark/release_sources/prowlarr/source.py b/shelfmark/release_sources/prowlarr/source.py index 0205d35..527e9d9 100644 --- a/shelfmark/release_sources/prowlarr/source.py +++ b/shelfmark/release_sources/prowlarr/source.py @@ -16,6 +16,7 @@ from shelfmark.core.languages import normalize_language from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import normalize_optional_text from shelfmark.core.search_plan import ReleaseSearchVariant +from shelfmark.core.utils import AUDIOBOOK_FORMATS as CORE_AUDIOBOOK_FORMATS from shelfmark.core.utils import normalize_http_url from shelfmark.release_sources import ( ColumnAlign, @@ -222,7 +223,7 @@ EBOOK_FORMATS = [ ] # Common audiobook formats -AUDIOBOOK_FORMATS = ["m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus"] +AUDIOBOOK_FORMATS = list(CORE_AUDIOBOOK_FORMATS) # Combined list for format detection (audiobook formats first for priority) ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS diff --git a/tests/config/test_audiobook_format_migration.py b/tests/config/test_audiobook_format_migration.py new file mode 100644 index 0000000..816b5cf --- /dev/null +++ b/tests/config/test_audiobook_format_migration.py @@ -0,0 +1,91 @@ +"""Tests for widening the audiobook format list on existing installs. + +`initialize_default_configs()` only writes field defaults when a tab has no config file +yet, so widening the default alone would have reached fresh installs only - exactly not +the installs already carrying the narrow m4b/mp3 list that loses FLAC/OPUS releases. +""" + +import logging + +import pytest + +from shelfmark.config.migrations import migrate_audiobook_formats +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS + +WIDENED = [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS] + + +@pytest.fixture +def migrate(): + """Run the migration over an in-memory config, returning the resulting config.""" + + def run(config: dict | None) -> dict: + stored = {} if config is None else dict(config) + saved: dict = {} + + def save(values: dict) -> None: + saved.update(values) + stored.update(values) + + migrate_audiobook_formats( + load_general_config=lambda: stored, + save_general_config=save, + widened_formats=WIDENED, + logger=logging.getLogger("test"), + ) + return stored + + return run + + +def test_legacy_default_is_widened(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"]}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED + + +def test_legacy_default_is_widened_regardless_of_order_or_case(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["MP3", " m4b "]}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED + + +def test_other_settings_are_preserved(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"], "SUPPORTED_FORMATS": ["epub"]}) + + assert result["SUPPORTED_FORMATS"] == ["epub"] + + +@pytest.mark.parametrize( + "customized", + [ + ["m4b"], # deliberately narrowed - re-enabling formats would override the choice + ["mp3", "flac"], + ["m4b", "mp3", "zip"], + [], + ], +) +def test_customized_lists_are_left_alone(migrate, customized): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": customized}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == customized + + +def test_absent_key_is_left_alone(migrate): + """Nothing persisted means the field default already applies - don't write one.""" + result = migrate({"SUPPORTED_FORMATS": ["epub"]}) + + assert "SUPPORTED_AUDIOBOOK_FORMATS" not in result + + +def test_unexpected_type_is_left_alone(migrate): + result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": "m4b,mp3"}) + + assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == "m4b,mp3" + + +def test_migration_is_idempotent(migrate): + once = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"]}) + twice = migrate(once) + + assert twice["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED diff --git a/tests/core/test_audiobook_format_consistency.py b/tests/core/test_audiobook_format_consistency.py new file mode 100644 index 0000000..a4bfcac --- /dev/null +++ b/tests/core/test_audiobook_format_consistency.py @@ -0,0 +1,38 @@ +"""The audiobook format list must stay in agreement across every layer that gates on it. + +These lists were maintained by hand in four places and drifted: the settings UI offered +only m4b/mp3/m4a, so FLAC/OPUS/OGG could never be enabled even though the parsers +recognized them, the sorter ranked them, and archive extraction knew them. The result was +a FLAC audiobook that was invisible in search and rejected after download. They now all +derive from `shelfmark.core.utils.AUDIOBOOK_FORMATS`; this test fails if one drifts again. +""" + +from shelfmark.config.settings import _AUDIOBOOK_FORMAT_OPTIONS +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS +from shelfmark.download.archive import ALL_AUDIO_EXTENSIONS +from shelfmark.release_sources.irc import parser +from shelfmark.release_sources.prowlarr.source import AUDIOBOOK_FORMATS as PROWLARR_FORMATS + + +def test_archive_extraction_knows_every_audiobook_format(): + assert ALL_AUDIO_EXTENSIONS == {f".{fmt}" for fmt in AUDIOBOOK_FORMATS} + + +def test_prowlarr_knows_every_audiobook_format(): + assert PROWLARR_FORMATS == list(AUDIOBOOK_FORMATS) + + +def test_irc_parser_knows_every_audiobook_format(): + assert set(AUDIOBOOK_FORMATS) <= set(parser.ALL_RECOGNIZED_FORMATS) + + +def test_every_audiobook_format_is_selectable_in_settings(): + """The settings list is the only one a user's config can be built from.""" + selectable = {option["value"] for option in _AUDIOBOOK_FORMAT_OPTIONS} + + assert selectable == {*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS} + + +def test_audiobook_and_ebook_formats_do_not_overlap(): + """Overlap would make content-type classification by extension ambiguous.""" + assert not set(AUDIOBOOK_FORMATS) & set(parser.EBOOK_FORMATS) diff --git a/tests/irc/test_parser.py b/tests/irc/test_parser.py index 32841e5..ade17f3 100644 --- a/tests/irc/test_parser.py +++ b/tests/irc/test_parser.py @@ -1,5 +1,96 @@ +import pytest + +from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS from shelfmark.release_sources.irc import parser +# What a stock install actually filters with, so these tests fail if the defaults regress. +_DEFAULT_CONFIG = { + "SUPPORTED_FORMATS": ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"], + "SUPPORTED_AUDIOBOOK_FORMATS": [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS], +} + + +@pytest.fixture +def default_formats(monkeypatch): + """Filter with the shipped default format lists.""" + monkeypatch.setattr( + parser.config, "get", lambda key, default=None: _DEFAULT_CONFIG.get(key, default) + ) + + +def test_audiobook_archives_are_found_with_default_settings(default_formats): + """Regression for #1129: audiobooks ship as .rar/.zip and were dropped by both buckets. + + A single "@search" answers with one file holding every format. These are the lines an + audiobook actually occupies in it - the extension is a container, and the release name + is the only thing saying what is inside. + """ + content = "\n".join( + [ + "!Oatmeal Andy Weir - Project Hail Mary (Audiobook) [MP3 64kbps].rar ::INFO:: 620.5MB", + "!DV8 Andy Weir - Project Hail Mary - Audiobook.zip ::INFO:: 700MB", + "!Horla Andy Weir - Project Hail Mary [Unabridged].m4b ::INFO:: 850.1MB", + ] + ) + + results = parser.parse_results_file(content, content_type="audiobook") + + assert [result.format for result in results] == ["rar", "zip", "m4b"] + + +def test_ebook_archive_does_not_leak_into_audiobook_results(default_formats): + """An ebook .rar must not be offered as an audiobook just because it is an archive.""" + content = "!bald Andy Weir - Project Hail Mary (retail).rar ::INFO:: 2.1MB" + + assert parser.parse_results_file(content, content_type="audiobook") == [] + + +def test_flac_audiobook_is_reachable_with_default_settings(default_formats): + """FLAC was recognized by the parser and ranked by the sorter, but never selectable.""" + content = "!Ook Andy Weir - Project Hail Mary.flac ::INFO:: 1.1GB" + + results = parser.parse_results_file(content, content_type="audiobook") + + assert [result.format for result in results] == ["flac"] + + +def test_decimal_size_is_not_mistaken_for_a_file_extension(): + """A line with no extension used to parse as format="5mb" out of "::INFO:: 620.5MB".""" + line = "!Ook Andy Weir - Project Hail Mary (2021) Audiobook ::INFO:: 620.5MB" + + result = parser.parse_result_line(line) + + assert result.format == "unknown" + assert result.title == "Project Hail Mary (2021) Audiobook" + assert result.size == "620.5MB" + + +@pytest.mark.parametrize( + ("line", "expected"), + [ + ("!s A - T.epub ::INFO:: 1MB", "ebook"), + ("!s A - T.mp3 ::INFO:: 1MB", "audiobook"), + ("!s A - T.flac ::INFO:: 1MB", "audiobook"), + # Archives carry no format information, so the name has to decide. + ("!s A - T (Audiobook).rar ::INFO:: 1MB", "audiobook"), + ("!s A - T [Unabridged].zip ::INFO:: 1MB", "audiobook"), + ("!s A - T (Narrated by Someone).rar ::INFO:: 1MB", "audiobook"), + ("!s A - T [64kbps].zip ::INFO:: 1MB", "audiobook"), + ("!s A - T (retail).rar ::INFO:: 1MB", "ebook"), + ("!s A - T.zip ::INFO:: 1MB", "ebook"), + ], +) +def test_detect_content_type(line, expected): + assert parser.detect_content_type(parser.parse_result_line(line)) == expected + + +def test_recognized_formats_order_is_deterministic(): + """This was a set, so which format won for a multi-extension line varied per restart.""" + assert parser.ALL_RECOGNIZED_FORMATS == tuple(parser.ALL_RECOGNIZED_FORMATS) + # Longest-first, so ".azw3" cannot be truncated to "azw" (nor ".docx" to "doc"). + lengths = [len(fmt) for fmt in parser.ALL_RECOGNIZED_FORMATS] + assert lengths == sorted(lengths, reverse=True) + def test_parse_results_file_uses_audiobook_format_settings(monkeypatch): values = {