refactor: one lookup for the supported formats, public names for two shared helpers (#1576)

Six places read the Supported Book Formats and Supported Audiobook
Formats settings on their own: post-processing, the post-process scan,
IRC, slskd, Prowlarr and direct download. For ebooks they already
agreed, since they all fall back to `DEFAULT_EBOOK_FORMATS`. For
audiobooks they did not. The settings registry and slskd fall back to
every audio format plus zip and rar, while post-processing and IRC fell
back to m4b and mp3 only.

They now share `shelfmark/core/formats.py`, with
`get_supported_formats`, `get_supported_audiobook_formats` and a
content-type aware `get_supported_formats_for`, and a shared
`DEFAULT_AUDIOBOOK_FORMATS` in `core/utils.py` next to the ebook one.
`policy.py` re-exports the same functions, so nothing that imports them
from there changes. It is not in `policy.py` itself because importing
the `postprocess` package pulls in the router, which imports the release
sources, so a source could not import from it without a cycle.

One behaviour difference, and only in the fallback: with the audiobook
setting unset, or emptied, post-processing and IRC used to accept m4b
and mp3 and now accept the full default list like slskd and the settings
page already did. A configured list is read exactly as before. Direct
download's own helper used the raw configured list and now reads the
same normalized one, which is identical for any list the settings page
can save.

Also public now: `parse_size` in the Prowlarr source (slskd and Newznab
imported it with the underscore) and
`coerce_completed_path_timeout_seconds` in the client base handler
(slskd imported that too).

Not changed on purpose: slskd's `_coerce_int` looks like a copy of
`coerce_int` but is not. It reads `"12.0"` as 12 and `True` as 0, where
`coerce_int` gives 0 and 1. And the `_parse_size` in the IRC source is a
different function, string to bytes.

Tests: a new set checks that post-processing, IRC and slskd all fall
back to the shared defaults and that the settings default is the shared
one, and the scan-archive and IRC tests now patch the shared config
instead of the per-module copies. Full suite passes.
This commit is contained in:
splitsec2 authored and GitHub committed 2026-10-24 16:31:07 -06:00
1 parent 606e94fde2
commit 9011319ade
16 files changed
+154 -88

No files matched your search

+7 -2
View File
@@ -36,7 +36,12 @@ from shelfmark.core.settings_registry import (
register_on_save,
register_settings,
)
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS, DEFAULT_EBOOK_FORMATS
from shelfmark.core.utils import (
ARCHIVE_FORMATS,
AUDIOBOOK_FORMATS,
DEFAULT_AUDIOBOOK_FORMATS,
DEFAULT_EBOOK_FORMATS,
)
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600
@@ -503,7 +508,7 @@ def general_settings() -> list[SettingsField]:
label="Supported Audiobook Formats",
description="Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.",
options=_AUDIOBOOK_FORMAT_OPTIONS,
default=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS],
default=list(DEFAULT_AUDIOBOOK_FORMATS),
),
]
+35
View File
@@ -0,0 +1,35 @@
"""Which file formats are enabled, by content type.
The one place that reads the Supported Book Formats and Supported Audiobook Formats
settings. It sits in ``core`` and imports nothing heavy, so release sources and
post-processing can both use it without an import cycle.
"""
from __future__ import annotations
import shelfmark.core.config as core_config
from shelfmark.core.utils import (
DEFAULT_AUDIOBOOK_FORMATS,
DEFAULT_EBOOK_FORMATS,
is_audiobook,
normalize_format_list,
)
def get_supported_formats() -> list[str]:
"""Get current supported book formats from the config singleton."""
formats = core_config.config.get("SUPPORTED_FORMATS", None)
return normalize_format_list(formats, DEFAULT_EBOOK_FORMATS)
def get_supported_audiobook_formats() -> list[str]:
"""Get current supported audiobook formats from the config singleton."""
formats = core_config.config.get("SUPPORTED_AUDIOBOOK_FORMATS", None)
return normalize_format_list(formats, DEFAULT_AUDIOBOOK_FORMATS)
def get_supported_formats_for(content_type: str | None) -> list[str]:
"""Supported formats for a task's content type: audiobook formats or book formats."""
if is_audiobook(content_type):
return get_supported_audiobook_formats()
return get_supported_formats()
+4
View File
@@ -146,6 +146,10 @@ ARCHIVE_FORMATS = ("zip", "rar")
# registry, post-processing and every release source fall back to this one list.
DEFAULT_EBOOK_FORMATS = ("epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr")
# What the Supported Audiobook Formats setting holds until someone changes it: every audio
# format, and the archives a multi-file audiobook usually comes in. Same rule as above.
DEFAULT_AUDIOBOOK_FORMATS = (*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS)
def normalize_format_list(value: object, default: tuple[str, ...] | list[str]) -> list[str]:
"""Read a configured format list, or a comma or space separated string of them.
+2 -2
View File
@@ -123,7 +123,7 @@ class _CompletedPathResolution:
retryable: bool
def _coerce_completed_path_timeout_seconds(value: object, default: float) -> float:
def coerce_completed_path_timeout_seconds(value: object, default: float) -> float:
if isinstance(value, bool) or value is None:
return default
if isinstance(value, (int, float)):
@@ -332,7 +332,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
"""Total time to wait for completed files to appear on disk."""
fallback = self._completed_path_retry_interval() * self._completed_path_max_attempts()
configured = config.get(COMPLETED_PATH_TIMEOUT_SETTING, fallback)
return _coerce_completed_path_timeout_seconds(configured, fallback)
return coerce_completed_path_timeout_seconds(configured, fallback)
def _refresh_download_request_after_add_failure(
self,
+10 -12
View File
@@ -17,7 +17,11 @@ circular imports (`archive` is used by the pipeline).
from __future__ import annotations
import shelfmark.core.config as core_config
from shelfmark.core.utils import DEFAULT_EBOOK_FORMATS, normalize_format_list
from shelfmark.core.formats import (
get_supported_audiobook_formats,
get_supported_formats,
get_supported_formats_for,
)
def _config_text(value: object) -> str:
@@ -26,17 +30,11 @@ def _config_text(value: object) -> str:
return ""
def get_supported_formats() -> list[str]:
"""Get current supported formats from config singleton."""
formats = core_config.config.get("SUPPORTED_FORMATS", None)
return normalize_format_list(formats, DEFAULT_EBOOK_FORMATS)
def get_supported_audiobook_formats() -> list[str]:
"""Get current supported audiobook formats from config singleton."""
default_formats = ["m4b", "mp3"]
formats = core_config.config.get("SUPPORTED_AUDIOBOOK_FORMATS", default_formats)
return normalize_format_list(formats, default_formats)
__all__ = [
"get_supported_audiobook_formats",
"get_supported_formats",
"get_supported_formats_for",
]
def get_file_organization(*, is_audiobook: bool) -> str:
+1 -11
View File
@@ -13,10 +13,7 @@ from shelfmark.download.archive import ArchiveExtractionError, extract_archive,
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.permissions_debug import log_path_permission_context
from shelfmark.download.postprocess.policy import (
get_supported_audiobook_formats,
)
from shelfmark.download.postprocess.policy import (
get_supported_formats as get_book_formats,
get_supported_formats_for as get_supported_formats,
)
from shelfmark.download.staging import build_staging_dir
@@ -28,13 +25,6 @@ if TYPE_CHECKING:
logger = setup_logger("shelfmark.download.postprocess.pipeline")
def get_supported_formats(content_type: str | None = None) -> list[str]:
"""Return supported file extensions for the requested content type."""
if check_audiobook(content_type):
return get_supported_audiobook_formats()
return get_book_formats()
def _format_not_supported_error(rejected_files: list[Path], task: DownloadTask) -> str:
content_type = task.content_type
file_type_label = "audiobook" if check_audiobook(content_type) else "book"
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable
from bs4 import BeautifulSoup, Tag
from shelfmark.core.config import config
from shelfmark.core.formats import get_supported_formats as _configured_book_formats
from shelfmark.core.languages import language_alias_map
from shelfmark.release_sources import BrowseRecord, SourceUnavailableError
@@ -40,7 +40,7 @@ def coerce_str_list(value: object) -> list[str]:
def get_supported_formats() -> list[str]:
"""Return configured supported formats as a clean string list."""
return coerce_str_list(config.SUPPORTED_FORMATS)
return _configured_book_formats()
def html_response_text(response: str | tuple[str, str]) -> str:
+3 -11
View File
@@ -8,14 +8,9 @@ import zipfile
from dataclasses import dataclass
from typing import TYPE_CHECKING
from shelfmark.core.config import config
from shelfmark.core.formats import get_supported_formats_for
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import (
ARCHIVE_FORMATS,
AUDIOBOOK_FORMATS,
DEFAULT_EBOOK_FORMATS,
normalize_format_list,
)
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
from shelfmark.core.utils import is_audiobook as check_audiobook
if TYPE_CHECKING:
@@ -58,10 +53,7 @@ ALL_RECOGNIZED_FORMATS = tuple(
def _get_supported_formats(content_type: str | None = None) -> set[str]:
"""Get the supported formats for the requested content type."""
if check_audiobook(content_type):
formats = config.get("SUPPORTED_AUDIOBOOK_FORMATS", None)
return set(normalize_format_list(formats, ["m4b", "mp3"]))
return set(normalize_format_list(config.get("SUPPORTED_FORMATS", None), DEFAULT_EBOOK_FORMATS))
return set(get_supported_formats_for(content_type))
# Regex to parse result lines
+2 -2
View File
@@ -38,7 +38,7 @@ from shelfmark.release_sources.prowlarr.source import (
# Re-use the Prowlarr source helpers — they operate on generic result dicts.
from shelfmark.release_sources.prowlarr.source import (
_detect_content_type_from_categories,
_parse_size,
parse_size,
)
logger = setup_logger(__name__)
@@ -246,7 +246,7 @@ def _newznab_result_to_release(
title=raw_title,
format=None,
language=None,
size=_parse_size(size_bytes),
size=parse_size(size_bytes),
size_bytes=size_bytes,
download_url=None,
info_url=info_url,
+5 -5
View File
@@ -15,12 +15,13 @@ if TYPE_CHECKING:
from shelfmark.core.author_match import AUTHOR_UNKNOWN, author_affinity
from shelfmark.core.config import config
from shelfmark.core.formats import get_supported_formats
from shelfmark.core.languages import normalize_language
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import coerce_int, normalize_optional_text
from shelfmark.core.search_plan import ReleaseSearchVariant
from shelfmark.core.utils import AUDIOBOOK_FORMATS as CORE_AUDIOBOOK_FORMATS
from shelfmark.core.utils import DEFAULT_EBOOK_FORMATS, normalize_format_list, normalize_http_url
from shelfmark.core.utils import normalize_http_url
from shelfmark.release_sources import (
ColumnAlign,
ColumnColorHint,
@@ -309,7 +310,7 @@ def _is_over_size(result: dict, max_bytes: int | None) -> bool:
return size > max_bytes
def _parse_size(size_bytes: int | None) -> str | None:
def parse_size(size_bytes: int | None) -> str | None:
"""Convert bytes to human-readable size string."""
if size_bytes is None or size_bytes <= 0:
return None
@@ -395,8 +396,7 @@ def _unsupported_title_extension(title: str, content_type: str) -> str | None:
extension = match.group(1)
if extension not in _KNOWN_FILE_EXTENSIONS:
return None
supported = normalize_format_list(config.get("SUPPORTED_FORMATS", None), DEFAULT_EBOOK_FORMATS)
if extension in supported:
if extension in get_supported_formats():
return None
return extension
@@ -720,7 +720,7 @@ def _prowlarr_result_to_release(
title=title,
format=format_detected,
language=language_detected,
size=_parse_size(size_bytes),
size=parse_size(size_bytes),
size_bytes=size_bytes,
download_url=None,
info_url=result.get("infoUrl") or result.get("guid"),
+2 -2
View File
@@ -35,7 +35,7 @@ from shelfmark.download.clients.base_handler import (
)
from shelfmark.download.clients.base_handler import (
COMPLETED_PATH_TIMEOUT_SETTING,
_coerce_completed_path_timeout_seconds,
coerce_completed_path_timeout_seconds,
)
from shelfmark.download.fs import run_blocking_io
from shelfmark.download.postprocess.packs import PackFile
@@ -426,7 +426,7 @@ class SlskdHandler(DownloadHandler):
def _completed_path_timeout_seconds(self) -> float:
fallback = self._completed_path_retry_interval() * COMPLETED_PATH_MAX_ATTEMPTS
return _coerce_completed_path_timeout_seconds(
return coerce_completed_path_timeout_seconds(
config.get(COMPLETED_PATH_TIMEOUT_SETTING, fallback), fallback
)
+5 -14
View File
@@ -18,12 +18,10 @@ if TYPE_CHECKING:
from shelfmark.metadata_providers import BookMetadata
from shelfmark.core.config import config
from shelfmark.core.formats import get_supported_formats_for
from shelfmark.core.logger import setup_logger
from shelfmark.core.utils import (
ARCHIVE_FORMATS,
AUDIOBOOK_FORMATS,
DEFAULT_EBOOK_FORMATS,
normalize_format_list,
normalize_http_url,
)
from shelfmark.core.utils import is_audiobook as check_audiobook
@@ -43,7 +41,7 @@ from shelfmark.release_sources import (
SourceUnavailableError,
register_source,
)
from shelfmark.release_sources.prowlarr.source import _parse_size
from shelfmark.release_sources.prowlarr.source import parse_size
from shelfmark.release_sources.slskd.api import SlskdClient
logger = setup_logger(__name__)
@@ -60,7 +58,6 @@ MAX_QUERY_VARIANTS = 3
# Whole-book audio files: one file holds the entire book, unlike chapter files.
CONTAINER_FORMATS = frozenset({"m4b", "m4a"})
_DEFAULT_AUDIOBOOK_FORMATS = [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS]
@dataclass(frozen=True)
@@ -133,13 +130,7 @@ def make_source_id(username: str, remote_path: str) -> str:
def get_supported_formats(content_type: str | None) -> set[str]:
"""Return the configured formats for the requested content type."""
if check_audiobook(content_type):
return set(
normalize_format_list(
config.get("SUPPORTED_AUDIOBOOK_FORMATS", None), _DEFAULT_AUDIOBOOK_FORMATS
)
)
return set(normalize_format_list(config.get("SUPPORTED_FORMATS", None), DEFAULT_EBOOK_FORMATS))
return set(get_supported_formats_for(content_type))
def _coerce_int(value: object) -> int:
@@ -232,7 +223,7 @@ def file_release(peer_file: PeerFile, response: dict[str, Any], content_type: st
title=_strip_extension(peer_file.basename),
format=peer_file.extension or None,
language=None,
size=_parse_size(peer_file.size),
size=parse_size(peer_file.size),
size_bytes=peer_file.size or None,
download_url=None,
info_url=None,
@@ -272,7 +263,7 @@ def directory_release(
title=f"{directory_name} ({len(files)} files)",
format=formats[0] if formats else None,
language=None,
size=_parse_size(total_size),
size=parse_size(total_size),
size_bytes=total_size or None,
download_url=None,
info_url=None,
+52 -1
View File
@@ -3,7 +3,12 @@
import pytest
from shelfmark.config import settings as app_settings
from shelfmark.core.utils import DEFAULT_EBOOK_FORMATS, normalize_format_list
from shelfmark.core.formats import get_supported_formats_for
from shelfmark.core.utils import (
DEFAULT_AUDIOBOOK_FORMATS,
DEFAULT_EBOOK_FORMATS,
normalize_format_list,
)
from shelfmark.download.postprocess import policy
DEFAULT = ["epub", "mobi"]
@@ -44,3 +49,49 @@ def test_post_processing_falls_back_to_the_shared_default(monkeypatch):
monkeypatch.setattr("shelfmark.core.config.config", _Config())
assert policy.get_supported_formats() == list(DEFAULT_EBOOK_FORMATS)
def test_the_audiobook_registry_default_is_the_shared_default():
field = next(
f
for f in app_settings.general_settings()
if getattr(f, "key", "") == "SUPPORTED_AUDIOBOOK_FORMATS"
)
assert field.default == list(DEFAULT_AUDIOBOOK_FORMATS)
class TestEveryReaderFallsBackToTheSameDefaults:
"""Post-processing and the sources each read the formats; none may keep its own default."""
@pytest.fixture(autouse=True)
def unset_config(self, monkeypatch):
class _Config:
def get(self, key, default=None, user_id=None):
return default
monkeypatch.setattr("shelfmark.core.config.config", _Config())
def test_post_processing(self):
assert policy.get_supported_formats() == list(DEFAULT_EBOOK_FORMATS)
assert policy.get_supported_audiobook_formats() == list(DEFAULT_AUDIOBOOK_FORMATS)
@pytest.mark.parametrize(
("content_type", "expected"),
[
(None, DEFAULT_EBOOK_FORMATS),
("book (fiction)", DEFAULT_EBOOK_FORMATS),
("audiobook", DEFAULT_AUDIOBOOK_FORMATS),
],
)
def test_the_lookup_follows_the_content_type(self, content_type, expected):
assert get_supported_formats_for(content_type) == list(expected)
def test_irc_and_slskd(self):
from shelfmark.release_sources.irc.parser import _get_supported_formats
from shelfmark.release_sources.slskd.source import get_supported_formats
assert _get_supported_formats("audiobook") == set(DEFAULT_AUDIOBOOK_FORMATS)
assert get_supported_formats("audiobook") == set(DEFAULT_AUDIOBOOK_FORMATS)
assert _get_supported_formats("ebook") == set(DEFAULT_EBOOK_FORMATS)
assert get_supported_formats("ebook") == set(DEFAULT_EBOOK_FORMATS)
@@ -2,13 +2,14 @@ from __future__ import annotations
import zipfile
from shelfmark.core import formats as formats_mod
from shelfmark.core.models import DownloadTask
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
from shelfmark.core.utils import DEFAULT_AUDIOBOOK_FORMATS
from shelfmark.download import archive as archive_mod
from shelfmark.download.postprocess import scan as scan_mod
# The shipped default for SUPPORTED_AUDIOBOOK_FORMATS, which includes archives.
_DEFAULT_AUDIOBOOK_FORMATS = [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS]
_DEFAULT_AUDIOBOOK_FORMATS = list(DEFAULT_AUDIOBOOK_FORMATS)
def _audiobook_task() -> DownloadTask:
@@ -17,7 +18,7 @@ def _audiobook_task() -> DownloadTask:
def _use_default_audiobook_formats(monkeypatch, tmp_path) -> None:
monkeypatch.setattr(
scan_mod, "get_supported_audiobook_formats", lambda: list(_DEFAULT_AUDIOBOOK_FORMATS)
formats_mod, "get_supported_audiobook_formats", lambda: list(_DEFAULT_AUDIOBOOK_FORMATS)
)
monkeypatch.setattr(
archive_mod, "get_supported_audiobook_formats", lambda: list(_DEFAULT_AUDIOBOOK_FORMATS)
+4 -5
View File
@@ -1,5 +1,6 @@
import pytest
from shelfmark.core.config import config
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
from shelfmark.release_sources.irc import parser
@@ -13,9 +14,7 @@ _DEFAULT_CONFIG = {
@pytest.fixture
def default_formats(monkeypatch):
"""Filter with the shipped default format lists."""
monkeypatch.setattr(
parser.config, "get", lambda key, default=None: _DEFAULT_CONFIG.get(key, default)
)
monkeypatch.setattr(config, "get", lambda key, default=None: _DEFAULT_CONFIG.get(key, default))
def test_audiobook_archives_are_found_with_default_settings(default_formats):
@@ -98,7 +97,7 @@ def test_parse_results_file_uses_audiobook_format_settings(monkeypatch):
"SUPPORTED_AUDIOBOOK_FORMATS": ["zip", "mp3"],
}
monkeypatch.setattr(parser.config, "get", lambda key, default=None: values.get(key, default))
monkeypatch.setattr(config, "get", lambda key, default=None: values.get(key, default))
content = "\n".join(
[
@@ -119,7 +118,7 @@ def test_parse_results_file_uses_book_format_settings_for_ebooks(monkeypatch):
"SUPPORTED_AUDIOBOOK_FORMATS": ["zip", "mp3"],
}
monkeypatch.setattr(parser.config, "get", lambda key, default=None: values.get(key, default))
monkeypatch.setattr(config, "get", lambda key, default=None: values.get(key, default))
content = "\n".join(
[
+16 -16
View File
@@ -22,11 +22,11 @@ from shelfmark.release_sources.prowlarr.source import (
_extract_mam_language,
_fetch_indexer_seed_settings,
_last_known_seed_settings,
_parse_size,
_release_identity,
_result_dedup_key,
_search_budget_seconds,
_unsupported_title_extension,
parse_size,
)
from shelfmark.release_sources.prowlarr.utils import (
build_source_id,
@@ -50,45 +50,45 @@ def source_available_by_default(monkeypatch):
class TestParseSize:
"""Tests for the _parse_size function."""
"""Tests for the parse_size function."""
def test_parse_size_bytes(self):
"""Test parsing small byte sizes."""
assert _parse_size(100) == "100 B"
assert _parse_size(512) == "512 B"
assert parse_size(100) == "100 B"
assert parse_size(512) == "512 B"
def test_parse_size_kilobytes(self):
"""Test parsing kilobyte sizes."""
assert _parse_size(1024) == "1.0 KB"
assert _parse_size(2048) == "2.0 KB"
assert _parse_size(1536) == "1.5 KB"
assert parse_size(1024) == "1.0 KB"
assert parse_size(2048) == "2.0 KB"
assert parse_size(1536) == "1.5 KB"
def test_parse_size_megabytes(self):
"""Test parsing megabyte sizes."""
assert _parse_size(1048576) == "1.0 MB"
assert _parse_size(5242880) == "5.0 MB"
assert _parse_size(1572864) == "1.5 MB"
assert parse_size(1048576) == "1.0 MB"
assert parse_size(5242880) == "5.0 MB"
assert parse_size(1572864) == "1.5 MB"
def test_parse_size_gigabytes(self):
"""Test parsing gigabyte sizes."""
assert _parse_size(1073741824) == "1.0 GB"
assert _parse_size(2147483648) == "2.0 GB"
assert parse_size(1073741824) == "1.0 GB"
assert parse_size(2147483648) == "2.0 GB"
def test_parse_size_terabytes(self):
"""Test parsing terabyte sizes."""
assert _parse_size(1099511627776) == "1.0 TB"
assert parse_size(1099511627776) == "1.0 TB"
def test_parse_size_none(self):
"""Test that None returns None."""
assert _parse_size(None) is None
assert parse_size(None) is None
def test_parse_size_zero(self):
"""Test that zero returns None."""
assert _parse_size(0) is None
assert parse_size(0) is None
def test_parse_size_negative(self):
"""Test that negative values return None."""
assert _parse_size(-100) is None
assert parse_size(-100) is None
class TestExtractFormat: