diff --git a/shelfmark/download/clients/base_handler.py b/shelfmark/download/clients/base_handler.py index 016f473e..f385f41e 100644 --- a/shelfmark/download/clients/base_handler.py +++ b/shelfmark/download/clients/base_handler.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Protocol, TypeGuard from shelfmark.core.config import config from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import coerce_int, normalize_optional_text -from shelfmark.core.utils import is_audiobook +from shelfmark.core.utils import AUDIOBOOK_FORMATS, DEFAULT_EBOOK_FORMATS, is_audiobook from shelfmark.download.activity import release_activity_grace, request_activity_grace from shelfmark.download.clients import ( DownloadClient, @@ -288,6 +288,56 @@ def _count_expected_files_present(path: Path, expected: Sequence[ExpectedFile]) return present +# Extensions that mean a finished book or recording, whichever content type the task is for. +# A file with one of these that the task does not accept (an .m4b in an ebook task, a .pdf +# with the default format list) is a wrong-type result, not a sidecar still waiting for the +# real files to sync. +_PAYLOAD_SUFFIXES = frozenset( + f".{fmt.lstrip('.').lower()}" for fmt in (*AUDIOBOOK_FORMATS, *DEFAULT_EBOOK_FORMATS, "pdf") +) + + +def _unsupported_payload_signature( + path: Path, content_type: str | None +) -> tuple[tuple[str, int, int], ...] | None: + """Sizes and mtimes of book or audio files in a path that this task does not accept. + + None when there are none. Used to tell a finished directory that holds the wrong kind of + files apart from one that has only cover art and notes so far. + """ + from shelfmark.download.postprocess.policy import ( + get_supported_audiobook_formats, + get_supported_formats, + ) + + formats = ( + get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats() + ) + supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats} + + try: + if not path.is_dir(): + return None + signature = [] + for candidate in path.rglob("*"): + suffix = candidate.suffix.lower() + if suffix not in _PAYLOAD_SUFFIXES or suffix in supported_suffixes: + continue + try: + stat = candidate.stat() + except OSError: + continue + if not candidate.is_file() or stat.st_size <= 0: + continue + signature.append( + (candidate.relative_to(path).as_posix(), stat.st_size, stat.st_mtime_ns) + ) + except OSError: + return None + + return tuple(sorted(signature)) or None + + class ExternalClientHandler(DownloadHandler, ABC): """Shared lifecycle handler for sources that hand off to torrent/usenet clients.""" @@ -911,6 +961,7 @@ class ExternalClientHandler(DownloadHandler, ABC): previous_signature: tuple[tuple[str, int, int], ...] | None = None expected_files: list[ExpectedFile] | None = None expected_files_asked = False + previous_unsupported: tuple[tuple[str, int, int], ...] | None = None retry_interval = self._completed_path_retry_interval() timeout_seconds = self._completed_path_timeout_seconds() if retry_interval <= 0 or timeout_seconds <= 0: @@ -960,6 +1011,20 @@ class ExternalClientHandler(DownloadHandler, ABC): ): return result.path, None previous_signature = signature + if signature is None: + # Nothing this task accepts. If the directory instead holds finished + # files of another kind and has stopped changing, waiting longer will + # not help: hand it on so post-processing names the format problem. + unsupported = run_blocking_io( + _unsupported_payload_signature, + result.path, + content_type, + ) + if unsupported is not None and ( + unsupported == previous_unsupported or max_attempts == 1 + ): + return result.path, None + previous_unsupported = unsupported last_error = "Completed download has no stable supported files yet." if attempt < max_attempts: status_callback("locating", "Waiting for completed files...") diff --git a/tests/prowlarr/test_remote_path_mappings.py b/tests/prowlarr/test_remote_path_mappings.py index 00eafcdc..9ca4b68a 100644 --- a/tests/prowlarr/test_remote_path_mappings.py +++ b/tests/prowlarr/test_remote_path_mappings.py @@ -9,6 +9,8 @@ from pathlib import Path from threading import Event from unittest.mock import MagicMock, patch +import pytest + from shelfmark.core.models import DownloadTask from shelfmark.download.clients import DownloadState, DownloadStatus from shelfmark.release_sources.prowlarr.handler import ProwlarrHandler @@ -366,7 +368,13 @@ def test_waits_for_supported_file_to_stop_growing_in_remapped_directory(monkeypa def _wait_for_local_directory( - monkeypatch, directory: Path, *, timeout: object, on_sleep=None, expected=None + monkeypatch, + directory: Path, + *, + timeout: object, + content_type: str | None = None, + on_sleep=None, + expected=None, ): import shelfmark.download.clients.base_handler as base_handler @@ -391,6 +399,7 @@ def _wait_for_local_directory( "download_id", cancel_flag=None, status_callback=ProgressRecorder().status_callback, + content_type=content_type, ) @@ -549,6 +558,66 @@ def test_a_client_without_a_file_list_keeps_the_old_behaviour(monkeypatch, tmp_p assert error is None +@pytest.mark.parametrize( + ("content_type", "filename"), + [ + (None, "book.m4b"), + ("book (fiction)", "recording.mp3"), + ("audiobook", "book.epub"), + (None, "book.pdf"), + ], +) +def test_a_finished_directory_of_the_wrong_kind_is_returned_without_waiting_out_the_timeout( + monkeypatch, tmp_path, content_type, filename +): + """Post-processing can name the format problem; polling for 60 s to say it later cannot (F3).""" + release = tmp_path / "Release" + release.mkdir() + (release / filename).write_bytes(b"finished") + + resolved_path, error = _wait_for_local_directory( + monkeypatch, release, timeout=1, content_type=content_type + ) + + assert resolved_path == release + assert error is None + + +def test_a_directory_with_only_cover_art_and_notes_keeps_waiting(monkeypatch, tmp_path): + """The book may still be syncing: sidecar files alone are not a finished wrong-type result.""" + release = tmp_path / "Release" + release.mkdir() + (release / "cover.jpg").write_bytes(b"jpg") + (release / "book.nfo").write_text("nfo") + + resolved_path, error = _wait_for_local_directory(monkeypatch, release, timeout=1) + + assert resolved_path is None + assert error == "Completed download has no stable supported files yet." + + +def test_a_wrong_kind_file_that_is_still_growing_keeps_waiting(monkeypatch, tmp_path): + release = tmp_path / "Release" + release.mkdir() + growing = release / "book.m4b" + growing.write_bytes(b"a") + sleeps = 0 + + def grow(_seconds: float) -> None: + nonlocal sleeps + sleeps += 1 + growing.write_bytes(b"a" * (sleeps + 1)) + + resolved_path, error = _wait_for_local_directory( + monkeypatch, release, timeout=0.05, on_sleep=grow + ) + + # The file changes on every poll, so it is never stable and the wait runs out. + assert resolved_path is None + assert error == "Completed download has no stable supported files yet." + assert sleeps >= 2 + + def test_wait_for_completed_path_fails_fast_for_unsafe_mapping(monkeypatch, tmp_path): import shelfmark.download.clients.base_handler as base_handler