From 525b094b5728bbb852638a55810fd3ebcfd8b9aa Mon Sep 17 00:00:00 2001 From: splitsec2 <35583321+splitsec2@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:41:21 -0600 Subject: [PATCH] fix(clients): hand a finished wrong-type directory on instead of polling out the timeout (#1573) A torrent that finishes with the wrong kind of files in it holds the download for the whole completed-path timeout (60 seconds by default, up to an hour) and then fails with "Completed download has no stable supported files yet." An ebook task whose torrent holds only an `.m4b`, an audiobook task that got an `.epub`, or a `.pdf`-only release with the default format list all land here. In v1.4.0 these failed with the specific format error from post-processing, but the wait for the sync to settle (#1464) only counts files of the task's own type, so it keeps polling for files that are never going to appear. If the directory has no supported files, `_wait_for_completed_path` now looks for finished book or audio files that this task does not accept (the audiobook formats, the default ebook formats and pdf). When it finds some and they are the same on two polls in a row, it returns the path so post-processing can say what is wrong with the formats. Cover art and notes alone do not count, because that is what a directory looks like while the real files are still syncing over a remote mount, so those keep waiting as before. A wrong-type file that is still growing keeps waiting too. With no wait configured there is only one sample, and a wrong-type file is handed on straight away. The existing test that pins "no book files, no wait configured" uses a jpg, so it stays as it is. I extended the test helper to take a content type, which the wait tests never passed before. Four new cases fail without the change (a wrong-type directory for ebook, audiobook and pdf tasks), and two more check that sidecar-only and still-growing directories keep waiting. Full suite passes. --- shelfmark/download/clients/base_handler.py | 67 ++++++++++++++++++- tests/prowlarr/test_remote_path_mappings.py | 71 ++++++++++++++++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/shelfmark/download/clients/base_handler.py b/shelfmark/download/clients/base_handler.py index 016f473e..f385f41e 100644 --- a/shelfmark/download/clients/base_handler.py +++ b/shelfmark/download/clients/base_handler.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Protocol, TypeGuard from shelfmark.core.config import config from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import coerce_int, normalize_optional_text -from shelfmark.core.utils import is_audiobook +from shelfmark.core.utils import AUDIOBOOK_FORMATS, DEFAULT_EBOOK_FORMATS, is_audiobook from shelfmark.download.activity import release_activity_grace, request_activity_grace from shelfmark.download.clients import ( DownloadClient, @@ -288,6 +288,56 @@ def _count_expected_files_present(path: Path, expected: Sequence[ExpectedFile]) return present +# Extensions that mean a finished book or recording, whichever content type the task is for. +# A file with one of these that the task does not accept (an .m4b in an ebook task, a .pdf +# with the default format list) is a wrong-type result, not a sidecar still waiting for the +# real files to sync. +_PAYLOAD_SUFFIXES = frozenset( + f".{fmt.lstrip('.').lower()}" for fmt in (*AUDIOBOOK_FORMATS, *DEFAULT_EBOOK_FORMATS, "pdf") +) + + +def _unsupported_payload_signature( + path: Path, content_type: str | None +) -> tuple[tuple[str, int, int], ...] | None: + """Sizes and mtimes of book or audio files in a path that this task does not accept. + + None when there are none. Used to tell a finished directory that holds the wrong kind of + files apart from one that has only cover art and notes so far. + """ + from shelfmark.download.postprocess.policy import ( + get_supported_audiobook_formats, + get_supported_formats, + ) + + formats = ( + get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats() + ) + supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats} + + try: + if not path.is_dir(): + return None + signature = [] + for candidate in path.rglob("*"): + suffix = candidate.suffix.lower() + if suffix not in _PAYLOAD_SUFFIXES or suffix in supported_suffixes: + continue + try: + stat = candidate.stat() + except OSError: + continue + if not candidate.is_file() or stat.st_size <= 0: + continue + signature.append( + (candidate.relative_to(path).as_posix(), stat.st_size, stat.st_mtime_ns) + ) + except OSError: + return None + + return tuple(sorted(signature)) or None + + class ExternalClientHandler(DownloadHandler, ABC): """Shared lifecycle handler for sources that hand off to torrent/usenet clients.""" @@ -911,6 +961,7 @@ class ExternalClientHandler(DownloadHandler, ABC): previous_signature: tuple[tuple[str, int, int], ...] | None = None expected_files: list[ExpectedFile] | None = None expected_files_asked = False + previous_unsupported: tuple[tuple[str, int, int], ...] | None = None retry_interval = self._completed_path_retry_interval() timeout_seconds = self._completed_path_timeout_seconds() if retry_interval <= 0 or timeout_seconds <= 0: @@ -960,6 +1011,20 @@ class ExternalClientHandler(DownloadHandler, ABC): ): return result.path, None previous_signature = signature + if signature is None: + # Nothing this task accepts. If the directory instead holds finished + # files of another kind and has stopped changing, waiting longer will + # not help: hand it on so post-processing names the format problem. + unsupported = run_blocking_io( + _unsupported_payload_signature, + result.path, + content_type, + ) + if unsupported is not None and ( + unsupported == previous_unsupported or max_attempts == 1 + ): + return result.path, None + previous_unsupported = unsupported last_error = "Completed download has no stable supported files yet." if attempt < max_attempts: status_callback("locating", "Waiting for completed files...") diff --git a/tests/prowlarr/test_remote_path_mappings.py b/tests/prowlarr/test_remote_path_mappings.py index 00eafcdc..9ca4b68a 100644 --- a/tests/prowlarr/test_remote_path_mappings.py +++ b/tests/prowlarr/test_remote_path_mappings.py @@ -9,6 +9,8 @@ from pathlib import Path from threading import Event from unittest.mock import MagicMock, patch +import pytest + from shelfmark.core.models import DownloadTask from shelfmark.download.clients import DownloadState, DownloadStatus from shelfmark.release_sources.prowlarr.handler import ProwlarrHandler @@ -366,7 +368,13 @@ def test_waits_for_supported_file_to_stop_growing_in_remapped_directory(monkeypa def _wait_for_local_directory( - monkeypatch, directory: Path, *, timeout: object, on_sleep=None, expected=None + monkeypatch, + directory: Path, + *, + timeout: object, + content_type: str | None = None, + on_sleep=None, + expected=None, ): import shelfmark.download.clients.base_handler as base_handler @@ -391,6 +399,7 @@ def _wait_for_local_directory( "download_id", cancel_flag=None, status_callback=ProgressRecorder().status_callback, + content_type=content_type, ) @@ -549,6 +558,66 @@ def test_a_client_without_a_file_list_keeps_the_old_behaviour(monkeypatch, tmp_p assert error is None +@pytest.mark.parametrize( + ("content_type", "filename"), + [ + (None, "book.m4b"), + ("book (fiction)", "recording.mp3"), + ("audiobook", "book.epub"), + (None, "book.pdf"), + ], +) +def test_a_finished_directory_of_the_wrong_kind_is_returned_without_waiting_out_the_timeout( + monkeypatch, tmp_path, content_type, filename +): + """Post-processing can name the format problem; polling for 60 s to say it later cannot (F3).""" + release = tmp_path / "Release" + release.mkdir() + (release / filename).write_bytes(b"finished") + + resolved_path, error = _wait_for_local_directory( + monkeypatch, release, timeout=1, content_type=content_type + ) + + assert resolved_path == release + assert error is None + + +def test_a_directory_with_only_cover_art_and_notes_keeps_waiting(monkeypatch, tmp_path): + """The book may still be syncing: sidecar files alone are not a finished wrong-type result.""" + release = tmp_path / "Release" + release.mkdir() + (release / "cover.jpg").write_bytes(b"jpg") + (release / "book.nfo").write_text("nfo") + + resolved_path, error = _wait_for_local_directory(monkeypatch, release, timeout=1) + + assert resolved_path is None + assert error == "Completed download has no stable supported files yet." + + +def test_a_wrong_kind_file_that_is_still_growing_keeps_waiting(monkeypatch, tmp_path): + release = tmp_path / "Release" + release.mkdir() + growing = release / "book.m4b" + growing.write_bytes(b"a") + sleeps = 0 + + def grow(_seconds: float) -> None: + nonlocal sleeps + sleeps += 1 + growing.write_bytes(b"a" * (sleeps + 1)) + + resolved_path, error = _wait_for_local_directory( + monkeypatch, release, timeout=0.05, on_sleep=grow + ) + + # The file changes on every poll, so it is never stable and the wait runs out. + assert resolved_path is None + assert error == "Completed download has no stable supported files yet." + assert sleeps >= 2 + + def test_wait_for_completed_path_fails_fast_for_unsafe_mapping(monkeypatch, tmp_path): import shelfmark.download.clients.base_handler as base_handler