fix(clients): hand a finished wrong-type directory on instead of polling out the timeout (#1573)

A torrent that finishes with the wrong kind of files in it holds the
download for the whole completed-path timeout (60 seconds by default, up
to an hour) and then fails with "Completed download has no stable
supported files yet." An ebook task whose torrent holds only an `.m4b`,
an audiobook task that got an `.epub`, or a `.pdf`-only release with the
default format list all land here. In v1.4.0 these failed with the
specific format error from post-processing, but the wait for the sync to
settle (#1464) only counts files of the task's own type, so it keeps
polling for files that are never going to appear.

If the directory has no supported files, `_wait_for_completed_path` now
looks for finished book or audio files that this task does not accept
(the audiobook formats, the default ebook formats and pdf). When it
finds some and they are the same on two polls in a row, it returns the
path so post-processing can say what is wrong with the formats. Cover
art and notes alone do not count, because that is what a directory looks
like while the real files are still syncing over a remote mount, so
those keep waiting as before. A wrong-type file that is still growing
keeps waiting too. With no wait configured there is only one sample, and
a wrong-type file is handed on straight away.

The existing test that pins "no book files, no wait configured" uses a
jpg, so it stays as it is. I extended the test helper to take a content
type, which the wait tests never passed before. Four new cases fail
without the change (a wrong-type directory for ebook, audiobook and pdf
tasks), and two more check that sidecar-only and still-growing
directories keep waiting. Full suite passes.
This commit is contained in:
splitsec2 authored and GitHub committed 2026-10-24 20:41:21 -06:00
1 parent 47f0acdd31
commit 525b094b57
2 files changed
+136 -2

No files matched your search

+66 -1
View File
@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Protocol, TypeGuard
from shelfmark.core.config import config
from shelfmark.core.logger import setup_logger
from shelfmark.core.request_helpers import coerce_int, normalize_optional_text
from shelfmark.core.utils import is_audiobook
from shelfmark.core.utils import AUDIOBOOK_FORMATS, DEFAULT_EBOOK_FORMATS, is_audiobook
from shelfmark.download.activity import release_activity_grace, request_activity_grace
from shelfmark.download.clients import (
DownloadClient,
@@ -288,6 +288,56 @@ def _count_expected_files_present(path: Path, expected: Sequence[ExpectedFile])
return present
# Extensions that mean a finished book or recording, whichever content type the task is for.
# A file with one of these that the task does not accept (an .m4b in an ebook task, a .pdf
# with the default format list) is a wrong-type result, not a sidecar still waiting for the
# real files to sync.
_PAYLOAD_SUFFIXES = frozenset(
f".{fmt.lstrip('.').lower()}" for fmt in (*AUDIOBOOK_FORMATS, *DEFAULT_EBOOK_FORMATS, "pdf")
)
def _unsupported_payload_signature(
path: Path, content_type: str | None
) -> tuple[tuple[str, int, int], ...] | None:
"""Sizes and mtimes of book or audio files in a path that this task does not accept.
None when there are none. Used to tell a finished directory that holds the wrong kind of
files apart from one that has only cover art and notes so far.
"""
from shelfmark.download.postprocess.policy import (
get_supported_audiobook_formats,
get_supported_formats,
)
formats = (
get_supported_audiobook_formats() if is_audiobook(content_type) else get_supported_formats()
)
supported_suffixes = {f".{fmt.lstrip('.').lower()}" for fmt in formats}
try:
if not path.is_dir():
return None
signature = []
for candidate in path.rglob("*"):
suffix = candidate.suffix.lower()
if suffix not in _PAYLOAD_SUFFIXES or suffix in supported_suffixes:
continue
try:
stat = candidate.stat()
except OSError:
continue
if not candidate.is_file() or stat.st_size <= 0:
continue
signature.append(
(candidate.relative_to(path).as_posix(), stat.st_size, stat.st_mtime_ns)
)
except OSError:
return None
return tuple(sorted(signature)) or None
class ExternalClientHandler(DownloadHandler, ABC):
"""Shared lifecycle handler for sources that hand off to torrent/usenet clients."""
@@ -911,6 +961,7 @@ class ExternalClientHandler(DownloadHandler, ABC):
previous_signature: tuple[tuple[str, int, int], ...] | None = None
expected_files: list[ExpectedFile] | None = None
expected_files_asked = False
previous_unsupported: tuple[tuple[str, int, int], ...] | None = None
retry_interval = self._completed_path_retry_interval()
timeout_seconds = self._completed_path_timeout_seconds()
if retry_interval <= 0 or timeout_seconds <= 0:
@@ -960,6 +1011,20 @@ class ExternalClientHandler(DownloadHandler, ABC):
):
return result.path, None
previous_signature = signature
if signature is None:
# Nothing this task accepts. If the directory instead holds finished
# files of another kind and has stopped changing, waiting longer will
# not help: hand it on so post-processing names the format problem.
unsupported = run_blocking_io(
_unsupported_payload_signature,
result.path,
content_type,
)
if unsupported is not None and (
unsupported == previous_unsupported or max_attempts == 1
):
return result.path, None
previous_unsupported = unsupported
last_error = "Completed download has no stable supported files yet."
if attempt < max_attempts:
status_callback("locating", "Waiting for completed files...")
+70 -1
View File
@@ -9,6 +9,8 @@ from pathlib import Path
from threading import Event
from unittest.mock import MagicMock, patch
import pytest
from shelfmark.core.models import DownloadTask
from shelfmark.download.clients import DownloadState, DownloadStatus
from shelfmark.release_sources.prowlarr.handler import ProwlarrHandler
@@ -366,7 +368,13 @@ def test_waits_for_supported_file_to_stop_growing_in_remapped_directory(monkeypa
def _wait_for_local_directory(
monkeypatch, directory: Path, *, timeout: object, on_sleep=None, expected=None
monkeypatch,
directory: Path,
*,
timeout: object,
content_type: str | None = None,
on_sleep=None,
expected=None,
):
import shelfmark.download.clients.base_handler as base_handler
@@ -391,6 +399,7 @@ def _wait_for_local_directory(
"download_id",
cancel_flag=None,
status_callback=ProgressRecorder().status_callback,
content_type=content_type,
)
@@ -549,6 +558,66 @@ def test_a_client_without_a_file_list_keeps_the_old_behaviour(monkeypatch, tmp_p
assert error is None
@pytest.mark.parametrize(
("content_type", "filename"),
[
(None, "book.m4b"),
("book (fiction)", "recording.mp3"),
("audiobook", "book.epub"),
(None, "book.pdf"),
],
)
def test_a_finished_directory_of_the_wrong_kind_is_returned_without_waiting_out_the_timeout(
monkeypatch, tmp_path, content_type, filename
):
"""Post-processing can name the format problem; polling for 60 s to say it later cannot (F3)."""
release = tmp_path / "Release"
release.mkdir()
(release / filename).write_bytes(b"finished")
resolved_path, error = _wait_for_local_directory(
monkeypatch, release, timeout=1, content_type=content_type
)
assert resolved_path == release
assert error is None
def test_a_directory_with_only_cover_art_and_notes_keeps_waiting(monkeypatch, tmp_path):
"""The book may still be syncing: sidecar files alone are not a finished wrong-type result."""
release = tmp_path / "Release"
release.mkdir()
(release / "cover.jpg").write_bytes(b"jpg")
(release / "book.nfo").write_text("nfo")
resolved_path, error = _wait_for_local_directory(monkeypatch, release, timeout=1)
assert resolved_path is None
assert error == "Completed download has no stable supported files yet."
def test_a_wrong_kind_file_that_is_still_growing_keeps_waiting(monkeypatch, tmp_path):
release = tmp_path / "Release"
release.mkdir()
growing = release / "book.m4b"
growing.write_bytes(b"a")
sleeps = 0
def grow(_seconds: float) -> None:
nonlocal sleeps
sleeps += 1
growing.write_bytes(b"a" * (sleeps + 1))
resolved_path, error = _wait_for_local_directory(
monkeypatch, release, timeout=0.05, on_sleep=grow
)
# The file changes on every poll, so it is never stable and the wait runs out.
assert resolved_path is None
assert error == "Completed download has no stable supported files yet."
assert sleeps >= 2
def test_wait_for_completed_path_fails_fast_for_unsafe_mapping(monkeypatch, tmp_path):
import shelfmark.download.clients.base_handler as base_handler