starred/shelfmark
1
0
Fork 0
mirror of https://github.com/calibrain/shelfmark.git synced 2026-10-05 08:11:11 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

fix(clients): correct Debrid-Link file listing and rate-limit handling (#1425)

Browse Source 
Follow-up to #1380.

- Fetch the file list with ?ids=, the documented parameter. There is no
  ?id=, so a many-file torrent was not expanded and the call could fail
  with badArguments at the last step of every download.
- Back off status checks for 60s after floodDetected, doubling to a 240s
cap, instead of an hour. The orchestrator cancels a download after five
minutes without a change, so the hour-long pause cancelled every active
  download. Only a flood on the status check starts it, and a successful
  check resets it.
- Drop the client-side one-hour stall timer. The orchestrator stall
timer
  always fires first.
- Keep progress at 50% when file retrieval starts instead of dropping to
  0% until the first file arrives.
- Report queued, paused and verifying torrents as QUEUED, PAUSED and
  CHECKING, so a queued torrent gets the handler queue grace.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
CaliBrain
2026-10-03 08:08:32 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent cad18b8019
commit 9321096032
2 changed files with 135 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
shelfmark/download/clients/debridlink.py
+43 -54
View File
@@ -75,21 +75,22 @@ _STATUS_FINISHED = 100
_DONE_STATUSES = frozenset({_STATUS_SEEDING, _STATUS_FINISHED})
# The download handler polls every couple of seconds. Debrid-Link rate-limits per
# endpoint and answers a breach with floodDetected, which blocks the API for an hour,
# so the seedbox is asked at most this often per torrent and cached status is served
# in between.
# endpoint and answers a breach with floodDetected, so the seedbox is asked at most
# this often per torrent and cached status is served in between.
_MIN_REFRESH_SECONDS = 15.0
_FLOOD_BACKOFF_SECONDS = 3600.0
# floodDetected only says to retry later; the docs give no duration. Status checks
# back off for a minute, doubling while the flood lasts. Each new back-off changes the
# status message, and the cap keeps the next check inside the orchestrator's
# five-minute stall window, so waiting out a flood doesn't get a download cancelled.
_FLOOD_BACKOFF_SECONDS = 60.0
_FLOOD_BACKOFF_MAX_SECONDS = 240.0
# Errors Debrid-Link documents as temporary. They, network failures and 5xx
# responses are retried until the torrent has been failing for this long.
_TRANSIENT_ERROR_CODES = frozenset({"internalError", "server_error", "freeServerOverload"})
_TRANSIENT_GRACE_SECONDS = 600.0
# A torrent whose progress has not moved for this long is treated as dead (no seeds,
# or left paused), because the API documents no error status for a torrent.
_STALL_TIMEOUT_SECONDS = 3600.0
# How long remove() waits for the retrieval thread to notice a cancel.
_WORKER_JOIN_TIMEOUT = 10.0
@@ -118,8 +119,7 @@ _BOOK_EXTENSIONS = (
".rtf",
".txt",
".wma",
# A torrent with many files can come back as a single ZIP of all of them; the
# post-processing step extracts it.
# A zipped book or audiobook; the post-processing step extracts it.
".zip",
)
@@ -154,11 +154,8 @@ class _DownloadState:
last_fetch_at: float = 0.0
# Transient-failure tracking: when the current run of failures started.
failing_since: float | None = None
# Stall tracking: the highest progress seen and when it last moved.
best_percent: float = -1.0
# Set once a torrent held for file selection (``wait``) has been told to start.
start_requested: bool = False
progress_moved_at: float = field(default_factory=time.monotonic)
@register_client("torrent")
@@ -177,8 +174,10 @@ class DebridLinkClient(DownloadClient):
_downloads: ClassVar[dict[str, _DownloadState]] = {}
_downloads_lock = threading.Lock()
# floodDetected blocks the whole account's API use, so the back-off is shared.
# A flood on the status endpoint applies to every torrent on the account, so the
# status back-off is shared.
_flood_until: ClassVar[float] = 0.0
_flood_backoff: ClassVar[float] = 0.0
def __init__(self) -> None:
self._api_key = config_text(config.get("DEBRIDLINK_API_KEY", ""))
@@ -333,13 +332,14 @@ class DebridLinkClient(DownloadClient):
cached, f"Debrid-Link rate limit reached, retrying at {resume}"
)
if fresh and cached is not None:
return self._handle_torrent_info(cached, state, observed=False)
return self._handle_torrent_info(cached, state)
try:
torrent = self._fetch_torrent(download_id)
except _DEBRIDLINK_CLIENT_ERRORS as e:
return self._handle_fetch_failure(e, state)
type(self)._flood_backoff = 0.0
with state.lock:
state.failing_since = None
state.last_fetch_at = time.monotonic()
@@ -456,8 +456,6 @@ class DebridLinkClient(DownloadClient):
# The documented error body is just {success: false, error: <code>}.
code = payload.get("error")
code = code if isinstance(code, str) and code else None
if code == "floodDetected":
type(self)._flood_until = time.time() + _FLOOD_BACKOFF_SECONDS
detail = code or ("an unsuccessful response" if ok_status else f"HTTP {status_code}")
msg = f"Debrid-Link {operation} failed: {detail}"
raise DebridLinkAPIError(msg, code or (None if ok_status else f"http{status_code}"))
@@ -483,19 +481,12 @@ class DebridLinkClient(DownloadClient):
def _fetch_file_list(self, download_id: str) -> list[dict[str, Any]]:
"""Return the torrent's individual files.
A torrent with many files is listed as one ZIP (``isZip``) under ``ids``;
Debrid-Link documents ``?id=TORRENT_ID`` as the way to see the files
themselves. Fetched once, after the torrent is complete, so the list is
final rather than the partial one an early status check can return.
The plain list shows a torrent with many files as one ZIP (``isZip``);
asking for it by ``ids``, as ``_fetch_torrent`` does, lists every file.
Fetched once, after the torrent is complete, so the list is final rather
than the partial one an early status check can return.
"""
value = self._request_value(
"GET",
"/seedbox/list",
operation="file listing",
params={"id": download_id},
timeout=_STATUS_TIMEOUT,
)
torrent = self._pick_torrent(value, download_id)
torrent = self._fetch_torrent(download_id)
if torrent is None:
msg = f"Torrent {download_id} is no longer on Debrid-Link"
_raise_runtime_error(msg)
@@ -513,7 +504,7 @@ class DebridLinkClient(DownloadClient):
"""Fail on a permanent error; ride out a temporary one for a grace period."""
code = getattr(error, "code", None)
if code == "floodDetected":
resume = time.strftime("%H:%M", time.localtime(type(self)._flood_until))
resume = self._start_flood_backoff()
logger.warning("Debrid-Link rate limit reached; pausing status checks until %s", resume)
return self._waiting_status(
state.last_info, f"Debrid-Link rate limit reached, retrying at {resume}"
@@ -539,6 +530,15 @@ class DebridLinkClient(DownloadClient):
logger.warning("Debrid-Link status check failed, will retry: %s", error)
return self._waiting_status(state.last_info, "Debrid-Link unreachable, retrying")
@classmethod
def _start_flood_backoff(cls) -> str:
"""Pause status checks after floodDetected and return the resume time as HH:MM."""
cls._flood_backoff = min(
max(cls._flood_backoff * 2, _FLOOD_BACKOFF_SECONDS), _FLOOD_BACKOFF_MAX_SECONDS
)
cls._flood_until = time.time() + cls._flood_backoff
return time.strftime("%H:%M", time.localtime(cls._flood_until))
@staticmethod
def _waiting_status(info: dict[str, Any] | None, message: str) -> DownloadStatus:
percent = _as_float(info.get("downloadPercent")) if info else 0.0
@@ -575,17 +575,11 @@ class DebridLinkClient(DownloadClient):
self._downloads[download_id] = state
return state
def _handle_torrent_info(
self,
info: dict[str, Any],
state: _DownloadState,
*,
observed: bool = True,
) -> DownloadStatus:
def _handle_torrent_info(self, info: dict[str, Any], state: _DownloadState) -> DownloadStatus:
"""Map a Debrid-Link seedbox torrent to a DownloadStatus.
``observed`` is False when ``info`` is a cached record being re-served
between throttled fetches, so it doesn't count toward stall detection.
The API documents no error status for a torrent, so a dead one is left to the
orchestrator's stall timer, as with the other torrent clients.
"""
percent = _as_float(info.get("downloadPercent"))
name = str(info.get("name") or state.name)
@@ -607,20 +601,7 @@ class DebridLinkClient(DownloadClient):
file_path=None,
)
now = time.monotonic()
with state.lock:
if percent > state.best_percent:
state.best_percent = percent
state.progress_moved_at = now
stalled_for = now - state.progress_moved_at
if observed and stalled_for >= _STALL_TIMEOUT_SECONDS:
minutes = int(_STALL_TIMEOUT_SECONDS // 60)
return self._set_error(
state,
f"Debrid-Link made no progress on {name} for {minutes} minutes "
f"({int(_as_float(info.get('peersConnected')))} peers connected)",
)
download_state = DownloadState.DOWNLOADING
if info.get("wait"):
# Held for file selection. Shelfmark wants every file, so start it.
self._start_waiting_torrent(state)
@@ -628,10 +609,15 @@ class DebridLinkClient(DownloadClient):
elif info.get("srvMaint"):
message = "Debrid-Link server maintenance, waiting"
elif status == _STATUS_PAUSED:
download_state = DownloadState.PAUSED
message = f"Paused on Debrid-Link ({name})"
elif status == _STATUS_QUEUED:
# Reported as queued so the handler gives it the queue grace, not the
# stall window for a torrent that should be moving.
download_state = DownloadState.QUEUED
message = f"Queued on Debrid-Link ({name})"
elif status == _STATUS_VERIFYING:
download_state = DownloadState.CHECKING
message = f"Debrid-Link verifying torrent ({name})"
else:
message = f"Debrid-Link downloading torrent ({name})"
@@ -639,7 +625,7 @@ class DebridLinkClient(DownloadClient):
# Still being fetched by the seedbox: the first half of the progress bar.
return DownloadStatus(
progress=min(percent, _COMPLETE_PERCENT) * 0.5,
state=DownloadState.DOWNLOADING,
state=download_state,
message=message,
complete=False,
file_path=None,
@@ -679,6 +665,9 @@ class DebridLinkClient(DownloadClient):
if already_running or thread_alive or state.cancel_event.is_set():
return
state.phase = "downloading_http"
# The seedbox half is done; without this the bar drops to 0% until the
# first file arrives.
state.progress = 50.0
t = threading.Thread(
target=self._process_and_download,
args=(state,),
tests/prowlarr/test_debridlink_client.py
+92 -47
View File
@@ -9,6 +9,7 @@ step to mock.
import hashlib
import io
import threading
from unittest.mock import MagicMock
import pytest
@@ -66,9 +67,11 @@ def _mock_request(monkeypatch, payload, *, status_code=200):
def _reset_shared_state():
"""The flood back-off and the download table are shared across instances."""
DebridLinkClient._flood_until = 0.0
DebridLinkClient._flood_backoff = 0.0
DebridLinkClient._downloads.clear()
yield
DebridLinkClient._flood_until = 0.0
DebridLinkClient._flood_backoff = 0.0
DebridLinkClient._downloads.clear()
@@ -147,13 +150,13 @@ class TestEnvelope:
assert "debrid-link.com" not in str(err.value)
assert "ConnectionError" in str(err.value)
def test_flood_detected_starts_the_shared_back_off(self, monkeypatch):
def test_flood_detected_is_raised_with_its_code(self, monkeypatch):
_mock_request(monkeypatch, {"success": False, "error": "floodDetected"}, status_code=429)
client = _client(monkeypatch)
with pytest.raises(DebridLinkAPIError):
with pytest.raises(DebridLinkAPIError) as err:
client._request_value("GET", "/seedbox/list", operation="status check")
assert DebridLinkClient._flood_until > 0
assert err.value.code == "floodDetected"
class TestAdd:
@@ -265,23 +268,24 @@ class TestStatus:
started.assert_not_called()
@pytest.mark.parametrize(
("info", "expected"),
("info", "expected_state", "expected_message"),
[
({"status": 0}, "Paused on Debrid-Link"),
({"status": 1}, "Queued on Debrid-Link"),
({"status": 2}, "verifying"),
({"status": 4}, "downloading torrent"),
({"status": 6}, "downloading torrent"), # the docs' own example value
({"srvMaint": True}, "server maintenance"),
({"status": 0}, DownloadState.PAUSED, "Paused on Debrid-Link"),
({"status": 1}, DownloadState.QUEUED, "Queued on Debrid-Link"),
({"status": 2}, DownloadState.CHECKING, "verifying"),
({"status": 4}, DownloadState.DOWNLOADING, "downloading torrent"),
# The docs' own example value.
({"status": 6}, DownloadState.DOWNLOADING, "downloading torrent"),
({"status": 1, "srvMaint": True}, DownloadState.DOWNLOADING, "server maintenance"),
],
)
def test_documented_states_are_reported(self, tmp_path, info, expected):
def test_documented_states_are_reported(self, tmp_path, info, expected_state, expected_message):
status = self._bare_client()._handle_torrent_info(
{"downloadPercent": 10, **info}, _state(tmp_path)
)
assert status.state == DownloadState.DOWNLOADING
assert expected in (status.message or "")
assert status.state == expected_state
assert expected_message in (status.message or "")
@pytest.mark.parametrize("done_status", [8, 100])
def test_seeding_or_finished_counts_as_done(self, tmp_path, monkeypatch, done_status):
@@ -317,40 +321,23 @@ class TestStatus:
assert args[1].endswith("/seedbox/DL1/config")
assert kwargs["json"] == {"files-unwanted": []}
def test_a_torrent_that_stops_moving_fails_after_the_stall_timeout(self, tmp_path, monkeypatch):
def test_retrieval_starts_at_half_progress_rather_than_zero(self, tmp_path, monkeypatch):
client = self._bare_client()
state = _state(tmp_path)
clock = [1000.0]
monkeypatch.setattr(debridlink.time, "monotonic", lambda: clock[0])
state.progress_moved_at = clock[0]
DebridLinkClient._downloads["DL1"] = state
release = threading.Event()
monkeypatch.setattr(client, "_process_and_download", lambda _state: release.wait(5))
client._handle_torrent_info({"downloadPercent": 12, "peersConnected": 0}, state)
clock[0] += debridlink._STALL_TIMEOUT_SECONDS - 1
assert client._handle_torrent_info({"downloadPercent": 12}, state).state == (
DownloadState.DOWNLOADING
)
client._maybe_start_download_thread(state)
try:
status = client.get_status("DL1")
finally:
release.set()
assert state.download_thread is not None
state.download_thread.join(5)
clock[0] += 2
status = client._handle_torrent_info({"downloadPercent": 12, "peersConnected": 0}, state)
assert status.state == DownloadState.ERROR
assert "no progress" in (status.message or "")
def test_progress_resets_the_stall_clock(self, tmp_path, monkeypatch):
client = self._bare_client()
state = _state(tmp_path)
clock = [1000.0]
monkeypatch.setattr(debridlink.time, "monotonic", lambda: clock[0])
state.progress_moved_at = clock[0]
client._handle_torrent_info({"downloadPercent": 12}, state)
clock[0] += debridlink._STALL_TIMEOUT_SECONDS - 1
client._handle_torrent_info({"downloadPercent": 13}, state)
clock[0] += debridlink._STALL_TIMEOUT_SECONDS - 1
assert client._handle_torrent_info({"downloadPercent": 13}, state).state == (
DownloadState.DOWNLOADING
)
assert status.message == "Downloading files via HTTP..."
assert status.progress == 50.0
def test_a_vanished_torrent_becomes_an_error_rather_than_a_stall(self, monkeypatch, tmp_path):
monkeypatch.setattr("shelfmark.download.clients.debridlink.TMP_DIR", tmp_path)
@@ -394,6 +381,57 @@ class TestPolling:
assert second.state == DownloadState.DOWNLOADING
assert request.call_count == 1
def test_the_rate_limit_back_off_doubles_and_stays_under_the_stall_window(
self, monkeypatch, tmp_path
):
# The orchestrator cancels a download after five minutes without a change,
# so no single back-off may outlast that.
monkeypatch.setattr("shelfmark.download.clients.debridlink.TMP_DIR", tmp_path)
_mock_request(monkeypatch, {"success": False, "error": "floodDetected"}, status_code=429)
clock = [1_000_000.0]
monkeypatch.setattr(debridlink.time, "time", lambda: clock[0])
client = _client(monkeypatch)
waits = []
for _ in range(4):
client.get_status("DL1")
waits.append(DebridLinkClient._flood_until - clock[0])
clock[0] = DebridLinkClient._flood_until + 1
assert waits == [60.0, 120.0, 240.0, 240.0]
assert max(waits) < 300
def test_a_successful_check_resets_the_back_off(self, monkeypatch, tmp_path):
monkeypatch.setattr("shelfmark.download.clients.debridlink.TMP_DIR", tmp_path)
clock = [1_000_000.0]
monkeypatch.setattr(debridlink.time, "time", lambda: clock[0])
client = _client(monkeypatch)
_mock_request(monkeypatch, {"success": False, "error": "floodDetected"}, status_code=429)
client.get_status("DL1")
clock[0] = DebridLinkClient._flood_until + 1
_mock_request(
monkeypatch, {"success": True, "value": [{"id": "DL1", "downloadPercent": 30}]}
)
client.get_status("DL1")
assert DebridLinkClient._flood_backoff == 0.0
def test_a_flood_on_another_endpoint_does_not_pause_status_checks(self, monkeypatch, tmp_path):
# floodDetected is per endpoint: hammering Test Connection must not stop polling.
monkeypatch.setattr("shelfmark.download.clients.debridlink.TMP_DIR", tmp_path)
_mock_request(monkeypatch, {"success": False, "error": "floodDetected"}, status_code=429)
client = _client(monkeypatch)
assert client.test_connection()[0] is False
request = _mock_request(
monkeypatch, {"success": True, "value": [{"id": "DL1", "downloadPercent": 30}]}
)
status = client.get_status("DL1")
assert request.call_count == 1
assert status.progress == 15.0
def test_a_temporary_error_is_ridden_out_then_fails(self, monkeypatch, tmp_path):
monkeypatch.setattr("shelfmark.download.clients.debridlink.TMP_DIR", tmp_path)
_mock_request(monkeypatch, {"success": False, "error": "internalError"}, status_code=500)
@@ -453,9 +491,9 @@ class TestFetchTorrent:
assert request.call_args.kwargs["params"] == {"ids": "DL1"}
def test_the_file_list_uses_the_singular_id_so_zips_are_expanded(self, monkeypatch):
# Debrid-Link lists a many-file torrent as one ZIP under ``ids``; ``?id=``
# returns the individual files.
def test_the_file_list_asks_by_ids_so_zips_are_expanded(self, monkeypatch):
# The plain list shows a many-file torrent as one ZIP; the docs say
# ``/seedbox/list?ids=TORRENT_ID`` lists every file. There is no ``id``.
request = _mock_request(
monkeypatch,
{
@@ -467,9 +505,16 @@ class TestFetchTorrent:
files = client._fetch_file_list("DL1")
assert request.call_args.kwargs["params"] == {"id": "DL1"}
assert request.call_args.kwargs["params"] == {"ids": "DL1"}
assert [f["name"] for f in files] == ["a.mp3", "b.mp3"]
def test_the_file_list_of_a_vanished_torrent_is_an_error(self, monkeypatch):
_mock_request(monkeypatch, {"success": False, "error": "badId"}, status_code=400)
client = _client(monkeypatch)
with pytest.raises(RuntimeError, match="no longer on Debrid-Link"):
client._fetch_file_list("DL1")
class TestConnection:
def test_missing_key_is_reported_without_a_call(self, monkeypatch):