Fix: Orchestrator timeout and exception handling (#832)

Fixes #823
2026-10-05 22:05:50 +01:00 · 2026-04-03 09:38:58 +01:00
parent c1143f808a
commit ff094bed56
3 changed files with 267 additions and 57 deletions
@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from unittest.mock import ANY, MagicMock
+
+import pytest
+
+
+class _StopLoop(BaseException):
+    """Sentinel used to stop the infinite coordinator loop during tests."""
+
+
+class _FakeExecutor:
+    def __init__(self, *args, **kwargs) -> None:
+        self.args = args
+        self.kwargs = kwargs
+
+    def __enter__(self) -> _FakeExecutor:
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        return False
+
+    def submit(self, *args, **kwargs):  # pragma: no cover - not expected in these tests
+        raise AssertionError("submit() should not be called in this test")
+
+
+class _StopCoordinator(BaseException):
+    """Sentinel used to stop a real coordinator thread cleanly in tests."""
+
+
+def test_concurrent_download_loop_logs_and_recovers_after_loop_error(monkeypatch):
+    import shelfmark.download.orchestrator as orchestrator
+
+    call_count = 0
+
+    def fake_get_next():
+        nonlocal call_count
+        call_count += 1
+        if call_count == 1:
+            raise RuntimeError("boom")
+        return None
+
+    sleep_delays: list[float] = []
+
+    def fake_sleep(delay: float) -> None:
+        sleep_delays.append(delay)
+        if len(sleep_delays) >= 2:
+            raise _StopLoop()
+
+    mock_queue = MagicMock()
+    mock_queue.get_next.side_effect = fake_get_next
+
+    error_trace = MagicMock()
+
+    monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
+    monkeypatch.setattr(orchestrator, "ThreadPoolExecutor", _FakeExecutor)
+    monkeypatch.setattr(orchestrator.time, "sleep", fake_sleep)
+    monkeypatch.setattr(orchestrator.logger, "error_trace", error_trace)
+
+    with pytest.raises(_StopLoop):
+        orchestrator.concurrent_download_loop()
+
+    assert mock_queue.get_next.call_count == 2
+    error_trace.assert_called_once_with("Download coordinator loop error: %s", ANY)
+    assert sleep_delays == [
+        orchestrator.COORDINATOR_LOOP_ERROR_RETRY_DELAY,
+        orchestrator.config.MAIN_LOOP_SLEEP_TIME,
+    ]
+
+
+def test_concurrent_download_loop_recovers_and_processes_task_after_transient_loop_error(monkeypatch):
+    import threading
+
+    import shelfmark.download.orchestrator as orchestrator
+
+    processed = threading.Event()
+
+    class FlakyQueue:
+        def __init__(self) -> None:
+            self.calls = 0
+
+        def get_next(self):
+            self.calls += 1
+            if self.calls == 1:
+                raise RuntimeError("boom")
+            if self.calls == 2:
+                return ("task-1", threading.Event())
+            if processed.is_set():
+                raise _StopCoordinator()
+            return None
+
+        def cancel_download(self, task_id: str) -> None:  # pragma: no cover - unused
+            raise AssertionError(f"cancel_download unexpectedly called for {task_id}")
+
+        def update_status_message(self, task_id: str, message: str) -> None:  # pragma: no cover - unused
+            raise AssertionError(f"update_status_message unexpectedly called for {task_id}: {message}")
+
+    queue = FlakyQueue()
+    error_trace = MagicMock()
+
+    monkeypatch.setattr(orchestrator, "book_queue", queue)
+    monkeypatch.setattr(
+        orchestrator,
+        "_process_single_download",
+        lambda task_id, cancel_flag: processed.set(),
+    )
+    monkeypatch.setattr(orchestrator.logger, "error_trace", error_trace)
+    monkeypatch.setattr(orchestrator, "COORDINATOR_LOOP_ERROR_RETRY_DELAY", 0.01)
+    monkeypatch.setattr(orchestrator.config, "MAX_CONCURRENT_DOWNLOADS", 1, raising=False)
+    monkeypatch.setattr(orchestrator.config, "MAIN_LOOP_SLEEP_TIME", 0.01, raising=False)
+
+    def run_loop() -> None:
+        try:
+            orchestrator.concurrent_download_loop()
+        except _StopCoordinator:
+            pass
+
+    thread = threading.Thread(target=run_loop, daemon=True, name="TestDownloadCoordinator")
+    thread.start()
+
+    assert processed.wait(timeout=1.0) is True
+    thread.join(timeout=1.0)
+
+    assert thread.is_alive() is False
+    assert queue.calls >= 3
+    error_trace.assert_called_once_with("Download coordinator loop error: %s", ANY)
+
+
+def test_start_replaces_dead_coordinator_thread(monkeypatch):
+    import shelfmark.download.orchestrator as orchestrator
+
+    dead_thread = MagicMock()
+    dead_thread.is_alive.return_value = False
+
+    new_thread = MagicMock()
+    new_thread.is_alive.return_value = True
+
+    thread_factory = MagicMock(return_value=new_thread)
+
+    monkeypatch.setattr(orchestrator, "_coordinator_thread", dead_thread)
+    monkeypatch.setattr(orchestrator.threading, "Thread", thread_factory)
+
+    orchestrator.start()
+
+    thread_factory.assert_called_once_with(
+        target=orchestrator.concurrent_download_loop,
+        daemon=True,
+        name="DownloadCoordinator",
+    )
+    new_thread.start.assert_called_once_with()
+    assert orchestrator._coordinator_thread is new_thread
@@ -8,6 +8,7 @@ def test_update_download_status_dedupes_identical_events(monkeypatch):

    # Ensure clean module-level state
    orchestrator._last_activity.clear()
+    orchestrator._last_progress_value.clear()
    orchestrator._last_status_event.clear()

    mock_queue = MagicMock()
@@ -28,6 +29,52 @@ def test_update_download_status_dedupes_identical_events(monkeypatch):
    assert mock_queue.update_status_message.call_count == 1
    assert mock_ws.broadcast_status_update.call_count == 1

-    # Activity timestamp should still be updated on the duplicate keep-alive call.
-    assert orchestrator._last_activity[book_id] == 2.0
+    # Duplicate keep-alives should not refresh stall activity.
+    assert orchestrator._last_activity[book_id] == 1.0

+
+def test_update_download_progress_dedupes_identical_progress_for_activity(monkeypatch):
+    import shelfmark.download.orchestrator as orchestrator
+
+    book_id = "test-progress-book"
+
+    orchestrator._last_activity.clear()
+    orchestrator._last_progress_value.clear()
+    orchestrator._last_status_event.clear()
+
+    mock_queue = MagicMock()
+    monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
+    monkeypatch.setattr(orchestrator, "ws_manager", None)
+
+    times = iter([10.0, 20.0])
+    monkeypatch.setattr(orchestrator.time, "time", lambda: next(times))
+
+    orchestrator.update_download_progress(book_id, 0.0)
+    orchestrator.update_download_progress(book_id, 0.0)
+
+    assert mock_queue.update_progress.call_count == 2
+    assert orchestrator._last_activity[book_id] == 10.0
+    assert orchestrator._last_progress_value[book_id] == 0.0
+
+
+def test_update_download_progress_refreshes_activity_when_progress_changes(monkeypatch):
+    import shelfmark.download.orchestrator as orchestrator
+
+    book_id = "test-progress-change"
+
+    orchestrator._last_activity.clear()
+    orchestrator._last_progress_value.clear()
+    orchestrator._last_status_event.clear()
+
+    mock_queue = MagicMock()
+    monkeypatch.setattr(orchestrator, "book_queue", mock_queue)
+    monkeypatch.setattr(orchestrator, "ws_manager", None)
+
+    times = iter([30.0, 40.0])
+    monkeypatch.setattr(orchestrator.time, "time", lambda: next(times))
+
+    orchestrator.update_download_progress(book_id, 0.0)
+    orchestrator.update_download_progress(book_id, 0.5)
+
+    assert orchestrator._last_activity[book_id] == 40.0
+    assert orchestrator._last_progress_value[book_id] == 0.5