mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-15 10:18:31 +00:00
splitsec2
88cc734db2
Two things that go wrong when a second request picks a release a first request already used. A task id comes from the release, so the history row is reused. **The row stays with the first request.** Recording a download kept the first request's id, notes and state. On my instance a card for one book showed up under the request for a different book, with that request's note on it. Recording a download now moves the row to the request that queued it and restarts its queue time. A retry with no request keeps the request the row had. **Old rows of a delivered request show as failures forever.** The startup sweep leaves an old request-linked row alone so its retry stays, but one whose request already has its book (delivered, or another download of it finished) is now closed as "Superseded by a completed download" instead of showing as an interrupted failure for ever. Also in the first commit: History cards used the saved cover URL as it was, an external address the page's own-origin image policy blocks, while running downloads use the cover proxy. They use it too. The second commit is about restarts. After a restart nothing is in flight, but a request in downloading, resolving or locating whose row had moved to another request was stranded, because only queued was reopened. Those states are reopened too. Tests cover the row takeover and the retry case, the cover through the proxy (and a missing cover), the superseded cases (delivered request, another row finished, an undelivered request is left alone) and the in-flight states with no row. Ruff and the backend suite pass.
324 lines
12 KiB
Python
324 lines
12 KiB
Python
"""Startup reconciliation of downloads the previous process left unfinished.
|
|
|
|
The download queue lives in memory. A restart mid-download leaves the history row "active" and
|
|
the request "queued" with nothing working on either, and nothing ever revisits them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from shelfmark.core import startup_reconcile
|
|
from shelfmark.core.activity_routes import _build_download_status_from_db
|
|
from shelfmark.core.download_history_service import DownloadHistoryService
|
|
from shelfmark.core.requests_service import sync_delivery_states_from_queue_status
|
|
from shelfmark.core.user_db import UserDB
|
|
|
|
NOW = datetime(2026, 9, 30, 12, 0, tzinfo=UTC)
|
|
|
|
|
|
@pytest.fixture
|
|
def db(tmp_path):
|
|
path = str(tmp_path / "users.db")
|
|
user_db = UserDB(path)
|
|
user_db.initialize()
|
|
return user_db, DownloadHistoryService(path), path
|
|
|
|
|
|
def _stamp(hours_ago: float) -> str:
|
|
"""A SQLite CURRENT_TIMESTAMP style value, as download_history stores queued_at."""
|
|
return (NOW - timedelta(hours=hours_ago)).strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
|
def _request(user_db, user_id, *, state="queued", status="fulfilled", hours_ago=1.0):
|
|
row = user_db.create_request(
|
|
user_id=user_id,
|
|
content_type="audiobook",
|
|
request_level="book",
|
|
policy_mode="request_book",
|
|
book_data={"title": "T", "author": "A", "provider": "hardcover", "provider_id": "1"},
|
|
)
|
|
return user_db.update_request(
|
|
row["id"],
|
|
status=status,
|
|
delivery_state=state,
|
|
delivery_updated_at=(NOW - timedelta(hours=hours_ago)).isoformat(timespec="seconds"),
|
|
release_data={"source": "audiobookbay", "source_id": "rel-1"},
|
|
)
|
|
|
|
|
|
def _download(
|
|
history,
|
|
path,
|
|
*,
|
|
task_id,
|
|
user_id,
|
|
request_id=None,
|
|
hours_ago=1.0,
|
|
final_status="active",
|
|
retry_payload=None,
|
|
):
|
|
history.record_download(
|
|
task_id=task_id,
|
|
user_id=user_id,
|
|
username="reader",
|
|
request_id=request_id,
|
|
source="audiobookbay",
|
|
source_display_name=None,
|
|
title=f"Title {task_id}",
|
|
author=None,
|
|
file_format=None,
|
|
size=None,
|
|
preview=None,
|
|
content_type="audiobook",
|
|
downloads=None,
|
|
origin="requested" if request_id else "direct",
|
|
retry_payload=retry_payload,
|
|
)
|
|
if final_status != "active":
|
|
history.finalize_download(task_id=task_id, final_status=final_status, status_message="x")
|
|
conn = sqlite3.connect(path)
|
|
conn.execute(
|
|
"UPDATE download_history SET queued_at = ? WHERE task_id = ?", (_stamp(hours_ago), task_id)
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
def _run(user_db, history, **kwargs):
|
|
return startup_reconcile.reconcile_interrupted_downloads(user_db, history, now=NOW, **kwargs)
|
|
|
|
|
|
def _read_activity(user_db, history):
|
|
"""Sync request delivery states the way an admin's activity snapshot does."""
|
|
status = _build_download_status_from_db(
|
|
db_rows=history.list_recent(user_id=None), queue_status={}
|
|
)
|
|
sync_delivery_states_from_queue_status(user_db, queue_status=status)
|
|
|
|
|
|
def test_a_recently_interrupted_download_is_closed_out_and_its_request_reopened(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"])
|
|
_download(history, path, task_id="t1", user_id=user["id"], request_id=request["id"])
|
|
|
|
assert _run(user_db, history) == {"marked": 1, "reopened": 1}
|
|
|
|
row = history.get_by_task_id("t1")
|
|
assert row["final_status"] == "error"
|
|
assert row["status_message"] == "Interrupted"
|
|
stored = user_db.get_request(request["id"])
|
|
assert stored["status"] == "pending"
|
|
assert stored["delivery_state"] == "none"
|
|
assert stored["release_data"] is None
|
|
assert "restart" in stored["last_failure_reason"]
|
|
|
|
|
|
def test_an_older_request_linked_orphan_is_left_active_and_keeps_its_retry(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], hours_ago=72)
|
|
_download(
|
|
history,
|
|
path,
|
|
task_id="t1",
|
|
user_id=user["id"],
|
|
request_id=request["id"],
|
|
hours_ago=72,
|
|
retry_payload={"source": "audiobookbay", "source_id": "rel-1"},
|
|
)
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 0}
|
|
|
|
row = history.get_by_task_id("t1")
|
|
assert row["final_status"] == "active"
|
|
assert history.is_retry_available(row) is True
|
|
|
|
# Closing the row out as an error would end its retry, and the activity read would then
|
|
# reopen the request regardless of the window.
|
|
_read_activity(user_db, history)
|
|
assert user_db.get_request(request["id"])["status"] == "fulfilled"
|
|
|
|
|
|
def test_a_download_with_no_request_is_closed_out_and_keeps_its_manual_retry(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
payload = {"source": "direct_download", "source_id": "abc"}
|
|
_download(history, path, task_id="t1", user_id=user["id"], retry_payload=payload)
|
|
|
|
assert _run(user_db, history) == {"marked": 1, "reopened": 0}
|
|
|
|
row = history.get_by_task_id("t1")
|
|
assert row["final_status"] == "error"
|
|
assert row["retry_payload"] == payload
|
|
assert history.is_retry_available(row) is True
|
|
|
|
|
|
def test_a_download_whose_request_already_completed_is_closed_without_reopening_it(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], state="complete")
|
|
_download(history, path, task_id="t1", user_id=user["id"], request_id=request["id"])
|
|
|
|
assert _run(user_db, history) == {"marked": 1, "reopened": 0}
|
|
|
|
# Closed as superseded rather than left "active", which the activity list would show as an
|
|
# interrupted failure on a request that has its book.
|
|
assert history.get_by_task_id("t1")["final_status"] == "cancelled"
|
|
assert user_db.get_request(request["id"])["delivery_state"] == "complete"
|
|
|
|
|
|
def test_finished_downloads_are_not_touched(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
for status in ("complete", "cancelled", "error"):
|
|
_download(history, path, task_id=status, user_id=user["id"], final_status=status)
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 0}
|
|
|
|
assert {
|
|
history.get_by_task_id(s)["status_message"] for s in ("complete", "cancelled", "error")
|
|
} == {"x"}
|
|
|
|
|
|
def test_a_recent_request_stuck_queued_with_no_history_is_reopened(db):
|
|
user_db, history, _path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
recent = _request(user_db, user["id"], hours_ago=2)
|
|
old = _request(user_db, user["id"], hours_ago=200)
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 1}
|
|
|
|
assert user_db.get_request(recent["id"])["status"] == "pending"
|
|
assert user_db.get_request(old["id"])["status"] == "fulfilled"
|
|
|
|
|
|
def test_a_request_whose_download_finished_is_not_reopened(db):
|
|
"""delivery_state only moves while the queue is polled, so it can still say "queued"."""
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], hours_ago=2)
|
|
_download(
|
|
history,
|
|
path,
|
|
task_id="t1",
|
|
user_id=user["id"],
|
|
request_id=request["id"],
|
|
hours_ago=2,
|
|
final_status="complete",
|
|
)
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 0}
|
|
|
|
stored = user_db.get_request(request["id"])
|
|
assert (stored["status"], stored["delivery_state"]) == ("fulfilled", "queued")
|
|
assert stored["release_data"] is not None
|
|
|
|
|
|
def test_a_request_that_is_not_fulfilled_keeps_its_status(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], status="pending", state="none")
|
|
_download(history, path, task_id="t1", user_id=user["id"], request_id=request["id"])
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 0}
|
|
|
|
assert history.get_by_task_id("t1")["final_status"] == "active"
|
|
assert user_db.get_request(request["id"])["status"] == "pending"
|
|
|
|
|
|
def test_running_it_twice_changes_nothing_the_second_time(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"])
|
|
_download(history, path, task_id="t1", user_id=user["id"], request_id=request["id"])
|
|
|
|
assert _run(user_db, history) == {"marked": 1, "reopened": 1}
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 0}
|
|
|
|
|
|
def test_the_window_is_configurable(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], hours_ago=30)
|
|
_download(
|
|
history, path, task_id="t1", user_id=user["id"], request_id=request["id"], hours_ago=30
|
|
)
|
|
|
|
assert _run(user_db, history, reopen_window=timedelta(hours=48)) == {"marked": 1, "reopened": 1}
|
|
|
|
|
|
def test_a_stale_row_of_a_request_that_was_delivered_is_closed_as_superseded(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], state="complete", hours_ago=300)
|
|
_download(
|
|
history, path, task_id="old", user_id=user["id"], request_id=request["id"], hours_ago=500
|
|
)
|
|
|
|
assert _run(user_db, history) == {"marked": 1, "reopened": 0}
|
|
|
|
row = history.get_by_task_id("old")
|
|
assert row["final_status"] == "cancelled"
|
|
assert row["status_message"] == startup_reconcile.SUPERSEDED_MESSAGE
|
|
assert user_db.get_request(request["id"])["status"] == "fulfilled" # not reopened
|
|
|
|
|
|
def test_a_stale_row_is_superseded_when_another_row_of_the_request_finished(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], state="queued", hours_ago=300)
|
|
_download(
|
|
history, path, task_id="old", user_id=user["id"], request_id=request["id"], hours_ago=500
|
|
)
|
|
_download(
|
|
history,
|
|
path,
|
|
task_id="done",
|
|
user_id=user["id"],
|
|
request_id=request["id"],
|
|
hours_ago=400,
|
|
final_status="complete",
|
|
)
|
|
|
|
assert _run(user_db, history) == {"marked": 1, "reopened": 0}
|
|
assert history.get_by_task_id("old")["final_status"] == "cancelled"
|
|
|
|
|
|
def test_an_old_row_of_an_undelivered_request_is_still_left_alone(db):
|
|
user_db, history, path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], state="queued", hours_ago=300)
|
|
_download(
|
|
history, path, task_id="old", user_id=user["id"], request_id=request["id"], hours_ago=500
|
|
)
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 0}
|
|
assert history.get_by_task_id("old")["final_status"] == "active"
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["queued", "resolving", "locating", "downloading"])
|
|
def test_a_request_with_no_row_that_says_it_is_in_flight_is_reopened(db, state):
|
|
user_db, history, _path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], state=state)
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 1}
|
|
|
|
assert user_db.get_request(request["id"])["status"] == "pending"
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["complete", "error", "cancelled", "none"])
|
|
def test_a_request_with_no_row_in_any_other_state_is_left_alone(db, state):
|
|
user_db, history, _path = db
|
|
user = user_db.create_user(username="reader", role="user")
|
|
request = _request(user_db, user["id"], state=state)
|
|
|
|
assert _run(user_db, history) == {"marked": 0, "reopened": 0}
|
|
|
|
assert user_db.get_request(request["id"])["status"] == "fulfilled"
|