mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-10-05 22:05:50 +01:00
Add Download Queue Management & Concurrent Downloads (#231)
## Summary
This PR completely overhauls the download system to support concurrent
downloads, cancellation, and advanced queue
management. No more waiting for stuck downloads to block the entire
queue!
## New Features
### **Concurrent Downloads**
- **3 simultaneous downloads** by default (configurable)
- ThreadPoolExecutor-based architecture replaces single-threaded
bottleneck
- Downloads no longer block each other
### **Download Cancellation**
- **Cancel button** for active/queued downloads
- Clean cancellation with partial file cleanup
- Thread-safe cancellation flags
### **Queue Priority & Reordering**
- **Priority-based queue** (lower number = higher priority)
- **Editable priority inputs** in UI for queued items
- **Bulk reordering** API support
## UI Enhancements
- **Enhanced status table** with Priority, Progress, and Actions columns
- **Progress bars** for active downloads
- **Cancel buttons** for downloads
- **Queue management controls**: Refresh, Clear Completed, Active
Counter
- **Real-time updates** every 60 seconds
## Implementation
### New Architecture
- `queue.PriorityQueue` replaces simple `set()` for proper ordering
- `concurrent.futures.ThreadPoolExecutor` handles worker threads
- `threading.Event` flags enable clean cancellation
- Thread-safe operations with proper locking
### New API Endpoints
DELETE /api/download/{id}/cancel # Cancel downloads
PUT /api/queue/{id}/priority # Set priorityPOST /api/queue/reorder #
Bulk reorder
GET /api/downloads/active # Active tracking
DELETE /api/queue/clear # Clear completed
### New Environment Variables
```bash
MAX_CONCURRENT_DOWNLOADS=3 # Worker thread count
DOWNLOAD_PROGRESS_UPDATE_INTERVAL=5 # Progress frequency
```
### Fully backward compatible - existing functionality unchanged
### Fixes
- Resolves: Downloads blocking when one gets stuck
- Resolves: No way to cancel problematic downloads
- Resolves: No queue management or prioritization
#183
---------
Co-authored-by: CaliBrain <calibrain@l4n.xyz>
This commit is contained in:
co-authored by
CaliBrain
parent
09bd5ae9f0
commit
5e04b6bfb8
@@ -245,9 +245,10 @@ def api_download() -> Union[Response, Tuple[Response, int]]:
|
||||
return jsonify({"error": "No book ID provided"}), 400
|
||||
|
||||
try:
|
||||
success = backend.queue_book(book_id)
|
||||
priority = int(request.args.get('priority', 0))
|
||||
success = backend.queue_book(book_id, priority)
|
||||
if success:
|
||||
return jsonify({"status": "queued"})
|
||||
return jsonify({"status": "queued", "priority": priority})
|
||||
return jsonify({"error": "Failed to queue book"}), 500
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Download error: {e}")
|
||||
@@ -306,6 +307,142 @@ def api_local_download() -> Union[Response, Tuple[Response, int]]:
|
||||
logger.error_trace(f"Local download error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/download/<book_id>/cancel', methods=['DELETE'])
|
||||
@login_required
|
||||
def api_cancel_download(book_id: str) -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
Cancel a download.
|
||||
|
||||
Path Parameters:
|
||||
book_id (str): Book identifier to cancel
|
||||
|
||||
Returns:
|
||||
flask.Response: JSON status indicating success or failure.
|
||||
"""
|
||||
try:
|
||||
success = backend.cancel_download(book_id)
|
||||
if success:
|
||||
return jsonify({"status": "cancelled", "book_id": book_id})
|
||||
return jsonify({"error": "Failed to cancel download or book not found"}), 404
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Cancel download error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/queue/<book_id>/priority', methods=['PUT'])
|
||||
@login_required
|
||||
def api_set_priority(book_id: str) -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
Set priority for a queued book.
|
||||
|
||||
Path Parameters:
|
||||
book_id (str): Book identifier
|
||||
|
||||
Request Body:
|
||||
priority (int): New priority level (lower number = higher priority)
|
||||
|
||||
Returns:
|
||||
flask.Response: JSON status indicating success or failure.
|
||||
"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data or 'priority' not in data:
|
||||
return jsonify({"error": "Priority not provided"}), 400
|
||||
|
||||
priority = int(data['priority'])
|
||||
success = backend.set_book_priority(book_id, priority)
|
||||
|
||||
if success:
|
||||
return jsonify({"status": "updated", "book_id": book_id, "priority": priority})
|
||||
return jsonify({"error": "Failed to update priority or book not found"}), 404
|
||||
except ValueError:
|
||||
return jsonify({"error": "Invalid priority value"}), 400
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Set priority error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/queue/reorder', methods=['POST'])
|
||||
@login_required
|
||||
def api_reorder_queue() -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
Bulk reorder queue by setting new priorities.
|
||||
|
||||
Request Body:
|
||||
book_priorities (dict): Mapping of book_id to new priority
|
||||
|
||||
Returns:
|
||||
flask.Response: JSON status indicating success or failure.
|
||||
"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data or 'book_priorities' not in data:
|
||||
return jsonify({"error": "book_priorities not provided"}), 400
|
||||
|
||||
book_priorities = data['book_priorities']
|
||||
if not isinstance(book_priorities, dict):
|
||||
return jsonify({"error": "book_priorities must be a dictionary"}), 400
|
||||
|
||||
# Validate all priorities are integers
|
||||
for book_id, priority in book_priorities.items():
|
||||
if not isinstance(priority, int):
|
||||
return jsonify({"error": f"Invalid priority for book {book_id}"}), 400
|
||||
|
||||
success = backend.reorder_queue(book_priorities)
|
||||
|
||||
if success:
|
||||
return jsonify({"status": "reordered", "updated_count": len(book_priorities)})
|
||||
return jsonify({"error": "Failed to reorder queue"}), 500
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Reorder queue error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/queue/order', methods=['GET'])
|
||||
@login_required
|
||||
def api_queue_order() -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
Get current queue order for display.
|
||||
|
||||
Returns:
|
||||
flask.Response: JSON array of queued books with their order and priorities.
|
||||
"""
|
||||
try:
|
||||
queue_order = backend.get_queue_order()
|
||||
return jsonify({"queue": queue_order})
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Queue order error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/downloads/active', methods=['GET'])
|
||||
@login_required
|
||||
def api_active_downloads() -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
Get list of currently active downloads.
|
||||
|
||||
Returns:
|
||||
flask.Response: JSON array of active download book IDs.
|
||||
"""
|
||||
try:
|
||||
active_downloads = backend.get_active_downloads()
|
||||
return jsonify({"active_downloads": active_downloads})
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Active downloads error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/queue/clear', methods=['DELETE'])
|
||||
@login_required
|
||||
def api_clear_completed() -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
Clear all completed, errored, or cancelled books from tracking.
|
||||
|
||||
Returns:
|
||||
flask.Response: JSON with count of removed books.
|
||||
"""
|
||||
try:
|
||||
removed_count = backend.clear_completed()
|
||||
return jsonify({"status": "cleared", "removed_count": removed_count})
|
||||
except Exception as e:
|
||||
logger.error_trace(f"Clear completed error: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.errorhandler(404)
|
||||
def not_found_error(error: Exception) -> Union[Response, Tuple[Response, int]]:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user