starred/shelfmark
1
0
Fork 0
mirror of https://github.com/calibrain/shelfmark.git synced 2026-10-06 08:54:40 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Direct source refactor (#895)

Browse Source 
- Updated mirror selection
- Removed built-in mirror options, users must provide their own
configurations
- Set Universal search to default, added ability to disable direct
source
- Updated documentation
- Updated makefile
This commit is contained in:
Alex
2026-04-15 18:50:13 +01:00
committed by GitHub
parent 18a3f0bf44
commit e35b4c47a7
40 changed files with 2176 additions and 918 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+53 -32
View File
@@ -8,8 +8,8 @@ permissions:
contents: read
jobs:
backend-quality:
name: Backend Quality
python-quality:
name: Python Quality
runs-on: ubuntu-latest
steps:
- name: Checkout
@@ -25,23 +25,17 @@ jobs:
- name: Sync dependencies
run: make install-python-dev
- name: Lint backend
- name: Lint
run: make python-lint
- name: Check backend formatting
run: make python-format-check
- name: Check formatting
run: make python-format
- name: Check backend dead code
- name: Check dead code
run: make python-dead-code
- name: Lint tests
run: make python-test-lint
- name: Check test formatting
run: make python-test-format-check
backend-typechecks:
name: Backend Typechecks
python-typechecks:
name: Python Typechecks
runs-on: ubuntu-latest
steps:
- name: Checkout
@@ -57,14 +51,11 @@ jobs:
- name: Sync dependencies
run: make install-python-dev
- name: Typecheck backend
- name: Typecheck
run: make python-typecheck
- name: Typecheck tests
run: make python-test-typecheck
backend-tests:
name: Backend Tests
python-tests:
name: Python Tests
runs-on: ubuntu-latest
steps:
- name: Checkout
@@ -81,7 +72,7 @@ jobs:
run: make install-python-dev
- name: Run tests
run: uv run pytest tests/ -x --tb=short -m "not integration and not e2e"
run: make python-test
docker-build-check:
runs-on: ubuntu-latest
@@ -103,7 +94,8 @@ jobs:
BUILD_VERSION=pr-${{ github.sha }}
RELEASE_VERSION=pr-${{ github.event.pull_request.number }}
frontend-checks:
frontend-quality:
name: Frontend Quality
runs-on: ubuntu-latest
steps:
- name: Checkout
@@ -117,21 +109,50 @@ jobs:
cache-dependency-path: src/frontend/package-lock.json
- name: Install dependencies
working-directory: src/frontend
run: npm ci
run: make install-ci
- name: Lint
working-directory: src/frontend
run: npm run lint
run: make frontend-lint
- name: Check formatting
working-directory: src/frontend
run: npm run format:check
run: make frontend-format
frontend-typechecks:
name: Frontend Typechecks
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: 24
cache: "npm"
cache-dependency-path: src/frontend/package-lock.json
- name: Install dependencies
run: make install-ci
- name: Typecheck
working-directory: src/frontend
run: npm run typecheck
run: make frontend-typecheck
frontend-tests:
name: Frontend Tests
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: 24
cache: "npm"
cache-dependency-path: src/frontend/package-lock.json
- name: Install dependencies
run: make install-ci
- name: Unit tests
working-directory: src/frontend
run: npm run test:unit
run: make frontend-test
Makefile
+48 -58
View File
@@ -1,4 +1,4 @@
.PHONY: help install install-python-dev dev build preview typecheck frontend-lint frontend-format frontend-format-check frontend-checks frontend-test clean up up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-check python-typecheck python-dead-code python-checks python-test-lint python-test-lint-fix python-test-format python-test-format-check python-test-typecheck python-test-checks python-coverage prek-install
.PHONY: help install install-ci install-python-dev dev build preview frontend-typecheck frontend-lint frontend-format frontend-format-fix frontend-checks frontend-test clean up down docker-build refresh restart build-serve python-lint python-lint-fix python-format python-format-fix python-typecheck python-dead-code python-checks python-test python-test-cov checks fix
# Frontend directory
FRONTEND_DIR := src/frontend
@@ -10,34 +10,34 @@ COMPOSE_FILE := docker-compose.dev.yml
help:
@echo "Available targets:"
@echo ""
@echo "Quality:"
@echo " checks - Run ALL static analysis checks (frontend + Python)"
@echo " fix - Auto-fix lint + format issues (frontend + Python)"
@echo ""
@echo "Frontend:"
@echo " install - Install frontend dependencies"
@echo " dev - Start development server"
@echo " build - Build frontend for production"
@echo " build-serve - Build and serve via Flask (test prod build without Docker)"
@echo " preview - Preview production build"
@echo " typecheck - Run TypeScript type checking"
@echo " frontend-typecheck - Run TypeScript type checking"
@echo " frontend-lint - Run Oxlint against frontend code"
@echo " frontend-format - Format frontend code with Oxfmt"
@echo " frontend-format-check - Check frontend formatting with Oxfmt"
@echo " frontend-format - Check frontend formatting with Oxfmt"
@echo " frontend-format-fix - Format frontend code with Oxfmt"
@echo " frontend-checks - Run all frontend static analysis checks"
@echo " frontend-test - Run frontend unit tests"
@echo ""
@echo "Python:"
@echo " install-python-dev - Sync Python runtime + dev tooling with uv"
@echo " python-lint - Run Ruff against Python backend code"
@echo " python-lint - Run Ruff against Python code (backend + tests)"
@echo " python-lint-fix - Run Ruff with safe auto-fixes"
@echo " python-format - Format Python backend code with Ruff"
@echo " python-format-check - Check Python backend formatting with Ruff"
@echo " python-typecheck - Run BasedPyright against Python backend code"
@echo " python-dead-code - Run Vulture against Python backend code"
@echo " python-format - Check Python formatting with Ruff"
@echo " python-format-fix - Format Python code with Ruff"
@echo " python-typecheck - Run BasedPyright against backend + tests"
@echo " python-dead-code - Run Vulture against backend code"
@echo " python-checks - Run all Python static analysis checks"
@echo " python-test-lint - Run Ruff against Python tests with the relaxed tests profile"
@echo " python-test-lint-fix - Run Ruff with safe auto-fixes against Python tests"
@echo " python-test-format - Format Python tests with Ruff"
@echo " python-test-format-check - Check Python test formatting with Ruff"
@echo " python-test-typecheck - Run lightweight BasedPyright checks against Python tests"
@echo " python-test-checks - Run all relaxed Python test static analysis checks"
@echo " python-coverage - Run tests with coverage report"
@echo " prek-install - Install prek git hooks"
@echo " python-test - Run unit tests"
@echo " python-test-cov - Run unit tests with coverage report"
@echo " clean - Remove node_modules and build artifacts"
@echo ""
@echo "Backend (Docker):"
@@ -52,6 +52,10 @@ install:
@echo "Installing frontend dependencies..."
cd $(FRONTEND_DIR) && npm install
install-ci:
@echo "Installing frontend dependencies (CI, lockfile-strict)..."
cd $(FRONTEND_DIR) && npm ci
# Install Python development dependencies
install-python-dev:
@echo "Syncing Python runtime and dev tooling with uv..."
@@ -82,67 +86,47 @@ preview:
cd $(FRONTEND_DIR) && npm run preview
# Type checking
typecheck:
frontend-typecheck:
@echo "Running TypeScript type checking..."
cd $(FRONTEND_DIR) && npm run typecheck
# Python linting
# Python linting (backend + tests)
python-lint:
@echo "Running Ruff..."
uv run ruff check shelfmark
uv run ruff check shelfmark tests
python-lint-fix:
@echo "Running Ruff with safe auto-fixes..."
uv run ruff check shelfmark --fix
uv run ruff check shelfmark tests --fix
python-format:
@echo "Formatting Python backend code with Ruff..."
uv run ruff format shelfmark
@echo "Checking Python formatting with Ruff..."
uv run ruff format --check shelfmark tests
python-format-check:
@echo "Checking Python backend formatting with Ruff..."
uv run ruff format --check shelfmark
python-format-fix:
@echo "Formatting Python code with Ruff..."
uv run ruff format shelfmark tests
python-typecheck:
@echo "Running BasedPyright..."
uv run basedpyright
@echo "Running BasedPyright against tests..."
uv run basedpyright tests --skipunannotated
python-dead-code:
@echo "Running Vulture..."
uv run vulture shelfmark
python-checks: python-lint python-format-check python-typecheck python-dead-code
python-checks: python-lint python-format python-typecheck python-dead-code
python-test-lint:
@echo "Running Ruff against tests with the relaxed tests profile..."
uv run ruff check tests
python-test:
@echo "Running tests..."
uv run pytest tests/ -x --tb=short -m "not integration and not e2e"
python-test-lint-fix:
@echo "Running Ruff with safe auto-fixes against tests..."
uv run ruff check tests --fix
python-test-format:
@echo "Formatting Python tests with Ruff..."
uv run ruff format tests
python-test-format-check:
@echo "Checking Python test formatting with Ruff..."
uv run ruff format --check tests
python-test-typecheck:
@echo "Running lightweight BasedPyright checks against tests..."
uv run basedpyright tests --skipunannotated
python-test-checks: python-test-lint python-test-format-check python-test-typecheck
python-coverage:
python-test-cov:
@echo "Running tests with coverage..."
uv run pytest tests/ -x --tb=short -m "not integration and not e2e" --cov --cov-report=term-missing
prek-install:
@echo "Installing prek git hooks..."
uv run prek install
# Frontend linting
frontend-lint:
@echo "Running Oxlint..."
@@ -150,21 +134,27 @@ frontend-lint:
# Frontend formatting
frontend-format:
@echo "Formatting frontend code with Oxfmt..."
cd $(FRONTEND_DIR) && npm run format
frontend-format-check:
@echo "Checking frontend formatting with Oxfmt..."
cd $(FRONTEND_DIR) && npm run format:check
frontend-format-fix:
@echo "Formatting frontend code with Oxfmt..."
cd $(FRONTEND_DIR) && npm run format
# All frontend static analysis
frontend-checks: frontend-lint frontend-format-check typecheck
frontend-checks: frontend-lint frontend-format frontend-typecheck
# Run frontend unit tests
frontend-test:
@echo "Running frontend unit tests..."
cd $(FRONTEND_DIR) && npm run test:unit
# All static analysis checks (frontend + Python)
checks: frontend-checks python-checks
# Auto-fix lint + format issues (frontend + Python)
fix: python-lint-fix python-format-fix frontend-format-fix
# Clean build artifacts and dependencies
clean:
@echo "Cleaning build artifacts and dependencies..."
docs/configuration.md
+2 -2
View File
@@ -18,9 +18,9 @@ Prowlarr -> Download client saves to <client path>
Key point: For torrent and usenet downloads, Shelfmark must see the same file path that your download client reports. The container path must match in both containers.
## Direct Download Setup
## Direct Download Volume Setup
Direct downloads do not use an external download client. A simple two-folder setup is enough.
If you plan to use Direct Download, it does not use an external download client. A simple two-folder setup is enough.
Required volumes:
docs/environment-variables.md
+93 -85
View File
@@ -21,7 +21,6 @@ This document lists all configuration options that can be set via environment va
- [Hardcover](#metadata-providers-hardcover)
- [Open Library](#metadata-providers-open-library)
- [Google Books](#metadata-providers-google-books)
- [Direct Download](#direct-download)
- [Download Sources](#direct-download-download-sources)
- [Cloudflare Bypass](#direct-download-cloudflare-bypass)
@@ -125,7 +124,7 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Booklore, etc). | string | _none_ |
| `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). | string | _none_ |
| `AUDIOBOOK_LIBRARY_URL` | Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text. | string | _none_ |
| `SUPPORTED_FORMATS` | Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found. | string (comma-separated) | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3` |
@@ -138,7 +137,7 @@ Show the onboarding wizard on first run. Set to false to skip (useful for epheme
**Library URL**
Adds a navigation button to your book library (Calibre-Web Automated, Booklore, etc).
Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc).
- **Type:** string
- **Default:** _none_
@@ -185,12 +184,14 @@ Default language filter for searches.
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `SEARCH_MODE` | How you want to search for and download books. | string (choice) | `direct` |
| `SEARCH_MODE` | How you want to search for and download books. | string (choice) | `universal` |
| `AA_DEFAULT_SORT` | Default sort order for search results. | string (choice) | `relevance` |
| `SHOW_RELEASE_SOURCE_LINKS` | Show clickable release-source links in release and details modals. Metadata provider links stay enabled. | boolean | `true` |
| `SHOW_COMBINED_SELECTOR` | Show the option to search for and download both a book and audiobook together. | boolean | `true` |
| `METADATA_PROVIDER` | Choose which metadata provider to use for book searches. | string (choice) | `openlibrary` |
| `METADATA_PROVIDER_AUDIOBOOK` | Metadata provider for audiobook searches. Uses the book provider if not set. | string (choice) | _empty string_ |
| `DEFAULT_RELEASE_SOURCE` | The release source tab to open by default in the release modal for books. | string (choice) | `direct_download` |
| `METADATA_PROVIDER_COMBINED` | Metadata provider for combined mode searches. Uses the book provider if not set. | string (choice) | _empty string_ |
| `DEFAULT_RELEASE_SOURCE` | The release source tab to open by default in the release modal for books. Leave unset to use the first available source. | string (choice) | _empty string_ |
| `DEFAULT_RELEASE_SOURCE_AUDIOBOOK` | The release source tab to open by default in the release modal for audiobooks. Uses the book release source if not set. | string (choice) | _empty string_ |
<details>
@@ -203,7 +204,7 @@ Default language filter for searches.
How you want to search for and download books.
- **Type:** string (choice)
- **Default:** `direct`
- **Default:** `universal`
- **Options:** `direct` (Direct), `universal` (Universal)
#### `AA_DEFAULT_SORT`
@@ -225,6 +226,15 @@ Show clickable release-source links in release and details modals. Metadata prov
- **Type:** boolean
- **Default:** `true`
#### `SHOW_COMBINED_SELECTOR`
**Show Combined Download Selector**
Show the option to search for and download both a book and audiobook together.
- **Type:** boolean
- **Default:** `true`
#### `METADATA_PROVIDER`
**Book Metadata Provider**
@@ -233,7 +243,7 @@ Choose which metadata provider to use for book searches.
- **Type:** string (choice)
- **Default:** `openlibrary`
- **Options:** `hardcover` (Hardcover), `openlibrary` (Open Library), `googlebooks` (Google Books)
- **Options:** `""` (No providers enabled)
#### `METADATA_PROVIDER_AUDIOBOOK`
@@ -243,17 +253,27 @@ Metadata provider for audiobook searches. Uses the book provider if not set.
- **Type:** string (choice)
- **Default:** _empty string_
- **Options:** `""` (Use book provider), `hardcover` (Hardcover), `openlibrary` (Open Library), `googlebooks` (Google Books)
- **Options:** `""` (Use book provider), `""` (No providers enabled)
#### `METADATA_PROVIDER_COMBINED`
**Combined Mode Metadata Provider**
Metadata provider for combined mode searches. Uses the book provider if not set.
- **Type:** string (choice)
- **Default:** _empty string_
- **Options:** `""` (Use book provider), `""` (No providers enabled)
#### `DEFAULT_RELEASE_SOURCE`
**Default Book Release Source**
The release source tab to open by default in the release modal for books.
The release source tab to open by default in the release modal for books. Leave unset to use the first available source.
- **Type:** string (choice)
- **Default:** `direct_download`
- **Options:** `direct_download` (Direct Download), `prowlarr` (Prowlarr)
- **Default:** _empty string_
- **Options:** `""` (Use first available source)
#### `DEFAULT_RELEASE_SOURCE_AUDIOBOOK`
@@ -263,7 +283,7 @@ The release source tab to open by default in the release modal for audiobooks. U
- **Type:** string (choice)
- **Default:** _empty string_
- **Options:** `""` (Use book release source), `prowlarr` (Prowlarr), `audiobookbay` (AudiobookBay)
- **Options:** `""` (Use book release source)
</details>
@@ -277,12 +297,12 @@ The release source tab to open by default in the release modal for audiobooks. U
| `TEMPLATE_RENAME` | Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title} ({Year})` |
| `TEMPLATE_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title} ({Year})` |
| `HARDLINK_TORRENTS` | Create hardlinks instead of copying. Preserves seeding but archives won't be extracted. Don't use if destination is a library ingest folder. | boolean | `false` |
| `BOOKLORE_HOST` | Base URL of your Booklore instance | string | _none_ |
| `BOOKLORE_USERNAME` | Booklore account username | string | _none_ |
| `BOOKLORE_PASSWORD` | Booklore account password | string (secret) | _none_ |
| `BOOKLORE_HOST` | Base URL of your Grimmory instance | string | _none_ |
| `BOOKLORE_USERNAME` | Grimmory account username | string | _none_ |
| `BOOKLORE_PASSWORD` | Grimmory account password | string (secret) | _none_ |
| `BOOKLORE_DESTINATION` | Choose whether uploads go directly to a specific library path or to Bookdrop for review. | string (choice) | `library` |
| `BOOKLORE_LIBRARY_ID` | Booklore library to upload into. | string (choice) | _none_ |
| `BOOKLORE_PATH_ID` | Booklore library path for uploads. | string (choice) | _none_ |
| `BOOKLORE_LIBRARY_ID` | Grimmory library to upload into. | string (choice) | _none_ |
| `BOOKLORE_PATH_ID` | Grimmory library path for uploads. | string (choice) | _none_ |
| `EMAIL_RECIPIENT` | Optional fallback email address when no per-user email recipient override is configured. | string | _none_ |
| `EMAIL_ATTACHMENT_SIZE_LIMIT_MB` | Maximum total attachment size per email. Email encoding adds overhead; keep this below your provider's limit. | number | `25` |
| `EMAIL_SMTP_HOST` | SMTP server hostname or IP (e.g., smtp.gmail.com). | string | _none_ |
@@ -315,7 +335,7 @@ Choose where completed book files are sent.
- **Type:** string (choice)
- **Default:** `folder`
- **Options:** `folder` (Folder), `email` (Email (SMTP)), `booklore` (Booklore (API))
- **Options:** `folder` (Folder), `email` (Email (SMTP)), `booklore` (Grimmory (API))
#### `INGEST_DIR`
@@ -366,9 +386,9 @@ Create hardlinks instead of copying. Preserves seeding but archives won't be ext
#### `BOOKLORE_HOST`
**Booklore URL**
**Grimmory URL**
Base URL of your Booklore instance
Base URL of your Grimmory instance
- **Type:** string
- **Default:** _none_
@@ -378,7 +398,7 @@ Base URL of your Booklore instance
**Username**
Booklore account username
Grimmory account username
- **Type:** string
- **Default:** _none_
@@ -388,7 +408,7 @@ Booklore account username
**Password**
Booklore account password
Grimmory account password
- **Type:** string (secret)
- **Default:** _none_
@@ -408,7 +428,7 @@ Choose whether uploads go directly to a specific library path or to Bookdrop for
**Library**
Booklore library to upload into.
Grimmory library to upload into.
- **Type:** string (choice)
- **Default:** _none_
@@ -418,7 +438,7 @@ Booklore library to upload into.
**Path**
Booklore library path for uploads.
Grimmory library path for uploads.
- **Type:** string (choice)
- **Default:** _none_
@@ -627,7 +647,7 @@ How long to keep completed/failed downloads in the queue display.
| `OIDC_DISCOVERY_URL` | OpenID Connect discovery endpoint URL. Usually ends with /.well-known/openid-configuration. | string | _none_ |
| `OIDC_CLIENT_ID` | OAuth2 client ID from your identity provider. | string | _none_ |
| `OIDC_CLIENT_SECRET` | OAuth2 client secret from your identity provider. | string (secret) | _none_ |
| `OIDC_SCOPES` | OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization. | string | `openid,email,profile` |
| `OIDC_SCOPES` | OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization. | string (comma-separated) | `openid,email,profile` |
| `OIDC_GROUP_CLAIM` | The name of the claim in the ID token that contains user groups. | string | `groups` |
| `OIDC_ADMIN_GROUP` | Users in this group will be given admin access (if enabled below). Leave empty to use database roles only. | string | _empty string_ |
| `OIDC_USE_ADMIN_GROUP` | When enabled, users in the Admin Group are granted admin access. When disabled, admin access is determined solely by database roles. | boolean | `true` |
@@ -719,7 +739,7 @@ OAuth2 client secret from your identity provider.
OAuth2 scopes to request from the identity provider. Managed automatically: includes essential scopes and the group claim when using admin group authorization.
- **Type:** string
- **Type:** string (comma-separated)
- **Default:** `openid,email,profile`
#### `OIDC_GROUP_CLAIM`
@@ -1253,7 +1273,7 @@ How long to keep cached search results before they expire.
| `QBITTORRENT_CATEGORY` | Category to assign to book downloads in qBittorrent | string | `books` |
| `QBITTORRENT_CATEGORY_AUDIOBOOK` | Category for audiobook downloads. Leave empty to use the book category. | string | _empty string_ |
| `QBITTORRENT_DOWNLOAD_DIR` | Server-side directory where torrents are downloaded (optional, uses qBittorrent default if not specified) | string | _none_ |
| `QBITTORRENT_TAG` | Tag(s) to assign to qBittorrent downloads. Leave empty for no tags. | string | _empty list_ |
| `QBITTORRENT_TAG` | Tag(s) to assign to qBittorrent downloads. Leave empty for no tags. | string (comma-separated) | _empty list_ |
| `TRANSMISSION_URL` | URL of your Transmission instance (use https:// for TLS) | string | _none_ |
| `TRANSMISSION_USERNAME` | Transmission RPC username (if authentication enabled) | string | _none_ |
| `TRANSMISSION_PASSWORD` | Transmission RPC password | string (secret) | _none_ |
@@ -1357,7 +1377,7 @@ Server-side directory where torrents are downloaded (optional, uses qBittorrent
Tag(s) to assign to qBittorrent downloads. Leave empty for no tags.
- **Type:** string
- **Type:** string (comma-separated)
- **Default:** _empty list_
#### `TRANSMISSION_URL`
@@ -1637,6 +1657,7 @@ Move deletes the job from your usenet client after import; Copy keeps it in the
| `HARDCOVER_DEFAULT_SORT` | Default sort order for Hardcover search results. | string (choice) | `relevance` |
| `HARDCOVER_EXCLUDE_COMPILATIONS` | Filter out compilations, anthologies, and omnibus editions from search results | boolean | `false` |
| `HARDCOVER_EXCLUDE_UNRELEASED` | Filter out books with a release year in the future | boolean | `false` |
| `HARDCOVER_AUTO_REMOVE_ON_DOWNLOAD` | Automatically remove a book from the active Hardcover list when you download it | boolean | `true` |
<details>
<summary>Detailed descriptions</summary>
@@ -1688,6 +1709,15 @@ Filter out books with a release year in the future
- **Type:** boolean
- **Default:** `false`
#### `HARDCOVER_AUTO_REMOVE_ON_DOWNLOAD`
**Auto-Remove from List on Download**
Automatically remove a book from the active Hardcover list when you download it
- **Type:** boolean
- **Default:** `true`
</details>
### Metadata Providers: Open Library
@@ -1769,6 +1799,7 @@ Default sort order for Google Books search results.
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `DIRECT_DOWNLOAD_ENABLED` | Show Direct Download in release-source lists and allow Direct mode searches. Add your own mirror URLs in the Mirrors tab before using it. | boolean | `false` |
| `AA_DONATOR_KEY` | Enables fast download access on AA. Get this from your donator account page. | string (secret) | _none_ |
| `FAST_SOURCES_DISPLAY` | Always tried first, no waiting or bypass required. | JSON array | _see UI for defaults_ |
| `SOURCE_PRIORITY` | Fallback sources, may have waiting. Requires bypasser. Drag to reorder. | JSON array | _see UI for defaults_ |
@@ -1787,6 +1818,15 @@ Default sort order for Google Books search results.
<details>
<summary>Detailed descriptions</summary>
#### `DIRECT_DOWNLOAD_ENABLED`
**Enable Direct Download Source**
Show Direct Download in release-source lists and allow Direct mode searches. Add your own mirror URLs in the Mirrors tab before using it.
- **Type:** boolean
- **Default:** `false`
#### `AA_DONATOR_KEY`
**Account Donator Key**
@@ -1971,14 +2011,11 @@ Timeout for external bypasser requests in milliseconds.
| Variable | Description | Type | Default |
|----------|-------------|------|---------|
| `AA_BASE_URL` | Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback). | string (choice) | `auto` |
| `AA_MIRROR_URLS` | Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation | string | `https://annas-archive.gl,https://annas-archive.pk,https://annas-archive.vg,https://annas-archive.gd` |
| `AA_ADDITIONAL_URLS` | Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables. | string | _none_ |
| `LIBGEN_ADDITIONAL_URLS` | Comma-separated list of custom LibGen mirrors to add to the defaults. | string | _none_ |
| `ZLIB_PRIMARY_URL` | Z-Library mirror to use for downloads. | string (choice) | `https://z-lib.fm` |
| `ZLIB_ADDITIONAL_URLS` | Comma-separated list of custom Z-Library mirror URLs. | string | _none_ |
| `WELIB_PRIMARY_URL` | Welib mirror to use for downloads. | string (choice) | `https://welib.org` |
| `WELIB_ADDITIONAL_URLS` | Comma-separated list of custom Welib mirror URLs. | string | _none_ |
| `AA_BASE_URL` | Select Auto to try mirrors from your list on startup and fail over on errors. Choosing a specific mirror pins Shelfmark to that URL. | string (choice) | `auto` |
| `AA_MIRROR_URLS` | List the Anna's Archive mirror URLs you want Shelfmark to use. Type a URL and press Enter to add it. Order matters when Auto is selected. | string (comma-separated) | _empty list_ |
| `LIBGEN_MIRROR_URLS` | Mirrors are tried in the order you add them until one works. | string (comma-separated) | _empty list_ |
| `ZLIB_MIRROR_URLS` | Only the first mirror in the list is used. | string (comma-separated) | _empty list_ |
| `WELIB_MIRROR_URLS` | Only the first mirror in the list is used. | string (comma-separated) | _empty list_ |
<details>
<summary>Detailed descriptions</summary>
@@ -1987,75 +2024,46 @@ Timeout for external bypasser requests in milliseconds.
**Primary Mirror**
Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback).
Select Auto to try mirrors from your list on startup and fail over on errors. Choosing a specific mirror pins Shelfmark to that URL.
- **Type:** string (choice)
- **Default:** `auto`
- **Options:** `auto` (Auto (Recommended)), `https://annas-archive.gl` (annas-archive.gl), `https://annas-archive.pk` (annas-archive.pk), `https://annas-archive.vg` (annas-archive.vg), `https://annas-archive.gd` (annas-archive.gd)
- **Options:** `auto` (Auto (Recommended))
#### `AA_MIRROR_URLS`
**Mirrors**
Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation
List the Anna's Archive mirror URLs you want Shelfmark to use. Type a URL and press Enter to add it. Order matters when Auto is selected.
- **Type:** string
- **Default:** `https://annas-archive.gl,https://annas-archive.pk,https://annas-archive.vg,https://annas-archive.gd`
- **Type:** string (comma-separated)
- **Default:** _empty list_
#### `AA_ADDITIONAL_URLS`
#### `LIBGEN_MIRROR_URLS`
**Additional Mirrors (Legacy)**
**LibGen**
Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables.
Mirrors are tried in the order you add them until one works.
- **Type:** string
- **Default:** _none_
- **Type:** string (comma-separated)
- **Default:** _empty list_
#### `LIBGEN_ADDITIONAL_URLS`
#### `ZLIB_MIRROR_URLS`
**Additional Mirrors**
**Z-Library**
Comma-separated list of custom LibGen mirrors to add to the defaults.
Only the first mirror in the list is used.
- **Type:** string
- **Default:** _none_
- **Type:** string (comma-separated)
- **Default:** _empty list_
#### `ZLIB_PRIMARY_URL`
#### `WELIB_MIRROR_URLS`
**Primary Mirror**
**Welib**
Z-Library mirror to use for downloads.
Only the first mirror in the list is used.
- **Type:** string (choice)
- **Default:** `https://z-lib.fm`
- **Options:** `https://z-lib.fm` (z-lib.fm), `https://z-lib.gs` (z-lib.gs), `https://z-lib.id` (z-lib.id), `https://z-library.sk` (z-library.sk), `https://zlibrary-global.se` (zlibrary-global.se)
#### `ZLIB_ADDITIONAL_URLS`
**Additional Mirrors**
Comma-separated list of custom Z-Library mirror URLs.
- **Type:** string
- **Default:** _none_
#### `WELIB_PRIMARY_URL`
**Primary Mirror**
Welib mirror to use for downloads.
- **Type:** string (choice)
- **Default:** `https://welib.org`
- **Options:** `https://welib.org` (welib.org)
#### `WELIB_ADDITIONAL_URLS`
**Additional Mirrors**
Comma-separated list of custom Welib mirror URLs.
- **Type:** string
- **Default:** _none_
- **Type:** string (comma-separated)
- **Default:** _empty list_
</details>
docs/index.md
+21 -1
View File
@@ -1,3 +1,23 @@
# Shelfmark Documentation
TODO
Shelfmark is a self-hosted interface for searching, requesting, and delivering books and audiobooks through the sources and services you choose to configure.
Use the guides below to set up the app, connect your library tools, and understand the main configuration areas.
## Getting Started
- [Installation](installation.md)
- [Directory and Volume Setup](configuration.md)
- [Environment Variables](environment-variables.md)
## Core Guides
- [Users & Requests](users-and-requests.md)
- [Reverse Proxy](reverse-proxy.md)
- [OIDC](oidc.md)
- [URL Search Parameters](url-search-parameters.md)
- [Custom Scripts](custom-scripts.md)
## Help
- [Troubleshooting](troubleshooting.md)
docs/installation.md
+31 -1
View File
@@ -1,3 +1,33 @@
# Installation
TODO
Shelfmark is typically deployed with Docker Compose.
## Quick Start
1. Download the compose file from the repository:
```bash
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.yml
```
2. Start the service:
```bash
docker compose up -d
```
3. Open `http://localhost:8084`
4. Configure the sources, metadata providers, and delivery settings you want to use
## Next Steps
- For volume and path setup, see [Directory and Volume Setup](configuration.md)
- For environment-based setup, see [Environment Variables](environment-variables.md)
- For authentication and user management, see [Users & Requests](users-and-requests.md) and [OIDC](oidc.md)
## Notes
- Universal search is the default mode for new installs
- Direct Download is optional and must be enabled and configured before it can be used
- Torrent and usenet setups require matching download paths between Shelfmark and your download client
docs/url-search-parameters.md
+2 -2
View File
@@ -65,9 +65,9 @@ Some parameters support multiple values by repeating the parameter:
## Search Mode Behavior
### Direct Download Mode (default)
### Direct Mode
All parameters are used to filter results from the direct download source.
When Search Mode is set to Direct, all parameters are used to filter results from the configured direct source.
`content_type` is ignored in Direct mode.
### Universal Mode
readme.md
+28 -37
View File
@@ -1,12 +1,10 @@
# 📚 Shelfmark: Book Downloader
Formerly *Calibre Web Automated Book Downloader (CWABD)*
# 📚 Shelfmark: Book Search & Request Tool
<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
Shelfmark is a self-hosted web interface for searching and downloading books and audiobooks from multiple sources. Works out of the box with popular web sources, no configuration required. Add metadata providers, additional release sources, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
**Fully standalone** - no external dependencies required. Works great alongside the following library tools, with support for automatic imports:
Works great alongside the following library tools, with support for automatic imports:
- [Calibre](https://calibre-ebook.com/)
- [Calibre-Web](https://github.com/janeczku/calibre-web)
- [Calibre-Web-Automated](https://github.com/crocodilestick/Calibre-Web-Automated)
@@ -15,16 +13,14 @@ Shelfmark is a self-hosted web interface for searching and downloading books and
## ✨ Features
- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple sources in one place
- **Multiple Sources** - Popular archive websites, Torrent, Usenet, and IRC download support
- **One-Stop Interface** - A clean, modern UI to search, browse, and download from multiple configured sources in one place
- **Multiple Sources** - Configurable web, torrent, usenet, and IRC source support
- **Audiobook Support** - Full audiobook search and download with dedicated processing
- **Two Search Modes**:
- **Direct** - Search popular web sources
- **Universal** - Search metadata providers (Hardcover, Open Library) for richer book and audiobook discovery, with multi-source downloads
- **Flexible Search** - Search metadata providers (Hardcover, Open Library, Google Books) for rich book and audiobook discovery, or query configured sources directly
- **Multi-User & Requests** - Share your instance with others, let users browse and request books, and manage approvals with configurable notifications
- **Authentication** - Built-in login, OIDC single sign-on, proxy auth, and Calibre-Web database support
- **Real-Time Progress** - Unified download queue with live status updates across all sources
- **Cloudflare Bypass** - Built-in bypasser for reliable access to protected sources
- **Network Flexibility** - Configurable proxy support, DNS settings, and optional Cloudflare handling for protected sources
## 🖼️ Screenshots
@@ -60,7 +56,7 @@ Shelfmark is a self-hosted web interface for searching and downloading books and
3. Open `http://localhost:8084`
That's it! Configure settings through the web interface as needed.
Open the web interface, then configure the sources and settings you want to use.
### Volume Setup
@@ -87,16 +83,13 @@ volumes:
### Search Modes
**Direct** (default)
- Works out of the box, no setup required
- Searches a huge library of books directly
- Returns downloadable releases immediately
**Direct**
- Queries configured sources directly
**Universal**
- Cleaner search results via metadata providers (Hardcover is recommended)
**Universal** (recommended)
- Search via metadata providers (Hardcover, Open Library, Google Books) for richer results
- Aggregates releases from multiple configured sources
- Full Audiobook support
- Requires manual setup (API keys, additional sources)
- Full audiobook support
### Environment Variables
@@ -108,19 +101,18 @@ Environment variables work for initial setup and Docker deployments. They serve
| `INGEST_DIR` | Book download directory | `/books` |
| `TZ` | Container timezone | `UTC` |
| `PUID` / `PGID` | Runtime user/group for the default root-startup flow (also supports legacy `UID`/`GID`) | `1000` / `1000` |
| `SEARCH_MODE` | `direct` or `universal` | `direct` |
| `SEARCH_MODE` | `direct` or `universal` | `universal` |
| `USING_TOR` | Enable Tor routing (requires root startup) | `false` |
See the full [Environment Variables Reference](docs/environment-variables.md) for all available options.
Some of the additional options available in Settings:
- **Fast Download Key** - Use your paid account to skip Cloudflare challenges entirely and use faster, direct downloads
- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
- **AudiobookBay** - Web scraping source for audiobook torrents (audiobooks only)
- **Additional audiobook sources** - Configure additional sources for audiobook discovery
- **IRC** - Add details for IRC book sources and download directly from the UI
- **Library Link** - Add a link to your Calibre-Web or Grimmory instance in the UI header
- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
- **Network Resilience** - Auto DNS rotation and mirror fallback when sources are unreachable. Custom proxy support (SOCK5 + HTTP/S), Tor routing.
- **Network Settings** - Custom proxy support (SOCKS5 + HTTP/S) and configurable DNS
- **Format & Language** - Filter downloads by preferred formats, languages and sorting order
- **Metadata Providers** - Configure API keys for Hardcover, Open Library, etc.
@@ -131,10 +123,10 @@ Some of the additional options available in Settings:
docker compose up -d
```
The full-featured image with built-in Cloudflare bypass.
The full-featured image with all network capabilities included.
#### Enable Tor Routing
Routes all traffic through Tor for enhanced privacy:
#### Tor Routing
Optional Tor support for network privacy:
```bash
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.tor.yml
docker compose -f docker-compose.tor.yml up -d
@@ -147,19 +139,18 @@ docker compose -f docker-compose.tor.yml up -d
- Custom DNS/proxy settings are ignored when Tor is active
### Lite
A smaller image without the built-in Cloudflare bypasser. Ideal for:
A lighter image without the built-in browser automation. Ideal for:
- **External bypassers** - Already running FlareSolverr or ByParr for other services
- **Fast downloads** - Using fast download sources
- **Alternative sources only** - Exclusively using Prowlarr, AudiobookBay, IRC, or other sources
- **Audiobooks** - Using Shelfmark exclusively for audiobooks
- **External services** - Already running FlareSolverr or similar for other applications
- **Alternative sources** - Using Prowlarr, IRC, or other configured sources
- **Audiobooks** - Using Shelfmark primarily for audiobooks
```bash
curl -O https://raw.githubusercontent.com/calibrain/shelfmark/main/compose/docker-compose.lite.yml
docker compose -f docker-compose.lite.yml up -d
```
If you need Cloudflare bypass with the Lite image, configure an external resolver (FlareSolverr/ByParr) in Settings under the Cloudflare tab.
If you need browser-based access with the Lite image, configure an external resolver in Settings.
## 🔐 Authentication
@@ -230,16 +221,16 @@ Log level is configurable via Settings or `LOG_LEVEL` environment variable.
## Development
```bash
# Python tooling
make install-python-dev # Sync Python runtime + dev tools with uv
# Quality checks
make checks # Run ALL static analysis (frontend + Python)
make python-checks # Run Ruff, BasedPyright, and Vulture
make python-test-checks # Run lightweight lint/type checks for tests
make install-python-dev # Sync Python runtime + dev tools with uv
# Frontend development
make install # Install dependencies
make dev # Start Vite dev server (localhost:5173)
make build # Production build
make typecheck # TypeScript checks
make frontend-typecheck # TypeScript checks
# Backend (Docker)
make up # Start backend via docker-compose.dev.yml
scripts/generate_env_docs.py
+3
View File
@@ -34,6 +34,7 @@ def get_field_type_name(field: Any) -> str:
OrderableListField,
PasswordField,
SelectField,
TagListField,
TextField,
)
@@ -45,6 +46,8 @@ def get_field_type_name(field: Any) -> str:
return "string (choice)"
if isinstance(field, MultiSelectField):
return "string (comma-separated)"
if isinstance(field, TagListField):
return "string (comma-separated)"
if isinstance(field, OrderableListField):
return "JSON array"
if isinstance(field, PasswordField):
shelfmark/bypass/internal_bypasser.py
+8 -4
View File
@@ -218,15 +218,19 @@ DDG_COOKIE_NAMES = {
"ddg_last_challenge",
}
# Domains requiring full session cookies (not just protection cookies)
FULL_COOKIE_DOMAINS = {"z-lib.fm", "z-lib.gs", "z-lib.id", "z-library.sk", "zlibrary-global.se"}
def _get_base_domain(domain: str) -> str:
"""Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
return ".".join(domain.split(".")[-2:]) if "." in domain else domain
def _get_full_cookie_domains() -> set[str]:
"""Return mirror domains that need full-session cookie extraction."""
from shelfmark.core.mirrors import get_zlib_cookie_domains
return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()}
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
"""Determine if a cookie should be extracted based on its name."""
if extract_all:
@@ -249,7 +253,7 @@ def _store_extracted_cookies(
return
base_domain = _get_base_domain(domain)
extract_all = base_domain in FULL_COOKIE_DOMAINS
extract_all = base_domain in _get_full_cookie_domains()
cookies_found: dict[str, dict[str, Any]] = {}
for cookie in cookies:
shelfmark/config/settings.py
+121 -159
View File
@@ -236,14 +236,18 @@ def _get_release_source_options_for_content_type(content_type: str) -> list[dict
return [
{"value": source["name"], "label": source["display_name"]}
for source in list_available_sources()
if source.get("can_be_default", True)
if source.get("enabled", True)
and source.get("can_be_default", True)
and content_type in source.get("supported_content_types", ["ebook", "audiobook"])
]
def _get_book_release_source_options() -> list[dict[str, str]]:
"""Build default release source options for book searches."""
return _get_release_source_options_for_content_type("ebook")
return [
{"value": "", "label": "Use first available source"},
*_get_release_source_options_for_content_type("ebook"),
]
def _get_audiobook_release_source_options() -> list[dict[str, str]]:
@@ -265,18 +269,15 @@ def _string_setting(value: object) -> str:
def _get_aa_base_url_options() -> list[dict[str, str]]:
"""Build AA URL options dynamically, including additional mirrors from config."""
"""Build AA URL options dynamically from user-supplied mirrors."""
from shelfmark.core.config import config
from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS, get_aa_mirrors
from shelfmark.core.mirrors import get_aa_mirrors
from shelfmark.core.utils import normalize_http_url
options = [{"value": "auto", "label": "Auto (Recommended)"}]
# Get all mirrors (defaults + custom)
all_mirrors = get_aa_mirrors()
# If AA_BASE_URL is configured to a custom mirror that isn't present in the
# defaults/additional list, include it so the UI can display the active value.
configured_url = normalize_http_url(
_string_setting(config.get("AA_BASE_URL", "auto")),
default_scheme="https",
@@ -287,63 +288,14 @@ def _get_aa_base_url_options() -> list[dict[str, str]]:
for url in all_mirrors:
domain = url.replace("https://", "").replace("http://", "")
is_custom = url not in DEFAULT_AA_MIRRORS
label = f"{domain} (custom)" if is_custom else domain
if configured_url and url == configured_url and is_custom:
label = domain
if configured_url and url == configured_url:
label = f"{domain} (configured)"
options.append({"value": url, "label": label})
return options
def _get_zlib_mirror_options() -> list[dict[str, str]]:
"""Build Z-Library mirror options for SelectField."""
from shelfmark.core.config import config
from shelfmark.core.mirrors import DEFAULT_ZLIB_MIRRORS
options = []
# Add default mirrors
for url in DEFAULT_ZLIB_MIRRORS:
domain = url.replace("https://", "").replace("http://", "")
options.append({"value": url, "label": domain})
# Add custom mirrors
additional = _string_setting(config.get("ZLIB_ADDITIONAL_URLS", ""))
if additional:
for raw_url in additional.split(","):
url = raw_url.strip()
if url and url not in DEFAULT_ZLIB_MIRRORS:
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
options.append({"value": url, "label": f"{domain} (custom)"})
return options
def _get_welib_mirror_options() -> list[dict[str, str]]:
"""Build Welib mirror options for SelectField."""
from shelfmark.core.config import config
from shelfmark.core.mirrors import DEFAULT_WELIB_MIRRORS
options = []
# Add default mirrors
for url in DEFAULT_WELIB_MIRRORS:
domain = url.replace("https://", "").replace("http://", "")
options.append({"value": url, "label": domain})
# Add custom mirrors
additional = _string_setting(config.get("WELIB_ADDITIONAL_URLS", ""))
if additional:
for raw_url in additional.split(","):
url = raw_url.strip()
if url and url not in DEFAULT_WELIB_MIRRORS:
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
options.append({"value": url, "label": f"{domain} (custom)"})
return options
def _clear_covers_cache(current_values: dict) -> dict:
"""Clear the cover image cache."""
try:
@@ -393,6 +345,13 @@ def _clear_metadata_cache(current_values: dict) -> dict:
def general_settings() -> list[SettingsField]:
"""Core application settings."""
return [
TextField(
key="SEARCH_PAGE_TITLE",
label="Search Page Title",
description="Title shown above the main search box on the homepage.",
default="Shelfmark",
placeholder="Shelfmark",
),
TextField(
key="CALIBRE_WEB_URL",
label="Library URL",
@@ -441,7 +400,10 @@ def search_mode_settings() -> list[SettingsField]:
HeadingField(
key="search_mode_heading",
title="Search Mode",
description="Direct mode searches web sources and downloads immediately. Universal mode supports Prowlarr, IRC and audiobooks with metadata-based searching.",
description=(
"Direct mode uses the optional Direct Download source. Universal mode uses "
"metadata search with whichever release sources you have enabled."
),
),
SelectField(
key="SEARCH_MODE",
@@ -451,7 +413,10 @@ def search_mode_settings() -> list[SettingsField]:
{
"value": "direct",
"label": "Direct",
"description": "Search web sources for books and download directly. Works out of the box.",
"description": (
"Search with the Direct Download source. Requires enabling the source "
"and adding your own mirror URLs."
),
},
{
"value": "universal",
@@ -459,7 +424,7 @@ def search_mode_settings() -> list[SettingsField]:
"description": "Metadata-based search with downloads from all sources. Book and Audiobook support.",
},
],
default="direct",
default="universal",
user_overridable=True,
),
SelectField(
@@ -523,9 +488,12 @@ def search_mode_settings() -> list[SettingsField]:
SelectField(
key="DEFAULT_RELEASE_SOURCE",
label="Default Book Release Source",
description="The release source tab to open by default in the release modal for books.",
description=(
"The release source tab to open by default in the release modal for books. "
"Leave unset to use the first available source."
),
options=_get_book_release_source_options, # Callable - evaluated lazily to avoid circular imports
default="direct_download",
default="",
show_when={"field": "SEARCH_MODE", "value": "universal"},
user_overridable=True,
),
@@ -1306,8 +1274,13 @@ register_on_save("downloads", _on_save_downloads)
def _get_fast_source_options() -> list[dict[str, str | bool | int | None]]:
"""Fast download sources - configurable list shown in settings."""
from shelfmark.core.config import config
from shelfmark.core.mirrors import get_download_source_missing_mirror_reason
has_donator_key = bool(config.get("AA_DONATOR_KEY", ""))
aa_fast_reason = get_download_source_missing_mirror_reason("aa-fast")
if not aa_fast_reason and not has_donator_key:
aa_fast_reason = "Requires Donator Key"
libgen_reason = get_download_source_missing_mirror_reason("libgen")
return [
{
@@ -1315,14 +1288,16 @@ def _get_fast_source_options() -> list[dict[str, str | bool | int | None]]:
"label": "AA Fast Downloads",
"description": "Fast downloads for donators",
"isPinned": True,
"isLocked": not has_donator_key,
"disabledReason": "Requires Donator Key" if not has_donator_key else None,
"isLocked": aa_fast_reason is not None,
"disabledReason": aa_fast_reason,
},
{
"id": "libgen",
"label": "Library Genesis",
"description": "Instant downloads, no bypass needed",
"isPinned": True,
"isLocked": libgen_reason is not None,
"disabledReason": libgen_reason,
},
]
@@ -1338,39 +1313,46 @@ def _get_fast_source_defaults() -> list[dict[str, str | bool]]:
def _get_slow_source_options() -> list[dict[str, str | bool | None]]:
"""Slow download sources - configurable order. All require bypasser."""
from shelfmark.core.config import config
from shelfmark.core.mirrors import get_download_source_missing_mirror_reason
bypass_enabled = config.get("USE_CF_BYPASS", True)
locked = not bypass_enabled
disabled_reason = "Requires Cloudflare bypass" if locked else None
def _get_reason(source_id: str) -> str | None:
mirror_reason = get_download_source_missing_mirror_reason(source_id)
if mirror_reason:
return mirror_reason
if not bypass_enabled:
return "Requires Cloudflare bypass"
return None
return [
{
"id": "aa-slow-nowait",
"label": "AA Slow Downloads (No Waitlist)",
"description": "Partner servers",
"isLocked": locked,
"disabledReason": disabled_reason,
"isLocked": _get_reason("aa-slow-nowait") is not None,
"disabledReason": _get_reason("aa-slow-nowait"),
},
{
"id": "aa-slow-wait",
"label": "AA Slow Downloads (Waitlist)",
"description": "Partner servers with countdown timer",
"isLocked": locked,
"disabledReason": disabled_reason,
"isLocked": _get_reason("aa-slow-wait") is not None,
"disabledReason": _get_reason("aa-slow-wait"),
},
{
"id": "welib",
"label": "Welib",
"description": "Alternative mirror",
"isLocked": locked,
"disabledReason": disabled_reason,
"isLocked": _get_reason("welib") is not None,
"disabledReason": _get_reason("welib"),
},
{
"id": "zlib",
"label": "Zlib",
"description": "Alternative mirror",
"isLocked": locked,
"disabledReason": disabled_reason,
"isLocked": _get_reason("zlib") is not None,
"disabledReason": _get_reason("zlib"),
},
]
@@ -1393,6 +1375,15 @@ def _get_slow_source_defaults() -> list[dict[str, str | bool]]:
def download_source_settings() -> list[SettingsField]:
"""Return settings for download source behavior."""
return [
CheckboxField(
key="DIRECT_DOWNLOAD_ENABLED",
label="Enable Direct Download Source",
description=(
"Show Direct Download in release-source lists and allow Direct mode "
"searches. Add your own mirror URLs in the Mirrors tab before using it."
),
default=False,
),
PasswordField(
key="AA_DONATOR_KEY",
label="Account Donator Key",
@@ -1401,7 +1392,7 @@ def download_source_settings() -> list[SettingsField]:
HeadingField(
key="source_priority_heading",
title="Source Priority",
description="Sources are tried in order until a download succeeds.",
description="Sources are tried in order until a download succeeds. Mirror-backed entries unlock automatically when you configure their mirrors.",
),
OrderableListField(
key="FAST_SOURCES_DISPLAY",
@@ -1548,37 +1539,37 @@ def cloudflare_bypass_settings() -> list[SettingsField]:
def _on_save_mirrors(values: dict[str, Any]) -> dict[str, Any]:
"""Normalize mirror list settings before persisting."""
from shelfmark.core.logger import setup_logger
from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS
from shelfmark.core.utils import normalize_http_url
logger = setup_logger(__name__)
mirror_list_keys = {
"AA_MIRROR_URLS",
"LIBGEN_MIRROR_URLS",
"ZLIB_MIRROR_URLS",
"WELIB_MIRROR_URLS",
}
raw_urls = values.get("AA_MIRROR_URLS")
if raw_urls is None:
return {"error": False, "values": values}
if isinstance(raw_urls, str):
parts = [p.strip() for p in raw_urls.split(",") if p.strip()]
elif isinstance(raw_urls, list):
parts = [str(p).strip() for p in raw_urls if str(p).strip()]
else:
parts = []
normalized: list[str] = []
for url in parts:
if url.lower() == "auto":
for key in mirror_list_keys:
raw_urls = values.get(key)
if raw_urls is None:
continue
norm = normalize_http_url(url, default_scheme="https")
if norm and norm not in normalized:
normalized.append(norm)
if not normalized:
logger.warning("AA_MIRROR_URLS saved empty/invalid; falling back to defaults")
normalized = [normalize_http_url(url, default_scheme="https") for url in DEFAULT_AA_MIRRORS]
normalized = [url for url in normalized if url]
if isinstance(raw_urls, str):
parts = [p.strip() for p in raw_urls.split(",") if p.strip()]
elif isinstance(raw_urls, list):
parts = [str(p).strip() for p in raw_urls if str(p).strip()]
else:
parts = []
normalized: list[str] = []
for url in parts:
if url.lower() == "auto":
continue
norm = normalize_http_url(url, default_scheme="https")
if norm and norm not in normalized:
normalized.append(norm)
values[key] = normalized
values["AA_MIRROR_URLS"] = normalized
return {"error": False, "values": values}
@@ -1589,85 +1580,56 @@ register_on_save("mirrors", _on_save_mirrors)
@register_settings("mirrors", "Mirrors", icon="globe", order=23, group="direct_download")
def mirror_settings() -> list[SettingsField]:
"""Configure download source mirrors."""
from shelfmark.core.mirrors import (
DEFAULT_AA_MIRRORS,
DEFAULT_WELIB_MIRRORS,
DEFAULT_ZLIB_MIRRORS,
)
return [
# === PRIMARY SOURCE ===
HeadingField(
key="aa_mirrors_heading",
title="Anna's Archive",
description="Choose a primary mirror, or use Auto to try mirrors from your list below. The mirror list controls which options appear in the dropdown and the order used in Auto mode.",
description=(
"Add your own Anna's Archive mirror URLs here. Auto mode will try them in the "
"order listed below."
),
),
SelectField(
key="AA_BASE_URL",
label="Primary Mirror",
description="Select 'Auto' to try mirrors from your list on startup and fall back on failures. Choosing a specific mirror locks Shelfmark to that mirror (no fallback).",
description=(
"Select Auto to try mirrors from your list on startup and fail over on errors. "
"Choosing a specific mirror pins Shelfmark to that URL."
),
options=_get_aa_base_url_options,
default="auto",
),
TagListField(
key="AA_MIRROR_URLS",
label="Mirrors",
description="Editable list of AA mirrors. Used to populate the Primary Mirror dropdown and the order used when Auto is selected. Type a URL and press Enter to add. Order matters for auto-rotation",
placeholder="https://annas-archive.gl",
default=DEFAULT_AA_MIRRORS,
),
TextField(
key="AA_ADDITIONAL_URLS",
label="Additional Mirrors (Legacy)",
description="Deprecated. Use Mirrors instead. This is kept for backwards compatibility with existing installs and environment variables.",
show_when={"field": "AA_ADDITIONAL_URLS", "notEmpty": True},
description=(
"List the Anna's Archive mirror URLs you want Shelfmark to use. Type a URL and "
"press Enter to add it. Order matters when Auto is selected."
),
placeholder="https://your-aa-mirror.example",
default=[],
),
# === LIBGEN ===
HeadingField(
key="libgen_mirrors_heading",
title="LibGen",
description="All mirrors are tried during download until one succeeds. Defaults: libgen.gl, libgen.li, libgen.bz, libgen.la, libgen.vg",
),
TextField(
key="LIBGEN_ADDITIONAL_URLS",
label="Additional Mirrors",
description="Comma-separated list of custom LibGen mirrors to add to the defaults.",
TagListField(
key="LIBGEN_MIRROR_URLS",
label="LibGen",
description="Mirrors are tried in the order you add them until one works.",
placeholder="https://your-libgen-mirror.example",
),
# === Z-LIBRARY ===
HeadingField(
key="zlib_mirrors_heading",
title="Z-Library",
description="Z-Library requires Cloudflare bypass. Only the primary mirror is used.",
),
SelectField(
key="ZLIB_PRIMARY_URL",
label="Primary Mirror",
description="Z-Library mirror to use for downloads.",
options=_get_zlib_mirror_options,
default=DEFAULT_ZLIB_MIRRORS[0],
),
TextField(
key="ZLIB_ADDITIONAL_URLS",
label="Additional Mirrors",
description="Comma-separated list of custom Z-Library mirror URLs.",
TagListField(
key="ZLIB_MIRROR_URLS",
label="Z-Library",
description="Only the first mirror in the list is used.",
placeholder="https://your-zlibrary-mirror.example",
),
# === WELIB ===
HeadingField(
key="welib_mirrors_heading",
title="Welib",
description="Welib requires Cloudflare bypass. Only the primary mirror is used.",
),
SelectField(
key="WELIB_PRIMARY_URL",
label="Primary Mirror",
description="Welib mirror to use for downloads.",
options=_get_welib_mirror_options,
default=DEFAULT_WELIB_MIRRORS[0],
),
TextField(
key="WELIB_ADDITIONAL_URLS",
label="Additional Mirrors",
description="Comma-separated list of custom Welib mirror URLs.",
TagListField(
key="WELIB_MIRROR_URLS",
label="Welib",
description="Only the first mirror in the list is used.",
placeholder="https://your-welib-mirror.example",
),
]
shelfmark/core/mirrors.py
+141 -152
View File
@@ -1,4 +1,4 @@
"""Centralized mirror configuration for all download sources."""
"""Centralized mirror configuration for direct-download sources."""
from __future__ import annotations
@@ -22,33 +22,21 @@ def _get_config() -> Config:
return _config_module
# Default mirror lists (hardcoded fallbacks)
DEFAULT_AA_MIRRORS = [
"https://annas-archive.gl",
"https://annas-archive.pk",
"https://annas-archive.vg",
"https://annas-archive.gd",
]
# Mirror URLs are intentionally user-supplied only.
DEFAULT_AA_MIRRORS: list[str] = []
DEFAULT_LIBGEN_MIRRORS: list[str] = []
DEFAULT_ZLIB_MIRRORS: list[str] = []
DEFAULT_WELIB_MIRRORS: list[str] = []
DEFAULT_LIBGEN_MIRRORS = [
"https://libgen.gl",
"https://libgen.li",
"https://libgen.bz",
"https://libgen.la",
"https://libgen.vg",
]
DEFAULT_ZLIB_MIRRORS = [
"https://z-lib.fm",
"https://z-lib.gs",
"https://z-lib.id",
"https://z-library.sk",
"https://zlibrary-global.se",
]
DEFAULT_WELIB_MIRRORS = [
"https://welib.org",
]
_DOWNLOAD_SOURCE_MIRROR_LABELS = {
"aa-fast": "Anna's Archive",
"aa-slow": "Anna's Archive",
"aa-slow-nowait": "Anna's Archive",
"aa-slow-wait": "Anna's Archive",
"libgen": "LibGen",
"zlib": "Z-Library",
"welib": "Welib",
}
def _normalize_mirror_url(url: str) -> str:
@@ -60,14 +48,51 @@ def _string_config_value(value: object) -> str:
return value if isinstance(value, str) else str(value or "")
def _normalize_configured_urls(value: object) -> list[str]:
"""Normalize list or comma-separated mirror config into unique URLs."""
if isinstance(value, list):
parts = value
elif isinstance(value, str) and value.strip():
parts = value.split(",")
else:
return []
normalized_urls: list[str] = []
for raw_url in parts:
normalized = _normalize_mirror_url(str(raw_url))
if normalized and normalized not in normalized_urls:
normalized_urls.append(normalized)
return normalized_urls
def _get_primary_mirror_url(key: str) -> str | None:
"""Return a configured primary mirror URL, if present."""
config = _get_config()
primary = _normalize_mirror_url(_string_config_value(config.get(key, "")))
return primary or None
def _build_primary_and_additional_mirrors(primary_key: str, additional_key: str) -> list[str]:
"""Build an ordered mirror list from primary + additional config values."""
config = _get_config()
mirrors: list[str] = []
primary = _get_primary_mirror_url(primary_key)
if primary:
mirrors.append(primary)
for url in _normalize_configured_urls(config.get(additional_key, "")):
if url not in mirrors:
mirrors.append(url)
return mirrors
def get_aa_mirrors() -> list[str]:
"""Get Anna's Archive mirrors.
Returns:
Ordered list of AA mirror URLs.
If AA_MIRROR_URLS is configured, it is treated as the full list.
Otherwise, defaults are used and AA_ADDITIONAL_URLS (legacy) is appended.
Ordered list of user-configured AA mirror URLs.
Notes:
- The list is used to populate the AA mirror dropdown in Settings.
@@ -75,172 +100,149 @@ def get_aa_mirrors() -> list[str]:
"""
config = _get_config()
configured_list = _normalize_configured_urls(config.get("AA_MIRROR_URLS", None))
if configured_list:
return configured_list
return _normalize_configured_urls(config.get("AA_ADDITIONAL_URLS", ""))
mirrors: list[str] = []
configured_list = config.get("AA_MIRROR_URLS", None)
if isinstance(configured_list, list):
for url in configured_list:
normalized = _normalize_mirror_url(str(url))
if normalized and normalized not in mirrors:
mirrors.append(normalized)
elif isinstance(configured_list, str) and configured_list.strip():
# Allow comma-separated env/manual configs.
for url in configured_list.split(","):
normalized = _normalize_mirror_url(url)
if normalized and normalized not in mirrors:
mirrors.append(normalized)
def has_aa_mirror_configuration() -> bool:
"""Return True when direct-download search has at least one AA base URL to use."""
if get_aa_mirrors():
return True
if not mirrors:
mirrors = [_normalize_mirror_url(url) for url in DEFAULT_AA_MIRRORS]
mirrors = [url for url in mirrors if url]
# Backwards-compatible append-only behavior for legacy configs/env.
additional = _string_config_value(config.get("AA_ADDITIONAL_URLS", ""))
if additional:
for url in additional.split(","):
normalized = _normalize_mirror_url(url)
if normalized and normalized not in mirrors:
mirrors.append(normalized)
return mirrors
configured_base_url = normalize_http_url(
_string_config_value(_get_config().get("AA_BASE_URL", "auto")),
default_scheme="https",
allow_special=("auto",),
)
return bool(configured_base_url and configured_base_url != "auto")
def get_libgen_mirrors() -> list[str]:
"""Get LibGen mirrors: defaults + any additional from config.
"""Get user-configured LibGen mirrors.
Returns:
List of LibGen mirror URLs (defaults first, then custom additions).
List of LibGen mirror URLs.
"""
mirrors = [_normalize_mirror_url(url) for url in DEFAULT_LIBGEN_MIRRORS]
mirrors = [url for url in mirrors if url]
config = _get_config()
configured_list = _normalize_configured_urls(config.get("LIBGEN_MIRROR_URLS", None))
if configured_list:
return configured_list
return _normalize_configured_urls(config.get("LIBGEN_ADDITIONAL_URLS", ""))
additional = _string_config_value(config.get("LIBGEN_ADDITIONAL_URLS", ""))
if additional:
for url in additional.split(","):
normalized = _normalize_mirror_url(url)
if normalized and normalized not in mirrors:
mirrors.append(normalized)
return mirrors
def has_libgen_mirror_configuration() -> bool:
"""Return True when at least one LibGen mirror URL is configured."""
return bool(get_libgen_mirrors())
def get_zlib_mirrors() -> list[str]:
"""Get Z-Library mirrors, with primary first.
"""Get user-configured Z-Library mirrors, with primary first.
Returns:
List of Z-Library mirror URLs, primary first.
"""
config = _get_config()
primary = _normalize_mirror_url(
_string_config_value(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
)
if not primary:
primary = _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
mirrors = [primary]
# Add other defaults (excluding primary)
for url in DEFAULT_ZLIB_MIRRORS:
normalized = _normalize_mirror_url(url)
if normalized and normalized != primary:
mirrors.append(normalized)
# Add custom mirrors
additional = _string_config_value(config.get("ZLIB_ADDITIONAL_URLS", ""))
if additional:
for url in additional.split(","):
normalized = _normalize_mirror_url(url)
if normalized and normalized not in mirrors:
mirrors.append(normalized)
return mirrors
configured_list = _normalize_configured_urls(config.get("ZLIB_MIRROR_URLS", None))
if configured_list:
return configured_list
return _build_primary_and_additional_mirrors("ZLIB_PRIMARY_URL", "ZLIB_ADDITIONAL_URLS")
def get_zlib_primary_url() -> str:
def has_zlib_mirror_configuration() -> bool:
"""Return True when at least one Z-Library mirror URL is configured."""
return bool(get_zlib_mirrors())
def get_zlib_primary_url() -> str | None:
"""Get the primary Z-Library mirror URL.
Returns:
Primary Z-Library mirror URL.
Primary Z-Library mirror URL, if configured.
"""
config = _get_config()
primary = _normalize_mirror_url(
_string_config_value(config.get("ZLIB_PRIMARY_URL", DEFAULT_ZLIB_MIRRORS[0]))
)
return primary or _normalize_mirror_url(DEFAULT_ZLIB_MIRRORS[0])
mirrors = get_zlib_mirrors()
return mirrors[0] if mirrors else None
def get_zlib_url_template() -> str:
def get_zlib_url_template() -> str | None:
"""Get Z-Library URL template using configured primary mirror.
Returns:
URL template with {md5} placeholder.
URL template with {md5} placeholder, if configured.
"""
primary = get_zlib_primary_url()
return f"{primary}/md5/{{md5}}"
return f"{primary}/md5/{{md5}}" if primary else None
def get_welib_mirrors() -> list[str]:
"""Get Welib mirrors, with primary first.
"""Get user-configured Welib mirrors, with primary first.
Returns:
List of Welib mirror URLs, primary first.
"""
config = _get_config()
primary = _normalize_mirror_url(
_string_config_value(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
)
if not primary:
primary = _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
mirrors = [primary]
# Add other defaults (excluding primary)
for url in DEFAULT_WELIB_MIRRORS:
normalized = _normalize_mirror_url(url)
if normalized and normalized != primary:
mirrors.append(normalized)
# Add custom mirrors
additional = _string_config_value(config.get("WELIB_ADDITIONAL_URLS", ""))
if additional:
for url in additional.split(","):
normalized = _normalize_mirror_url(url)
if normalized and normalized not in mirrors:
mirrors.append(normalized)
return mirrors
configured_list = _normalize_configured_urls(config.get("WELIB_MIRROR_URLS", None))
if configured_list:
return configured_list
return _build_primary_and_additional_mirrors("WELIB_PRIMARY_URL", "WELIB_ADDITIONAL_URLS")
def get_welib_primary_url() -> str:
def has_welib_mirror_configuration() -> bool:
"""Return True when at least one Welib mirror URL is configured."""
return bool(get_welib_mirrors())
def has_download_source_mirror_configuration(source_id: str) -> bool:
"""Return True when the requested direct-download source has mirror config."""
if source_id in {"aa-fast", "aa-slow", "aa-slow-nowait", "aa-slow-wait"}:
return has_aa_mirror_configuration()
if source_id == "libgen":
return has_libgen_mirror_configuration()
if source_id == "zlib":
return has_zlib_mirror_configuration()
if source_id == "welib":
return has_welib_mirror_configuration()
return False
def get_download_source_missing_mirror_reason(source_id: str) -> str | None:
"""Return a user-facing reason when a direct-download source has no mirror config."""
if has_download_source_mirror_configuration(source_id):
return None
label = _DOWNLOAD_SOURCE_MIRROR_LABELS.get(source_id)
if not label:
return None
return f"Add at least one {label} mirror in Mirrors"
def get_welib_primary_url() -> str | None:
"""Get the primary Welib mirror URL.
Returns:
Primary Welib mirror URL.
Primary Welib mirror URL, if configured.
"""
config = _get_config()
primary = _normalize_mirror_url(
_string_config_value(config.get("WELIB_PRIMARY_URL", DEFAULT_WELIB_MIRRORS[0]))
)
return primary or _normalize_mirror_url(DEFAULT_WELIB_MIRRORS[0])
mirrors = get_welib_mirrors()
return mirrors[0] if mirrors else None
def get_welib_url_template() -> str:
def get_welib_url_template() -> str | None:
"""Get Welib URL template using configured primary mirror.
Returns:
URL template with {md5} placeholder.
URL template with {md5} placeholder, if configured.
"""
primary = get_welib_primary_url()
return f"{primary}/md5/{{md5}}"
return f"{primary}/md5/{{md5}}" if primary else None
def get_zlib_cookie_domains() -> set:
@@ -254,21 +256,8 @@ def get_zlib_cookie_domains() -> set:
"""
domains = set()
# Add all default domains
for url in DEFAULT_ZLIB_MIRRORS:
normalized = _normalize_mirror_url(url)
if normalized:
domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
domains.add(domain)
# Add custom domains
config = _get_config()
additional = _string_config_value(config.get("ZLIB_ADDITIONAL_URLS", ""))
if additional:
for url in additional.split(","):
normalized = _normalize_mirror_url(url)
if normalized:
domain = normalized.replace("https://", "").replace("http://", "").split("/")[0]
domains.add(domain)
for url in get_zlib_mirrors():
domain = url.replace("https://", "").replace("http://", "").split("/")[0]
domains.add(domain)
return domains
shelfmark/core/onboarding.py
+399 -99
View File
@@ -12,8 +12,10 @@ from typing import Any
from shelfmark.core.logger import setup_logger
from shelfmark.core.settings_registry import (
HeadingField,
MultiSelectField,
SettingsField,
get_setting_value,
get_settings_field_map,
get_settings_tab,
save_config_file,
serialize_field,
@@ -23,6 +25,8 @@ logger = setup_logger(__name__)
ONBOARDING_STORAGE_KEY = "onboarding_complete"
ONBOARDING_RELEASE_SOURCES_KEY = "ONBOARDING_RELEASE_SOURCES"
_ONBOARDING_VIRTUAL_KEYS = {ONBOARDING_RELEASE_SOURCES_KEY}
def _get_config_dir() -> Path:
@@ -86,6 +90,20 @@ def _get_field_from_tab(tab_name: str, field_key: str) -> SettingsField | None:
return None
def _get_field_tab_name(field: SettingsField, fallback_tab_name: str) -> str:
"""Return the owning settings tab for a value field."""
field_key = getattr(field, "key", None)
if not field_key:
return fallback_tab_name
field_map = get_settings_field_map()
field_entry = field_map.get(field_key)
if field_entry is None:
return fallback_tab_name
return field_entry[1]
def _clone_field_with_overrides(field: SettingsField, **overrides: object) -> SettingsField:
"""Clone a field with optional attribute overrides.
@@ -94,6 +112,100 @@ def _clone_field_with_overrides(field: SettingsField, **overrides: object) -> Se
return replace(field, **overrides)
def _get_fields_from_tab(
tab_name: str,
field_keys: list[str],
*,
strip_show_when_keys: set[str] | None = None,
) -> list[SettingsField]:
"""Return the requested fields from a settings tab in the supplied order."""
fields: list[SettingsField] = []
for field_key in field_keys:
field = _get_field_from_tab(tab_name, field_key)
if field:
show_when = getattr(field, "show_when", None)
stripped_show_when = _strip_show_when_keys(show_when, strip_show_when_keys or set())
if stripped_show_when != show_when:
field = replace(field, show_when=stripped_show_when)
fields.append(field)
return fields
def _strip_show_when_keys(
show_when: dict[str, Any] | list[dict[str, Any]] | None,
field_keys: set[str],
) -> dict[str, Any] | list[dict[str, Any]] | None:
"""Remove conditions tied to fields that onboarding handles implicitly."""
if not show_when or not field_keys:
return show_when
if isinstance(show_when, list):
remaining = [
condition for condition in show_when if condition.get("field") not in field_keys
]
return remaining or None
if show_when.get("field") in field_keys:
return None
return show_when
def _is_release_source_selected(values: dict[str, Any], source_name: str) -> bool:
"""Return True when a release source has been chosen during onboarding."""
raw_sources = values.get(ONBOARDING_RELEASE_SOURCES_KEY, [])
if not isinstance(raw_sources, list):
return False
return source_name in raw_sources
def _evaluate_show_when_condition(condition: dict[str, Any], values: dict[str, Any]) -> bool:
"""Evaluate one onboarding show_when condition against submitted values."""
current_value = values.get(condition["field"])
expected_value = condition.get("value")
if condition.get("notEmpty"):
if isinstance(current_value, list):
return len(current_value) > 0
return current_value not in (None, "")
if isinstance(current_value, list):
if isinstance(expected_value, list):
return all(item in current_value for item in expected_value)
return expected_value in current_value
if isinstance(expected_value, list):
return current_value in expected_value
return current_value == expected_value
def _is_step_visible(step_config: dict[str, Any], values: dict[str, Any]) -> bool:
"""Return True when a step should be included for the provided values."""
show_when = step_config.get("show_when")
if not show_when:
return True
return all(_evaluate_show_when_condition(condition, values) for condition in show_when)
def _is_field_visible(field: SettingsField, values: dict[str, Any]) -> bool:
"""Return True when a field should be included in the onboarding save."""
if getattr(field, "hidden_in_ui", False):
return False
if getattr(field, "universal_only", False) and values.get("SEARCH_MODE") != "universal":
return False
show_when = getattr(field, "show_when", None)
if not show_when:
return True
if isinstance(show_when, list):
return all(_evaluate_show_when_condition(condition, values) for condition in show_when)
return _evaluate_show_when_condition(show_when, values)
# =============================================================================
# Step Definitions
# =============================================================================
@@ -217,126 +329,267 @@ def get_googlebooks_setup_fields() -> list[SettingsField]:
return fields
def get_release_source_selection_fields() -> list[SettingsField]:
"""Choose which release sources to configure during onboarding."""
fields: list[SettingsField] = [
HeadingField(
key="release_sources_heading",
title="Release Sources",
description=(
"Choose the release sources you want to configure now. You can always add or "
"change sources later in Settings."
),
),
MultiSelectField(
key=ONBOARDING_RELEASE_SOURCES_KEY,
label="Sources to Set Up",
description="Select one or more release sources to configure now.",
default=[],
variant="dropdown",
env_supported=False,
options=[
{
"value": "direct_download",
"label": "Direct Download",
"description": "Configure your own Anna's Archive mirror URLs for direct ebook downloads.",
},
{
"value": "prowlarr",
"label": "Prowlarr",
"description": "Search your torrent and Usenet indexers through Prowlarr.",
},
{
"value": "audiobookbay",
"label": "AudiobookBay",
"description": "Search AudiobookBay directly for audiobook releases.",
},
{
"value": "irc",
"label": "IRC",
"description": "Connect to IRC for ebook and audiobook release searches.",
},
],
),
]
return fields
def get_direct_download_setup_fields() -> list[SettingsField]:
"""Render trimmed direct-download essentials for onboarding."""
fields: list[SettingsField] = [
HeadingField(
key="direct_download_setup_onboarding_heading",
title="Direct Download Setup",
description=(
"Add at least one Anna's Archive mirror URL to enable Direct Download. If you "
"have an Anna's Archive donator key, you can add it here too. You can configure "
"alternative mirrors later in Settings."
),
)
]
fields.extend(_get_fields_from_tab("download_sources", ["AA_DONATOR_KEY"]))
fields.extend(_get_fields_from_tab("mirrors", ["AA_MIRROR_URLS"]))
return fields
def get_direct_download_bypass_fields() -> list[SettingsField]:
"""Render only the core Cloudflare bypass fields for onboarding."""
return _get_fields_from_tab(
"cloudflare_bypass",
[
"USE_CF_BYPASS",
"USING_EXTERNAL_BYPASSER",
"EXT_BYPASSER_URL",
"EXT_BYPASSER_PATH",
],
)
def get_prowlarr_fields() -> list[SettingsField]:
"""Step 4: Configure Prowlarr connection - uses actual Prowlarr fields."""
fields: list[SettingsField] = [
"""Render trimmed Prowlarr setup fields for onboarding."""
return _get_fields_from_tab(
"prowlarr_config",
[
"prowlarr_heading",
"PROWLARR_URL",
"PROWLARR_API_KEY",
"test_prowlarr",
"PROWLARR_INDEXERS",
],
strip_show_when_keys={"PROWLARR_ENABLED"},
)
def get_audiobookbay_fields() -> list[SettingsField]:
"""Render trimmed AudiobookBay setup fields for onboarding."""
return [
HeadingField(
key="prowlarr_heading",
title="Prowlarr Integration (Optional)",
description="Connect to Prowlarr to search your indexers for torrents and NZBs. Skip this step if you only want to use Direct Download.",
key="audiobookbay_onboarding_heading",
title="AudiobookBay",
description="Add the AudiobookBay domain you want Shelfmark to search.",
),
*_get_fields_from_tab(
"audiobookbay_config",
["ABB_HOSTNAME"],
strip_show_when_keys={"ABB_ENABLED"},
),
]
# Get actual Prowlarr connection fields
prowlarr_fields = ["PROWLARR_ENABLED", "PROWLARR_URL", "PROWLARR_API_KEY", "test_prowlarr"]
for field_key in prowlarr_fields:
field = _get_field_from_tab("prowlarr_config", field_key)
if field:
fields.append(field)
return fields
def get_irc_fields() -> list[SettingsField]:
"""Render trimmed IRC setup fields for onboarding."""
return _get_fields_from_tab(
"irc",
[
"heading",
"IRC_SERVER",
"IRC_PORT",
"IRC_USE_TLS",
"IRC_CHANNEL",
"IRC_NICK",
"IRC_SEARCH_BOT",
],
)
def get_prowlarr_indexers_fields() -> list[SettingsField]:
"""Step 5: Select Prowlarr indexers to search."""
fields: list[SettingsField] = [
HeadingField(
key="prowlarr_indexers_heading",
title="Select Indexers",
description="Choose which indexers to search for books. Leave empty to search all available indexers.",
),
def get_onboarding_steps() -> list[dict[str, Any]]:
"""Return the full onboarding step configuration."""
return [
{
"id": "search_mode",
"title": "Search Mode",
"tab": "search_mode",
"get_fields": get_search_mode_fields,
},
{
"id": "metadata_provider",
"title": "Metadata Provider",
"tab": "search_mode",
"get_fields": get_metadata_provider_fields,
"show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
},
{
"id": "hardcover_setup",
"title": "Hardcover Setup",
"tab": "hardcover",
"get_fields": get_hardcover_setup_fields,
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": "METADATA_PROVIDER", "value": "hardcover"},
],
},
{
"id": "googlebooks_setup",
"title": "Google Books Setup",
"tab": "googlebooks",
"get_fields": get_googlebooks_setup_fields,
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": "METADATA_PROVIDER", "value": "googlebooks"},
],
},
{
"id": "release_sources",
"title": "Release Sources",
"tab": "search_mode",
"get_fields": get_release_source_selection_fields,
"show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
"optional": True,
},
{
"id": "direct_download_setup_direct_mode",
"title": "Direct Download Setup",
"tab": "download_sources",
"get_fields": get_direct_download_setup_fields,
"show_when": [{"field": "SEARCH_MODE", "value": "direct"}],
},
{
"id": "direct_download_cloudflare_bypass_direct_mode",
"title": "Cloudflare Bypass",
"tab": "cloudflare_bypass",
"get_fields": get_direct_download_bypass_fields,
"show_when": [{"field": "SEARCH_MODE", "value": "direct"}],
},
{
"id": "direct_download_setup",
"title": "Direct Download Setup",
"tab": "download_sources",
"get_fields": get_direct_download_setup_fields,
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "direct_download"},
],
"optional": True,
},
{
"id": "direct_download_cloudflare_bypass",
"title": "Cloudflare Bypass",
"tab": "cloudflare_bypass",
"get_fields": get_direct_download_bypass_fields,
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "direct_download"},
],
"optional": True,
},
{
"id": "prowlarr",
"title": "Prowlarr",
"tab": "prowlarr_config",
"get_fields": get_prowlarr_fields,
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "prowlarr"},
],
"optional": True,
},
{
"id": "audiobookbay",
"title": "AudiobookBay",
"tab": "audiobookbay_config",
"get_fields": get_audiobookbay_fields,
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "audiobookbay"},
],
"optional": True,
},
{
"id": "irc",
"title": "IRC",
"tab": "irc",
"get_fields": get_irc_fields,
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": ONBOARDING_RELEASE_SOURCES_KEY, "value": "irc"},
],
"optional": True,
},
]
# Get the indexers multi-select field
indexers_field = _get_field_from_tab("prowlarr_config", "PROWLARR_INDEXERS")
if indexers_field:
fields.append(indexers_field)
return fields
# =============================================================================
# Step Configuration
# =============================================================================
ONBOARDING_STEPS = [
{
"id": "search_mode",
"title": "Search Mode",
"tab": "search_mode",
"get_fields": get_search_mode_fields,
},
{
"id": "metadata_provider",
"title": "Metadata Provider",
"tab": "search_mode",
"get_fields": get_metadata_provider_fields,
"show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
},
{
"id": "hardcover_setup",
"title": "Hardcover Setup",
"tab": "hardcover",
"get_fields": get_hardcover_setup_fields,
# Must be universal mode AND hardcover selected
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": "METADATA_PROVIDER", "value": "hardcover"},
],
},
{
"id": "googlebooks_setup",
"title": "Google Books Setup",
"tab": "googlebooks",
"get_fields": get_googlebooks_setup_fields,
# Must be universal mode AND googlebooks selected
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": "METADATA_PROVIDER", "value": "googlebooks"},
],
},
{
"id": "prowlarr",
"title": "Prowlarr",
"tab": "prowlarr_config",
"get_fields": get_prowlarr_fields,
"show_when": [{"field": "SEARCH_MODE", "value": "universal"}],
"optional": True,
},
{
"id": "prowlarr_indexers",
"title": "Indexers",
"tab": "prowlarr_config",
"get_fields": get_prowlarr_indexers_fields,
# Only show when Prowlarr is enabled
"show_when": [
{"field": "SEARCH_MODE", "value": "universal"},
{"field": "PROWLARR_ENABLED", "value": True},
],
"optional": True,
},
]
def get_onboarding_config() -> dict[str, Any]:
"""Get the full onboarding configuration including steps and current values."""
steps = []
all_values = {}
for step_config in ONBOARDING_STEPS:
for step_config in get_onboarding_steps():
fields = step_config["get_fields"]()
tab_name = step_config["tab"]
# Serialize fields with current values
serialized_fields = []
for field in fields:
serialized = serialize_field(field, tab_name, include_value=True)
field_tab_name = _get_field_tab_name(field, tab_name)
serialized = serialize_field(field, field_tab_name, include_value=True)
serialized_fields.append(serialized)
# Collect values (skip HeadingFields)
if hasattr(field, "key") and field.key and not isinstance(field, HeadingField):
value = get_setting_value(field, tab_name)
if hasattr(field, "env_supported") and getattr(field, "key", None):
if field.key in _ONBOARDING_VIRTUAL_KEYS:
value = getattr(field, "default", "")
else:
value = get_setting_value(field, field_tab_name)
all_values[field.key] = (
value if value is not None else getattr(field, "default", "")
)
@@ -376,8 +629,10 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]:
# Group values by their target tab
tab_values: dict[str, dict[str, Any]] = {}
for step_config in ONBOARDING_STEPS:
tab_name = step_config["tab"]
for step_config in get_onboarding_steps():
if not _is_step_visible(step_config, values):
continue
fields = step_config["get_fields"]()
for field in fields:
@@ -385,7 +640,12 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]:
continue
key = field.key
if key in _ONBOARDING_VIRTUAL_KEYS:
continue
if not _is_field_visible(field, values):
continue
if key in values:
tab_name = _get_field_tab_name(field, step_config["tab"])
if tab_name not in tab_values:
tab_values[tab_name] = {}
tab_values[tab_name][key] = values[key]
@@ -396,8 +656,7 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]:
save_config_file(tab_name, tab_data)
logger.info("Saved onboarding settings to %s: %s", tab_name, list(tab_data.keys()))
# Enable the selected metadata provider
search_mode = values.get("SEARCH_MODE", "direct")
search_mode = values.get("SEARCH_MODE", "universal")
if search_mode == "universal":
provider = values.get("METADATA_PROVIDER", "hardcover")
if provider:
@@ -425,6 +684,47 @@ def save_onboarding_settings(values: dict[str, Any]) -> dict[str, Any]:
list(provider_config.keys()),
)
selected_release_sources = values.get(ONBOARDING_RELEASE_SOURCES_KEY, [])
if not isinstance(selected_release_sources, list):
selected_release_sources = []
source_updates: dict[str, dict[str, Any]] = {}
if search_mode == "direct":
source_updates.setdefault("download_sources", {})["DIRECT_DOWNLOAD_ENABLED"] = True
else:
if _is_release_source_selected(values, "direct_download"):
source_updates.setdefault("download_sources", {})["DIRECT_DOWNLOAD_ENABLED"] = True
if _is_release_source_selected(values, "prowlarr"):
source_updates.setdefault("prowlarr_config", {})["PROWLARR_ENABLED"] = True
if _is_release_source_selected(values, "audiobookbay"):
source_updates.setdefault("audiobookbay_config", {})["ABB_ENABLED"] = True
if not values.get("DEFAULT_RELEASE_SOURCE"):
for source_name in selected_release_sources:
if source_name in {"direct_download", "prowlarr", "irc"}:
source_updates.setdefault("search_mode", {})["DEFAULT_RELEASE_SOURCE"] = (
source_name
)
break
if not values.get("DEFAULT_RELEASE_SOURCE_AUDIOBOOK"):
for source_name in selected_release_sources:
if source_name in {"prowlarr", "audiobookbay", "irc"}:
source_updates.setdefault("search_mode", {})[
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK"
] = source_name
break
for tab_name, tab_data in source_updates.items():
if tab_data:
save_config_file(tab_name, tab_data)
logger.info(
"Enabled onboarding release source settings for %s: %s",
tab_name,
list(tab_data.keys()),
)
# Mark onboarding as complete
mark_onboarding_complete()
shelfmark/core/settings_registry.py
+176 -103
View File
@@ -521,6 +521,15 @@ def initialize_default_configs() -> bool:
def sync_env_to_config() -> None:
"""Sync supported environment-backed settings into config files."""
config_dir = _get_config_dir()
plugins_dir = config_dir / "plugins"
had_existing_search_page_title = "SEARCH_PAGE_TITLE" in load_config_file("general")
had_existing_install_state = (
(config_dir / "settings.json").exists()
or (config_dir / "users.db").exists()
or (plugins_dir.exists() and any(plugins_dir.glob("*.json")))
)
# Initialize default configs first (for fresh installs)
initialize_default_configs()
@@ -532,13 +541,8 @@ def sync_env_to_config() -> None:
if not getattr(settings_field, "env_supported", True):
continue
# Check if ENV var is set
env_var_name = settings_field.get_env_var_name()
env_value = os.environ.get(env_var_name)
if env_value is not None:
# Parse the ENV value to the appropriate type
parsed_value = _parse_env_value(env_value, settings_field)
has_env_value, parsed_value = _get_env_value_for_field(settings_field)
if has_env_value:
values_to_sync[settings_field.key] = parsed_value
# Save synced values to config file (merge with existing)
@@ -554,21 +558,39 @@ def sync_env_to_config() -> None:
migrate_legacy_settings()
migrate_download_to_browser_settings()
migrate_mirror_settings()
migrate_direct_download_upgrade(existing_install=had_existing_install_state)
migrate_search_page_title(
existing_install=had_existing_install_state,
had_existing_value=had_existing_search_page_title,
)
def migrate_direct_download_upgrade(*, existing_install: bool) -> None:
"""Preserve the direct-download enabled flag for existing installs only."""
if not existing_install:
return
download_sources_config = load_config_file("download_sources")
download_updates: dict[str, Any] = {}
if "DIRECT_DOWNLOAD_ENABLED" not in download_sources_config:
download_updates["DIRECT_DOWNLOAD_ENABLED"] = True
if download_updates:
save_config_file("download_sources", download_updates)
def migrate_search_page_title(*, existing_install: bool, had_existing_value: bool) -> None:
"""Keep the legacy homepage search title for existing installs only."""
if not existing_install or had_existing_value or os.getenv("SEARCH_PAGE_TITLE") is not None:
return
save_config_file("general", {"SEARCH_PAGE_TITLE": "Book Search & Download"})
def migrate_mirror_settings() -> None:
"""Sync AA mirror list when code defaults change between versions.
On startup, compares a hash of DEFAULT_AA_MIRRORS against the hash stored
in the config file. If they differ (i.e., an update shipped new defaults),
the config is overwritten with the new defaults. If they match, the user's
customizations are left untouched.
Also handles legacy migration from AA_ADDITIONAL_URLS.
"""
import hashlib
from shelfmark.core.mirrors import DEFAULT_AA_MIRRORS
"""Normalize canonical mirror-list settings and migrate legacy config forward safely."""
from shelfmark.core.utils import normalize_http_url
def _normalize_list(values: list[str]) -> list[str]:
@@ -581,88 +603,75 @@ def migrate_mirror_settings() -> None:
out.append(norm)
return out
def _hash_mirrors(mirrors: list[str]) -> str:
return hashlib.sha256(",".join(mirrors).encode()).hexdigest()
normalized_defaults = _normalize_list(DEFAULT_AA_MIRRORS)
current_defaults_hash = _hash_mirrors(normalized_defaults)
mirrors_config = load_config_file("mirrors")
stored_hash = mirrors_config.get("_AA_MIRRORS_DEFAULTS_HASH")
raw_list = mirrors_config.get("AA_MIRROR_URLS")
raw_additional = mirrors_config.get("AA_ADDITIONAL_URLS", "")
mirror_updates: dict[str, Any] = {}
def _save_mirrors(values: dict[str, Any]) -> None:
merged = dict(mirrors_config)
merged.update(values)
save_config_file("mirrors", merged)
mirrors_config.update(values)
def _queue_update(key: str, normalized: list[str]) -> None:
current = mirrors_config.get(key)
if current != normalized:
mirror_updates[key] = normalized
# Defaults changed since last startup — push new mirrors to config
if stored_hash != current_defaults_hash:
_save_mirrors(
{
"AA_MIRROR_URLS": normalized_defaults,
"_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash,
}
)
return
def _resolve_canonical_list(
canonical_key: str,
*,
legacy_list_key: str | None = None,
legacy_primary_key: str | None = None,
legacy_additional_key: str | None = None,
) -> None:
raw_list = mirrors_config.get(canonical_key)
# --- Legacy migration (only runs if hash already matches / first time) ---
if isinstance(raw_list, list):
_queue_update(canonical_key, _normalize_list([str(v) for v in raw_list]))
return
# If already a proper list, just ensure it's non-empty.
if isinstance(raw_list, list):
normalized = _normalize_list([str(v) for v in raw_list])
if isinstance(raw_list, str) and raw_list.strip():
parts = [p.strip() for p in raw_list.split(",") if p.strip()]
_queue_update(canonical_key, _normalize_list(parts))
return
combined: list[str] = []
if legacy_primary_key:
raw_primary = mirrors_config.get(legacy_primary_key, "")
if isinstance(raw_primary, str) and raw_primary.strip():
combined.append(raw_primary.strip())
if legacy_list_key:
raw_legacy_list = mirrors_config.get(legacy_list_key, "")
if isinstance(raw_legacy_list, str) and raw_legacy_list.strip():
combined.extend([p.strip() for p in raw_legacy_list.split(",") if p.strip()])
elif isinstance(raw_legacy_list, list):
combined.extend([str(p).strip() for p in raw_legacy_list if str(p).strip()])
if legacy_additional_key:
raw_additional = mirrors_config.get(legacy_additional_key, "")
if isinstance(raw_additional, str) and raw_additional.strip():
combined.extend([p.strip() for p in raw_additional.split(",") if p.strip()])
elif isinstance(raw_additional, list):
combined.extend([str(p).strip() for p in raw_additional if str(p).strip()])
normalized = _normalize_list(combined)
if normalized:
return
_save_mirrors(
{
"AA_MIRROR_URLS": normalized_defaults,
"_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash,
}
)
return
_queue_update(canonical_key, normalized)
# If saved as a string, convert to list.
if isinstance(raw_list, str) and raw_list.strip():
parts = [p.strip() for p in raw_list.split(",") if p.strip()]
normalized = _normalize_list(parts)
if normalized:
_save_mirrors(
{
"AA_MIRROR_URLS": normalized,
"_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash,
}
)
return
_save_mirrors(
{
"AA_MIRROR_URLS": normalized_defaults,
"_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash,
}
)
return
# If there's legacy additional mirrors, seed the full list.
if isinstance(raw_additional, str) and raw_additional.strip():
additional_parts = [p.strip() for p in raw_additional.split(",") if p.strip()]
combined = _normalize_list(DEFAULT_AA_MIRRORS + additional_parts)
if combined:
_save_mirrors(
{
"AA_MIRROR_URLS": combined,
"_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash,
}
)
return
# No config at all yet — write defaults
_save_mirrors(
{
"AA_MIRROR_URLS": normalized_defaults,
"_AA_MIRRORS_DEFAULTS_HASH": current_defaults_hash,
}
_resolve_canonical_list(
"AA_MIRROR_URLS",
legacy_list_key="AA_ADDITIONAL_URLS",
)
_resolve_canonical_list(
"LIBGEN_MIRROR_URLS",
legacy_list_key="LIBGEN_ADDITIONAL_URLS",
)
_resolve_canonical_list(
"ZLIB_MIRROR_URLS",
legacy_primary_key="ZLIB_PRIMARY_URL",
legacy_additional_key="ZLIB_ADDITIONAL_URLS",
)
_resolve_canonical_list(
"WELIB_MIRROR_URLS",
legacy_primary_key="WELIB_PRIMARY_URL",
legacy_additional_key="WELIB_ADDITIONAL_URLS",
)
if mirror_updates:
save_config_file("mirrors", mirror_updates)
def migrate_legacy_settings() -> None:
@@ -813,11 +822,9 @@ def migrate_download_to_browser_settings() -> None:
def get_setting_value(field: FieldBase, tab_name: str) -> object:
"""Resolve the effective value for a settings field."""
# 1. Check environment variable (if supported for this field)
if field.env_supported:
env_var_name = field.get_env_var_name()
env_value = os.environ.get(env_var_name)
if env_value is not None:
return _parse_env_value(env_value, field)
has_env_value, parsed_env_value = _get_env_value_for_field(field)
if has_env_value:
return parsed_env_value
# 2. Check config file
config = load_config_file(tab_name)
@@ -860,12 +867,78 @@ def _parse_env_value(value: str, field: FieldBase) -> object:
return value
def _normalize_mirror_env_urls(values: list[str]) -> list[str]:
"""Normalize mirror URL env values to stable https URLs without duplicates."""
from shelfmark.core.utils import normalize_http_url
normalized: list[str] = []
for raw_value in values:
stripped = raw_value.strip()
if not stripped:
continue
normalized_url = normalize_http_url(stripped, default_scheme="https")
if normalized_url and normalized_url not in normalized:
normalized.append(normalized_url)
return normalized
def _get_env_value_for_field(field: FieldBase) -> tuple[bool, object | None]:
"""Return parsed env-backed value for a field, including legacy mirror aliases."""
if not field.env_supported:
return False, None
env_var_name = field.get_env_var_name()
env_value = os.environ.get(env_var_name)
if env_value is not None:
parsed = _parse_env_value(env_value, field)
if field.key in {
"AA_MIRROR_URLS",
"LIBGEN_MIRROR_URLS",
"ZLIB_MIRROR_URLS",
"WELIB_MIRROR_URLS",
} and isinstance(parsed, list):
parsed = _normalize_mirror_env_urls(parsed)
return True, parsed
if field.key == "AA_MIRROR_URLS":
legacy_additional = os.environ.get("AA_ADDITIONAL_URLS")
if legacy_additional is not None:
return True, _normalize_mirror_env_urls(legacy_additional.split(","))
if field.key == "LIBGEN_MIRROR_URLS":
legacy_additional = os.environ.get("LIBGEN_ADDITIONAL_URLS")
if legacy_additional is not None:
return True, _normalize_mirror_env_urls(legacy_additional.split(","))
if field.key == "ZLIB_MIRROR_URLS":
legacy_primary = os.environ.get("ZLIB_PRIMARY_URL")
legacy_additional = os.environ.get("ZLIB_ADDITIONAL_URLS")
if legacy_primary is not None or legacy_additional is not None:
combined = []
if legacy_primary is not None:
combined.append(legacy_primary)
if legacy_additional is not None:
combined.extend(legacy_additional.split(","))
return True, _normalize_mirror_env_urls(combined)
if field.key == "WELIB_MIRROR_URLS":
legacy_primary = os.environ.get("WELIB_PRIMARY_URL")
legacy_additional = os.environ.get("WELIB_ADDITIONAL_URLS")
if legacy_primary is not None or legacy_additional is not None:
combined = []
if legacy_primary is not None:
combined.append(legacy_primary)
if legacy_additional is not None:
combined.extend(legacy_additional.split(","))
return True, _normalize_mirror_env_urls(combined)
return False, None
def is_value_from_env(field: FieldBase) -> bool:
"""Check if a field's value comes from an environment variable."""
# UI-only settings never come from ENV (env_supported=False)
if not getattr(field, "env_supported", True):
return False
return field.get_env_var_name() in os.environ
has_env_value, _parsed = _get_env_value_for_field(field)
return has_env_value
def serialize_field(
@@ -1155,7 +1228,7 @@ def _apply_dns_settings(config: Config) -> None:
def _apply_aa_mirror_settings(config: Config) -> None:
"""Apply AA mirror settings changes to the network module.
This ensures AA_BASE_URL / AA_ADDITIONAL_URLS changes take effect immediately
This ensures AA_BASE_URL / AA_MIRROR_URLS changes take effect immediately
without requiring a container restart.
"""
try:
shelfmark/download/http.py
+21 -2
View File
@@ -204,7 +204,8 @@ def _try_rotation(
original_url: str, current_url: str, selector: network.AAMirrorSelector
) -> str | None:
"""Try mirror/DNS rotation. Returns new URL or None."""
if current_url.startswith(network.get_aa_base_url()):
aa_base_url = network.get_aa_base_url()
if aa_base_url and current_url.startswith(aa_base_url):
new_base, action = selector.next_mirror_or_rotate_dns()
if action in ("mirror", "dns") and new_base:
new_url = selector.rewrite(original_url)
@@ -538,7 +539,7 @@ def download_url(
and not zlib_cookie_refresh_attempted
):
parsed = urlparse(current_url)
if parsed.hostname and "z-lib" in parsed.hostname and referer:
if _is_configured_zlib_host(parsed.hostname) and referer:
zlib_cookie_refresh_attempted = True
logger.info("Z-Library 403 - refreshing cookies via referer: %s", referer)
try:
@@ -602,6 +603,24 @@ def download_url(
return None
def _is_configured_zlib_host(hostname: str | None) -> bool:
"""Return True when a hostname matches a configured Z-Library mirror."""
if not hostname:
return False
from shelfmark.core.mirrors import get_zlib_cookie_domains
hostname = hostname.lower()
base_domain = ".".join(hostname.split(".")[-2:]) if "." in hostname else hostname
for domain in get_zlib_cookie_domains():
candidate = str(domain).lower()
if hostname == candidate or hostname.endswith(f".{candidate}") or base_domain == candidate:
return True
return False
def _try_resume(
url: str,
buffer: BytesIO,
shelfmark/download/network.py
+13 -3
View File
@@ -924,9 +924,13 @@ def rotate_dns_and_reset_aa() -> bool:
if configured_url == "auto":
# Auto mode always resets to the first mirror to restart the cascade
_current_aa_url_index = 0
_aa_base_url = _aa_urls[0] if _aa_urls else "https://annas-archive.gl"
logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url)
_save_state(aa_url=_aa_base_url)
if _aa_urls:
_aa_base_url = _aa_urls[0]
logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url)
_save_state(aa_url=_aa_base_url)
else:
_aa_base_url = ""
logger.info("After DNS switch, AA URL remains unconfigured")
else:
# Keep the user's configured primary mirror (if it exists in the list),
# otherwise keep the configured URL as-is (custom/env).
@@ -1105,6 +1109,12 @@ def _initialize_aa_state() -> None:
if configured_url != "auto" and configured_url not in _aa_urls:
_aa_urls = [configured_url, *_aa_urls]
if not _aa_urls:
_aa_base_url = ""
_current_aa_url_index = 0
logger.info("AA_BASE_URL: unconfigured")
return
if configured_url == "auto":
if state.get("aa_base_url") and state["aa_base_url"] in _aa_urls:
_current_aa_url_index = _aa_urls.index(state["aa_base_url"])
shelfmark/main.py
+3 -2
View File
@@ -1113,10 +1113,10 @@ def api_config() -> Response | tuple[Response, int]:
db_user_id = get_session_db_user_id(session)
search_mode = app_config.get("SEARCH_MODE", "direct", user_id=db_user_id)
search_mode = app_config.get("SEARCH_MODE", "universal", user_id=db_user_id)
default_release_source = app_config.get(
"DEFAULT_RELEASE_SOURCE",
"direct_download",
"",
user_id=db_user_id,
)
default_release_source_audiobook = app_config.get(
@@ -1145,6 +1145,7 @@ def api_config() -> Response | tuple[Response, int]:
config = {
"calibre_web_url": app_config.get("CALIBRE_WEB_URL", ""),
"audiobook_library_url": app_config.get("AUDIOBOOK_LIBRARY_URL", ""),
"search_page_title": app_config.get("SEARCH_PAGE_TITLE", "Shelfmark"),
"debug": app_config.get("DEBUG", False),
"build_version": BUILD_VERSION,
"release_version": RELEASE_VERSION,
shelfmark/release_sources/direct_download.py
+61 -6
View File
@@ -7,7 +7,7 @@ import time
from dataclasses import replace
from http import HTTPStatus
from typing import TYPE_CHECKING, ClassVar, NoReturn, TypedDict
from urllib.parse import quote
from urllib.parse import quote, urlparse
import requests
from bs4 import BeautifulSoup, Tag
@@ -204,6 +204,24 @@ _CF_BYPASS_REQUIRED = frozenset({"aa-slow-nowait", "aa-slow-wait", "zlib", "weli
_AA_PAGE_SOURCES = frozenset({"aa-slow-nowait", "aa-slow-wait"})
def _is_configured_zlib_link(url: str) -> bool:
"""Return True when a URL belongs to a configured Z-Library mirror."""
from shelfmark.core.mirrors import get_zlib_cookie_domains
hostname = (urlparse(url).hostname or "").lower()
if not hostname:
return False
base_domain = ".".join(hostname.split(".")[-2:]) if "." in hostname else hostname
for domain in get_zlib_cookie_domains():
candidate = str(domain).lower()
if hostname == candidate or hostname.endswith(f".{candidate}") or base_domain == candidate:
return True
return False
def _get_md5_url_template(source_id: str) -> str | None:
"""Get URL template for MD5-based sources from centralized config."""
from shelfmark.core import mirrors
@@ -246,6 +264,8 @@ def _get_source_priority() -> list[SourcePriorityEntry]:
Fast sources come from user config (FAST_SOURCES_DISPLAY).
Slow sources come from user config.
"""
from shelfmark.core import mirrors
fast_sources = _parse_source_priority_entries(
config.get("FAST_SOURCES_DISPLAY"),
allowed_ids={"aa-fast", "libgen"},
@@ -253,13 +273,18 @@ def _get_source_priority() -> list[SourcePriorityEntry]:
has_donator_key = bool(config.get("AA_DONATOR_KEY"))
for source in fast_sources:
if source["id"] == "aa-fast" and not has_donator_key:
if (not mirrors.has_download_source_mirror_configuration(source["id"])) or (
source["id"] == "aa-fast" and not has_donator_key
):
source["enabled"] = False
slow_sources = _parse_source_priority_entries(
config.get("SOURCE_PRIORITY"),
excluded_ids={"aa-fast", "libgen"},
)
for source in slow_sources:
if not mirrors.has_download_source_mirror_configuration(source["id"]):
source["enabled"] = False
return fast_sources + slow_sources
@@ -275,6 +300,31 @@ def _is_source_enabled(source_id: str) -> bool:
return False
def _get_direct_download_unavailable_reason() -> str | None:
"""Return a user-facing reason when Direct Download cannot be used."""
from shelfmark.core import mirrors
if not config.get("DIRECT_DOWNLOAD_ENABLED", False):
return (
"Direct Download is disabled. Enable the source in Settings and add your mirror URLs."
)
if not mirrors.has_aa_mirror_configuration():
return (
"Direct Download is not configured. Add at least one Anna's Archive mirror URL in "
"Settings."
)
return None
def _ensure_direct_download_available() -> None:
"""Raise a source-unavailable error when Direct Download is disabled or unconfigured."""
reason = _get_direct_download_unavailable_reason()
if reason:
raise SearchUnavailableError(reason)
_SIZE_UNIT_PATTERN = re.compile(r"(kb|mb|gb|tb)", re.IGNORECASE)
@@ -907,7 +957,10 @@ def _get_download_urls_from_welib(
if not _is_source_enabled("welib"):
return []
url = mirrors.get_welib_url_template().format(md5=book_id)
template = mirrors.get_welib_url_template()
if not template:
return []
url = template.format(md5=book_id)
logger.info("Fetching welib download URLs for %s", book_id)
try:
html = downloader.html_get_page(
@@ -1144,7 +1197,7 @@ def _get_download_url(
url = ""
# Z-Library
if link.startswith("https://z-lib."):
if _is_configured_zlib_link(link):
dl = soup.find("a", href=True, class_="addDownloadedBook")
if not dl:
# Retry after delay if page not fully loaded
@@ -1454,6 +1507,7 @@ class DirectDownloadSource(ReleaseSource):
fetch_download_count: bool = True,
) -> BrowseRecord | None:
"""Resolve a direct-download record for direct-mode info/download flows."""
_ensure_direct_download_available()
return get_book_info(record_id, fetch_download_count=fetch_download_count)
def search_results_are_releases(self) -> bool:
@@ -1506,6 +1560,7 @@ class DirectDownloadSource(ReleaseSource):
content_type: Ignored - Direct download uses format filtering instead
"""
_ensure_direct_download_available()
lang_filter = plan.languages
# Reset search type tracking
@@ -1595,8 +1650,8 @@ class DirectDownloadSource(ReleaseSource):
return [_browse_record_to_release(record) for record in all_results]
def is_available(self) -> bool:
"""Direct download is always available."""
return True
"""Check if Direct Download has been explicitly enabled and configured."""
return _get_direct_download_unavailable_reason() is None
@register_handler("direct_download")
src/README.md
+2 -2
View File
@@ -42,7 +42,7 @@ make build
make preview
# Run type checking
make typecheck
make frontend-typecheck
```
Alternatively, from `src/frontend`:
@@ -95,7 +95,7 @@ Output is generated in `src/frontend/dist/`
Run TypeScript checks without building:
```bash
make typecheck
make frontend-typecheck
```
## Debugging
src/frontend/src/App.tsx
+1
View File
@@ -2470,6 +2470,7 @@ function App() {
onSearch={handleSearchDispatch}
isLoading={isSearching}
isInitialState={isInitialState}
searchPageTitle={config?.search_page_title || 'Shelfmark'}
bookLanguages={bookLanguages}
defaultLanguage={defaultLanguageCodes}
logoUrl={logoUrl}
src/frontend/src/components/Dropdown.tsx
+59 -76
View File
@@ -1,22 +1,9 @@
import type { ReactNode } from 'react';
import { useCallback, useLayoutEffect, useRef, useState } from 'react';
import { createPortal } from 'react-dom';
import { useDismiss } from '../hooks/useDismiss';
// Find the closest scrollable ancestor element
function getScrollableAncestor(element: HTMLElement | null): HTMLElement | null {
let current = element?.parentElement;
while (current) {
const style = getComputedStyle(current);
const overflowY = style.overflowY;
if (overflowY === 'auto' || overflowY === 'scroll' || overflowY === 'hidden') {
return current;
}
current = current.parentElement;
}
return null;
}
// Simple throttle function to limit how often a function can be called
function throttle<Args extends unknown[]>(
fn: (...args: Args) => void,
@@ -75,11 +62,11 @@ export const Dropdown = ({
}: DropdownProps) => {
const [isOpen, setIsOpen] = useState(false);
const containerRef = useRef<HTMLDivElement>(null);
const triggerRef = useRef<HTMLDivElement>(null);
const panelRef = useRef<HTMLDivElement>(null);
const [panelDirection, setPanelDirection] = useState<'down' | 'up'>('down');
const [resolvedAlign, setResolvedAlign] = useState<'left' | 'right'>(
align === 'right' ? 'right' : 'left',
);
const [panelPos, setPanelPos] = useState({ top: 0, left: 0, width: 0 });
let triggerBorderRadius = '0.5rem';
if (triggerChrome === 'minimal') {
triggerBorderRadius = '0';
@@ -87,13 +74,6 @@ export const Dropdown = ({
triggerBorderRadius = panelDirection === 'down' ? '0.5rem 0.5rem 0 0' : '0 0 0.5rem 0.5rem';
}
let panelOffsetClassName = '';
if (panelDirection === 'down') {
panelOffsetClassName = renderTrigger ? 'mt-2' : '';
} else {
panelOffsetClassName = renderTrigger ? 'bottom-full mb-2' : 'bottom-full';
}
let panelBorderRadius = '0.5rem';
if (!renderTrigger) {
panelBorderRadius = panelDirection === 'down' ? '0 0 0.5rem 0.5rem' : '0.5rem 0.5rem 0 0';
@@ -113,47 +93,47 @@ export const Dropdown = ({
onOpenChange?.(false);
}, [onOpenChange]);
useDismiss(isOpen, [containerRef], close);
useDismiss(isOpen, [containerRef, panelRef], close);
// Memoize the panel direction calculation
const updatePanelDirection = useCallback(() => {
if (!containerRef.current || !panelRef.current) {
return;
}
// Compute panel direction and fixed position relative to the trigger
const updatePanelPosition = useCallback(() => {
if (!triggerRef.current || !panelRef.current) return;
const rect = containerRef.current.getBoundingClientRect();
const rect = triggerRef.current.getBoundingClientRect();
const panelHeight = panelRef.current.offsetHeight || panelRef.current.scrollHeight;
const panelWidth = panelRef.current.offsetWidth || panelRef.current.scrollWidth;
// Check if we're inside a scrollable container and use its bounds
const scrollableAncestor = getScrollableAncestor(containerRef.current);
const containerBottom = scrollableAncestor
? scrollableAncestor.getBoundingClientRect().bottom
: window.innerHeight;
const containerTop = scrollableAncestor ? scrollableAncestor.getBoundingClientRect().top : 0;
const spaceBelow = containerBottom - rect.bottom - 8;
const spaceAbove = rect.top - containerTop - 8;
// Direction: flip up if not enough space below but enough above
const spaceBelow = window.innerHeight - rect.bottom - 8;
const spaceAbove = rect.top - 8;
const shouldOpenUp = spaceBelow < panelHeight && spaceAbove >= panelHeight;
setPanelDirection(shouldOpenUp ? 'up' : 'down');
// Auto horizontal alignment: check if panel overflows viewport right/left
// Vertical: seamless trigger uses -1px border overlap, custom trigger uses 8px gap
let top: number;
if (shouldOpenUp) {
top = renderTrigger ? rect.top - panelHeight - 8 : rect.top - panelHeight + 1;
} else {
top = renderTrigger ? rect.bottom + 8 : rect.bottom - 1;
}
// Horizontal alignment
let left: number;
if (align === 'auto') {
const panelWidth = panelRef.current.offsetWidth || panelRef.current.scrollWidth;
const overflowsRight = rect.left + panelWidth > window.innerWidth - 8;
const overflowsLeft = rect.right - panelWidth < 8;
if (overflowsRight && !overflowsLeft) {
setResolvedAlign('right');
} else if (overflowsLeft && !overflowsRight) {
setResolvedAlign('left');
} else {
setResolvedAlign('left');
}
left =
overflowsRight && !overflowsLeft
? rect.right - Math.max(panelWidth, rect.width)
: rect.left;
} else if (align === 'right') {
left = rect.right - Math.max(panelWidth, rect.width);
} else {
setResolvedAlign(align === 'right' ? 'right' : 'left');
left = rect.left;
}
}, [align]);
setPanelPos({ top, left, width: rect.width });
}, [align, renderTrigger]);
useLayoutEffect(() => {
if (!isOpen) {
@@ -161,9 +141,9 @@ export const Dropdown = ({
}
// Throttle scroll/resize handlers to reduce layout thrashing
const throttledUpdate = throttle(updatePanelDirection, 100);
const throttledUpdate = throttle(updatePanelPosition, 100);
updatePanelDirection();
updatePanelPosition();
window.addEventListener('resize', throttledUpdate);
window.addEventListener('scroll', throttledUpdate, true);
@@ -171,7 +151,7 @@ export const Dropdown = ({
window.removeEventListener('resize', throttledUpdate);
window.removeEventListener('scroll', throttledUpdate, true);
};
}, [isOpen, updatePanelDirection]);
}, [isOpen, updatePanelPosition]);
return (
<div className={widthClassName} ref={containerRef}>
@@ -183,7 +163,7 @@ export const Dropdown = ({
{label}
</label>
)}
<div className="relative">
<div ref={triggerRef}>
{renderTrigger ? (
renderTrigger({ isOpen, toggle: toggleOpen })
) : (
@@ -214,25 +194,28 @@ export const Dropdown = ({
</button>
)}
{isOpen && (
<div
ref={panelRef}
className={`absolute ${resolvedAlign === 'right' ? 'right-0' : 'left-0'} ${
panelOffsetClassName
} z-20 border ${panelDirection === 'down' ? 'shadow-lg' : ''} ${panelClassName || widthClassName}`}
style={{
background: 'var(--bg)',
borderColor: 'var(--border-muted)',
borderRadius: panelBorderRadius,
marginTop: !renderTrigger && panelDirection === 'down' ? '-1px' : undefined,
marginBottom: !renderTrigger && panelDirection === 'up' ? '-1px' : undefined,
}}
>
<div className={noScrollLimit ? '' : 'max-h-64 overflow-auto'}>
{children({ close })}
</div>
</div>
)}
{isOpen &&
createPortal(
<div
ref={panelRef}
className={`border ${panelDirection === 'down' ? 'shadow-lg' : ''} ${panelClassName ?? ''}`}
style={{
position: 'fixed',
top: panelPos.top,
left: panelPos.left,
width: panelClassName ? undefined : panelPos.width,
zIndex: 100,
background: 'var(--bg)',
borderColor: 'var(--border-muted)',
borderRadius: panelBorderRadius,
}}
>
<div className={noScrollLimit ? '' : 'max-h-64 overflow-auto'}>
{children({ close })}
</div>
</div>,
document.body,
)}
</div>
</div>
);
src/frontend/src/components/OnboardingModal.tsx
+88 -72
View File
@@ -3,7 +3,7 @@ import { useState, useCallback, useMemo } from 'react';
import { useBodyScrollLock } from '../hooks/useBodyScrollLock';
import { useEscapeKey } from '../hooks/useEscapeKey';
import { useMountEffect } from '../hooks/useMountEffect';
import type { OnboardingStep } from '../services/api';
import type { OnboardingStep, OnboardingStepCondition } from '../services/api';
import {
getOnboarding,
saveOnboarding,
@@ -32,8 +32,10 @@ interface OnboardingModalProps {
onShowToast?: (message: string, type: 'success' | 'error' | 'info') => void;
}
type VisibilityCondition = ShowWhenCondition | OnboardingStepCondition;
function evaluateShowWhenCondition(
showWhen: ShowWhenCondition,
showWhen: VisibilityCondition,
values: Record<string, unknown>,
): boolean {
const currentValue = values[showWhen.field];
@@ -45,9 +47,19 @@ function evaluateShowWhenCondition(
return currentValue !== undefined && currentValue !== null && currentValue !== '';
}
return Array.isArray(showWhen.value)
? typeof currentValue === 'string' && showWhen.value.includes(currentValue)
: currentValue === showWhen.value;
if (Array.isArray(currentValue)) {
if (Array.isArray(showWhen.value)) {
return showWhen.value.every((value) => currentValue.includes(value));
}
return showWhen.value !== undefined && currentValue.includes(showWhen.value);
}
if (Array.isArray(showWhen.value)) {
const currentStringValue = toStringValue(currentValue);
return currentStringValue !== undefined && showWhen.value.includes(currentStringValue);
}
return currentValue === showWhen.value;
}
// Check if a field should be visible based on showWhen condition
@@ -70,11 +82,7 @@ function isFieldVisible(field: SettingsField, values: Record<string, unknown>):
function isStepVisible(step: OnboardingStep, values: Record<string, unknown>): boolean {
if (!step.showWhen || step.showWhen.length === 0) return true;
// All conditions must be true (AND logic)
return step.showWhen.every((condition) => {
const currentValue = values[condition.field];
return currentValue === condition.value;
});
return step.showWhen.every((condition) => evaluateShowWhenCondition(condition, values));
}
// Render the appropriate field component based on type
@@ -240,8 +248,12 @@ const OnboardingModalSession = ({
return steps.filter((step) => isStepVisible(step, values));
}, [steps, values]);
// Clamp step index to valid range (handles steps becoming hidden)
const clampedStepIndex =
visibleSteps.length === 0 ? 0 : Math.min(currentStepIndex, visibleSteps.length - 1);
// Get current step
const currentStep = visibleSteps[currentStepIndex];
const currentStep = visibleSteps[clampedStepIndex];
// Get visible fields for current step
const visibleFields = useMemo(() => {
@@ -256,17 +268,17 @@ const OnboardingModalSession = ({
// Handle next step
const handleNext = useCallback(() => {
if (currentStepIndex < visibleSteps.length - 1) {
setCurrentStepIndex(currentStepIndex + 1);
if (clampedStepIndex < visibleSteps.length - 1) {
setCurrentStepIndex(clampedStepIndex + 1);
}
}, [currentStepIndex, visibleSteps.length]);
}, [clampedStepIndex, visibleSteps.length]);
// Handle previous step
const handleBack = useCallback(() => {
if (currentStepIndex > 0) {
setCurrentStepIndex(currentStepIndex - 1);
if (clampedStepIndex > 0) {
setCurrentStepIndex(clampedStepIndex - 1);
}
}, [currentStepIndex]);
}, [clampedStepIndex]);
// Handle skip
const handleSkip = useCallback(async () => {
@@ -399,9 +411,9 @@ const OnboardingModalSession = ({
);
}
const isFirstStep = currentStepIndex === 0;
const isLastStep = currentStepIndex === visibleSteps.length - 1;
const progress = ((currentStepIndex + 1) / visibleSteps.length) * 100;
const isFirstStep = clampedStepIndex === 0;
const isLastStep = clampedStepIndex === visibleSteps.length - 1;
const progress = ((clampedStepIndex + 1) / visibleSteps.length) * 100;
return (
<div className="fixed inset-0 z-50 flex items-center justify-center p-4">
@@ -412,72 +424,76 @@ const OnboardingModalSession = ({
{/* Modal */}
<div
className={`relative w-full max-w-xl rounded-xl border border-(--border-muted) shadow-2xl ${isClosing ? 'settings-modal-exit' : 'settings-modal-enter'}`}
className={`relative flex max-h-[min(85vh,750px)] w-full max-w-xl flex-col overflow-hidden rounded-xl border border-(--border-muted) shadow-2xl ${isClosing ? 'settings-modal-exit' : 'settings-modal-enter'}`}
style={{ background: 'var(--bg)' }}
role="dialog"
aria-modal="true"
aria-label="Setup Wizard"
>
{/* Header */}
<div className="flex items-center justify-between border-b border-(--border-muted) px-6 py-4">
<div className="flex items-center gap-3">
<div className="flex h-8 w-8 items-center justify-center rounded-full bg-sky-500/20 text-sm font-medium text-sky-500">
{currentStepIndex + 1}
<div className="flex-shrink-0">
{/* Header */}
<div className="flex items-center justify-between border-b border-(--border-muted) px-6 py-4">
<div className="flex items-center gap-3">
<div className="flex h-8 w-8 items-center justify-center rounded-full bg-sky-500/20 text-sm font-medium text-sky-500">
{clampedStepIndex + 1}
</div>
<div>
<h2 className="text-lg font-semibold">{currentStep?.title || 'Setup'}</h2>
<p className="text-xs opacity-60">
Step {clampedStepIndex + 1} of {visibleSteps.length}
</p>
</div>
</div>
<div>
<h2 className="text-lg font-semibold">{currentStep?.title || 'Setup'}</h2>
<p className="text-xs opacity-60">
Step {currentStepIndex + 1} of {visibleSteps.length}
</p>
</div>
</div>
<button
type="button"
onClick={handleClose}
className="rounded-lg p-1.5 transition-colors hover:bg-(--hover-surface)"
aria-label="Close"
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
strokeWidth={1.5}
stroke="currentColor"
className="h-5 w-5"
<button
type="button"
onClick={handleClose}
className="rounded-lg p-1.5 transition-colors hover:bg-(--hover-surface)"
aria-label="Close"
>
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18 18 6M6 6l12 12" />
</svg>
</button>
</div>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
strokeWidth={1.5}
stroke="currentColor"
className="h-5 w-5"
>
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18 18 6M6 6l12 12" />
</svg>
</button>
</div>
{/* Progress bar */}
<div className="h-1 bg-(--bg-soft)">
<div
className="h-full bg-sky-500 transition-all duration-300"
style={{ width: `${progress}%` }}
/>
{/* Progress bar */}
<div className="h-1 bg-(--bg-soft)">
<div
className="h-full bg-sky-500 transition-all duration-300"
style={{ width: `${progress}%` }}
/>
</div>
</div>
{/* Content */}
<div className="min-h-[280px] space-y-5 px-6 py-5">
{visibleFields.map((field) => {
const isDisabled = 'fromEnv' in field ? (field.fromEnv ?? false) : false;
return (
<FieldWrapper key={field.key} field={field}>
{renderField(
field,
values[field.key],
(v) => handleChange(field.key, v),
() => handleAction(field.key),
isDisabled,
)}
</FieldWrapper>
);
})}
<div className="min-h-0 flex-1 overflow-y-auto">
<div className="min-h-[280px] space-y-5 px-6 py-5">
{visibleFields.map((field) => {
const isDisabled = 'fromEnv' in field ? (field.fromEnv ?? false) : false;
return (
<FieldWrapper key={field.key} field={field}>
{renderField(
field,
values[field.key],
(v) => handleChange(field.key, v),
() => handleAction(field.key),
isDisabled,
)}
</FieldWrapper>
);
})}
</div>
</div>
{/* Footer */}
<div className="flex h-[68px] items-center justify-between border-t border-(--border-muted) px-6 py-4">
<div className="flex h-[68px] flex-shrink-0 items-center justify-between border-t border-(--border-muted) px-6 py-4">
<div>
<button
type="button"
src/frontend/src/components/SearchSection.tsx
+3 -1
View File
@@ -14,6 +14,7 @@ interface SearchSectionProps {
onSearch: () => void;
isLoading: boolean;
isInitialState: boolean;
searchPageTitle: string;
bookLanguages: Language[];
defaultLanguage: string[];
logoUrl: string;
@@ -45,6 +46,7 @@ export const SearchSection = ({
onSearch,
isLoading,
isInitialState,
searchPageTitle,
bookLanguages,
defaultLanguage,
logoUrl,
@@ -84,7 +86,7 @@ export const SearchSection = ({
}`}
>
<img src={logoUrl} alt="Logo" className="h-8 w-8" />
<h1 className="text-2xl font-semibold">Book Search & Download</h1>
<h1 className="text-2xl font-semibold">{searchPageTitle}</h1>
</div>
<div
className={`search-wrapper flex flex-col gap-3 transition-all duration-500 ${
src/frontend/src/components/UrlSearchBootstrapMount.tsx
+1 -1
View File
@@ -46,7 +46,7 @@ export const UrlSearchBootstrapMount = ({
useMountEffect(() => {
onComplete();
const parsedSearchMode = config.search_mode || 'direct';
const parsedSearchMode = config.search_mode || 'universal';
const urlContentTypeOverride =
parsedSearchMode === 'universal' ? parsedParams.contentType : undefined;
src/frontend/src/components/settings/users/UserSearchPreferencesSection.tsx
+2 -2
View File
@@ -153,11 +153,11 @@ export const UserSearchPreferencesSection = ({
});
};
const searchModeValue = readValue('SEARCH_MODE', 'direct');
const searchModeValue = readValue('SEARCH_MODE', 'universal');
const effectiveSearchMode = normalizeSearchMode(searchModeValue);
const metadataProviderValue = readValue('METADATA_PROVIDER');
const metadataProviderAudiobookValue = readValue('METADATA_PROVIDER_AUDIOBOOK');
const defaultReleaseSourceValue = readValue('DEFAULT_RELEASE_SOURCE', 'direct_download');
const defaultReleaseSourceValue = readValue('DEFAULT_RELEASE_SOURCE');
const defaultAudiobookReleaseSourceValue = readValue('DEFAULT_RELEASE_SOURCE_AUDIOBOOK');
const canOverrideSearchMode =
src/frontend/src/hooks/useSearch.ts
+2 -2
View File
@@ -165,7 +165,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn {
providerOverride?: string;
}) => {
const effectiveContentType = contentTypeOverride ?? contentType;
const searchMode = (searchModeOverride ?? config?.search_mode) || 'direct';
const searchMode = (searchModeOverride ?? config?.search_mode) || 'universal';
// In universal mode, check if we have either a query or field values
if (searchMode === 'universal') {
@@ -314,7 +314,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn {
// Load more results (universal mode pagination)
const loadMore = useCallback(
async (config: AppConfig | null, searchModeOverride?: SearchMode) => {
const searchMode = (searchModeOverride ?? config?.search_mode) || 'direct';
const searchMode = (searchModeOverride ?? config?.search_mode) || 'universal';
if (searchMode !== 'universal') return;
if (!lastSearchParamsRef.current) return;
if (isLoadingMore || !hasMore) return;
src/frontend/src/services/api.ts
+1
View File
@@ -697,6 +697,7 @@ export const executeSettingsAction = async (
export interface OnboardingStepCondition {
field: string;
value: unknown;
notEmpty?: boolean;
}
export interface OnboardingStep {
src/frontend/src/types/index.ts
+1
View File
@@ -265,6 +265,7 @@ export type BooksOutputMode = 'folder' | 'booklore' | 'email';
export interface AppConfig {
calibre_web_url: string;
audiobook_library_url: string;
search_page_title: string;
debug: boolean;
build_version: string;
release_version: string;
tests/README.md
+1 -4
View File
@@ -11,12 +11,9 @@ make install-python-dev
# Run all unit tests locally (fast, no external dependencies)
uv run pytest tests/ -v -m "not integration and not e2e"
# Run Python static analysis
# Run all Python static analysis (lint, format, typecheck, dead code)
make python-checks
# Run lightweight test lint/type checks
make python-test-checks
# Run E2E API tests against a running app stack
uv run pytest tests/e2e/ -v -m e2e
tests/bypass/test_internal_bypasser.py
+46
View File
@@ -79,6 +79,52 @@ def test_extract_cookies_from_cdp_filters_and_stores_ua():
assert internal_bypasser.get_cf_user_agent_for_domain("example.com") == "TestUA/1.0"
def test_extract_cookies_from_cdp_keeps_full_session_cookies_for_configured_zlib_domains(
monkeypatch,
):
import time
import shelfmark.bypass.internal_bypasser as internal_bypasser
class FakeCookie:
def __init__(self, name, value, domain, path, expires, secure=True):
self.name = name
self.value = value
self.domain = domain
self.path = path
self.expires = expires
self.secure = secure
class FakeCookies:
async def get_all(self, requests_cookie_format=False):
assert requests_cookie_format is True
return [
FakeCookie("cf_clearance", "abc", "z-lib.fm", "/", int(time.time()) + 3600),
FakeCookie("sessionid", "zzz", "z-lib.fm", "/", int(time.time()) + 3600),
]
class FakeDriver:
cookies = FakeCookies()
class FakePage:
async def evaluate(self, _expr):
return "TestUA/1.0"
monkeypatch.setattr(internal_bypasser, "_get_full_cookie_domains", lambda: {"z-lib.fm"})
internal_bypasser.clear_cf_cookies()
asyncio.run(
internal_bypasser._extract_cookies_from_cdp(
FakeDriver(),
FakePage(),
"https://z-lib.fm/books/example",
)
)
cookies = internal_bypasser.get_cf_cookies_for_domain("z-lib.fm")
assert cookies == {"cf_clearance": "abc", "sessionid": "zzz"}
def test_extract_cookies_from_cdp_normalizes_session_expiry():
import time
tests/config/test_download_settings.py
+76
View File
@@ -277,3 +277,79 @@ def test_execute_action_passes_unsaved_values_to_destination_test(tmp_path):
assert result["success"] is True
assert captured["path"] == destination
def test_search_mode_defaults_to_universal_and_direct_mentions_configuration():
from shelfmark.config.settings import search_mode_settings
fields = search_mode_settings()
search_mode_field = next(
field for field in fields if getattr(field, "key", None) == "SEARCH_MODE"
)
direct_option = next(
option for option in search_mode_field.options if option["value"] == "direct"
)
assert search_mode_field.default == "universal"
assert "mirror URLs" in direct_option["description"]
def test_download_source_settings_include_direct_download_toggle():
from shelfmark.config.settings import download_source_settings
fields = download_source_settings()
toggle_field = next(
field for field in fields if getattr(field, "key", None) == "DIRECT_DOWNLOAD_ENABLED"
)
assert toggle_field.default is False
assert "Add your own mirror URLs" in toggle_field.description
def test_fast_source_options_lock_entries_without_mirror_or_donator_requirements(monkeypatch):
from shelfmark.config.settings import _get_fast_source_options
def _fake_get(key: str, default=None, user_id=None):
del user_id
values = {
"AA_DONATOR_KEY": "",
}
return values.get(key, default)
monkeypatch.setattr("shelfmark.core.config.config.get", _fake_get)
monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: False)
monkeypatch.setattr("shelfmark.core.mirrors.has_libgen_mirror_configuration", lambda: True)
options = {option["id"]: option for option in _get_fast_source_options()}
assert options["aa-fast"]["isLocked"] is True
assert (
options["aa-fast"]["disabledReason"] == "Add at least one Anna's Archive mirror in Mirrors"
)
assert options["libgen"]["isLocked"] is False
assert options["libgen"]["disabledReason"] is None
def test_slow_source_options_lock_entries_until_mirror_dependencies_exist(monkeypatch):
from shelfmark.config.settings import _get_slow_source_options
def _fake_get(key: str, default=None, user_id=None):
del user_id
values = {
"USE_CF_BYPASS": True,
}
return values.get(key, default)
monkeypatch.setattr("shelfmark.core.config.config.get", _fake_get)
monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True)
monkeypatch.setattr("shelfmark.core.mirrors.has_welib_mirror_configuration", lambda: False)
monkeypatch.setattr("shelfmark.core.mirrors.has_zlib_mirror_configuration", lambda: False)
options = {option["id"]: option for option in _get_slow_source_options()}
assert options["aa-slow-nowait"]["isLocked"] is False
assert options["aa-slow-wait"]["isLocked"] is False
assert options["welib"]["isLocked"] is True
assert options["welib"]["disabledReason"] == "Add at least one Welib mirror in Mirrors"
assert options["zlib"]["isLocked"] is True
assert options["zlib"]["disabledReason"] == "Add at least one Z-Library mirror in Mirrors"
tests/config/test_generate_env_docs.py
+40
View File
@@ -0,0 +1,40 @@
from scripts.generate_env_docs import generate_env_docs
def test_generated_env_docs_use_canonical_mirror_env_vars() -> None:
docs = generate_env_docs()
for canonical_var in (
"AA_BASE_URL",
"AA_MIRROR_URLS",
"LIBGEN_MIRROR_URLS",
"ZLIB_MIRROR_URLS",
"WELIB_MIRROR_URLS",
):
assert f"`{canonical_var}`" in docs
for legacy_var in (
"AA_ADDITIONAL_URLS",
"LIBGEN_ADDITIONAL_URLS",
"ZLIB_PRIMARY_URL",
"ZLIB_ADDITIONAL_URLS",
"WELIB_PRIMARY_URL",
"WELIB_ADDITIONAL_URLS",
):
assert f"`{legacy_var}`" not in docs
assert "https://annas-archive.gl" not in docs
def test_generated_env_docs_describe_mirror_lists_as_comma_separated_strings() -> None:
docs = generate_env_docs()
assert (
"| `AA_MIRROR_URLS` | List the Anna's Archive mirror URLs you want Shelfmark to use. "
"Type a URL and press Enter to add it. Order matters when Auto is selected. | "
"string (comma-separated) | _empty list_ |"
) in docs
assert (
"| `LIBGEN_MIRROR_URLS` | Mirrors are tried in the order you add them until one works. | "
"string (comma-separated) | _empty list_ |"
) in docs
tests/config/test_mirror_settings_migration.py
+203
View File
@@ -0,0 +1,203 @@
from unittest.mock import MagicMock
def test_mirror_settings_use_canonical_tag_lists_for_all_non_base_mirror_sets():
import shelfmark.config.settings # noqa: F401
from shelfmark.core.settings_registry import TagListField, get_settings_tab
tab = get_settings_tab("mirrors")
assert tab is not None
fields = {field.key: field for field in tab.fields if hasattr(field, "key")}
assert isinstance(fields["AA_MIRROR_URLS"], TagListField)
assert isinstance(fields["LIBGEN_MIRROR_URLS"], TagListField)
assert isinstance(fields["ZLIB_MIRROR_URLS"], TagListField)
assert isinstance(fields["WELIB_MIRROR_URLS"], TagListField)
def test_migrate_mirror_settings_does_not_seed_defaults_on_fresh_install(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.setattr(
registry, "load_config_file", lambda tab_name: {} if tab_name == "mirrors" else {}
)
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_mirror_settings()
saves.assert_not_called()
def test_migrate_mirror_settings_converts_legacy_additional_urls_to_list(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.setattr(
registry,
"load_config_file",
lambda tab_name: (
{"AA_ADDITIONAL_URLS": "aa.one, https://aa.two/"} if tab_name == "mirrors" else {}
),
)
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_mirror_settings()
saves.assert_called_once_with(
"mirrors",
{
"AA_MIRROR_URLS": ["https://aa.one", "https://aa.two"],
},
)
def test_migrate_mirror_settings_preserves_existing_list_without_reseeding(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.setattr(
registry,
"load_config_file",
lambda tab_name: (
{
"AA_MIRROR_URLS": ["https://annas-archive.existing"],
"_AA_MIRRORS_DEFAULTS_HASH": "old-hash",
}
if tab_name == "mirrors"
else {}
),
)
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_mirror_settings()
saves.assert_not_called()
def test_migrate_mirror_settings_converts_legacy_split_fields_to_canonical_lists(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.setattr(
registry,
"load_config_file",
lambda tab_name: (
{
"LIBGEN_ADDITIONAL_URLS": "libgen.one, https://libgen.two/",
"ZLIB_PRIMARY_URL": "zlib.primary",
"ZLIB_ADDITIONAL_URLS": "https://zlib.primary, zlib.backup",
"WELIB_PRIMARY_URL": "welib.primary",
"WELIB_ADDITIONAL_URLS": "https://welib.backup",
}
if tab_name == "mirrors"
else {}
),
)
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_mirror_settings()
saves.assert_called_once_with(
"mirrors",
{
"LIBGEN_MIRROR_URLS": ["https://libgen.one", "https://libgen.two"],
"ZLIB_MIRROR_URLS": ["https://zlib.primary", "https://zlib.backup"],
"WELIB_MIRROR_URLS": ["https://welib.primary", "https://welib.backup"],
},
)
def test_canonical_mirror_fields_accept_legacy_env_vars(monkeypatch):
import shelfmark.config.settings # noqa: F401
from shelfmark.core.settings_registry import (
get_setting_value,
get_settings_tab,
is_value_from_env,
)
monkeypatch.setenv("ZLIB_PRIMARY_URL", "zlib.primary")
monkeypatch.setenv("ZLIB_ADDITIONAL_URLS", "https://zlib.primary, zlib.backup")
tab = get_settings_tab("mirrors")
assert tab is not None
fields = {field.key: field for field in tab.fields if hasattr(field, "key")}
zlib_field = fields["ZLIB_MIRROR_URLS"]
assert is_value_from_env(zlib_field) is True
assert get_setting_value(zlib_field, "mirrors") == [
"https://zlib.primary",
"https://zlib.backup",
]
def test_migrate_direct_download_upgrade_enables_direct_download_without_seeding_mirrors(
monkeypatch,
):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.setattr(registry, "load_config_file", lambda tab_name: {})
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_direct_download_upgrade(existing_install=True)
saves.assert_called_once_with("download_sources", {"DIRECT_DOWNLOAD_ENABLED": True})
def test_migrate_direct_download_upgrade_skips_fresh_install(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_direct_download_upgrade(existing_install=False)
saves.assert_not_called()
def test_migrate_search_page_title_preserves_legacy_title_for_existing_installs(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.delenv("SEARCH_PAGE_TITLE", raising=False)
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_search_page_title(existing_install=True, had_existing_value=False)
saves.assert_called_once_with("general", {"SEARCH_PAGE_TITLE": "Book Search & Download"})
def test_migrate_search_page_title_skips_when_value_already_exists(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.delenv("SEARCH_PAGE_TITLE", raising=False)
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_search_page_title(existing_install=True, had_existing_value=True)
saves.assert_not_called()
def test_migrate_search_page_title_skips_when_env_var_is_set(monkeypatch):
import shelfmark.core.settings_registry as registry
saves = MagicMock(return_value=True)
monkeypatch.setenv("SEARCH_PAGE_TITLE", "Shelfmark")
monkeypatch.setattr(registry, "save_config_file", saves)
registry.migrate_search_page_title(existing_install=True, had_existing_value=False)
saves.assert_not_called()
tests/core/test_config_api.py
+2
View File
@@ -42,6 +42,7 @@ def test_config_endpoint_uses_user_scope_and_runtime_flags(main_module, client):
"SHOW_RELEASE_SOURCE_LINKS": False,
"SHOW_COMBINED_SELECTOR": False,
"SEARCH_MODE": "universal",
"SEARCH_PAGE_TITLE": "Custom Shelfmark",
"METADATA_PROVIDER": "openlibrary",
"METADATA_PROVIDER_AUDIOBOOK": "",
"DEFAULT_RELEASE_SOURCE": "prowlarr",
@@ -68,6 +69,7 @@ def test_config_endpoint_uses_user_scope_and_runtime_flags(main_module, client):
assert data["show_release_source_links"] is False
assert data["show_combined_selector"] is False
assert data["search_mode"] == "universal"
assert data["search_page_title"] == "Custom Shelfmark"
assert data["metadata_sort_options"] == ["sort-a"]
assert data["metadata_search_fields"] == ["field-a"]
assert data["default_release_source"] == "prowlarr"
tests/core/test_mirrors_config.py
+109 -6
View File
@@ -24,7 +24,7 @@ def test_get_aa_mirrors_prefers_full_configured_list(monkeypatch):
]
def test_get_aa_mirrors_falls_back_to_defaults_and_legacy_additional(monkeypatch):
def test_get_aa_mirrors_uses_legacy_additional_when_no_explicit_list(monkeypatch):
dummy = _DummyConfig(
{
"AA_MIRROR_URLS": [],
@@ -33,8 +33,111 @@ def test_get_aa_mirrors_falls_back_to_defaults_and_legacy_additional(monkeypatch
)
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
aa = mirrors.get_aa_mirrors()
for default_mirror in mirrors.DEFAULT_AA_MIRRORS:
assert default_mirror in aa
assert "https://extra.example" in aa
assert "https://extra2.example" in aa
assert mirrors.get_aa_mirrors() == [
"https://extra.example",
"https://extra2.example",
]
def test_get_aa_mirrors_returns_empty_when_unconfigured(monkeypatch):
monkeypatch.setattr(mirrors, "_get_config", lambda: _DummyConfig({}))
assert mirrors.get_aa_mirrors() == []
def test_has_aa_mirror_configuration_accepts_custom_base_url_without_list(monkeypatch):
dummy = _DummyConfig(
{
"AA_BASE_URL": "https://custom-aa.example",
"AA_MIRROR_URLS": [],
}
)
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
assert mirrors.has_aa_mirror_configuration() is True
def test_get_libgen_mirrors_returns_user_supplied_urls_only(monkeypatch):
dummy = _DummyConfig({"LIBGEN_MIRROR_URLS": ["libgen.one", "https://libgen.two/"]})
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
assert mirrors.get_libgen_mirrors() == [
"https://libgen.one",
"https://libgen.two",
]
def test_get_libgen_mirrors_falls_back_to_legacy_config(monkeypatch):
dummy = _DummyConfig({"LIBGEN_ADDITIONAL_URLS": "libgen.legacy, https://libgen.backup/"})
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
assert mirrors.get_libgen_mirrors() == [
"https://libgen.legacy",
"https://libgen.backup",
]
def test_get_zlib_mirrors_prefers_canonical_list(monkeypatch):
dummy = _DummyConfig({"ZLIB_MIRROR_URLS": ["zlib.primary", "https://zlib.backup"]})
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
assert mirrors.get_zlib_mirrors() == [
"https://zlib.primary",
"https://zlib.backup",
]
assert mirrors.get_zlib_url_template() == "https://zlib.primary/md5/{md5}"
def test_get_zlib_mirrors_falls_back_to_primary_then_additional(monkeypatch):
dummy = _DummyConfig(
{
"ZLIB_PRIMARY_URL": "zlib.primary",
"ZLIB_ADDITIONAL_URLS": "https://zlib.primary, zlib.backup",
}
)
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
assert mirrors.get_zlib_mirrors() == [
"https://zlib.primary",
"https://zlib.backup",
]
assert mirrors.get_zlib_url_template() == "https://zlib.primary/md5/{md5}"
def test_get_zlib_url_template_returns_none_without_config(monkeypatch):
monkeypatch.setattr(mirrors, "_get_config", lambda: _DummyConfig({}))
assert mirrors.get_zlib_url_template() is None
def test_get_welib_mirrors_prefers_primary_then_additional(monkeypatch):
dummy = _DummyConfig({"WELIB_MIRROR_URLS": ["welib.primary", "https://welib.backup"]})
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
assert mirrors.get_welib_mirrors() == [
"https://welib.primary",
"https://welib.backup",
]
assert mirrors.get_welib_url_template() == "https://welib.primary/md5/{md5}"
def test_get_welib_mirrors_falls_back_to_primary_then_additional(monkeypatch):
dummy = _DummyConfig(
{
"WELIB_PRIMARY_URL": "welib.primary",
"WELIB_ADDITIONAL_URLS": "https://welib.backup",
}
)
monkeypatch.setattr(mirrors, "_get_config", lambda: dummy)
assert mirrors.get_welib_mirrors() == [
"https://welib.primary",
"https://welib.backup",
]
assert mirrors.get_welib_url_template() == "https://welib.primary/md5/{md5}"
def test_get_welib_url_template_returns_none_without_config(monkeypatch):
monkeypatch.setattr(mirrors, "_get_config", lambda: _DummyConfig({}))
assert mirrors.get_welib_url_template() is None
tests/core/test_onboarding.py
+180
View File
@@ -0,0 +1,180 @@
from __future__ import annotations
from collections import defaultdict
import shelfmark.config.settings as settings_config
import shelfmark.metadata_providers.googlebooks as googlebooks_provider
import shelfmark.metadata_providers.hardcover as hardcover_provider
import shelfmark.metadata_providers.openlibrary as openlibrary_provider
import shelfmark.release_sources.audiobookbay.settings as audiobookbay_settings
import shelfmark.release_sources.irc.settings as irc_settings
import shelfmark.release_sources.prowlarr.settings as prowlarr_settings
_REGISTERED_SETTINGS_MODULES = (
settings_config,
googlebooks_provider,
hardcover_provider,
openlibrary_provider,
audiobookbay_settings,
irc_settings,
prowlarr_settings,
)
def _group_save_calls(
save_calls: list[tuple[str, dict[str, object]]],
) -> dict[str, list[dict[str, object]]]:
grouped: dict[str, list[dict[str, object]]] = defaultdict(list)
for tab_name, payload in save_calls:
grouped[tab_name].append(payload)
return grouped
def test_get_onboarding_config_uses_release_source_settings_pages():
from shelfmark.core.onboarding import get_onboarding_config
config = get_onboarding_config()
steps = {step["id"]: step for step in config["steps"]}
assert "release_sources" in steps
assert "direct_download_setup" in steps
assert "direct_download_cloudflare_bypass" in steps
assert "prowlarr" in steps
assert "audiobookbay" in steps
assert "irc" in steps
direct_download_fields = {
field["key"] for field in steps["direct_download_setup"]["fields"] if "key" in field
}
direct_download_bypass_fields = {
field["key"]
for field in steps["direct_download_cloudflare_bypass"]["fields"]
if "key" in field
}
prowlarr_fields = {
field["key"]: field for field in steps["prowlarr"]["fields"] if "key" in field
}
audiobookbay_fields = {
field["key"]: field for field in steps["audiobookbay"]["fields"] if "key" in field
}
assert "DIRECT_DOWNLOAD_ENABLED" not in direct_download_fields
assert "FAST_SOURCES_DISPLAY" not in direct_download_fields
assert "SOURCE_PRIORITY" not in direct_download_fields
assert "AA_DONATOR_KEY" in direct_download_fields
assert "AA_MIRROR_URLS" in direct_download_fields
assert "LIBGEN_ADDITIONAL_URLS" not in direct_download_fields
assert "ZLIB_PRIMARY_URL" not in direct_download_fields
assert "EXT_BYPASSER_TIMEOUT" not in direct_download_bypass_fields
assert "PROWLARR_ENABLED" not in prowlarr_fields
assert "PROWLARR_AUTO_EXPAND" not in prowlarr_fields
assert "ABB_ENABLED" not in audiobookbay_fields
assert "ABB_PAGE_LIMIT" not in audiobookbay_fields
assert "showWhen" not in prowlarr_fields["PROWLARR_URL"]
assert "showWhen" not in audiobookbay_fields["ABB_HOSTNAME"]
def test_save_onboarding_settings_enables_selected_release_sources(monkeypatch):
import shelfmark.core.config as app_config
import shelfmark.core.onboarding as onboarding
save_calls: list[tuple[str, dict[str, object]]] = []
def _fake_save(tab_name: str, values: dict[str, object]) -> bool:
save_calls.append((tab_name, dict(values)))
return True
monkeypatch.setattr(onboarding, "save_config_file", _fake_save)
monkeypatch.setattr(onboarding, "mark_onboarding_complete", lambda: True)
monkeypatch.setattr(app_config.config, "refresh", lambda: None, raising=False)
result = onboarding.save_onboarding_settings(
{
"SEARCH_MODE": "universal",
"METADATA_PROVIDER": "hardcover",
"HARDCOVER_API_KEY": "hardcover-api-key",
onboarding.ONBOARDING_RELEASE_SOURCES_KEY: [
"direct_download",
"audiobookbay",
"prowlarr",
],
"AA_MIRROR_URLS": ["https://annas-archive.example"],
"PROWLARR_URL": "http://prowlarr:9696",
"PROWLARR_API_KEY": "secret-key",
"ABB_HOSTNAME": "audiobookbay.lu",
}
)
grouped_calls = _group_save_calls(save_calls)
assert result == {"success": True, "message": "Onboarding complete!"}
assert any(
payload.get("DIRECT_DOWNLOAD_ENABLED") is True
for payload in grouped_calls["download_sources"]
)
assert any(
payload.get("AA_MIRROR_URLS") == ["https://annas-archive.example"]
for payload in grouped_calls["mirrors"]
)
assert any(
payload.get("PROWLARR_ENABLED") is True for payload in grouped_calls["prowlarr_config"]
)
assert any(
payload.get("ABB_ENABLED") is True for payload in grouped_calls["audiobookbay_config"]
)
assert any(
payload.get("PROWLARR_URL") == "http://prowlarr:9696"
and payload.get("PROWLARR_API_KEY") == "secret-key"
for payload in grouped_calls["prowlarr_config"]
)
assert any(
payload.get("HARDCOVER_ENABLED") is True
and payload.get("HARDCOVER_API_KEY") == "hardcover-api-key"
for payload in grouped_calls["hardcover"]
)
assert any(
payload.get("DEFAULT_RELEASE_SOURCE") == "direct_download"
and payload.get("DEFAULT_RELEASE_SOURCE_AUDIOBOOK") == "audiobookbay"
for payload in grouped_calls["search_mode"]
)
assert not any(
onboarding.ONBOARDING_RELEASE_SOURCES_KEY in payload for _, payload in save_calls
)
def test_save_onboarding_settings_skips_hidden_fields(monkeypatch):
import shelfmark.core.config as app_config
import shelfmark.core.onboarding as onboarding
save_calls: list[tuple[str, dict[str, object]]] = []
def _fake_save(tab_name: str, values: dict[str, object]) -> bool:
save_calls.append((tab_name, dict(values)))
return True
monkeypatch.setattr(onboarding, "save_config_file", _fake_save)
monkeypatch.setattr(onboarding, "mark_onboarding_complete", lambda: True)
monkeypatch.setattr(app_config.config, "refresh", lambda: None, raising=False)
result = onboarding.save_onboarding_settings(
{
"SEARCH_MODE": "direct",
"USE_CF_BYPASS": True,
"USING_EXTERNAL_BYPASSER": False,
"EXT_BYPASSER_URL": "http://should-not-save.example",
"EXT_BYPASSER_PATH": "/v2",
"EXT_BYPASSER_TIMEOUT": 120000,
}
)
grouped_calls = _group_save_calls(save_calls)
bypass_payloads = grouped_calls["cloudflare_bypass"]
assert result == {"success": True, "message": "Onboarding complete!"}
assert any(
payload.get("USE_CF_BYPASS") is True and payload.get("USING_EXTERNAL_BYPASSER") is False
for payload in bypass_payloads
)
assert all("EXT_BYPASSER_URL" not in payload for payload in bypass_payloads)
assert all("EXT_BYPASSER_PATH" not in payload for payload in bypass_payloads)
assert all("EXT_BYPASSER_TIMEOUT" not in payload for payload in bypass_payloads)
tests/direct_download/test_search_queries.py
+20 -4
View File
@@ -9,6 +9,22 @@ def _browse_record(record_id: str, title: str) -> BrowseRecord:
return BrowseRecord(id=record_id, title=title, source="direct_download")
def _enable_direct_download(monkeypatch):
import shelfmark.release_sources.direct_download as dd
original_get = dd.config.get
def _fake_get(key: str, default=None, user_id=None):
del user_id
if key == "DIRECT_DOWNLOAD_ENABLED":
return True
return original_get(key, default)
monkeypatch.setattr(dd.config, "get", _fake_get)
monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True)
return dd
class TestDirectDownloadSearchQueries:
def test_uses_search_title_for_english_queries(self, monkeypatch):
captured: list[str] = []
@@ -17,7 +33,7 @@ class TestDirectDownloadSearchQueries:
captured.append(query)
return []
import shelfmark.release_sources.direct_download as dd
dd = _enable_direct_download(monkeypatch)
monkeypatch.setattr(dd, "search_books", fake_search_books)
@@ -59,7 +75,7 @@ class TestDirectDownloadSearchQueries:
captured.append((query, filters.lang))
return records_by_query[query]
import shelfmark.release_sources.direct_download as dd
dd = _enable_direct_download(monkeypatch)
monkeypatch.setattr(dd, "search_books", fake_search_books)
@@ -103,7 +119,7 @@ class TestDirectDownloadSearchQueries:
return []
return fallback_results[query]
import shelfmark.release_sources.direct_download as dd
dd = _enable_direct_download(monkeypatch)
monkeypatch.setattr(dd, "search_books", fake_search_books)
@@ -141,7 +157,7 @@ class TestDirectDownloadSearchQueries:
return []
return [_browse_record("manual-1", "Manual result")]
import shelfmark.release_sources.direct_download as dd
dd = _enable_direct_download(monkeypatch)
monkeypatch.setattr(dd, "search_books", fake_search_books)
tests/direct_download/test_source_availability.py
+97
View File
@@ -0,0 +1,97 @@
from types import SimpleNamespace
import pytest
from shelfmark.release_sources.direct_download import DirectDownloadSource, SearchUnavailableError
def _fake_config_get(values: dict[str, object]):
def _get(key: str, default=None, user_id=None):
del user_id
return values.get(key, default)
return _get
def test_direct_download_source_is_unavailable_when_disabled(monkeypatch):
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(dd.config, "get", _fake_config_get({"DIRECT_DOWNLOAD_ENABLED": False}))
monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True)
source = DirectDownloadSource()
assert source.is_available() is False
with pytest.raises(SearchUnavailableError, match="Direct Download is disabled"):
source.search(SimpleNamespace(), SimpleNamespace())
def test_direct_download_source_is_unavailable_without_aa_mirrors(monkeypatch):
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(dd.config, "get", _fake_config_get({"DIRECT_DOWNLOAD_ENABLED": True}))
monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: False)
source = DirectDownloadSource()
assert source.is_available() is False
with pytest.raises(SearchUnavailableError, match="not configured"):
source.get_record("md5-abc")
def test_direct_download_source_is_available_when_enabled_and_configured(monkeypatch):
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(dd.config, "get", _fake_config_get({"DIRECT_DOWNLOAD_ENABLED": True}))
monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: True)
source = DirectDownloadSource()
assert source.is_available() is True
def test_get_source_priority_disables_entries_without_required_mirrors(monkeypatch):
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(
dd.config,
"get",
_fake_config_get(
{
"AA_DONATOR_KEY": "donator-key",
"FAST_SOURCES_DISPLAY": [
{"id": "aa-fast", "enabled": True},
{"id": "libgen", "enabled": True},
],
"SOURCE_PRIORITY": [
{"id": "welib", "enabled": True},
{"id": "zlib", "enabled": True},
],
}
),
)
monkeypatch.setattr("shelfmark.core.mirrors.has_aa_mirror_configuration", lambda: False)
monkeypatch.setattr("shelfmark.core.mirrors.has_libgen_mirror_configuration", lambda: True)
monkeypatch.setattr("shelfmark.core.mirrors.has_welib_mirror_configuration", lambda: False)
monkeypatch.setattr("shelfmark.core.mirrors.has_zlib_mirror_configuration", lambda: True)
priority = {item["id"]: item["enabled"] for item in dd._get_source_priority()}
assert priority["aa-fast"] is False
assert priority["libgen"] is True
assert priority["welib"] is False
assert priority["zlib"] is True
def test_is_configured_zlib_link_uses_configured_mirror_domains(monkeypatch):
import shelfmark.release_sources.direct_download as dd
monkeypatch.setattr(
"shelfmark.core.mirrors.get_zlib_cookie_domains",
lambda: {"custom-zlib.example"},
)
assert dd._is_configured_zlib_link("https://custom-zlib.example/books/example") is True
assert dd._is_configured_zlib_link("https://other-zlib.example/books/example") is False
tests/download/test_http_bypasser_fallbacks.py
+1
View File
@@ -69,6 +69,7 @@ def test_download_url_ignores_zlib_cookie_refresh_failure(monkeypatch):
import shelfmark.download.http as http
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
monkeypatch.setattr(http, "_is_configured_zlib_host", lambda hostname: hostname == "z-lib.fm")
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
tests/download/test_network_dns_failover.py
+17
View File
@@ -81,6 +81,23 @@ def test_rotate_dns_provider_cycles_back_to_first_provider(monkeypatch):
]
def test_rotate_dns_and_reset_aa_keeps_aa_unconfigured_without_user_mirrors(monkeypatch):
network = _set_auto_dns_mode(monkeypatch)
events: list[tuple] = []
monkeypatch.setattr(network, "rotate_dns_provider", lambda: True)
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
monkeypatch.setattr(network, "_aa_urls", [])
monkeypatch.setattr(network, "_aa_base_url", "https://legacy-aa.example")
monkeypatch.setattr(network, "_current_aa_url_index", 3)
monkeypatch.setattr(network, "_save_state", lambda **kwargs: events.append(("save", kwargs)))
assert network.rotate_dns_and_reset_aa() is True
assert network._current_aa_url_index == 0
assert network._aa_base_url == ""
assert events == []
def test_system_failover_getaddrinfo_retries_after_dns_switch(monkeypatch):
network = _set_auto_dns_mode(monkeypatch)
calls: list[tuple] = []