mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 22:05:20 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
97e289ae13 | ||
|
|
c95ee72ad5 | ||
|
|
b25acdb2ad | ||
|
|
7569aaecc5 | ||
|
|
f441b85da2 | ||
|
|
02b7e9d958 | ||
|
|
ff06a1a581 | ||
|
|
463ef49ac3 | ||
|
|
a5595cf9f1 | ||
|
|
9bcf595111 | ||
|
|
65e2e3be20 | ||
|
|
ddc26f01b6 | ||
|
|
89104ae80f | ||
|
|
1e45add4d5 | ||
|
|
cb3f6fee82 | ||
|
|
0dc13c1ca4 | ||
|
|
d1f8527089 | ||
|
|
95e34670f7 | ||
|
|
7d56624ab6 | ||
|
|
f4421ff189 | ||
|
|
e7007865a4 | ||
|
|
5b3df2a463 | ||
|
|
5247ec6124 | ||
|
|
bd21ec1257 | ||
|
|
7b9c416df8 | ||
|
|
646b531669 | ||
|
|
eafb965662 | ||
|
|
7193036626 | ||
|
|
12d554a92f | ||
|
|
fae6140c6a | ||
|
|
63133097e4 | ||
|
|
82aeee387e | ||
|
|
4cd1091d16 | ||
|
|
651096ed7b | ||
|
|
ebb833a82c | ||
|
|
b7093f4594 | ||
|
|
b656f019be | ||
|
|
6e96ead519 | ||
|
|
7345f6be1a | ||
|
|
2b8b35bb52 | ||
|
|
58a5b5ed27 | ||
|
|
52c1702419 | ||
|
|
3e2a7a48d5 | ||
|
|
a178541561 | ||
|
|
78e1f4daba | ||
|
|
eeea92280c | ||
|
|
0a5256ecbb | ||
|
|
6d2af0ac28 | ||
|
|
056ddd372a | ||
|
|
d0e008adde | ||
|
|
03e219eb43 |
@@ -96,7 +96,7 @@ jobs:
|
||||
type=ref,event=tag
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Build and push ${{ matrix.target }} Docker image
|
||||
id: push
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Create legacy aliases
|
||||
run: |
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Build shelfmark-lite image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
|
||||
@@ -25,14 +25,14 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
|
||||
@@ -236,6 +236,7 @@ pyrightconfig.json
|
||||
*.local.*
|
||||
AGENTS.md
|
||||
.claude/
|
||||
CLAUDE.md
|
||||
.nvmrc
|
||||
.playwright-mcp/
|
||||
frontend-dist/
|
||||
|
||||
+18
-12
@@ -24,10 +24,14 @@ COPY src/frontend/ ./
|
||||
# Build the frontend
|
||||
RUN npm run build
|
||||
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14.7-slim@sha256:83c1cebb322d099ac9e3a3a532ba74b0146d702838b25e4c75c02fa81ffeb910 AS base
|
||||
# uv is a build-time tool only, so it is mounted into the RUNs that need it rather
|
||||
# than copied into the image. A COPY here would land ~24 MB in a `base` layer that
|
||||
# every published image inherits, and a later `rm` cannot take it back out again --
|
||||
# a RUN adds a layer, it does not rewrite the one underneath.
|
||||
FROM ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1 AS uv
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 /uv /uvx /bin/
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14.7-slim@sha256:ce40764625a4ff50df3548277632e7f96c4e77fe75fa848aae9885476e7df5a4 AS base
|
||||
|
||||
# Add build argument for version
|
||||
ARG BUILD_VERSION
|
||||
@@ -111,6 +115,7 @@ WORKDIR /app
|
||||
# Install core Python dependencies first for better layer caching
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=from=uv,source=/uv,target=/usr/local/bin/uv \
|
||||
uv sync --locked --no-default-groups
|
||||
|
||||
# Runtime dependencies are installed into /app/.venv during the build. Remove the
|
||||
@@ -147,9 +152,14 @@ RUN mkdir -p \
|
||||
EXPOSE ${FLASK_PORT}
|
||||
|
||||
# Add healthcheck for container status
|
||||
# Uses /api/health which doesn't require authentication
|
||||
HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
|
||||
CMD curl -s http://localhost:${FLASK_PORT}/api/health > /dev/null || exit 1
|
||||
# Uses /api/health which doesn't require authentication.
|
||||
# curl needs -f so an HTTP error status fails the probe instead of passing it:
|
||||
# plain `curl -s` exits 0 on a 500, which reported a broken app as healthy.
|
||||
# timeout stays well under interval so a hung probe cannot occupy a whole cycle.
|
||||
# --start-interval matches the daemon default (5s), made explicit so startup
|
||||
# probing does not depend on that default staying put.
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=90s --start-interval=5s --retries=3 \
|
||||
CMD curl -fsS http://localhost:${FLASK_PORT}/api/health > /dev/null || exit 1
|
||||
|
||||
# Use dumb-init as the entrypoint to handle signals properly
|
||||
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
||||
@@ -199,6 +209,7 @@ RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-
|
||||
|
||||
# Install the browser automation stack used by the full image
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=from=uv,source=/uv,target=/usr/local/bin/uv \
|
||||
uv sync --locked --no-default-groups --extra browser
|
||||
|
||||
# Deterministically resolve the Xlib namespace collision.
|
||||
@@ -212,13 +223,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
# and force python-xlib 0.33 to own the namespace. pyautogui runs fine against
|
||||
# 0.33 (superset API).
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=from=uv,source=/uv,target=/usr/local/bin/uv \
|
||||
uv pip uninstall --python /app/.venv/bin/python python3-xlib && \
|
||||
uv pip install --python /app/.venv/bin/python --reinstall python-xlib==0.33 && \
|
||||
/app/.venv/bin/python -c "import Xlib.X; assert hasattr(Xlib.X, 'FamilyServerInterpreted'), 'Xlib.X.FamilyServerInterpreted missing after fix'; print('Xlib namespace OK:', Xlib.__version__)"
|
||||
|
||||
# uv is only needed while building the image.
|
||||
RUN rm -f /usr/bin/uv /usr/bin/uvx
|
||||
|
||||
# Keep SeleniumBase's bundled driver cache writable for the fixed non-root user.
|
||||
RUN SELENIUMBASE_DRIVERS_DIR=$(/app/.venv/bin/python -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')") && \
|
||||
chown -R 1000:1000 "${SELENIUMBASE_DRIVERS_DIR}" && \
|
||||
@@ -235,7 +244,4 @@ FROM base AS shelfmark-lite
|
||||
|
||||
ENV USING_EXTERNAL_BYPASSER=true
|
||||
|
||||
# uv is only needed while building the image.
|
||||
RUN rm -f /usr/bin/uv /usr/bin/uvx
|
||||
|
||||
CMD ["/app/entrypoint.sh"]
|
||||
|
||||
@@ -276,6 +276,27 @@ class DownloadHandler(ABC):
|
||||
pass
|
||||
```
|
||||
|
||||
### Optional: Listing Files Before Download
|
||||
|
||||
Some releases bundle several books (a whole-series torrent). Shelfmark inspects a
|
||||
release before queueing it so the user can review how it will be split into books.
|
||||
Override `list_files` when your source can enumerate a release's files without
|
||||
downloading it; the default returns `None`, which the UI reports as "can't inspect":
|
||||
|
||||
```python
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""Return the release's files (release-relative paths + sizes), or None."""
|
||||
torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page
|
||||
return extract_file_list_from_torrent(torrent_bytes) # from download.clients.torrent_utils
|
||||
```
|
||||
|
||||
`release_data` is the same payload the frontend sends to `/api/releases/download`
|
||||
(`source_id`, `download_url`, `content_type`, `series_name`, ...). Built-in examples:
|
||||
Prowlarr parses the `.torrent` it already fetches (magnet-only releases return
|
||||
`None`), and AudiobookBay reads the file table off its detail page.
|
||||
|
||||
### Download Method Parameters
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|
||||
@@ -247,7 +247,7 @@ Seconds since the last WireGuard handshake before the healthcheck bounces the tu
|
||||
| `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). | string | _none_ |
|
||||
| `AUDIOBOOK_LIBRARY_URL` | Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text. | string | _none_ |
|
||||
| `SUPPORTED_FORMATS` | Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found. | string (comma-separated) | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
|
||||
| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3` |
|
||||
| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3,m4a,mp4,flac,ogg,wma,aac,wav,opus,zip,rar` |
|
||||
| `BOOK_LANGUAGE` | Default language filter for searches. | string (comma-separated) | `en` |
|
||||
|
||||
<details>
|
||||
@@ -296,16 +296,7 @@ Book formats to include in search results. ZIP/RAR archives are extracted automa
|
||||
Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `m4b,mp3`
|
||||
|
||||
#### `BOOK_LANGUAGE`
|
||||
|
||||
**Default Book Languages**
|
||||
|
||||
Default language filter for searches.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `en`
|
||||
- **Default:** `m4b,mp3,m4a,mp4,flac,ogg,wma,aac,wav,opus,zip,rar`
|
||||
|
||||
</details>
|
||||
|
||||
@@ -314,6 +305,7 @@ Default language filter for searches.
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `SEARCH_MODE` | How you want to search for and download books. | string (choice) | `universal` |
|
||||
| `BOOK_LANGUAGE` | Default language filter for searches. Users can override this for their own account. | string (comma-separated) | `en` |
|
||||
| `AA_DEFAULT_SORT` | Default sort order for search results. | string (choice) | `relevance` |
|
||||
| `SHOW_RELEASE_SOURCE_LINKS` | Show clickable release-source links in release and details modals. Metadata provider links stay enabled. | boolean | `true` |
|
||||
| `SHOW_COMBINED_SELECTOR` | Show the option to search for and download both a book and audiobook together. | boolean | `true` |
|
||||
@@ -337,6 +329,15 @@ How you want to search for and download books.
|
||||
- **Default:** `universal`
|
||||
- **Options:** `direct` (Direct), `universal` (Universal)
|
||||
|
||||
#### `BOOK_LANGUAGE`
|
||||
|
||||
**Default Book Languages**
|
||||
|
||||
Default language filter for searches. Users can override this for their own account.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `en`
|
||||
|
||||
#### `AA_DEFAULT_SORT`
|
||||
|
||||
**Default Sort Order**
|
||||
@@ -704,7 +705,7 @@ Choose how downloaded audiobook files are named and organized.
|
||||
|
||||
- **Type:** string (choice)
|
||||
- **Default:** `rename`
|
||||
- **Options:** `none` (None), `rename` (Rename Only), `organize` (Rename and Organize)
|
||||
- **Options:** `none` (None), `rename` (Rename Only), `organize` (Rename and Organize), `rename_and_group` (Rename and Group)
|
||||
|
||||
#### `TEMPLATE_AUDIOBOOK_RENAME`
|
||||
|
||||
@@ -749,6 +750,7 @@ Automatically open the downloads sidebar when a new download is queued.
|
||||
Automatically download completed files to your browser for the selected content types.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
|
||||
- **Default:** _empty list_
|
||||
|
||||
#### `MAX_CONCURRENT_DOWNLOADS`
|
||||
@@ -1223,6 +1225,7 @@ How long to cache individual book details. Default: 600 (10 minutes). Max: 60480
|
||||
| `PROWLARR_URL` | Base URL of your Prowlarr instance | string | _none_ |
|
||||
| `PROWLARR_API_KEY` | Found in Prowlarr: Settings > General > API Key | string (secret) | _none_ |
|
||||
| `PROWLARR_INDEXERS` | Select which indexers to search. 📚 = has book categories. Leave empty to search all. | string (comma-separated) | _empty list_ |
|
||||
| `PROWLARR_INDEXER_TIMEOUT` | How long to wait for a single indexer to answer a search. Indexers behind FlareSolverr can need 90 seconds or more while a cold Cloudflare challenge is solved; raise this if searches come back empty and the Prowlarr log shows the search still running. | number | `90` |
|
||||
| `PROWLARR_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
|
||||
| `PROWLARR_COLLAPSE_DUPLICATES` | Collapse a release that several indexer entries returned down to a single row, keeping the entry with the best Prowlarr priority. Turn this off to see every entry that carried it, which is what makes results from filter-specific entries (freeleech and the like) visible. | boolean | `true` |
|
||||
| `PROWLARR_USE_SEED_PREFERENCES` | Apply per-indexer seed time and ratio preferences from Prowlarr when sending torrents to the download client | boolean | `false` |
|
||||
@@ -1268,6 +1271,16 @@ Select which indexers to search. 📚 = has book categories. Leave empty to sear
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** _empty list_
|
||||
|
||||
#### `PROWLARR_INDEXER_TIMEOUT`
|
||||
|
||||
**Indexer Search Timeout (seconds)**
|
||||
|
||||
How long to wait for a single indexer to answer a search. Indexers behind FlareSolverr can need 90 seconds or more while a cold Cloudflare challenge is solved; raise this if searches come back empty and the Prowlarr log shows the search still running.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `90`
|
||||
- **Constraints:** min: 5, max: 300
|
||||
|
||||
#### `PROWLARR_AUTO_EXPAND`
|
||||
|
||||
**Auto-expand search on no results**
|
||||
@@ -1302,8 +1315,11 @@ Apply per-indexer seed time and ratio preferences from Prowlarr when sending tor
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `NEWZNAB_ENABLED` | Enable searching for books via a Newznab-compatible indexer | boolean | `false` |
|
||||
| `NEWZNAB_URL` | Base URL of your Newznab indexer or aggregator | string | _none_ |
|
||||
| `NEWZNAB_API_KEY` | Your Newznab API key (leave blank if not required) | string (secret) | _none_ |
|
||||
| `NEWZNAB_INDEXERS` | Named Newznab connections. Each row accepts `name`, `url`, and `api_key`. | JSON array | `[]` |
|
||||
| `NEWZNAB_URL` | Legacy single-indexer URL, used when `NEWZNAB_INDEXERS` is empty | string | _none_ |
|
||||
| `NEWZNAB_API_KEY` | Legacy single-indexer API key | string (secret) | _none_ |
|
||||
| `NEWZNAB_EBOOK_CATEGORIES` | Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000. | string (comma-separated) | `7000` |
|
||||
| `NEWZNAB_AUDIOBOOK_CATEGORIES` | Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030. | string (comma-separated) | `3030` |
|
||||
| `NEWZNAB_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
|
||||
|
||||
<details>
|
||||
@@ -1318,25 +1334,58 @@ Enable searching for books via a Newznab-compatible indexer
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `NEWZNAB_INDEXERS`
|
||||
|
||||
**Named Indexers**
|
||||
|
||||
Configure multiple named Newznab-compatible indexers. The name is shown beside each search result. For environment-based configuration, provide a JSON array:
|
||||
|
||||
```json
|
||||
[
|
||||
{"name":"NZBGeek","url":"https://api.nzbgeek.info","api_key":"..."},
|
||||
{"name":"DrunkenSlug","url":"https://drunkenslug.com","api_key":"..."}
|
||||
]
|
||||
```
|
||||
|
||||
- **Type:** JSON array
|
||||
- **Default:** `[]`
|
||||
|
||||
#### `NEWZNAB_URL`
|
||||
|
||||
**Newznab URL**
|
||||
**Legacy Newznab URL**
|
||||
|
||||
Base URL of your Newznab indexer or aggregator
|
||||
Single-indexer fallback used only when `NEWZNAB_INDEXERS` is empty.
|
||||
|
||||
- **Type:** string
|
||||
- **Default:** _none_
|
||||
- **Required:** Yes
|
||||
|
||||
#### `NEWZNAB_API_KEY`
|
||||
|
||||
**API Key**
|
||||
**Legacy API Key**
|
||||
|
||||
Your Newznab API key (leave blank if not required)
|
||||
API key for the legacy Newznab URL.
|
||||
|
||||
- **Type:** string (secret)
|
||||
- **Default:** _none_
|
||||
|
||||
#### `NEWZNAB_EBOOK_CATEGORIES`
|
||||
|
||||
**Ebook Categories**
|
||||
|
||||
Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `7000`
|
||||
|
||||
#### `NEWZNAB_AUDIOBOOK_CATEGORIES`
|
||||
|
||||
**Audiobook Categories**
|
||||
|
||||
Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `3030`
|
||||
|
||||
#### `NEWZNAB_AUTO_EXPAND`
|
||||
|
||||
**Auto-expand search on no results**
|
||||
@@ -1421,7 +1470,7 @@ Delay between requests in seconds to avoid rate limiting (0-10).
|
||||
| `IRC_CHANNEL` | Channel name without the # prefix. Used for all searches unless a separate audiobook channel is configured below. | string | _none_ |
|
||||
| `IRC_NICK` | Your IRC nickname (required). Must be unique on the IRC network. | string | _none_ |
|
||||
| `IRC_SEARCH_BOT` | The search bot to address queries to (required). Searches are sent as "@<bot> <query>". | string | _none_ |
|
||||
| `IRC_AUDIOBOOK_CHANNEL` | Optional. Channel name (without the # prefix) to use for audiobook searches. Leave blank to use the main channel above for audiobooks too. | string | _none_ |
|
||||
| `IRC_AUDIOBOOK_CHANNEL` | Optional. Channel name (without the # prefix) for networks that index audiobooks separately, such as Undernet's bookz. Leave blank (the usual setting) to search the main channel above for audiobooks too. | string | _none_ |
|
||||
| `IRC_AUDIOBOOK_SEARCH_BOT` | Optional. Search bot for the audiobook channel. Leave blank to reuse the main search bot above. Only used when an audiobook channel is set. | string | _none_ |
|
||||
| `IRC_CACHE_TTL` | How long to keep cached search results before they expire. | string (choice) | `2592000` |
|
||||
|
||||
@@ -1490,7 +1539,7 @@ The search bot to address queries to (required). Searches are sent as "@<bot> <q
|
||||
|
||||
**Audiobook channel**
|
||||
|
||||
Optional. Channel name (without the # prefix) to use for audiobook searches. Leave blank to use the main channel above for audiobooks too.
|
||||
Optional. Channel name (without the # prefix) for networks that index audiobooks separately, such as Undernet's bookz. Leave blank (the usual setting) to search the main channel above for audiobooks too.
|
||||
|
||||
- **Type:** string
|
||||
- **Default:** _none_
|
||||
@@ -1957,7 +2006,7 @@ Move deletes the job from your usenet client after import; Copy keeps it in the
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `HARDCOVER_ENABLED` | Enable Hardcover as a metadata provider for book searches | boolean | `false` |
|
||||
| `HARDCOVER_API_KEY` | Get your API key from hardcover.app/account/api | string (secret) | _none_ |
|
||||
| `HARDCOVER_API_KEY` | Get your API key from hardcover.app/account/api (starts with hc_pat_) | string (secret) | _none_ |
|
||||
| `HARDCOVER_DEFAULT_SORT` | Default sort order for Hardcover search results. | string (choice) | `relevance` |
|
||||
| `HARDCOVER_EXCLUDE_COMPILATIONS` | Filter out compilations, anthologies, and omnibus editions from search results | boolean | `false` |
|
||||
| `HARDCOVER_EXCLUDE_UNRELEASED` | Filter out books with a release year in the future | boolean | `false` |
|
||||
@@ -1979,7 +2028,7 @@ Enable Hardcover as a metadata provider for book searches
|
||||
|
||||
**API Key**
|
||||
|
||||
Get your API key from hardcover.app/account/api
|
||||
Get your API key from hardcover.app/account/api (starts with hc_pat_)
|
||||
|
||||
- **Type:** string (secret)
|
||||
- **Default:** _none_
|
||||
@@ -2130,6 +2179,7 @@ Enable Moly.hu as a metadata provider for book searches
|
||||
| `SOURCE_PRIORITY` | Fallback sources, may have waiting. Requires bypasser. Drag to reorder. | JSON array | _see UI for defaults_ |
|
||||
| `MAX_RETRY` | Maximum retry attempts for failed downloads. | number | `10` |
|
||||
| `DEFAULT_SLEEP` | Wait time between download retry attempts. | number | `5` |
|
||||
| `RELEASE_SEARCH_TIMEOUT` | How long one release search may run before it gives up and reports why. A first search on a cold start pays for a browser solve, so leave room for one. If you use a reverse proxy, its read timeout should be at least this high or it will cut the search off with a 504 first. | number | `300` |
|
||||
| `AA_CONTENT_TYPE_ROUTING` | Override destination based on content type metadata. | boolean | `false` |
|
||||
| `AA_CONTENT_TYPE_DIR_FICTION` | Fiction Books | string | _none_ |
|
||||
| `AA_CONTENT_TYPE_DIR_NON_FICTION` | Non-Fiction Books | string | _none_ |
|
||||
@@ -2208,6 +2258,16 @@ Wait time between download retry attempts.
|
||||
- **Default:** `5`
|
||||
- **Constraints:** min: 1, max: 60
|
||||
|
||||
#### `RELEASE_SEARCH_TIMEOUT`
|
||||
|
||||
**Release Search Timeout (seconds)**
|
||||
|
||||
How long one release search may run before it gives up and reports why. A first search on a cold start pays for a browser solve, so leave room for one. If you use a reverse proxy, its read timeout should be at least this high or it will cut the search off with a 504 first.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `300`
|
||||
- **Constraints:** min: 30, max: 1800
|
||||
|
||||
#### `AA_CONTENT_TYPE_ROUTING`
|
||||
|
||||
**Enable Content-Type Routing**
|
||||
@@ -2284,6 +2344,7 @@ Override destination based on content type metadata.
|
||||
| `EXT_BYPASSER_URL` | URL of the external bypasser service (e.g., FlareSolverr). | string | `http://flaresolverr:8191` |
|
||||
| `EXT_BYPASSER_PATH` | API path for the external bypasser. | string | `/v1` |
|
||||
| `EXT_BYPASSER_TIMEOUT` | Timeout for external bypasser requests in milliseconds. | number | `60000` |
|
||||
| `BYPASS_BROWSER_IDLE_TIMEOUT` | How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner. | number | `180` |
|
||||
|
||||
<details>
|
||||
<summary>Detailed descriptions</summary>
|
||||
@@ -2339,6 +2400,17 @@ Timeout for external bypasser requests in milliseconds.
|
||||
- **Requires restart:** Yes
|
||||
- **Constraints:** min: 10000, max: 300000
|
||||
|
||||
#### `BYPASS_BROWSER_IDLE_TIMEOUT`
|
||||
|
||||
**Bypasser Idle Timeout (seconds)**
|
||||
|
||||
How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `180`
|
||||
- **Requires restart:** Yes
|
||||
- **Constraints:** min: 30, max: 3600
|
||||
|
||||
</details>
|
||||
|
||||
### Direct Download: Mirrors
|
||||
|
||||
@@ -30,7 +30,7 @@ Requires mounting your Calibre-Web `app.db` to `/auth/app.db`.
|
||||
|
||||
Admins can configure per-user settings by editing a user in the user management panel. Non-admin users can also edit their own settings through **My Account** (accessible from the user menu). Admins control which sections are visible in My Account via the **Visible Self-Settings Sections** option.
|
||||
|
||||
There are three categories of per-user settings:
|
||||
There are four categories of per-user settings:
|
||||
|
||||
### Delivery Preferences
|
||||
|
||||
@@ -42,6 +42,15 @@ Override where a user's downloads are sent. Options depend on the global output
|
||||
- **BookLore library/path** — Per-user BookLore target (when using BookLore output mode)
|
||||
- **Email recipient** — Per-user email address (when using Email output mode)
|
||||
|
||||
### Search Preferences
|
||||
|
||||
Override how a user searches, on top of the global search defaults:
|
||||
|
||||
- **Search mode** — Direct or Universal for this user
|
||||
- **Default book languages** — The languages a user's searches fall back to when they don't pick one themselves. Useful for a shared instance where readers want different languages.
|
||||
- **Metadata providers** — Book, audiobook, and combined-mode provider for this user
|
||||
- **Default release sources** — The release tab opened first for books and audiobooks
|
||||
|
||||
### Notifications
|
||||
|
||||
Users can configure personal notification routes, separate from the global notification settings. Each route targets a URL (e.g. an Apprise-compatible endpoint) and can be scoped to specific event types or all events.
|
||||
|
||||
+8
-5
@@ -19,28 +19,31 @@ dependencies = [
|
||||
"psutil",
|
||||
"emoji",
|
||||
"rarfile",
|
||||
"qbittorrent-api>=2026.8.0",
|
||||
"qbittorrent-api>=2026.8.1",
|
||||
"transmission-rpc",
|
||||
"authlib>=1.7.2,<1.8",
|
||||
"apprise>=1.12.0",
|
||||
"apprise>=1.13.0",
|
||||
# HTTP/2 client for RFC 8484 DoH: quad9 rejects HTTP/1.1 outright (505), which
|
||||
# requests cannot speak. See shelfmark/download/doh_wireformat.py.
|
||||
"httpx[http2]>=0.28.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
browser = [
|
||||
"pyvirtualdisplay",
|
||||
"pyautogui",
|
||||
"seleniumbase==4.51.11",
|
||||
"seleniumbase==4.52.2",
|
||||
"python-xlib",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"basedpyright>=1.39.9",
|
||||
"basedpyright>=1.39.10",
|
||||
"prek",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pytest-xdist>=3.8.0",
|
||||
"ruff==0.16.2",
|
||||
"ruff==0.16.4",
|
||||
"vulture>=2.14",
|
||||
]
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
|
||||
|
||||
> [!NOTE]
|
||||
> This project is in a stable state as of May 2026 but is not under active maintenance.
|
||||
> Shelfmark is feature stable and maintained on a best-effort basis. Bug fixes, security updates, and small quality-of-life improvements are still shipped, and pull requests are reviewed — including new features. There is no roadmap for new features for now.
|
||||
|
||||
Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
|
||||
|
||||
@@ -95,6 +95,30 @@ volumes:
|
||||
- Aggregates releases from multiple configured sources
|
||||
- Full audiobook support
|
||||
|
||||
### Hardcover API Key
|
||||
|
||||
Hardcover powers metadata search in Universal mode. Create a token at
|
||||
[hardcover.app/account/api](https://hardcover.app/account/api) — current keys start with `hc_pat_`
|
||||
and are far shorter than the JWTs Hardcover issued before August 2026.
|
||||
|
||||
Tick these seven scopes on the token screen:
|
||||
|
||||
| Scope | Used for |
|
||||
|-------|----------|
|
||||
| `read:catalog` | Metadata search, plus book, edition, author and series lookups |
|
||||
| `read:library` | Your reading status and shelf counts |
|
||||
| `read:lists` | Your lists and the books on them |
|
||||
| `read:me:content` | Test Connection and the "Connected as" label |
|
||||
| `read:users` | Usernames shown alongside lists |
|
||||
| `write:library` | Setting a book's reading status from Shelfmark |
|
||||
| `write:lists` | Adding and removing books from lists, including auto-remove on download |
|
||||
|
||||
The two `write:` scopes matter only if you set reading status from Shelfmark or leave
|
||||
**Auto-Remove from List on Download** enabled (it is on by default) — without them those actions
|
||||
fail silently. Everything else Hardcover offers (journal, goals, reviews, prompts, notifications,
|
||||
account) can stay unticked. The `all` scope works too, but it grants full account access including
|
||||
deletion, so prefer the list above.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
|
||||
@@ -123,7 +147,8 @@ See the full [Environment Variables Reference](docs/environment-variables.md) fo
|
||||
Some of the additional options available in Settings:
|
||||
- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
|
||||
- **Additional audiobook sources** - Configure additional sources for audiobook discovery
|
||||
- **IRC** - Add details for IRC book sources and download directly from the UI
|
||||
- **Direct Download mirrors** - Supply your own Anna's Archive mirror URLs; Auto mode tries them in the order listed. The `annas-archive.is` domain does not currently work as a source — use `annas-archive.gl` instead (checked August 2026; mirror availability changes)
|
||||
- **IRC** - Add details for IRC book sources and download directly from the UI. Most networks serve audiobooks from the same channel as ebooks (on `irc.irchighway.net` that's `#ebooks`, while `#bookz` is effectively inactive), so leave the separate audiobook channel blank unless your network actually indexes one. IRC audiobooks usually arrive as ZIP/RAR archives — keep those enabled under Supported Audiobook Formats or the releases are filtered out of results
|
||||
- **Library Link** - Add a link to your Calibre-Web or Grimmory instance in the UI header
|
||||
- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
|
||||
- **Network Settings** - Custom proxy support (SOCKS5 + HTTP/S) and configurable DNS
|
||||
@@ -238,9 +263,11 @@ These are non-goals, not missing features.
|
||||
|
||||
## Contributing
|
||||
|
||||
Shelfmark's core feature set is complete. Development focuses on stability, bug fixes, quality-of-life improvements, and refining the search experience. Contributions in these areas are welcome, please file issues or submit pull requests on GitHub.
|
||||
Shelfmark's core feature set is complete.
|
||||
|
||||
Feature requests that fall outside the project scope (library integration, automation, collection management) will be closed. If you're unsure whether something fits, open a discussion first.
|
||||
Pull requests are welcome and all of them get reviewed, new features included. If you want a feature, the fastest path is to send a PR for it rather than to file a request.
|
||||
|
||||
Feature requests that fall outside the project scope (library integration, automation, collection management) will be closed, and PRs implementing them won't be merged. If you're unsure whether something fits, open a discussion first.
|
||||
|
||||
## Health Monitoring
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Challenge-page detection shared by the bypassers and the HTTP retry path.
|
||||
|
||||
Kept out of `internal_bypasser` so the HTTP layer can recognise an interstitial
|
||||
without importing SeleniumBase: that module is imported lazily precisely because its
|
||||
browser dependencies are optional, and external-bypasser setups run without them.
|
||||
"""
|
||||
|
||||
# Matched against lowercased text, so every entry must be lowercase.
|
||||
CLOUDFLARE_INDICATORS = [
|
||||
"just a moment",
|
||||
"verify you are human",
|
||||
"verifying you are human",
|
||||
"cloudflare.com/products/turnstile",
|
||||
]
|
||||
|
||||
DDOS_GUARD_INDICATORS = [
|
||||
"ddos-guard",
|
||||
"ddos guard",
|
||||
"checking your browser before accessing",
|
||||
"complete the manual check to continue",
|
||||
"could not verify your browser automatically",
|
||||
]
|
||||
|
||||
# Markers that exist only in raw markup: the bypassers scan rendered innerText, where
|
||||
# a script src or a <title> never appears. The title match is scoped to the tag on
|
||||
# purpose - hosts word the rest of that sentence differently, and matching "checking
|
||||
# your browser" as free text would trip on any page that merely discusses a challenge.
|
||||
_RAW_HTML_MARKERS = (
|
||||
"<title>checking your browser",
|
||||
"/cdn-cgi/challenge-platform",
|
||||
"/.well-known/ddos-guard/",
|
||||
)
|
||||
|
||||
# An interstitial is a few KB of markup. Past that it is a real page that happens to
|
||||
# mention a marker - a protected site links its own DDoS-Guard endpoints on every page.
|
||||
MAX_CHALLENGE_HTML_CHARS = 64 * 1024
|
||||
|
||||
|
||||
def challenge_marker(html: str) -> str | None:
|
||||
"""Return the marker proving `html` is an unsolved challenge page, or None.
|
||||
|
||||
Only meaningful for a response that already carries a challenge status: the
|
||||
markers appear on protected sites' real pages too, so the status is what
|
||||
separates "blocked" from "served".
|
||||
"""
|
||||
if not html or len(html) > MAX_CHALLENGE_HTML_CHARS:
|
||||
return None
|
||||
lowered = html.lower()
|
||||
for marker in (*_RAW_HTML_MARKERS, *DDOS_GUARD_INDICATORS, *CLOUDFLARE_INDICATORS):
|
||||
if marker in lowered:
|
||||
return marker
|
||||
return None
|
||||
@@ -0,0 +1,281 @@
|
||||
"""Clearance cookies won by a bypass, shared by every bypasser implementation.
|
||||
|
||||
Kept in its own module rather than inside a bypasser because both of them feed it and
|
||||
both read from it. The internal bypasser cannot host it: it imports seleniumbase at
|
||||
module scope, which is exactly the dependency an external-bypasser deployment is
|
||||
entitled not to have installed.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Cookie storage - shared with requests library for Cloudflare bypass
|
||||
# Nested mapping of domain to cookie name to cookie metadata.
|
||||
_cf_cookies: dict[str, dict] = {}
|
||||
_cf_cookies_lock = threading.Lock()
|
||||
|
||||
# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
|
||||
_cf_user_agents: dict[str, str] = {}
|
||||
|
||||
# Protection cookie names we care about (Cloudflare and DDoS-Guard)
|
||||
CF_COOKIE_NAMES = {"cf_clearance", "__cf_bm", "cf_chl_2", "cf_chl_prog"}
|
||||
DDG_COOKIE_NAMES = {
|
||||
"__ddg1_",
|
||||
"__ddg2_",
|
||||
"__ddg5_",
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"__ddgid_",
|
||||
"__ddgmark_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
# DDoS-Guard cookies that describe *one* check rather than granting clearance, and so
|
||||
# must never be replayed on a later request. Observed live on Anna's Archive:
|
||||
#
|
||||
# __ddg9_ the client IP address
|
||||
# __ddg10_ the unix timestamp the check was issued
|
||||
# __ddg8_ an opaque token issued with them, same ~40 minute expiry
|
||||
#
|
||||
# Clearance itself lives in __ddg1_/__ddg2_/__ddgid_ (roughly a year) and __ddg5_.
|
||||
# Replaying the trio is actively harmful: once the timestamp ages out - or the egress
|
||||
# IP changes, which happens routinely behind a VPN - the values no longer describe the
|
||||
# caller, DDoS-Guard re-arms its check and answers every request with a ?check=1
|
||||
# redirect. That is the redirect loop, and it is self-inflicted. Dropping them simply
|
||||
# lets DDoS-Guard issue a fresh set, exactly as it does for a browser.
|
||||
DDG_EPHEMERAL_COOKIE_NAMES = {
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
|
||||
def _get_base_domain(domain: str) -> str:
|
||||
"""Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
|
||||
return ".".join(domain.split(".")[-2:]) if "." in domain else domain
|
||||
|
||||
|
||||
def _get_full_cookie_domains() -> set[str]:
|
||||
"""Return mirror domains that need full-session cookie extraction."""
|
||||
from shelfmark.core.mirrors import get_zlib_cookie_domains
|
||||
|
||||
return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()}
|
||||
|
||||
|
||||
def _replay_per_check_cookies() -> bool:
|
||||
"""Whether the per-check trio is kept rather than dropped (see env.py)."""
|
||||
from shelfmark.config import env
|
||||
|
||||
return env.DDG_REPLAY_PER_CHECK_COOKIES
|
||||
|
||||
|
||||
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
|
||||
"""Determine if a cookie should be extracted based on its name."""
|
||||
# Checked before extract_all: a per-check token is wrong to replay for every
|
||||
# domain, including the full-session ones.
|
||||
if name in DDG_EPHEMERAL_COOKIE_NAMES and not _replay_per_check_cookies():
|
||||
return False
|
||||
if extract_all:
|
||||
return True
|
||||
is_cf = name in CF_COOKIE_NAMES or name.startswith("cf_")
|
||||
is_ddg = name in DDG_COOKIE_NAMES or name.startswith("__ddg")
|
||||
return is_cf or is_ddg
|
||||
|
||||
|
||||
def _cookie_field(cookie: Any, name: str) -> Any:
|
||||
"""Read one field from a cookie in either shape we are handed.
|
||||
|
||||
The internal bypasser extracts CDP cookie objects; an external bypasser returns
|
||||
the same fields as JSON objects, so the difference is attribute versus key access.
|
||||
"""
|
||||
if isinstance(cookie, Mapping):
|
||||
return cookie.get(name)
|
||||
return getattr(cookie, name, None)
|
||||
|
||||
|
||||
def _cookie_expiry(cookie: Any) -> float | None:
|
||||
"""A cookie's absolute expiry, or None when it is a session cookie.
|
||||
|
||||
The two spellings are not interchangeable and both reach this store. CDP and
|
||||
Playwright cookies carry `expires`; the WebDriver cookie object - what a
|
||||
Selenium-based solver such as FlareSolverr returns - carries `expiry`. Reading
|
||||
only one silently turns every cookie from the other into a never-expiring one,
|
||||
which is exactly how dead clearance ends up replayed forever (see
|
||||
get_cf_cookies_for_domain).
|
||||
|
||||
The value is coerced rather than trusted: it arrives as JSON from a service we
|
||||
do not control, and a string here used to raise straight out of the store.
|
||||
"""
|
||||
for field in ("expires", "expiry"):
|
||||
raw = _cookie_field(cookie, field)
|
||||
if raw is None:
|
||||
continue
|
||||
try:
|
||||
expiry = float(raw)
|
||||
except TypeError, ValueError:
|
||||
logger.debug("Unreadable cookie expiry %r; treating as a session cookie", raw)
|
||||
return None
|
||||
# <= 0 is how both shapes spell "session cookie", not "expired in 1970".
|
||||
return expiry if expiry > 0 else None
|
||||
return None
|
||||
|
||||
|
||||
def store_extracted_cookies(
|
||||
*,
|
||||
url: str,
|
||||
cookies: list[Any],
|
||||
user_agent: str | None = None,
|
||||
) -> None:
|
||||
"""Store filtered bypass cookies (and optional UA) for a URL domain."""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.hostname or ""
|
||||
if not domain:
|
||||
return
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
extract_all = base_domain in _get_full_cookie_domains()
|
||||
|
||||
cookies_found: dict[str, dict[str, Any]] = {}
|
||||
dropped: list[str] = []
|
||||
for cookie in cookies:
|
||||
name = _cookie_field(cookie, "name") or ""
|
||||
if not _should_extract_cookie(name, extract_all=extract_all):
|
||||
dropped.append(name)
|
||||
continue
|
||||
secure = _cookie_field(cookie, "secure")
|
||||
cookies_found[name] = {
|
||||
"value": _cookie_field(cookie, "value") or "",
|
||||
"domain": _cookie_field(cookie, "domain") or domain,
|
||||
"path": _cookie_field(cookie, "path") or "/",
|
||||
"expiry": _cookie_expiry(cookie),
|
||||
"secure": True if secure is None else bool(secure),
|
||||
"httpOnly": True,
|
||||
}
|
||||
|
||||
# Names only, never values. Which cookies a solve won, and which of them were held
|
||||
# back, is the evidence needed to settle what DDoS-Guard actually treats as clearance
|
||||
# (issue #1276) - and without it a debug log shows a solve succeeding and the next
|
||||
# request being challenged with nothing in between to explain why.
|
||||
logger.debug(
|
||||
"Solve on %s won %s; keeping %s; dropping %s",
|
||||
base_domain,
|
||||
sorted({_cookie_field(c, "name") or "" for c in cookies}),
|
||||
sorted(cookies_found),
|
||||
sorted(set(dropped)) or "nothing",
|
||||
)
|
||||
|
||||
if not cookies_found:
|
||||
return
|
||||
|
||||
with _cf_cookies_lock:
|
||||
_cf_cookies[base_domain] = cookies_found
|
||||
if user_agent:
|
||||
_cf_user_agents[base_domain] = user_agent
|
||||
logger.debug("Stored UA for %s: %s...", base_domain, str(user_agent)[:60])
|
||||
else:
|
||||
logger.debug("No UA captured for %s", base_domain)
|
||||
|
||||
cookie_type = "all" if extract_all else "protection"
|
||||
logger.debug("Extracted %s %s cookies for %s", len(cookies_found), cookie_type, base_domain)
|
||||
|
||||
|
||||
def _is_cookie_expired(cookie: dict[str, Any]) -> bool:
|
||||
"""Whether a stored cookie's expiry has passed. Session cookies never expire here."""
|
||||
expiry = cookie.get("expiry")
|
||||
if expiry is None:
|
||||
expiry = cookie.get("expires")
|
||||
if not expiry or expiry <= 0:
|
||||
return False
|
||||
return time.time() > expiry
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get stored cookies for a domain. Returns empty dict if none available."""
|
||||
if not domain:
|
||||
return {}
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
|
||||
with _cf_cookies_lock:
|
||||
cookies = _cf_cookies.get(base_domain, {})
|
||||
if not cookies:
|
||||
return {}
|
||||
|
||||
cf_clearance = cookies.get("cf_clearance", {})
|
||||
if cf_clearance and _is_cookie_expired(cf_clearance):
|
||||
logger.debug("CF cookies expired for %s", base_domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
return {}
|
||||
|
||||
# Expiry applies to every cookie, not just Cloudflare's. DDoS-Guard domains
|
||||
# have no cf_clearance, so the check above never fired for them and dead
|
||||
# cookies were replayed indefinitely - the server answers those with a
|
||||
# challenge, which is indistinguishable from having sent nothing at all.
|
||||
live = {name: c for name, c in cookies.items() if not _is_cookie_expired(c)}
|
||||
if len(live) != len(cookies):
|
||||
expired = sorted(set(cookies) - set(live))
|
||||
logger.debug("Dropping expired cookies for %s: %s", base_domain, expired)
|
||||
if live:
|
||||
_cf_cookies[base_domain] = live
|
||||
else:
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
|
||||
return {name: c["value"] for name, c in live.items()}
|
||||
|
||||
|
||||
def has_valid_cf_cookies(domain: str) -> bool:
|
||||
"""Check if we have valid Cloudflare cookies for a domain."""
|
||||
return bool(get_cf_cookies_for_domain(domain))
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get the User-Agent that was used during bypass for a domain."""
|
||||
if not domain:
|
||||
return None
|
||||
with _cf_cookies_lock:
|
||||
return _cf_user_agents.get(_get_base_domain(domain))
|
||||
|
||||
|
||||
def export_store() -> tuple[dict[str, dict], dict[str, str]]:
|
||||
"""Snapshot the whole store, for handing to another process.
|
||||
|
||||
The internal bypasser's Docker helper solves in a subprocess, so the clearance it
|
||||
wins has to be serialized back to the parent or the solve is lost with the child.
|
||||
"""
|
||||
with _cf_cookies_lock:
|
||||
return (
|
||||
{domain: dict(cookies) for domain, cookies in _cf_cookies.items()},
|
||||
dict(_cf_user_agents),
|
||||
)
|
||||
|
||||
|
||||
def import_store(cookies: object, user_agents: object) -> None:
|
||||
"""Merge a snapshot produced by :func:`export_store` into this process's store."""
|
||||
with _cf_cookies_lock:
|
||||
if isinstance(cookies, dict):
|
||||
_cf_cookies.update(cookies)
|
||||
if isinstance(user_agents, dict):
|
||||
_cf_user_agents.update(
|
||||
{str(domain): str(agent) for domain, agent in user_agents.items()}
|
||||
)
|
||||
|
||||
|
||||
def clear_cf_cookies(domain: str | None = None) -> None:
|
||||
"""Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
|
||||
with _cf_cookies_lock:
|
||||
if domain:
|
||||
base_domain = _get_base_domain(domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
_cf_user_agents.pop(base_domain, None)
|
||||
else:
|
||||
_cf_cookies.clear()
|
||||
_cf_user_agents.clear()
|
||||
@@ -2,17 +2,19 @@
|
||||
|
||||
import random
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass.cookie_store import store_extracted_cookies
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
from threading import Event
|
||||
|
||||
from shelfmark.download import network
|
||||
@@ -63,6 +65,31 @@ def max_duration_seconds() -> float:
|
||||
return MAX_RETRY * read_timeout + backoff_total
|
||||
|
||||
|
||||
def _store_solution_clearance(target_url: str, solution: Mapping[str, Any]) -> None:
|
||||
"""Keep the clearance the solver won, so later requests do not re-solve.
|
||||
|
||||
A solve is the expensive part of an external bypass - tens of seconds of real
|
||||
browser - and FlareSolverr-compatible services hand back the cookies and the
|
||||
User-Agent that earned it. Dropping them meant every single request paid a 403
|
||||
plus a full solve, and a file download (which the solver cannot proxy, being
|
||||
binary) never presented clearance at all.
|
||||
|
||||
The UA matters as much as the cookies: Cloudflare ties cf_clearance to the UA
|
||||
that solved the challenge, so replaying the cookie under our own UA is rejected.
|
||||
"""
|
||||
cookies = solution.get("cookies") or []
|
||||
if not isinstance(cookies, list):
|
||||
logger.debug("External bypasser returned no usable cookie list for '%s'", target_url)
|
||||
return
|
||||
|
||||
user_agent = solution.get("userAgent")
|
||||
store_extracted_cookies(
|
||||
url=target_url,
|
||||
cookies=cookies,
|
||||
user_agent=user_agent if isinstance(user_agent, str) else None,
|
||||
)
|
||||
|
||||
|
||||
def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
"""Make a single request to the external bypasser service. Returns HTML or None."""
|
||||
raw_bypasser_url = _coerce_config_str(
|
||||
@@ -116,6 +143,15 @@ def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
logger.warning("External bypasser returned empty response for '%s'", target_url)
|
||||
return None
|
||||
|
||||
try:
|
||||
_store_solution_clearance(target_url, solution)
|
||||
except AttributeError, KeyError, TypeError, ValueError:
|
||||
# Storing clearance is an optimisation; the page is the product. The
|
||||
# solution JSON comes from a service we do not control, so a surprise in
|
||||
# its cookie shape must not discard HTML that already cost a ~30s solve
|
||||
# and send the caller round for up to MAX_RETRY more of them.
|
||||
logger.debug("Could not store bypass clearance for '%s'", target_url, exc_info=True)
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning(
|
||||
"External bypasser timed out for '%s' (connect: %ss, read: %.0fs)",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -203,6 +203,21 @@ ONBOARDING = string_to_bool(os.getenv("ONBOARDING", "true"))
|
||||
_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
|
||||
DEBUG_SKIP_SOURCES = {s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip()}
|
||||
|
||||
# Debug: keep DDoS-Guard's __ddg8_/__ddg9_/__ddg10_ in the clearance store instead of
|
||||
# dropping them after a solve.
|
||||
#
|
||||
# Which of DDoS-Guard's cookies actually *are* clearance is not settled. The store treats
|
||||
# the trio as describing one check (client IP, timestamp, token) and drops them, on the
|
||||
# reasoning that replaying a stale IP/timestamp is what re-arms the ?check=1 loop - see
|
||||
# shelfmark.bypass.cookie_store. Field reports on issue #1276 point the other way: every
|
||||
# request after a successful solve was challenged again, which is only consistent with
|
||||
# what the store keeps not being sufficient clearance on its own.
|
||||
#
|
||||
# Deliberately env-only and off by default: this is a knob for reproducing the question
|
||||
# against a live host, not a setting to offer users. Set it to true, solve once, and watch
|
||||
# whether the next search still logs "Redirect loop detected".
|
||||
DDG_REPLAY_PER_CHECK_COOKIES = string_to_bool(os.getenv("DDG_REPLAY_PER_CHECK_COOKIES", "false"))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Legacy migration support - will be removed in future version
|
||||
|
||||
@@ -7,7 +7,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Sequence
|
||||
from os import PathLike
|
||||
|
||||
_DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
|
||||
@@ -16,6 +16,13 @@ _DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
|
||||
"RESTRICT_SETTINGS_TO_ADMIN",
|
||||
)
|
||||
|
||||
# The audiobook format list shipped as the default until the format sets were unified.
|
||||
# It only covered m4b/mp3, so FLAC/OPUS/OGG/M4A releases were dropped from search results
|
||||
# and rejected after download - and the wider default alone would never reach existing
|
||||
# installs, because initialize_default_configs() only writes defaults when the config
|
||||
# file does not exist yet.
|
||||
_LEGACY_AUDIOBOOK_FORMATS_DEFAULT = ("m4b", "mp3")
|
||||
|
||||
|
||||
class MigrationLogger(Protocol):
|
||||
"""Logger surface used by config migration helpers."""
|
||||
@@ -57,6 +64,54 @@ def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None:
|
||||
return None
|
||||
|
||||
|
||||
def migrate_audiobook_formats(
|
||||
*,
|
||||
load_general_config: Callable[[], dict[str, Any]],
|
||||
# `object` rather than `None`: the result is discarded, and savers that report
|
||||
# success (settings_registry.save_config_file returns bool) are not assignable to a
|
||||
# `-> None` callable.
|
||||
save_general_config: Callable[[dict[str, Any]], object],
|
||||
widened_formats: Sequence[str],
|
||||
logger: MigrationLogger,
|
||||
) -> None:
|
||||
"""Widen an untouched audiobook format list to the current, fuller default.
|
||||
|
||||
Only a list that still matches the old default exactly is rewritten. Any other value
|
||||
means someone chose it deliberately, and a migration that "helpfully" re-enabled
|
||||
formats a user had turned off would be worse than leaving them on the narrow list.
|
||||
"""
|
||||
try:
|
||||
config = load_general_config()
|
||||
|
||||
if "SUPPORTED_AUDIOBOOK_FORMATS" not in config:
|
||||
# Nothing persisted, so the field default already applies.
|
||||
logger.debug("No persisted audiobook formats - the current default applies")
|
||||
return
|
||||
|
||||
current = config.get("SUPPORTED_AUDIOBOOK_FORMATS")
|
||||
if not isinstance(current, list):
|
||||
return
|
||||
|
||||
normalized = {str(fmt).strip().lower() for fmt in current if str(fmt).strip()}
|
||||
if normalized != set(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT):
|
||||
logger.debug(
|
||||
"Audiobook formats were customized (%s) - left unchanged", sorted(normalized)
|
||||
)
|
||||
return
|
||||
|
||||
save_general_config({"SUPPORTED_AUDIOBOOK_FORMATS": list(widened_formats)})
|
||||
logger.info(
|
||||
"Widened audiobook formats from the legacy default %s to %s",
|
||||
list(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT),
|
||||
list(widened_formats),
|
||||
)
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.debug("No existing general config file found - nothing to migrate")
|
||||
except Exception:
|
||||
logger.exception("Failed to migrate audiobook formats")
|
||||
|
||||
|
||||
def migrate_security_settings(
|
||||
*,
|
||||
load_security_config: Callable[[], dict[str, Any]],
|
||||
|
||||
@@ -14,6 +14,7 @@ from shelfmark.config.download_settings_handlers import (
|
||||
check_books_destination,
|
||||
)
|
||||
from shelfmark.config.email_settings import check_email_connection
|
||||
from shelfmark.config.migrations import migrate_audiobook_formats
|
||||
from shelfmark.core.languages import supported_book_languages
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.settings_registry import (
|
||||
@@ -35,6 +36,7 @@ from shelfmark.core.settings_registry import (
|
||||
register_on_save,
|
||||
register_settings,
|
||||
)
|
||||
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
|
||||
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600
|
||||
@@ -134,6 +136,20 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def migrate_audiobook_format_settings() -> None:
|
||||
"""Bring installs created before the audiobook format sets were unified up to date."""
|
||||
from shelfmark.core.settings_registry import load_config_file, save_config_file
|
||||
|
||||
migrate_audiobook_formats(
|
||||
load_general_config=lambda: load_config_file("general"),
|
||||
save_general_config=lambda values: save_config_file("general", values),
|
||||
widened_formats=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS],
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
|
||||
_SMTP_PORT_MAX = 65535
|
||||
_EMAIL_ATTACHMENT_LIMIT_MB_MAX = 600
|
||||
|
||||
@@ -215,11 +231,7 @@ _FORMAT_OPTIONS = [
|
||||
]
|
||||
|
||||
_AUDIOBOOK_FORMAT_OPTIONS = [
|
||||
{"value": "m4b", "label": "M4B"},
|
||||
{"value": "mp3", "label": "MP3"},
|
||||
{"value": "m4a", "label": "M4A"},
|
||||
{"value": "zip", "label": "ZIP"},
|
||||
{"value": "rar", "label": "RAR"},
|
||||
{"value": fmt, "label": fmt.upper()} for fmt in (*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS)
|
||||
]
|
||||
|
||||
_DOWNLOAD_TO_BROWSER_CONTENT_TYPE_OPTIONS = [
|
||||
@@ -416,14 +428,7 @@ def general_settings() -> list[SettingsField]:
|
||||
label="Supported Audiobook Formats",
|
||||
description="Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.",
|
||||
options=_AUDIOBOOK_FORMAT_OPTIONS,
|
||||
default=["m4b", "mp3"],
|
||||
),
|
||||
MultiSelectField(
|
||||
key="BOOK_LANGUAGE",
|
||||
label="Default Book Languages",
|
||||
description="Default language filter for searches.",
|
||||
options=_LANGUAGE_OPTIONS,
|
||||
default=["en"],
|
||||
default=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS],
|
||||
),
|
||||
]
|
||||
|
||||
@@ -462,6 +467,17 @@ def search_mode_settings() -> list[SettingsField]:
|
||||
default="universal",
|
||||
user_overridable=True,
|
||||
),
|
||||
MultiSelectField(
|
||||
key="BOOK_LANGUAGE",
|
||||
label="Default Book Languages",
|
||||
description=(
|
||||
"Default language filter for searches. Users can override this for their "
|
||||
"own account."
|
||||
),
|
||||
options=_LANGUAGE_OPTIONS,
|
||||
default=["en"],
|
||||
user_overridable=True,
|
||||
),
|
||||
SelectField(
|
||||
key="AA_DEFAULT_SORT",
|
||||
label="Default Sort Order",
|
||||
@@ -754,7 +770,10 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
# Audiobooks are always folder output.
|
||||
if effective.get("FILE_ORGANIZATION_AUDIOBOOK", "rename") == "rename":
|
||||
if effective.get("FILE_ORGANIZATION_AUDIOBOOK", "rename") in {
|
||||
"rename",
|
||||
"rename_and_group",
|
||||
}:
|
||||
template = effective.get("TEMPLATE_AUDIOBOOK_RENAME", "")
|
||||
if _contains_path_separators(template):
|
||||
return {
|
||||
@@ -1282,6 +1301,11 @@ def download_settings() -> list[SettingsField]:
|
||||
"label": "Rename and Organize",
|
||||
"description": "Create folders and rename files using a template. Recommended for Audiobookshelf. Do not use with ingest folders.",
|
||||
},
|
||||
{
|
||||
"value": "rename_and_group",
|
||||
"label": "Rename and Group",
|
||||
"description": "Rename single-file downloads; keep multi-file downloads grouped in their source folder. Do not use with ingest folders.",
|
||||
},
|
||||
],
|
||||
default="rename",
|
||||
universal_only=True,
|
||||
@@ -1300,7 +1324,10 @@ def download_settings() -> list[SettingsField]:
|
||||
),
|
||||
default="{Author} - {Title}",
|
||||
placeholder="{Author} - {Title}{ - Part }{PartNumber}",
|
||||
show_when={"field": "FILE_ORGANIZATION_AUDIOBOOK", "value": "rename"},
|
||||
show_when={
|
||||
"field": "FILE_ORGANIZATION_AUDIOBOOK",
|
||||
"value": ["rename", "rename_and_group"],
|
||||
},
|
||||
universal_only=True,
|
||||
),
|
||||
# Organize mode template - folders allowed
|
||||
@@ -1533,6 +1560,19 @@ def download_source_settings() -> list[SettingsField]:
|
||||
min_value=1,
|
||||
max_value=60,
|
||||
),
|
||||
NumberField(
|
||||
key="RELEASE_SEARCH_TIMEOUT",
|
||||
label="Release Search Timeout (seconds)",
|
||||
description=(
|
||||
"How long one release search may run before it gives up and reports why. "
|
||||
"A first search on a cold start pays for a browser solve, so leave room "
|
||||
"for one. If you use a reverse proxy, its read timeout should be at least "
|
||||
"this high or it will cut the search off with a 504 first."
|
||||
),
|
||||
default=300,
|
||||
min_value=30,
|
||||
max_value=1800,
|
||||
),
|
||||
HeadingField(
|
||||
key="content_type_routing_heading",
|
||||
title="Content-Type Routing",
|
||||
@@ -1643,6 +1683,19 @@ def cloudflare_bypass_settings() -> list[SettingsField]:
|
||||
requires_restart=True,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
|
||||
),
|
||||
NumberField(
|
||||
key="BYPASS_BROWSER_IDLE_TIMEOUT",
|
||||
label="Bypasser Idle Timeout (seconds)",
|
||||
description=(
|
||||
"How long the bypass helper process may sit unused before it is shut down. "
|
||||
"Higher keeps more searches fast, lower frees memory sooner."
|
||||
),
|
||||
default=180,
|
||||
min_value=30,
|
||||
max_value=3600,
|
||||
requires_restart=True,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": False},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ that talks to /api/admin/users endpoints.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from shelfmark.core.languages import normalize_language
|
||||
from shelfmark.core.request_policy import (
|
||||
get_source_content_type_capabilities,
|
||||
parse_policy_mode,
|
||||
@@ -61,7 +62,7 @@ _SELF_SETTINGS_SECTION_OPTIONS = [
|
||||
{
|
||||
"value": "search",
|
||||
"label": "Search Preferences",
|
||||
"description": "Show personal search mode and provider settings.",
|
||||
"description": "Show personal search mode, language, and provider settings.",
|
||||
},
|
||||
{
|
||||
"value": "notifications",
|
||||
@@ -77,8 +78,9 @@ _SEARCH_PREFERENCE_PROVIDER_KEYS = {
|
||||
"METADATA_PROVIDER_AUDIOBOOK",
|
||||
"METADATA_PROVIDER_COMBINED",
|
||||
}
|
||||
_SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
|
||||
SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
|
||||
"SEARCH_MODE",
|
||||
"BOOK_LANGUAGE",
|
||||
"DEFAULT_RELEASE_SOURCE",
|
||||
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK",
|
||||
"SHOW_COMBINED_SELECTOR",
|
||||
@@ -178,14 +180,43 @@ def _get_request_policy_rule_columns() -> list[dict[str, object]]:
|
||||
]
|
||||
|
||||
|
||||
def _validate_book_languages(value: Any) -> tuple[Any, str | None]:
|
||||
"""Validate a per-user default language list against the known languages.
|
||||
|
||||
Accepts the list the settings UI sends as well as a comma-separated string, so an
|
||||
API client can spell the value the way the env var does. Blank entries are skipped
|
||||
rather than rejected, which makes "" and "en," mean the same as [] and ["en"]. An
|
||||
empty result is a deliberate override meaning "no default language filter", so it
|
||||
is kept as-is; ``None`` clears the override further up the chain.
|
||||
"""
|
||||
entries = value.split(",") if isinstance(value, str) else value
|
||||
if not isinstance(entries, (list, tuple)):
|
||||
return value, "BOOK_LANGUAGE must be a list of language codes"
|
||||
|
||||
normalized: list[str] = []
|
||||
for entry in entries:
|
||||
if entry is None or (isinstance(entry, str) and not entry.strip()):
|
||||
continue
|
||||
code = normalize_language(entry)
|
||||
if code is None:
|
||||
return value, f"BOOK_LANGUAGE contains an unsupported language: {entry}"
|
||||
if code not in normalized:
|
||||
normalized.append(code)
|
||||
|
||||
return normalized, None
|
||||
|
||||
|
||||
def validate_search_preference_value(key: str, value: Any) -> tuple[Any, str | None]:
|
||||
"""Validate and normalize a search preference value for user overrides."""
|
||||
if key not in _SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
if key not in SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
return value, None
|
||||
|
||||
if value is None:
|
||||
return None, None
|
||||
|
||||
if key == "BOOK_LANGUAGE":
|
||||
return _validate_book_languages(value)
|
||||
|
||||
normalized_value = str(value).strip()
|
||||
|
||||
if key == "SEARCH_MODE":
|
||||
@@ -298,7 +329,7 @@ def _on_save_users(values: dict[str, object]) -> dict[str, object]:
|
||||
}
|
||||
values["REQUEST_POLICY_RULES"] = normalized_rules
|
||||
|
||||
for key in _SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
for key in SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
if key not in values:
|
||||
continue
|
||||
normalized_value, validation_error = validate_search_preference_value(key, values[key])
|
||||
|
||||
@@ -11,7 +11,10 @@ from shelfmark.config.notifications_settings import (
|
||||
is_valid_notification_url,
|
||||
normalize_notification_routes,
|
||||
)
|
||||
from shelfmark.config.users_settings import validate_search_preference_value
|
||||
from shelfmark.config.users_settings import (
|
||||
SEARCH_PREFERENCE_VALIDATABLE_KEYS,
|
||||
validate_search_preference_value,
|
||||
)
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.request_policy import parse_policy_mode, validate_policy_rules
|
||||
from shelfmark.core.settings_registry import load_config_file
|
||||
@@ -91,13 +94,9 @@ def validate_user_settings(
|
||||
if search_validation_error:
|
||||
errors.append(search_validation_error)
|
||||
continue
|
||||
if key in {
|
||||
"SEARCH_MODE",
|
||||
"METADATA_PROVIDER",
|
||||
"METADATA_PROVIDER_AUDIOBOOK",
|
||||
"DEFAULT_RELEASE_SOURCE",
|
||||
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK",
|
||||
}:
|
||||
# Every key the search validator recognises keeps its normalized value;
|
||||
# a hand-maintained subset here silently dropped normalization for the rest.
|
||||
if key in SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
valid[key] = normalized_search_value
|
||||
continue
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ def upsert_cwa_user(
|
||||
email=normalized_email,
|
||||
role=role,
|
||||
allow_email_link=True,
|
||||
sync_username=True,
|
||||
collision_strategy=collision_strategy,
|
||||
alias_suffix=_CWA_ALIAS_SUFFIX,
|
||||
context=context,
|
||||
|
||||
@@ -108,6 +108,7 @@ def _build_updates(
|
||||
auth_source: str,
|
||||
role: str,
|
||||
sync_role: bool,
|
||||
username: str | object,
|
||||
email: str | None | object,
|
||||
display_name: str | None | object,
|
||||
subject_field: str | None,
|
||||
@@ -116,6 +117,8 @@ def _build_updates(
|
||||
updates: dict[str, Any] = {"auth_source": auth_source}
|
||||
if sync_role:
|
||||
updates["role"] = _normalize_role(role)
|
||||
if username is not UNSET:
|
||||
updates["username"] = _normalize_username(username)
|
||||
if email is not UNSET:
|
||||
updates["email"] = _normalize_email(email)
|
||||
if display_name is not UNSET:
|
||||
@@ -125,10 +128,17 @@ def _build_updates(
|
||||
return updates
|
||||
|
||||
|
||||
def _next_suffix_username(user_db: UserDB, base_username: str) -> str:
|
||||
def _next_suffix_username(
|
||||
user_db: UserDB,
|
||||
base_username: str,
|
||||
*,
|
||||
exclude_user_id: int | None = None,
|
||||
) -> str:
|
||||
candidate = base_username
|
||||
suffix = 1
|
||||
while user_db.get_user(username=candidate):
|
||||
while existing := user_db.get_user(username=candidate):
|
||||
if exclude_user_id is not None and int(existing.get("id") or 0) == exclude_user_id:
|
||||
return candidate
|
||||
candidate = f"{base_username}_{suffix}"
|
||||
suffix += 1
|
||||
return candidate
|
||||
@@ -185,6 +195,38 @@ def _resolve_create_username(
|
||||
return _next_suffix_username(user_db, alias_base), None, "username_collision_alias"
|
||||
|
||||
|
||||
def _resolve_update_username(
|
||||
user_db: UserDB,
|
||||
*,
|
||||
current_user: dict[str, Any],
|
||||
requested_username: str,
|
||||
strategy: CollisionStrategy,
|
||||
alias_suffix: str,
|
||||
) -> str:
|
||||
current_user_id = int(current_user["id"])
|
||||
existing = user_db.get_user(username=requested_username)
|
||||
if existing is None or int(existing.get("id") or 0) == current_user_id:
|
||||
return requested_username
|
||||
|
||||
if strategy == "suffix":
|
||||
return _next_suffix_username(
|
||||
user_db,
|
||||
requested_username,
|
||||
exclude_user_id=current_user_id,
|
||||
)
|
||||
if strategy == "alias":
|
||||
return _next_suffix_username(
|
||||
user_db,
|
||||
f"{requested_username}{alias_suffix}",
|
||||
exclude_user_id=current_user_id,
|
||||
)
|
||||
|
||||
# `takeover` can select an existing row during creation, but once an
|
||||
# identity is already matched it must never replace a different username
|
||||
# owner. Preserve the matched row's current collision-free name instead.
|
||||
return str(current_user["username"])
|
||||
|
||||
|
||||
def upsert_external_user(
|
||||
user_db: UserDB,
|
||||
*,
|
||||
@@ -197,6 +239,7 @@ def upsert_external_user(
|
||||
subject: str | None = None,
|
||||
allow_email_link: bool = False,
|
||||
sync_role: bool = True,
|
||||
sync_username: bool = False,
|
||||
allow_create: bool = True,
|
||||
collision_strategy: CollisionStrategy = "takeover",
|
||||
alias_suffix: str | None = None,
|
||||
@@ -229,10 +272,26 @@ def upsert_external_user(
|
||||
subject=subject,
|
||||
allow_email_link=allow_email_link,
|
||||
)
|
||||
resolved_alias_suffix = alias_suffix or f"__{auth_source}"
|
||||
update_username: str | object = UNSET
|
||||
if (
|
||||
matched is not None
|
||||
and sync_username
|
||||
and normalize_auth_source(matched.get("auth_source"), matched.get("oidc_subject"))
|
||||
== auth_source
|
||||
):
|
||||
update_username = _resolve_update_username(
|
||||
user_db,
|
||||
current_user=matched,
|
||||
requested_username=normalized_username,
|
||||
strategy=collision_strategy,
|
||||
alias_suffix=resolved_alias_suffix,
|
||||
)
|
||||
updates = _build_updates(
|
||||
auth_source=auth_source,
|
||||
role=normalized_role,
|
||||
sync_role=sync_role,
|
||||
username=update_username,
|
||||
email=normalized_email if email is not UNSET else UNSET,
|
||||
display_name=normalized_display_name if display_name is not UNSET else UNSET,
|
||||
subject_field=subject_field,
|
||||
@@ -261,7 +320,6 @@ def upsert_external_user(
|
||||
)
|
||||
return None, "not_found"
|
||||
|
||||
resolved_alias_suffix = alias_suffix or f"__{auth_source}"
|
||||
create_username, takeover_target, create_reason = _resolve_create_username(
|
||||
user_db,
|
||||
auth_source=auth_source,
|
||||
|
||||
@@ -136,6 +136,12 @@ class DownloadTask:
|
||||
default_factory=dict
|
||||
) # Per-output parameters (e.g. email recipient)
|
||||
|
||||
# Multi-book packs: one release holding several books. `book_plan` is the split the
|
||||
# user approved before download (list of {title, series_position, year, files});
|
||||
# `multi_book` asks post-processing to split heuristically when no plan exists.
|
||||
multi_book: bool = False
|
||||
book_plan: list[dict[str, Any]] | None = None
|
||||
|
||||
# User association (multi-user support)
|
||||
user_id: int | None = None # DB user ID who queued this download
|
||||
username: str | None = None # Username for {User} template variable
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Pre-download release inspection: list a release's files and plan a multi-book split."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import is_audiobook
|
||||
from shelfmark.download.postprocess.packs import PackFile, PackPlan, plan_pack
|
||||
from shelfmark.download.postprocess.policy import (
|
||||
get_supported_audiobook_formats,
|
||||
get_supported_formats,
|
||||
)
|
||||
from shelfmark.release_sources import get_handler
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from flask import Flask, Response
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
_INSPECT_ERRORS = (OSError, RuntimeError, ValueError, TypeError, KeyError, AttributeError)
|
||||
NOT_INSPECTABLE_REASON = "This source cannot list the release's files before downloading"
|
||||
|
||||
|
||||
def _serialize_plan(plan: PackPlan) -> dict[str, Any]:
|
||||
return {
|
||||
"is_pack": plan.is_pack,
|
||||
"ignored": plan.ignored,
|
||||
"books": [
|
||||
{
|
||||
"title": book.title,
|
||||
"series_position": book.series_position,
|
||||
"year": book.year,
|
||||
"files": book.files,
|
||||
}
|
||||
for book in plan.books
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def inspect_release(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Build the inspect response for a release payload (same shape as a download)."""
|
||||
source = str(data["source"])
|
||||
handler = get_handler(source)
|
||||
try:
|
||||
files: list[PackFile] | None = handler.list_files(data)
|
||||
except _INSPECT_ERRORS as exc:
|
||||
logger.warning(
|
||||
"Could not list files for %s release %s: %s", source, data.get("source_id"), exc
|
||||
)
|
||||
return {"inspected": False, "reason": str(exc), "files": [], "plan": None}
|
||||
|
||||
if files is None:
|
||||
return {"inspected": False, "reason": NOT_INSPECTABLE_REASON, "files": [], "plan": None}
|
||||
|
||||
content_type = data.get("content_type")
|
||||
supported = (
|
||||
get_supported_audiobook_formats()
|
||||
if is_audiobook(content_type if isinstance(content_type, str) else None)
|
||||
else get_supported_formats()
|
||||
)
|
||||
series_name = data.get("series_name")
|
||||
author_name = data.get("author")
|
||||
plan = plan_pack(
|
||||
files,
|
||||
supported_extensions=set(supported),
|
||||
series_name=series_name if isinstance(series_name, str) else None,
|
||||
author_name=author_name if isinstance(author_name, str) else None,
|
||||
)
|
||||
return {
|
||||
"inspected": True,
|
||||
"reason": None,
|
||||
"files": [{"path": f.path, "size": f.size} for f in files],
|
||||
"plan": _serialize_plan(plan),
|
||||
}
|
||||
|
||||
|
||||
def register_release_inspect_routes(
|
||||
app: Flask,
|
||||
login_required: Callable[..., Any],
|
||||
) -> None:
|
||||
"""Register POST /api/releases/inspect."""
|
||||
|
||||
@app.route("/api/releases/inspect", methods=["POST"])
|
||||
@login_required
|
||||
def api_inspect_release() -> Response | tuple[Response, int]:
|
||||
data = request.get_json(silent=True)
|
||||
if not isinstance(data, dict):
|
||||
return jsonify({"error": "No data provided"}), 400
|
||||
if not data.get("source_id"):
|
||||
return jsonify({"error": "source_id is required"}), 400
|
||||
if not data.get("source"):
|
||||
return jsonify({"error": "source is required"}), 400
|
||||
try:
|
||||
get_handler(str(data["source"]))
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify(inspect_release(data))
|
||||
@@ -0,0 +1,134 @@
|
||||
"""A wall-clock budget for one release search, enforced through the existing cancel flag.
|
||||
|
||||
`/api/releases` is synchronous: the browser waits on it while the search runs. Nothing
|
||||
bounded that wait, and the bypasser's own worst case is minutes long
|
||||
(`internal_bypasser.max_duration_seconds()`), so a search that ran into an unsolvable
|
||||
protection challenge outlived every reverse proxy in front of it. The user then saw
|
||||
"Server unavailable (504)" - a gateway timeout that says nothing about what went wrong
|
||||
and points the blame at their proxy config. See issue #1276.
|
||||
|
||||
The budget is expressed as the cancel flag the download path already understands: an
|
||||
Event armed by a timer. `html_get_page`, the bypassers and the helper subprocess all poll
|
||||
it, so an expired budget stops a solve already in flight rather than only refusing the
|
||||
next one. When it trips, the search fails with a message that names the real cause.
|
||||
|
||||
Scoped to a context variable so it applies to the request that set it and to nothing else
|
||||
- a queued download must keep its own, much longer, budget.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# What one search may spend. A first search on a cold start legitimately pays for a
|
||||
# browser solve - jfmlima measured 60-120s for a successful one on Anna's Archive - so
|
||||
# this cannot be as tight as a proxy's default read timeout without breaking working
|
||||
# setups. It is instead well below the ~840s the bypass path could previously reach,
|
||||
# which is what turned a failing challenge into a gateway timeout.
|
||||
DEFAULT_SEARCH_BUDGET_SECONDS = 300.0
|
||||
|
||||
_MIN_SEARCH_BUDGET_SECONDS = 30.0
|
||||
_MAX_SEARCH_BUDGET_SECONDS = 1800.0
|
||||
|
||||
# Raised to the caller when the budget runs out, so the API can say so plainly.
|
||||
SEARCH_DEADLINE_MESSAGE = (
|
||||
"The release search ran out of time (%.0fs). Anna's Archive is behind a protection "
|
||||
"challenge the bypasser could not solve in that window. Raise the release search "
|
||||
"timeout if your setup is simply slow."
|
||||
)
|
||||
|
||||
|
||||
class SearchDeadline:
|
||||
"""A budget with an Event that trips when it expires."""
|
||||
|
||||
def __init__(self, budget_seconds: float) -> None:
|
||||
self.budget_seconds = budget_seconds
|
||||
self.expires_at = time.monotonic() + budget_seconds
|
||||
# A plain threading.Event on purpose: this is handed on as a cancel flag, and
|
||||
# that is the type the download path, the CDP worker thread and the bypass helper
|
||||
# already poll.
|
||||
self.event = threading.Event()
|
||||
self._timer = threading.Timer(budget_seconds, self.event.set)
|
||||
self._timer.daemon = True
|
||||
|
||||
def start(self) -> None:
|
||||
self._timer.start()
|
||||
|
||||
def cancel(self) -> None:
|
||||
self._timer.cancel()
|
||||
|
||||
@property
|
||||
def remaining(self) -> float:
|
||||
return max(0.0, self.expires_at - time.monotonic())
|
||||
|
||||
@property
|
||||
def expired(self) -> bool:
|
||||
return self.event.is_set() or self.remaining <= 0
|
||||
|
||||
|
||||
_current: ContextVar[SearchDeadline | None] = ContextVar("search_deadline", default=None)
|
||||
|
||||
|
||||
def budget_seconds() -> float:
|
||||
"""The configured budget for one release search."""
|
||||
from shelfmark.core.config import config as app_config
|
||||
|
||||
raw = app_config.get("RELEASE_SEARCH_TIMEOUT", DEFAULT_SEARCH_BUDGET_SECONDS)
|
||||
if isinstance(raw, bool) or not isinstance(raw, int | float | str):
|
||||
return DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
try:
|
||||
value = float(raw)
|
||||
except TypeError, ValueError:
|
||||
return DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
if value <= 0:
|
||||
return DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
return min(max(value, _MIN_SEARCH_BUDGET_SECONDS), _MAX_SEARCH_BUDGET_SECONDS)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def search_deadline(budget: float | None = None) -> Iterator[SearchDeadline]:
|
||||
"""Apply a budget to everything the calling context does."""
|
||||
deadline = SearchDeadline(budget if budget is not None else budget_seconds())
|
||||
token = _current.set(deadline)
|
||||
deadline.start()
|
||||
logger.debug("Release search budget: %.0fs", deadline.budget_seconds)
|
||||
try:
|
||||
yield deadline
|
||||
finally:
|
||||
deadline.cancel()
|
||||
_current.reset(token)
|
||||
|
||||
|
||||
def current() -> SearchDeadline | None:
|
||||
"""The budget in force, or None outside a search."""
|
||||
return _current.get()
|
||||
|
||||
|
||||
def expired() -> bool:
|
||||
"""Whether the budget in force has run out. False when there is no budget."""
|
||||
deadline = _current.get()
|
||||
return deadline is not None and deadline.expired
|
||||
|
||||
|
||||
def cancel_event() -> threading.Event | None:
|
||||
"""The Event that trips when the budget runs out, for use as a cancel flag."""
|
||||
deadline = _current.get()
|
||||
return deadline.event if deadline is not None else None
|
||||
|
||||
|
||||
def deadline_message() -> str:
|
||||
"""The failure to report when the budget has run out."""
|
||||
deadline = _current.get()
|
||||
budget = deadline.budget_seconds if deadline else DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
return SEARCH_DEADLINE_MESSAGE % budget
|
||||
@@ -7,6 +7,7 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.metadata_providers import (
|
||||
BookMetadata,
|
||||
build_localized_search_titles,
|
||||
@@ -16,6 +17,8 @@ from shelfmark.metadata_providers import (
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.models import SearchFilters
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
MANUAL_QUERY_MAX_LEN = 256
|
||||
|
||||
|
||||
@@ -52,30 +55,61 @@ class ReleaseSearchPlan:
|
||||
return self.title_variants[0].query if self.title_variants else ""
|
||||
|
||||
|
||||
def _normalize_languages(languages: list[str] | None) -> list[str] | None:
|
||||
def _to_language_codes(values: Iterable[object], *, source: str) -> list[str] | None:
|
||||
"""Resolve any spelling of a language to the ISO code the sources expect.
|
||||
|
||||
Anna's Archive matches `lang=` against ISO codes: `lang=english` is not a loose
|
||||
spelling of `lang=en`, it is a facet value AA does not have, and it filters every
|
||||
search down to nothing. Only the *per-user* override was normalised
|
||||
(config.users_settings.validate), so a global BOOK_LANGUAGE=english - the spelling
|
||||
the old docs used - reached the query verbatim and silently emptied every search
|
||||
with no error anywhere. See issue #1276.
|
||||
|
||||
An entry that resolves to nothing is dropped with a warning rather than passed
|
||||
through: searching unfiltered and saying so beats reporting "no results" for a book
|
||||
the source is full of.
|
||||
"""
|
||||
from shelfmark.core.languages import normalize_language
|
||||
|
||||
codes: list[str] = []
|
||||
unresolved: list[str] = []
|
||||
for value in values:
|
||||
text = str(value).strip() if value is not None else ""
|
||||
if not text:
|
||||
continue
|
||||
if text.lower() == "all":
|
||||
# An explicit "search every language", not a language.
|
||||
return None
|
||||
code = normalize_language(text)
|
||||
if code is None:
|
||||
unresolved.append(text)
|
||||
continue
|
||||
if code not in codes:
|
||||
codes.append(code)
|
||||
|
||||
if unresolved:
|
||||
logger.warning(
|
||||
"Ignoring unrecognised language(s) in %s: %s. Use an ISO code such as 'en', "
|
||||
"a three-letter code, or an English name like 'English'.",
|
||||
source,
|
||||
", ".join(unresolved),
|
||||
)
|
||||
|
||||
return codes or None
|
||||
|
||||
|
||||
def _normalize_languages(languages: list[str] | None, user_id: int | None) -> list[str] | None:
|
||||
if not languages:
|
||||
default = getattr(config, "BOOK_LANGUAGE", None)
|
||||
default = config.get("BOOK_LANGUAGE", None, user_id=user_id)
|
||||
if isinstance(default, str):
|
||||
default_values: list[object] = [default]
|
||||
elif isinstance(default, Iterable) and not isinstance(default, (bytes, bytearray, dict)):
|
||||
default_values = list(default)
|
||||
else:
|
||||
return None
|
||||
return [str(lang).strip() for lang in default_values if str(lang).strip()]
|
||||
return _to_language_codes(default_values, source="BOOK_LANGUAGE")
|
||||
|
||||
normalized: list[str] = []
|
||||
for lang in languages:
|
||||
if not lang:
|
||||
continue
|
||||
s = str(lang).strip()
|
||||
if not s:
|
||||
continue
|
||||
normalized.append(s)
|
||||
|
||||
if any(lang.lower() == "all" for lang in normalized):
|
||||
return None
|
||||
|
||||
return normalized or None
|
||||
return _to_language_codes(languages, source="the search request")
|
||||
|
||||
|
||||
def _pick_search_author(book: BookMetadata) -> str:
|
||||
@@ -102,9 +136,15 @@ def build_release_search_plan(
|
||||
manual_query: str | None = None,
|
||||
indexers: list[str] | None = None,
|
||||
source_filters: SearchFilters | None = None,
|
||||
user_id: int | None = None,
|
||||
) -> ReleaseSearchPlan:
|
||||
"""Build normalized search variants shared across release sources."""
|
||||
resolved_languages = _normalize_languages(languages)
|
||||
"""Build normalized search variants shared across release sources.
|
||||
|
||||
``user_id`` picks up that user's default languages when the caller does not
|
||||
filter explicitly, so a search started without a language filter uses the
|
||||
reader's own default rather than the instance-wide one.
|
||||
"""
|
||||
resolved_languages = _normalize_languages(languages, user_id)
|
||||
|
||||
resolved_manual_query = None
|
||||
if manual_query:
|
||||
|
||||
@@ -344,6 +344,7 @@ class UserDB:
|
||||
|
||||
_ALLOWED_UPDATE_COLUMNS: ClassVar[frozenset[str]] = frozenset(
|
||||
{
|
||||
"username",
|
||||
"email",
|
||||
"display_name",
|
||||
"password_hash",
|
||||
@@ -353,6 +354,7 @@ class UserDB:
|
||||
}
|
||||
)
|
||||
_USER_UPDATE_STATEMENTS: ClassVar[dict[str, str]] = {
|
||||
"username": "UPDATE users SET username = ? WHERE id = ?",
|
||||
"email": "UPDATE users SET email = ? WHERE id = ?",
|
||||
"display_name": "UPDATE users SET display_name = ? WHERE id = ?",
|
||||
"password_hash": "UPDATE users SET password_hash = ? WHERE id = ?",
|
||||
|
||||
@@ -115,6 +115,26 @@ def is_audiobook(content_type: str | None) -> bool:
|
||||
return bool(content_type and "audiobook" in content_type.lower())
|
||||
|
||||
|
||||
# Every audio format an audiobook can legitimately arrive in, and the single source of
|
||||
# truth for that list. The settings UI, release-source parsing, archive extraction and
|
||||
# post-download scanning all derive from it, so a format added here becomes selectable,
|
||||
# searchable AND downloadable at once. These used to be four hand-maintained copies that
|
||||
# had drifted apart: the settings UI only offered m4b/mp3/m4a, which meant a FLAC
|
||||
# audiobook could never be enabled, was silently dropped from every search result, and
|
||||
# was rejected after download as "format not supported".
|
||||
#
|
||||
# "mp4" is here because some trackers (MyAnonamouse in particular) ship AAC audiobooks
|
||||
# as per-chapter .mp4 files - the same ISO-BMFF container as .m4a/.m4b, just with the
|
||||
# generic extension. Without it those releases downloaded fine and then failed
|
||||
# post-processing with "No book files found in download".
|
||||
AUDIOBOOK_FORMATS = ("m4b", "mp3", "m4a", "mp4", "flac", "ogg", "wma", "aac", "wav", "opus")
|
||||
|
||||
# Multi-file audiobooks are almost always distributed as an archive. These are containers
|
||||
# rather than formats: they are what a *release* looks like, and the formats above are
|
||||
# what comes out of one after extraction.
|
||||
ARCHIVE_FORMATS = ("zip", "rar")
|
||||
|
||||
|
||||
CONTENT_TYPES = [
|
||||
"book (fiction)",
|
||||
"book (non-fiction)",
|
||||
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.fs import atomic_move
|
||||
from shelfmark.download.postprocess.policy import (
|
||||
@@ -98,7 +99,7 @@ ALL_EBOOK_EXTENSIONS = {
|
||||
}
|
||||
|
||||
# All known audio extensions (superset of what user might enable for audiobooks)
|
||||
ALL_AUDIO_EXTENSIONS = {".m4b", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".wma", ".wav", ".opus"}
|
||||
ALL_AUDIO_EXTENSIONS = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS}
|
||||
|
||||
|
||||
def _filter_files(
|
||||
|
||||
@@ -26,6 +26,11 @@ from shelfmark.download.clients import (
|
||||
register_client,
|
||||
)
|
||||
from shelfmark.download.clients._coercion import config_text
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
DebridMagnet,
|
||||
DebridUpload,
|
||||
resolve_debrid_upload,
|
||||
)
|
||||
from shelfmark.download.http import download_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
|
||||
@@ -71,6 +76,7 @@ _BOOK_EXTENSIONS = (
|
||||
".m4b",
|
||||
".mobi",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".ogg",
|
||||
".opus",
|
||||
".pdf",
|
||||
@@ -202,41 +208,19 @@ class AllDebridClient(DownloadClient):
|
||||
expected_hash: str | None = None,
|
||||
**kwargs: object,
|
||||
) -> str:
|
||||
"""Upload a magnet link to AllDebrid and return the magnet ID."""
|
||||
"""Send a torrent to AllDebrid and return the magnet ID.
|
||||
|
||||
Accepts a magnet link, a .torrent URL, or an indexer proxy URL; anything
|
||||
that is not already a magnet is resolved first, since an HTTP URL posted
|
||||
as a magnet is rejected rather than downloaded (#1250).
|
||||
"""
|
||||
if not self._api_key:
|
||||
msg = "AllDebrid API key is not configured"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
magnet_link = url
|
||||
if not magnet_link.startswith("magnet:") and expected_hash:
|
||||
magnet_link = f"magnet:?xt=urn:btih:{expected_hash}"
|
||||
|
||||
api_url = f"{_API_BASE}/magnet/upload"
|
||||
try:
|
||||
resp = requests.post(
|
||||
api_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnets[]": magnet_link},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(api_url),
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("status") != "success":
|
||||
code = data.get("error", {}).get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid upload failed: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
magnets = data.get("data", {}).get("magnets", [])
|
||||
if not magnets:
|
||||
msg = "No magnet returned from AllDebrid"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
info = magnets[0]
|
||||
if info.get("error"):
|
||||
code = info["error"].get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid magnet error: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
upload = resolve_debrid_upload(url, expected_hash=expected_hash)
|
||||
info = self._send_torrent(upload)
|
||||
|
||||
magnet_id = str(info.get("id", ""))
|
||||
if not magnet_id:
|
||||
@@ -262,12 +246,65 @@ class AllDebridClient(DownloadClient):
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Failed to upload magnet to AllDebrid")
|
||||
logger.exception("Failed to add torrent to AllDebrid")
|
||||
raise
|
||||
|
||||
else:
|
||||
return magnet_id
|
||||
|
||||
def _send_torrent(self, upload: DebridUpload) -> dict[str, Any]:
|
||||
"""Hand the torrent to AllDebrid, as a magnet or as a file upload.
|
||||
|
||||
Both endpoints answer with the same envelope and the same per-entry
|
||||
error shape, differing only in which key holds the entries.
|
||||
"""
|
||||
if isinstance(upload, DebridMagnet):
|
||||
api_url = f"{_API_BASE}/magnet/upload"
|
||||
entries_key = "magnets"
|
||||
resp = requests.post(
|
||||
api_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnets[]": upload.magnet_url},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(api_url),
|
||||
)
|
||||
else:
|
||||
api_url = f"{_API_BASE}/magnet/upload/file"
|
||||
entries_key = "files"
|
||||
resp = requests.post(
|
||||
api_url,
|
||||
headers=self._auth_headers(),
|
||||
files={
|
||||
"files[]": (
|
||||
"release.torrent",
|
||||
upload.torrent_data,
|
||||
"application/x-bittorrent",
|
||||
)
|
||||
},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(api_url),
|
||||
)
|
||||
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("status") != "success":
|
||||
code = data.get("error", {}).get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid upload failed: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
entries = data.get("data", {}).get(entries_key, [])
|
||||
if not entries:
|
||||
msg = "AllDebrid accepted the upload but returned no torrent"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
info = entries[0]
|
||||
if info.get("error"):
|
||||
code = info["error"].get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid rejected the torrent: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
return info
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""Poll AllDebrid for magnet status and drive the download."""
|
||||
state = self._ensure_state(download_id)
|
||||
|
||||
@@ -843,6 +843,9 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
expected_hash=request.expected_hash,
|
||||
seeding_time_limit=request.seeding_time_limit,
|
||||
ratio_limit=request.ratio_limit,
|
||||
# rTorrent has no category concept, so its audiobook label
|
||||
# can only be chosen from the content type (#1235).
|
||||
content_type=task.content_type,
|
||||
)
|
||||
except Exception as e:
|
||||
if not refresh_attempted:
|
||||
|
||||
@@ -45,6 +45,10 @@ _HASH_LENGTH_ED2K = 32
|
||||
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
|
||||
_HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
|
||||
_METADATA_DOWNLOAD_STATES = {"forcedMetaDL", "metaDL"}
|
||||
# How long add_download waits for magnet metadata before falling back to the info
|
||||
# hash it already knows, rather than holding the download queue on a thin swarm.
|
||||
_METADATA_WAIT_POLLS = 20
|
||||
_METADATA_WAIT_INTERVAL_SECONDS = 0.5
|
||||
_ONE_WEEK_IN_SECONDS = 604800
|
||||
|
||||
|
||||
@@ -221,6 +225,9 @@ class QBittorrentClient(DownloadClient):
|
||||
self._category = config_text(config.get("QBITTORRENT_CATEGORY", "books"))
|
||||
self._download_dir = config_text(config.get("QBITTORRENT_DOWNLOAD_DIR", ""))
|
||||
self._tags = _normalize_tags(config.get("QBITTORRENT_TAG", []))
|
||||
# download_id -> qBittorrent's current primary hash, for identities that no
|
||||
# longer match it directly. See _resolve_torrent().
|
||||
self._primary_hashes: dict[str, str] = {}
|
||||
|
||||
@property
|
||||
def _can_reauthenticate(self) -> bool:
|
||||
@@ -307,13 +314,31 @@ class QBittorrentClient(DownloadClient):
|
||||
params = {"category": category} if category else {}
|
||||
return self._request_torrent_info_records(params)
|
||||
|
||||
def _remember_primary_hash(self, download_id: str, torrent: SimpleNamespace) -> None:
|
||||
"""Note the primary hash a listing scan found, so later lookups skip the scan."""
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
self._primary_hashes[download_id.lower()] = torrent_hash.lower()
|
||||
|
||||
def _resolve_torrent(
|
||||
self, download_id: str, category: str | None = None
|
||||
) -> tuple[SimpleNamespace | None, str | None]:
|
||||
"""Resolve any known torrent identity to its current qBittorrent record."""
|
||||
torrent, error = self._get_torrent_info(download_id)
|
||||
if error or torrent:
|
||||
return torrent, error
|
||||
"""Resolve any known torrent identity to its current qBittorrent record.
|
||||
|
||||
A hybrid torrent's primary hash switches from the v1 hash to the truncated v2
|
||||
hash once metadata resolves, so a download tracked by its v1 hash misses the
|
||||
`hashes=` lookup and falls through to a full listing. Since `get_status()`
|
||||
polls every couple of seconds for the life of the download, remember the
|
||||
primary hash a scan finds and try it first.
|
||||
"""
|
||||
cached = self._primary_hashes.get(download_id.lower())
|
||||
for candidate in (item for item in dict.fromkeys((cached, download_id)) if item):
|
||||
torrent, error = self._get_torrent_info(candidate)
|
||||
if error:
|
||||
return None, error
|
||||
if torrent:
|
||||
self._remember_primary_hash(download_id, torrent)
|
||||
return torrent, None
|
||||
|
||||
categories = [candidate for candidate in (category, self._category) if candidate]
|
||||
for candidate in dict.fromkeys(categories):
|
||||
@@ -325,18 +350,41 @@ class QBittorrentClient(DownloadClient):
|
||||
None,
|
||||
)
|
||||
if torrent:
|
||||
self._remember_primary_hash(download_id, torrent)
|
||||
return torrent, None
|
||||
|
||||
torrents, error = self._list_torrents_by_category(None)
|
||||
if error:
|
||||
return None, error
|
||||
return (
|
||||
next(
|
||||
(item for item in torrents if _torrent_matches_download_id(item, download_id)),
|
||||
None,
|
||||
),
|
||||
torrent = next(
|
||||
(item for item in torrents if _torrent_matches_download_id(item, download_id)),
|
||||
None,
|
||||
)
|
||||
if torrent:
|
||||
self._remember_primary_hash(download_id, torrent)
|
||||
else:
|
||||
# The torrent is gone; drop the note so a re-add is not looked up by a
|
||||
# hash that no longer exists.
|
||||
self._primary_hashes.pop(download_id.lower(), None)
|
||||
return torrent, None
|
||||
|
||||
def _current_hash(self, download_id: str) -> str:
|
||||
"""qBittorrent's current primary hash for any identity we know the torrent by.
|
||||
|
||||
Falls back to the given ID when the torrent cannot be found, so callers
|
||||
still address the hash they were handed and surface the client's error.
|
||||
"""
|
||||
try:
|
||||
torrent, error = self._resolve_torrent(download_id)
|
||||
except _QBITTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("Could not resolve current hash for %s: %s", download_id, e)
|
||||
return download_id
|
||||
if error or not torrent:
|
||||
return download_id
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
return torrent_hash
|
||||
return download_id
|
||||
|
||||
def _list_category_hashes(self, category: str | None) -> set[str] | None:
|
||||
"""Snapshot the hashes qBittorrent currently reports for a category."""
|
||||
@@ -495,9 +543,13 @@ class QBittorrentClient(DownloadClient):
|
||||
message = f"{message} (torrent file fetch failed: {torrent_info.fetch_error})"
|
||||
_raise_runtime_error(message)
|
||||
|
||||
# Wait until qBittorrent has resolved magnet metadata so the returned
|
||||
# hash is its stable primary torrent ID, which may differ from the v1 hash.
|
||||
for _ in range(20):
|
||||
# Prefer qBittorrent's primary torrent ID, which for hybrid torrents
|
||||
# switches from the v1 hash to the truncated v2 hash once metadata
|
||||
# resolves. A magnet with few peers can take minutes to fetch metadata,
|
||||
# and the torrent is worth keeping in the meantime: every lookup goes
|
||||
# through `_resolve_torrent`, which still matches the v1 hash against
|
||||
# `infohash_v1` after the primary ID has changed.
|
||||
for _ in range(_METADATA_WAIT_POLLS):
|
||||
torrent, error = self._resolve_torrent(expected_hash, category)
|
||||
if error:
|
||||
logger.debug("qBittorrent add_download: %s", error)
|
||||
@@ -506,17 +558,18 @@ class QBittorrentClient(DownloadClient):
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
logger.info("Added torrent: %s", torrent_hash)
|
||||
return torrent_hash.lower()
|
||||
time.sleep(0.5)
|
||||
time.sleep(_METADATA_WAIT_INTERVAL_SECONDS)
|
||||
|
||||
_raise_runtime_error(
|
||||
"Torrent metadata resolution was not confirmed within the visibility grace period "
|
||||
f"(response={result_text})"
|
||||
logger.info(
|
||||
"Added torrent %s; metadata still pending after %.0fs, tracking it by info hash",
|
||||
expected_hash,
|
||||
_METADATA_WAIT_POLLS * _METADATA_WAIT_INTERVAL_SECONDS,
|
||||
)
|
||||
except _QBITTORRENT_CLIENT_ERRORS:
|
||||
logger.exception("qBittorrent add failed")
|
||||
raise
|
||||
else:
|
||||
return expected_hash
|
||||
return expected_hash.lower()
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""Get torrent status by hash.
|
||||
@@ -529,7 +582,7 @@ class QBittorrentClient(DownloadClient):
|
||||
|
||||
"""
|
||||
try:
|
||||
torrent, error = self._get_torrent_info(download_id)
|
||||
torrent, error = self._resolve_torrent(download_id)
|
||||
if error:
|
||||
return DownloadStatus.error(error)
|
||||
if not torrent:
|
||||
@@ -613,7 +666,9 @@ class QBittorrentClient(DownloadClient):
|
||||
|
||||
"""
|
||||
try:
|
||||
self._client.torrents_delete(torrent_hashes=download_id, delete_files=delete_files)
|
||||
torrent_hash = self._current_hash(download_id)
|
||||
self._client.torrents_delete(torrent_hashes=torrent_hash, delete_files=delete_files)
|
||||
self._primary_hashes.pop(download_id.lower(), None)
|
||||
logger.info(
|
||||
"Removed torrent from qBittorrent: %s%s",
|
||||
download_id,
|
||||
@@ -635,7 +690,7 @@ class QBittorrentClient(DownloadClient):
|
||||
logger.debug("Could not create category '%s': %s", category, e)
|
||||
|
||||
self._client.torrents_set_category(
|
||||
torrent_hashes=download_id,
|
||||
torrent_hashes=self._current_hash(download_id),
|
||||
category=category,
|
||||
)
|
||||
logger.info("Set qBittorrent category for %s to '%s'", download_id, category)
|
||||
@@ -657,7 +712,7 @@ class QBittorrentClient(DownloadClient):
|
||||
- join `save_path` with the torrent's top-level directory
|
||||
"""
|
||||
try:
|
||||
torrent, error = self._get_torrent_info(download_id)
|
||||
torrent, error = self._resolve_torrent(download_id)
|
||||
if error:
|
||||
logger.debug("qBittorrent get_download_path: %s", error)
|
||||
return None
|
||||
@@ -758,6 +813,33 @@ class QBittorrentClient(DownloadClient):
|
||||
)
|
||||
return None
|
||||
|
||||
def _await_existing_torrent(
|
||||
self, info_hash: str, category: str | None
|
||||
) -> tuple[str, DownloadStatus] | None:
|
||||
"""Report a torrent already in qBittorrent, waiting out magnet metadata first."""
|
||||
for _ in range(_METADATA_WAIT_POLLS):
|
||||
torrent, error = self._resolve_torrent(info_hash, category)
|
||||
if error:
|
||||
logger.debug("qBittorrent find_existing: %s", error)
|
||||
return None
|
||||
if not torrent:
|
||||
return None
|
||||
if getattr(torrent, "state", None) not in _METADATA_DOWNLOAD_STATES:
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
torrent_hash = torrent_hash.lower()
|
||||
return (torrent_hash, self.get_status(torrent_hash))
|
||||
time.sleep(_METADATA_WAIT_INTERVAL_SECONDS)
|
||||
|
||||
# Metadata is still pending, but the torrent is here and `add_download` keeps
|
||||
# one in this state rather than giving up. Report it by info hash so the
|
||||
# caller joins the download in progress instead of adding a duplicate.
|
||||
logger.info(
|
||||
"Existing torrent %s is still fetching metadata; joining it by info hash",
|
||||
info_hash,
|
||||
)
|
||||
return (info_hash.lower(), self.get_status(info_hash))
|
||||
|
||||
def find_existing(
|
||||
self, url: str, category: str | None = None
|
||||
) -> tuple[str, DownloadStatus] | None:
|
||||
@@ -767,21 +849,9 @@ class QBittorrentClient(DownloadClient):
|
||||
if not torrent_info.info_hash:
|
||||
return None
|
||||
|
||||
for _ in range(20):
|
||||
torrent, error = self._resolve_torrent(torrent_info.info_hash, category)
|
||||
if error:
|
||||
logger.debug("qBittorrent find_existing: %s", error)
|
||||
return None
|
||||
if not torrent:
|
||||
return None
|
||||
if getattr(torrent, "state", None) not in _METADATA_DOWNLOAD_STATES:
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
torrent_hash = torrent_hash.lower()
|
||||
return (torrent_hash, self.get_status(torrent_hash))
|
||||
time.sleep(0.5)
|
||||
existing = self._await_existing_torrent(torrent_info.info_hash, category)
|
||||
except _QBITTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("Error checking for existing torrent: %s", e)
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
return existing
|
||||
|
||||
@@ -24,6 +24,11 @@ from shelfmark.download.clients import (
|
||||
register_client,
|
||||
)
|
||||
from shelfmark.download.clients._coercion import config_text
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
DebridMagnet,
|
||||
DebridUpload,
|
||||
resolve_debrid_upload,
|
||||
)
|
||||
from shelfmark.download.http import download_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
|
||||
@@ -75,6 +80,7 @@ _BOOK_EXTENSIONS = (
|
||||
".m4b",
|
||||
".mobi",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".ogg",
|
||||
".opus",
|
||||
".pdf",
|
||||
@@ -173,26 +179,19 @@ class RealDebridClient(DownloadClient):
|
||||
expected_hash: str | None = None,
|
||||
**kwargs: object,
|
||||
) -> str:
|
||||
"""Upload a magnet link to Real-Debrid and select all files."""
|
||||
"""Send a torrent to Real-Debrid and select all files.
|
||||
|
||||
Accepts a magnet link, a .torrent URL, or an indexer proxy URL; anything
|
||||
that is not already a magnet is resolved first, because Real-Debrid
|
||||
answers a non-magnet body on addMagnet with a bare 404 (#1250).
|
||||
"""
|
||||
if not self._api_key:
|
||||
msg = "Real-Debrid API key is not configured"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
magnet_link = url
|
||||
if not magnet_link.startswith("magnet:") and expected_hash:
|
||||
magnet_link = f"magnet:?xt=urn:btih:{expected_hash}"
|
||||
|
||||
add_url = f"{_API_BASE}/torrents/addMagnet"
|
||||
try:
|
||||
resp = requests.post(
|
||||
add_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnet": magnet_link},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(add_url),
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
upload = resolve_debrid_upload(url, expected_hash=expected_hash)
|
||||
data = self._send_torrent(upload)
|
||||
|
||||
torrent_id = str(data.get("id", ""))
|
||||
if not torrent_id:
|
||||
@@ -229,12 +228,41 @@ class RealDebridClient(DownloadClient):
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Failed to upload magnet to Real-Debrid")
|
||||
logger.exception("Failed to add torrent to Real-Debrid")
|
||||
raise
|
||||
|
||||
else:
|
||||
return torrent_id
|
||||
|
||||
def _send_torrent(self, upload: DebridUpload) -> dict[str, Any]:
|
||||
"""Hand the torrent to Real-Debrid, as a magnet or as a file upload."""
|
||||
if isinstance(upload, DebridMagnet):
|
||||
add_url = f"{_API_BASE}/torrents/addMagnet"
|
||||
resp = requests.post(
|
||||
add_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnet": upload.magnet_url},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(add_url),
|
||||
)
|
||||
else:
|
||||
# addTorrent is a PUT that takes the raw file as the request body,
|
||||
# not a form field: https://api.real-debrid.com/
|
||||
add_url = f"{_API_BASE}/torrents/addTorrent"
|
||||
resp = requests.put(
|
||||
add_url,
|
||||
headers={
|
||||
**self._auth_headers(),
|
||||
"Content-Type": "application/x-bittorrent",
|
||||
},
|
||||
data=upload.torrent_data,
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(add_url),
|
||||
)
|
||||
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""Poll Real-Debrid for torrent status and drive the download."""
|
||||
state = self._ensure_state(download_id)
|
||||
|
||||
@@ -11,7 +11,12 @@ from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import get_hardened_xmlrpc_client
|
||||
from shelfmark.core.utils import (
|
||||
get_hardened_xmlrpc_client,
|
||||
)
|
||||
from shelfmark.core.utils import (
|
||||
is_audiobook as check_audiobook,
|
||||
)
|
||||
from shelfmark.download.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
@@ -173,7 +178,8 @@ class RTorrentClient(DownloadClient):
|
||||
|
||||
commands = []
|
||||
|
||||
is_audiobook = kwargs.get("content_type") == "audiobook"
|
||||
content_type = kwargs.get("content_type")
|
||||
is_audiobook = check_audiobook(content_type if isinstance(content_type, str) else None)
|
||||
default_label = (
|
||||
self._audiobook_label if is_audiobook and self._audiobook_label else self._label
|
||||
)
|
||||
|
||||
@@ -248,6 +248,15 @@ class SABnzbdClient(DownloadClient):
|
||||
if trusted_url and _url_origin(trusted_url) == target_origin:
|
||||
return True
|
||||
|
||||
named_indexers = config.get("NEWZNAB_INDEXERS", [])
|
||||
if isinstance(named_indexers, list):
|
||||
for row in named_indexers:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
trusted_url = normalize_http_config_url(row.get("url"))
|
||||
if trusted_url and _url_origin(trusted_url) == target_origin:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _get_prowlarr_headers(self, url: str) -> dict:
|
||||
|
||||
@@ -17,6 +17,7 @@ from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -82,6 +83,60 @@ class TorrentInfo:
|
||||
return self
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebridMagnet:
|
||||
"""A magnet link, ready to hand to a debrid service as-is."""
|
||||
|
||||
magnet_url: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebridTorrentFile:
|
||||
"""Raw .torrent bytes, for a debrid service's file-upload endpoint."""
|
||||
|
||||
torrent_data: bytes
|
||||
|
||||
|
||||
# A debrid service takes one or the other, never an indexer page or a proxy URL.
|
||||
type DebridUpload = DebridMagnet | DebridTorrentFile
|
||||
|
||||
|
||||
def resolve_debrid_upload(url: str, *, expected_hash: str | None = None) -> DebridUpload:
|
||||
"""Resolve a release download URL into a magnet link or .torrent bytes.
|
||||
|
||||
Prowlarr hands out a proxy URL, with no magnetUrl and no infoHash, for any
|
||||
indexer that only publishes torrent files - 1337x among them. Posting that
|
||||
URL to a debrid service as if it were a magnet is what produced a bare 404
|
||||
from the service instead of a download (#1250).
|
||||
|
||||
The torrent file is preferred over a synthesized `urn:btih:` magnet because
|
||||
it carries the tracker list, which is how the service finds a swarm that is
|
||||
not already cached. Fetches are shared with the rest of the add path through
|
||||
the torrent fetch cache, so resolving here costs at most one request.
|
||||
|
||||
Raises:
|
||||
ValueError: The URL resolved to neither form, so there is nothing to send.
|
||||
|
||||
"""
|
||||
if url.startswith("magnet:"):
|
||||
return DebridMagnet(magnet_url=url)
|
||||
|
||||
info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
|
||||
if info.is_magnet and info.magnet_url:
|
||||
# The download URL redirected to, or returned, a magnet link.
|
||||
return DebridMagnet(magnet_url=info.magnet_url)
|
||||
if info.torrent_data:
|
||||
return DebridTorrentFile(torrent_data=info.torrent_data)
|
||||
if info.info_hash:
|
||||
# No file to upload, but the hash alone still identifies the torrent.
|
||||
return DebridMagnet(magnet_url=f"magnet:?xt=urn:btih:{info.info_hash}")
|
||||
|
||||
reason = info.fetch_error or "no magnet link, info hash, or torrent file was available"
|
||||
msg = f"Could not resolve a torrent to send from {url[:120]} ({reason})"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def extract_torrent_info(
|
||||
url: str,
|
||||
*,
|
||||
@@ -379,6 +434,56 @@ def extract_info_hash_from_torrent(torrent_data: bytes) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _decode_torrent_text(value: object) -> str | None:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def extract_file_list_from_torrent(torrent_data: bytes) -> list[PackFile] | None:
|
||||
"""List the files a .torrent describes, release-relative, without downloading it.
|
||||
|
||||
Multi-file torrents nest every path under the torrent name (which becomes the
|
||||
client's save folder); single-file torrents are just the named file.
|
||||
"""
|
||||
try:
|
||||
decoded, _ = bencode_decode(torrent_data)
|
||||
except _TORRENT_PARSE_ERRORS as e:
|
||||
logger.debug("Failed to parse torrent file list: %s", e)
|
||||
return None
|
||||
if not isinstance(decoded, dict):
|
||||
return None
|
||||
info = decoded.get(b"info")
|
||||
if not isinstance(info, dict):
|
||||
return None
|
||||
|
||||
name = _decode_torrent_text(info.get(b"name")) or ""
|
||||
raw_files = info.get(b"files")
|
||||
if not isinstance(raw_files, list):
|
||||
length = info.get(b"length")
|
||||
if not name:
|
||||
return None
|
||||
return [PackFile(name, length if isinstance(length, int) else None)]
|
||||
|
||||
files: list[PackFile] = []
|
||||
for entry in raw_files:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
raw_path = entry.get(b"path")
|
||||
if not isinstance(raw_path, list):
|
||||
continue
|
||||
segments = [seg for seg in (_decode_torrent_text(part) for part in raw_path) if seg]
|
||||
if not segments:
|
||||
continue
|
||||
if name:
|
||||
segments.insert(0, name)
|
||||
length = entry.get(b"length")
|
||||
files.append(PackFile("/".join(segments), length if isinstance(length, int) else None))
|
||||
return files
|
||||
|
||||
|
||||
def extract_hash_from_magnet(magnet_url: str) -> str | None:
|
||||
"""Extract info_hash from a magnet URL."""
|
||||
if not magnet_url.startswith("magnet:"):
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""RFC 8484 DNS wireformat encoding/decoding for DoH providers.
|
||||
|
||||
Providers split into two incompatible camps and the difference is not cosmetic:
|
||||
|
||||
* **JSON** (Cloudflare, Google) - ``?name=<host>&type=A`` returning a JSON body. A
|
||||
convention, not a standard, and the only one Shelfmark used to speak.
|
||||
* **Wireformat** (Quad9, OpenDNS) - RFC 8484 proper: a base64url-encoded DNS message
|
||||
in ``?dns=``, answered with ``application/dns-message``. Quad9 additionally
|
||||
*requires HTTP/2* per RFC 8484 section 5.2 and answers HTTP/1.1 with 505.
|
||||
|
||||
This module carries the codec only; the transport choice lives in the resolver.
|
||||
Encoding a query is a handful of bytes, and parsing an answer needs message
|
||||
compression support (RFC 1035 section 4.1.4) because answer names are almost always
|
||||
pointers back into the question.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import secrets
|
||||
import struct
|
||||
|
||||
# Record types we resolve.
|
||||
TYPE_A = 1
|
||||
TYPE_AAAA = 28
|
||||
|
||||
_CLASS_IN = 1
|
||||
_HEADER = struct.Struct(">HHHHHH")
|
||||
_RR_FIXED = struct.Struct(">HHIH") # type, class, ttl, rdlength
|
||||
_FLAG_RECURSION_DESIRED = 0x0100
|
||||
_MAX_LABEL_JUMPS = 64 # cap pointer-following so a malicious answer cannot loop
|
||||
_MAX_NAME_LENGTH = 255
|
||||
|
||||
|
||||
class WireformatError(ValueError):
|
||||
"""Raised when a DNS wireformat message cannot be parsed."""
|
||||
|
||||
|
||||
def encode_query(hostname: str, record_type: int) -> bytes:
|
||||
"""Build a DNS query message for ``hostname``.
|
||||
|
||||
The ID is zero because RFC 8484 section 4.1 requires it for cacheability, but the
|
||||
caller may randomise it when not using a cache.
|
||||
"""
|
||||
if not hostname:
|
||||
msg = "hostname must not be empty"
|
||||
raise WireformatError(msg)
|
||||
|
||||
question = bytearray()
|
||||
for label in hostname.rstrip(".").split("."):
|
||||
encoded = label.encode("idna") if not label.isascii() else label.encode("ascii")
|
||||
if not encoded or len(encoded) > 63:
|
||||
msg = f"invalid DNS label in {hostname!r}"
|
||||
raise WireformatError(msg)
|
||||
question.append(len(encoded))
|
||||
question.extend(encoded)
|
||||
question.append(0)
|
||||
question.extend(struct.pack(">HH", record_type, _CLASS_IN))
|
||||
|
||||
header = _HEADER.pack(0, _FLAG_RECURSION_DESIRED, 1, 0, 0, 0)
|
||||
return header + bytes(question)
|
||||
|
||||
|
||||
def encode_query_param(hostname: str, record_type: int) -> str:
|
||||
"""Return the base64url ``dns=`` parameter value for a query (padding stripped)."""
|
||||
return base64.urlsafe_b64encode(encode_query(hostname, record_type)).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def _read_name(message: bytes, offset: int) -> int:
|
||||
"""Skip over a (possibly compressed) name, returning the offset after it."""
|
||||
jumps = 0
|
||||
length = 0
|
||||
while True:
|
||||
if offset >= len(message):
|
||||
msg = "truncated DNS name"
|
||||
raise WireformatError(msg)
|
||||
label_len = message[offset]
|
||||
if label_len == 0:
|
||||
return offset + 1
|
||||
if label_len & 0xC0 == 0xC0:
|
||||
# A pointer ends this name; the rest of the record follows the 2 bytes.
|
||||
if offset + 1 >= len(message):
|
||||
msg = "truncated DNS name pointer"
|
||||
raise WireformatError(msg)
|
||||
return offset + 2
|
||||
offset += 1 + label_len
|
||||
length += 1 + label_len
|
||||
jumps += 1
|
||||
if jumps > _MAX_LABEL_JUMPS or length > _MAX_NAME_LENGTH:
|
||||
msg = "malformed DNS name"
|
||||
raise WireformatError(msg)
|
||||
|
||||
|
||||
def decode_answer(message: bytes, record_type: int) -> list[str]:
|
||||
"""Extract the IP addresses of ``record_type`` from a DNS response message.
|
||||
|
||||
Returns an empty list for a well-formed response that carries no matching record
|
||||
(NXDOMAIN, or only CNAMEs), and raises WireformatError for a malformed one - the
|
||||
caller treats those differently.
|
||||
"""
|
||||
if len(message) < _HEADER.size:
|
||||
msg = "DNS response shorter than its header"
|
||||
raise WireformatError(msg)
|
||||
|
||||
_id, _flags, qdcount, ancount, _ns, _ar = _HEADER.unpack_from(message, 0)
|
||||
offset = _HEADER.size
|
||||
|
||||
for _ in range(qdcount):
|
||||
offset = _read_name(message, offset)
|
||||
offset += 4 # QTYPE + QCLASS
|
||||
|
||||
results: list[str] = []
|
||||
for _ in range(ancount):
|
||||
offset = _read_name(message, offset)
|
||||
if offset + _RR_FIXED.size > len(message):
|
||||
msg = "truncated resource record"
|
||||
raise WireformatError(msg)
|
||||
rtype, rclass, _ttl, rdlength = _RR_FIXED.unpack_from(message, offset)
|
||||
offset += _RR_FIXED.size
|
||||
rdata = message[offset : offset + rdlength]
|
||||
if len(rdata) != rdlength:
|
||||
msg = "truncated record data"
|
||||
raise WireformatError(msg)
|
||||
offset += rdlength
|
||||
|
||||
if rclass != _CLASS_IN or rtype != record_type:
|
||||
continue
|
||||
if rtype == TYPE_A and rdlength == 4:
|
||||
results.append(".".join(str(b) for b in rdata))
|
||||
elif rtype == TYPE_AAAA and rdlength == 16:
|
||||
groups = struct.unpack(">8H", rdata)
|
||||
results.append(_compress_ipv6(groups))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _compress_ipv6(groups: tuple[int, ...]) -> str:
|
||||
"""Render an IPv6 address with the longest zero run collapsed to '::'."""
|
||||
best_start = best_len = -1
|
||||
run_start = -1
|
||||
for i, group in enumerate([*list(groups), 1]): # sentinel closes a trailing run
|
||||
if group == 0 and i < len(groups):
|
||||
if run_start < 0:
|
||||
run_start = i
|
||||
elif run_start >= 0:
|
||||
if i - run_start > best_len:
|
||||
best_start, best_len = run_start, i - run_start
|
||||
run_start = -1
|
||||
|
||||
parts = [format(g, "x") for g in groups]
|
||||
if best_len > 1:
|
||||
return ":".join(parts[:best_start]) + "::" + ":".join(parts[best_start + best_len :])
|
||||
return ":".join(parts)
|
||||
|
||||
|
||||
def random_query_id() -> int:
|
||||
"""A random DNS message ID, for callers that do not want the RFC 8484 zero."""
|
||||
return secrets.randbelow(0x10000)
|
||||
+349
-58
@@ -10,7 +10,9 @@ from urllib.parse import urljoin, urlparse
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass import BypassCancelledError, cookie_store
|
||||
from shelfmark.bypass.challenge import challenge_marker
|
||||
from shelfmark.core import search_deadline
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int
|
||||
@@ -51,6 +53,7 @@ _BYPASSER_ERRORS = (
|
||||
RuntimeError,
|
||||
TypeError,
|
||||
ValueError,
|
||||
network.RateLimitedError,
|
||||
requests.exceptions.RequestException,
|
||||
)
|
||||
|
||||
@@ -145,19 +148,13 @@ def get_bypassed_page(
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get CF cookies - only available with internal bypasser."""
|
||||
if _is_using_external_bypasser():
|
||||
logger.debug("External bypasser in use, CF cookies not available for %s", domain)
|
||||
return {}
|
||||
return _get_internal_bypasser().get_cf_cookies_for_domain(domain)
|
||||
"""Get the clearance cookies won by whichever bypasser solved this domain."""
|
||||
return cookie_store.get_cf_cookies_for_domain(domain)
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get CF user agent - only available with internal bypasser."""
|
||||
if _is_using_external_bypasser():
|
||||
logger.debug("External bypasser in use, CF user agent not available for %s", domain)
|
||||
return None
|
||||
return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
|
||||
"""Get the User-Agent that solved this domain's challenge, if one is stored."""
|
||||
return cookie_store.get_cf_user_agent_for_domain(domain)
|
||||
|
||||
|
||||
def _apply_cf_bypass(url: str, headers: dict) -> dict:
|
||||
@@ -234,13 +231,66 @@ def _is_retryable_error(e: Exception) -> bool:
|
||||
return status is not None and status in RETRYABLE_CODES
|
||||
|
||||
|
||||
# Statuses that mean the host is gone rather than busy: 410 Gone and 451 Unavailable
|
||||
# For Legal Reasons are what a seized domain answers with.
|
||||
_DEAD_MIRROR_CODES = (410, 451)
|
||||
|
||||
|
||||
def _response_challenge_marker(response: requests.Response) -> str | None:
|
||||
"""The challenge marker in a response body, or None if it carries no challenge.
|
||||
|
||||
Content type is checked first so a JSON or octet-stream error body is never
|
||||
decoded just to be scanned; a missing header is scanned anyway, since an
|
||||
interstitial served without one is still an interstitial.
|
||||
"""
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if content_type and "html" not in content_type.lower():
|
||||
return None
|
||||
try:
|
||||
return challenge_marker(response.text)
|
||||
except UnicodeDecodeError, ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _fatal_mirror_reason(e: Exception) -> str | None:
|
||||
"""Return why ``e`` proves the mirror is unusable, or None if it may recover.
|
||||
|
||||
Hard evidence only - the name does not resolve, nothing is listening, or the host
|
||||
says it is gone for good. A timeout, a 5xx or a challenge all mean the mirror is
|
||||
alive, and rotating off it discards the bypass clearance held for that domain.
|
||||
"""
|
||||
status = _get_status_code(e)
|
||||
if status is not None and status in _DEAD_MIRROR_CODES:
|
||||
return f"HTTP {status}"
|
||||
|
||||
# requests wraps the real cause; a read timeout subclasses ConnectionError for
|
||||
# some adapters, so exclude timeouts explicitly before inspecting the message.
|
||||
if isinstance(e, requests.exceptions.Timeout):
|
||||
return None
|
||||
if not isinstance(e, requests.exceptions.ConnectionError):
|
||||
return None
|
||||
|
||||
text = str(e).lower()
|
||||
if "nameresolutionerror" in text or "failed to resolve" in text or "name or service" in text:
|
||||
return "DNS does not resolve"
|
||||
if "connection refused" in text or "no route to host" in text:
|
||||
return "connection refused"
|
||||
return None
|
||||
|
||||
|
||||
def _try_rotation(
|
||||
original_url: str, current_url: str, selector: network.AAMirrorSelector
|
||||
original_url: str,
|
||||
current_url: str,
|
||||
selector: network.AAMirrorSelector,
|
||||
*,
|
||||
fatal_reason: str | None = None,
|
||||
) -> str | None:
|
||||
"""Try mirror/DNS rotation. Returns new URL or None."""
|
||||
aa_base_url = network.get_aa_base_url()
|
||||
if aa_base_url and current_url.startswith(aa_base_url):
|
||||
new_base, action = selector.next_mirror_or_rotate_dns()
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
fatal=fatal_reason is not None, reason=fatal_reason or ""
|
||||
)
|
||||
if action in ("mirror", "dns") and new_base:
|
||||
new_url = selector.rewrite(original_url)
|
||||
logger.info("[%s] switching to: %s", action, new_url)
|
||||
@@ -272,8 +322,11 @@ def html_get_page(
|
||||
selector: Mirror selector used for AA mirror and DNS rotation.
|
||||
cancel_flag: Optional event used to abort retries early.
|
||||
status_callback: Optional callback for UI status updates.
|
||||
allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
|
||||
instead of switching to the bypasser. Use for search operations.
|
||||
allow_bypasser_fallback: Whether a challenge may be handed to the bypasser.
|
||||
If False, a 403 triggers mirror rotation instead, and an AA redirect loop
|
||||
gives up immediately rather than waiting on a browser solve. Use False for
|
||||
best-effort fetches whose result is optional (e.g. the download count on
|
||||
the details modal); search and detail pages pass True.
|
||||
use_bypasser: Whether to start with the bypasser instead of direct HTTP.
|
||||
include_response_url: If True, return `(html, final_url)` to expose the
|
||||
resolved response URL after redirects.
|
||||
@@ -282,55 +335,157 @@ def html_get_page(
|
||||
|
||||
"""
|
||||
|
||||
# Normalise before the closures below capture it: they touch selector.last_failure,
|
||||
# so it must be a concrete selector, not the Optional parameter.
|
||||
selector = selector or network.AAMirrorSelector()
|
||||
|
||||
# A release search runs under a wall-clock budget (see shelfmark.core.search_deadline).
|
||||
# Adopting it as the cancel flag is what makes the budget bite on a solve already in
|
||||
# flight: the bypassers and the helper subprocess poll this flag but know nothing about
|
||||
# deadlines. Only when the caller has no flag of its own - a queued download brings one
|
||||
# and must keep it, and runs outside any search context anyway.
|
||||
if cancel_flag is None:
|
||||
cancel_flag = search_deadline.cancel_event()
|
||||
|
||||
def _result(html: str, response_url: str) -> str | tuple[str, str]:
|
||||
if include_response_url:
|
||||
return html, response_url
|
||||
return html
|
||||
|
||||
def _fail(reason: str, response_url: str) -> str | tuple[str, str]:
|
||||
"""Record why the fetch is giving up, then return the empty result.
|
||||
|
||||
Every give-up path returns an empty page, which is all the caller used to
|
||||
see. Stashing the concrete reason on the shared selector lets the caller
|
||||
surface it (see release_sources.direct_download) rather than reporting the
|
||||
same generic "network restricted or mirrors blocked" for every cause.
|
||||
"""
|
||||
selector.last_failure = reason
|
||||
return _result("", response_url)
|
||||
|
||||
def _run_bypasser(bypass_url: str) -> str | tuple[str, str]:
|
||||
"""Run the active bypasser for one URL and return its result.
|
||||
|
||||
Factored out so the redirect-loop handoff below can invoke it directly. That
|
||||
call site sits inside the inner redirect `while`, so it cannot reach the
|
||||
retry-loop branch above with `continue`, and with MAX_RETRY=1 there is no
|
||||
later attempt for that branch to run on either.
|
||||
"""
|
||||
# Never start a minutes-long browser solve on a budget that has already run out:
|
||||
# nothing downstream would get to report the real reason before the caller's
|
||||
# deadline (or its reverse proxy) cut the request off.
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent; not starting a bypass for %s", bypass_url)
|
||||
return _fail(search_deadline.deadline_message(), bypass_url)
|
||||
|
||||
if status_callback:
|
||||
status_callback("resolving", "Bypassing protection...")
|
||||
try:
|
||||
# A bypass is one long blocking call with no incremental progress, so
|
||||
# tell the orchestrator up front how long it may legitimately take
|
||||
# instead of trying to fake activity while it runs. Inside the try so a
|
||||
# bypasser that fails to load is still reported as a bypasser error.
|
||||
request_activity_grace(status_callback, _bypass_grace_seconds())
|
||||
result = get_bypassed_page(bypass_url, selector, cancel_flag)
|
||||
if result:
|
||||
return _result(result, bypass_url)
|
||||
return _fail(
|
||||
"The protection bypasser returned an empty page — the challenge was "
|
||||
"not solved. Check that FlareSolverr/the CF bypasser is reachable.",
|
||||
bypass_url,
|
||||
)
|
||||
except network.RateLimitedError as e:
|
||||
# Not a bypasser malfunction: the host is throttling this IP and a solve
|
||||
# cannot help. Surface the wait as a plain failure so the search ends cleanly
|
||||
# instead of looping another minutes-long solve against a 429.
|
||||
logger.info("Skipping bypass (rate-limited): %s", e)
|
||||
if status_callback:
|
||||
try:
|
||||
status_callback("resolving", "Rate limited, try again shortly")
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Rate-limit status callback failed", exc_info=True)
|
||||
return _fail(str(e), bypass_url)
|
||||
except _BYPASSER_ERRORS as e:
|
||||
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
|
||||
# Surface the real reason. Without this the caller only sees an empty
|
||||
# page and the download dies with a generic failure, hiding e.g. a
|
||||
# FlareSolverr 500 behind a silent wait.
|
||||
if status_callback and not isinstance(e, BypassCancelledError):
|
||||
try:
|
||||
status_callback("error", f"Bypass failed: {type(e).__name__}: {e}")
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Bypass error status callback failed", exc_info=True)
|
||||
if isinstance(e, BypassCancelledError):
|
||||
# The budget trips the same cancel flag a user's cancel does, so tell them
|
||||
# apart here - "cancelled" is a confusing thing to read when nobody did.
|
||||
if search_deadline.expired():
|
||||
return _fail(search_deadline.deadline_message(), bypass_url)
|
||||
return _fail("The protection bypass was cancelled.", bypass_url)
|
||||
return _fail(f"The protection bypasser failed: {type(e).__name__}: {e}", bypass_url)
|
||||
finally:
|
||||
release_activity_grace(status_callback)
|
||||
|
||||
def _bypass_handoff_allowed() -> bool:
|
||||
"""Whether a challenge on the current URL may be handed to the bypasser.
|
||||
|
||||
allow_bypasser_fallback is honoured for the same reason the 403 path honours it:
|
||||
callers such as the /dyn/md5/summary fetch behind the details modal pass False
|
||||
precisely so a best-effort request fails fast instead of holding the UI open for
|
||||
a minutes-long browser solve.
|
||||
"""
|
||||
return allow_bypasser_fallback and _is_cf_bypass_enabled() and not use_bypasser_now
|
||||
|
||||
def _purge_clearance(target_url: str) -> None:
|
||||
"""Drop the host's stored clearance cookies.
|
||||
|
||||
Called whenever the protection answered a request that *carried* cookies:
|
||||
being challenged while presenting them proves they no longer work, so keeping
|
||||
them only guarantees the same rejection on every later request. Applies to
|
||||
either bypasser, since both fill the same store.
|
||||
"""
|
||||
hostname = urlparse(target_url).hostname or ""
|
||||
# An empty domain means "clear every host" to the store, so skip the purge
|
||||
# rather than wipe clearance for sites that are working fine.
|
||||
if hostname:
|
||||
cookie_store.clear_cf_cookies(hostname)
|
||||
|
||||
def _redirect_loop_handoff(bypass_url: str) -> str | tuple[str, str]:
|
||||
"""Drop the host's stale clearance cookies, then bypass `bypass_url`.
|
||||
|
||||
A `?check=1` loop is how DDoS-Guard answers a clearance cookie that has gone
|
||||
stale, so the dead cookie has to go before the solve — otherwise it is merged
|
||||
back over the fresh one on the next request and the loop simply resumes.
|
||||
"""
|
||||
_purge_clearance(bypass_url)
|
||||
return _run_bypasser(bypass_url)
|
||||
|
||||
configured_retry = normalize_positive_int(app_config.MAX_RETRY)
|
||||
retry_limit = (
|
||||
retry if retry is not None else (configured_retry if configured_retry is not None else 1)
|
||||
)
|
||||
selector = selector or network.AAMirrorSelector()
|
||||
original_url = url
|
||||
current_url = selector.rewrite(original_url)
|
||||
use_bypasser_now = use_bypasser
|
||||
# Survives across attempts so a cookie won once is still presented on later retries.
|
||||
handshake_cookies: dict[str, str] = {}
|
||||
handshake_retries = 0
|
||||
# Last transport error seen, so the exhausted-retries path can name the real
|
||||
# cause (timeout, connection refused, DNS, ...) instead of a generic message.
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, retry_limit + 1):
|
||||
# Check for cancellation before each attempt
|
||||
if cancel_flag and cancel_flag.is_set():
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent before attempt %s", attempt)
|
||||
return _fail(search_deadline.deadline_message(), current_url)
|
||||
logger.info("html_get_page cancelled before attempt %s", attempt)
|
||||
return _result("", current_url)
|
||||
return _fail("The request was cancelled.", current_url)
|
||||
|
||||
cookies: dict[str, str] = {}
|
||||
try:
|
||||
if use_bypasser_now and _is_cf_bypass_enabled():
|
||||
if status_callback:
|
||||
status_callback("resolving", "Bypassing protection...")
|
||||
try:
|
||||
# A bypass is one long blocking call with no incremental progress, so
|
||||
# tell the orchestrator up front how long it may legitimately take
|
||||
# instead of trying to fake activity while it runs. Inside the try so a
|
||||
# bypasser that fails to load is still reported as a bypasser error.
|
||||
request_activity_grace(status_callback, _bypass_grace_seconds())
|
||||
result = get_bypassed_page(current_url, selector, cancel_flag)
|
||||
return _result(result or "", current_url)
|
||||
except _BYPASSER_ERRORS as e:
|
||||
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
|
||||
# Surface the real reason. Without this the caller only sees an empty
|
||||
# page and the download dies with a generic failure, hiding e.g. a
|
||||
# FlareSolverr 500 behind a silent wait.
|
||||
if status_callback and not isinstance(e, BypassCancelledError):
|
||||
try:
|
||||
status_callback("error", f"Bypass failed: {type(e).__name__}: {e}")
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Bypass error status callback failed", exc_info=True)
|
||||
return _result("", current_url)
|
||||
finally:
|
||||
release_activity_grace(status_callback)
|
||||
return _run_bypasser(current_url)
|
||||
|
||||
logger.debug("GET: %s", current_url)
|
||||
|
||||
@@ -352,8 +507,15 @@ def html_get_page(
|
||||
current_url,
|
||||
proxies=get_proxies(current_url),
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
# Bypasser-derived cookies win: they came from a real solved challenge.
|
||||
cookies={**handshake_cookies, **cookies},
|
||||
# Handshake cookies win. They were issued by *this* exchange, so by
|
||||
# definition they are fresher than anything the store holds, and the
|
||||
# server is waiting to see them echoed back on the very next hop.
|
||||
# Letting the store overwrite them meant a stored cookie of the same
|
||||
# name (DDoS-Guard reuses __ddg1_/__ddg2_ for both) was replayed on
|
||||
# every hop and the freshly issued value never left this process - the
|
||||
# ?check=1 probe could then never terminate, so every request ended in
|
||||
# the redirect-loop handoff and paid for a full browser solve.
|
||||
cookies={**cookies, **handshake_cookies},
|
||||
headers=headers,
|
||||
allow_redirects=allow_redirects,
|
||||
verify=get_ssl_verify(current_url),
|
||||
@@ -377,6 +539,35 @@ def html_get_page(
|
||||
)
|
||||
continue
|
||||
|
||||
# A 503 still serving a challenge is protection, not a busy origin. The
|
||||
# handshake above has nothing left to echo back, and 503 is in
|
||||
# RETRYABLE_CODES, so without this the request spends every attempt on
|
||||
# the same wall: the bypasser is only ever reached from the 403 branch
|
||||
# and the AA redirect rescues. Gate on the body, not the status, so a
|
||||
# genuine overloaded-origin 503 keeps its retry path.
|
||||
if response.status_code == _HTTP_STATUS_SERVICE_UNAVAILABLE:
|
||||
marker = _response_challenge_marker(response)
|
||||
if marker and _bypass_handoff_allowed():
|
||||
if cookies:
|
||||
# Challenged while presenting clearance means those cookies
|
||||
# are dead; same reasoning as the 403 branch below.
|
||||
logger.debug(
|
||||
"503 challenge with cookies presented; purging: %s", current_url
|
||||
)
|
||||
_purge_clearance(current_url)
|
||||
logger.info(
|
||||
"503 challenge detected (%s); switching to bypasser: %s",
|
||||
marker,
|
||||
current_url,
|
||||
)
|
||||
return _run_bypasser(current_url)
|
||||
if marker:
|
||||
logger.debug(
|
||||
"503 challenge (%s) but no bypasser handoff available: %s",
|
||||
marker,
|
||||
current_url,
|
||||
)
|
||||
|
||||
if is_aa_url and response.is_redirect:
|
||||
location = response.headers.get("Location", "")
|
||||
if not location:
|
||||
@@ -398,7 +589,12 @@ def html_get_page(
|
||||
redirect_host,
|
||||
current_url,
|
||||
)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
f"The configured mirror {current_host} redirected to "
|
||||
f"{redirect_host}; it may be down or seized. Point MIRROR at "
|
||||
"a working host or switch to auto mode.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
new_url = _try_rotation(original_url, current_url, selector)
|
||||
if new_url:
|
||||
@@ -417,12 +613,47 @@ def html_get_page(
|
||||
redirect_host,
|
||||
current_url,
|
||||
)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
"Every Anna's Archive mirror redirected away to a dead host — "
|
||||
"all configured mirrors are unreachable.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
# Same-host redirect (relative or absolute) - follow manually.
|
||||
# DDoS-Guard gates AA /search behind a cookie probe: the 302 to
|
||||
# ?check=1 carries Set-Cookie (__ddg*) which must be echoed back on
|
||||
# the next hop, or the server just re-issues the redirect forever.
|
||||
issued = _new_cookies(response, handshake_cookies)
|
||||
if issued:
|
||||
handshake_cookies.update(issued)
|
||||
redirects_followed += 1
|
||||
if redirects_followed > _MAX_REDIRECTS:
|
||||
_raise_too_many_redirects(f"Too many redirects for {current_url}")
|
||||
# A same-host redirect loop on AA is not a network fault — it is
|
||||
# how DDoS-Guard presents a handshake that is unsolved, or whose
|
||||
# clearance cookie has gone stale: /search redirects to
|
||||
# /search&check=1, which redirects back, indefinitely. Hand it
|
||||
# straight to the bypasser rather than raising, which would send it
|
||||
# down the retry path to re-run the whole loop on every attempt
|
||||
# (10 x 6 = ~60 requests to AA) without ever offering the URL to the
|
||||
# bypasser. `continue` is no use here either — it would target this
|
||||
# inner redirect loop rather than the retry branch below.
|
||||
if _bypass_handoff_allowed():
|
||||
logger.info(
|
||||
"Redirect loop detected; switching to bypasser: %s", current_url
|
||||
)
|
||||
return _redirect_loop_handoff(current_url)
|
||||
# No bypasser to hand it to. Every AA mirror shares the challenge,
|
||||
# so rotating only collects another loop — give up now instead of
|
||||
# raising and burning the same ~60 requests over the retry budget.
|
||||
logger.warning(
|
||||
"Redirect loop and no bypasser available, giving up: %s", current_url
|
||||
)
|
||||
return _fail(
|
||||
"Anna's Archive is behind a protection challenge (endless "
|
||||
"redirect loop) and no bypasser is enabled to solve it. Enable "
|
||||
"FlareSolverr/the CF bypasser.",
|
||||
current_url,
|
||||
)
|
||||
current_url = redirect_url
|
||||
continue
|
||||
|
||||
@@ -432,8 +663,24 @@ def html_get_page(
|
||||
return _result(response.text, response.url)
|
||||
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
status = _get_status_code(e)
|
||||
|
||||
# The same DDoS-Guard rescue, for the loops the manual AA follower above hands
|
||||
# back rather than resolving inline — an AA redirect missing its Location
|
||||
# header. TooManyRedirects carries no status, so the 403 rescue below never
|
||||
# fires and every retry would re-send the dead cookies. Scoped to the hosts
|
||||
# whose redirects we follow manually: elsewhere `requests` follows them itself,
|
||||
# and a loop there is an ordinary misconfiguration that a cookie purge and a
|
||||
# minutes-long browser solve would be the wrong answer to.
|
||||
if (
|
||||
isinstance(e, requests.exceptions.TooManyRedirects)
|
||||
and network.should_rotate_dns_for_url(current_url)
|
||||
and _bypass_handoff_allowed()
|
||||
):
|
||||
logger.info("Redirect loop detected; switching to bypasser: %s", current_url)
|
||||
return _redirect_loop_handoff(current_url)
|
||||
|
||||
# 403 = Cloudflare/DDoS-Guard protection
|
||||
if status == _HTTP_STATUS_FORBIDDEN:
|
||||
# If bypasser fallback is disabled, try mirrors instead
|
||||
@@ -443,36 +690,70 @@ def html_get_page(
|
||||
current_url = new_url
|
||||
continue
|
||||
logger.warning("403 error, mirrors exhausted: %s", current_url)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
"Anna's Archive returned 403 (blocked) and all mirrors are exhausted.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
if _is_cf_bypass_enabled() and not use_bypasser_now:
|
||||
# Before switching to bypasser, check if cookies have become available
|
||||
# (another concurrent download may have completed bypass and extracted cookies)
|
||||
parsed = urlparse(current_url)
|
||||
fresh_cookies = get_cf_cookies_for_domain(parsed.hostname or "")
|
||||
if fresh_cookies and not cookies:
|
||||
# Cookies are now available - retry with cookies before using bypasser
|
||||
if fresh_cookies and not cookies and attempt < retry_limit:
|
||||
# Cookies are now available - retry with cookies before using bypasser.
|
||||
# Guarded on there being a next attempt: `continue` on the last one
|
||||
# ends the retry loop and abandons the request without ever offering
|
||||
# the URL to the bypasser, and MAX_RETRY=1 is the supported setting.
|
||||
# Same reasoning as the bypasser invocation below.
|
||||
logger.debug(
|
||||
"403 but cookies now available - retrying with cookies: %s",
|
||||
current_url,
|
||||
)
|
||||
continue
|
||||
if cookies:
|
||||
# Challenged *while presenting* clearance: those cookies are
|
||||
# dead. Without this they survive the solve and get merged back
|
||||
# over the fresh ones, so every later request re-presents a
|
||||
# known-rejected cookie and is challenged again - the stale
|
||||
# retry that never ends.
|
||||
logger.debug("403 with cookies presented; purging: %s", current_url)
|
||||
_purge_clearance(current_url)
|
||||
logger.info("403 detected; switching to bypasser: %s", current_url)
|
||||
if status_callback:
|
||||
status_callback("resolving", "Bypassing protection...")
|
||||
use_bypasser_now = True
|
||||
continue
|
||||
# Invoke it here rather than setting use_bypasser_now and continuing.
|
||||
# The branch that acts on that flag runs at the top of the *next* retry
|
||||
# attempt, so under the supported MAX_RETRY=1 there is no next attempt
|
||||
# and the bypasser was never reached — a 403 simply ended the search.
|
||||
# Same reasoning as the redirect-loop handoffs.
|
||||
return _run_bypasser(current_url)
|
||||
logger.warning("403 error, giving up: %s", current_url)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
"Anna's Archive returned 403 (blocked) and no bypasser is enabled "
|
||||
"to solve the protection challenge.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
# 404 = Not found
|
||||
if status == _HTTP_STATUS_NOT_FOUND:
|
||||
logger.warning("404 error: %s", current_url)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
f"Anna's Archive returned 404 Not Found for {current_url}.", current_url
|
||||
)
|
||||
|
||||
# Try mirror/DNS rotation on retryable errors
|
||||
if _is_retryable_error(e):
|
||||
new_url = _try_rotation(original_url, current_url, selector)
|
||||
# 429 = origin throttling this IP. Arm the per-host backoff so selection and
|
||||
# the bypasser stop hammering it, then fall through to normal rotation onto a
|
||||
# mirror that is not (yet) rate-limited.
|
||||
if status == _HTTP_STATUS_RATE_LIMITED:
|
||||
network.note_rate_limited(current_url)
|
||||
|
||||
# Try mirror/DNS rotation on retryable errors. A failure that proves the
|
||||
# mirror is unusable also drops it from this process's rotation, so the
|
||||
# next search does not pay for it again.
|
||||
fatal_reason = _fatal_mirror_reason(e)
|
||||
if fatal_reason or _is_retryable_error(e):
|
||||
new_url = _try_rotation(
|
||||
original_url, current_url, selector, fatal_reason=fatal_reason
|
||||
)
|
||||
if new_url:
|
||||
current_url = new_url
|
||||
handshake_cookies.clear()
|
||||
@@ -492,7 +773,16 @@ def html_get_page(
|
||||
else:
|
||||
logger.exception("Giving up after %s attempts: %s", retry_limit, current_url)
|
||||
|
||||
return _result("", current_url)
|
||||
if last_error is not None:
|
||||
return _fail(
|
||||
f"Could not reach Anna's Archive after {retry_limit} attempt(s): "
|
||||
f"{type(last_error).__name__}: {last_error}",
|
||||
current_url,
|
||||
)
|
||||
return _fail(
|
||||
"Could not reach Anna's Archive — all mirrors were exhausted without a usable response.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
|
||||
def download_url(
|
||||
@@ -609,6 +899,7 @@ def download_url(
|
||||
# Rate limited - skip to next source immediately
|
||||
# (waiting doesn't help with concurrent downloads hitting the same server)
|
||||
if status == _HTTP_STATUS_RATE_LIMITED:
|
||||
network.note_rate_limited(current_url)
|
||||
logger.info("Rate limited (429) - trying next source")
|
||||
if status_callback:
|
||||
status_callback("resolving", "Server busy, trying next")
|
||||
|
||||
+334
-55
@@ -3,14 +3,16 @@
|
||||
import fnmatch
|
||||
import ipaddress
|
||||
import socket
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
from socket import AddressFamily, SocketKind
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
||||
|
||||
import dns.resolver
|
||||
import httpx
|
||||
import requests
|
||||
from dns.exception import DNSException
|
||||
|
||||
@@ -277,6 +279,116 @@ _current_aa_url_index = 0
|
||||
_aa_urls: list[str] = [] # Initialized lazily in _initialize_aa_state()
|
||||
_aa_base_url: str = "" # Current active AA URL
|
||||
|
||||
# Mirrors quarantined for this process: domains that are not a working AA mirror at
|
||||
# all (NXDOMAIN, refused, or a 200 that isn't AA - seized/parked/for-sale domains all
|
||||
# land here). Kept separate from ordinary failures: a 403 challenge or a 5xx means the
|
||||
# mirror is alive and rotating away from it only discards the DDoS-Guard clearance we
|
||||
# hold for it. Deliberately in-memory only, so a restart re-probes everything.
|
||||
_dead_aa_urls: set[str] = set()
|
||||
_dead_aa_urls_lock = _RLock()
|
||||
|
||||
|
||||
# Per-host rate-limit backoff. A 429 is the origin throttling *this IP*, not a challenge:
|
||||
# a DDoS-Guard/Cloudflare solve still renders, so the bypass "succeeds" yet the cleared
|
||||
# request is rejected again and the throttle is only renewed. The single answer is to
|
||||
# wait, so a 429 sidelines the host for a growing window - mirror selection and the
|
||||
# bypasser both skip a cooling-down host until its deadline passes. The wait escalates
|
||||
# 2 -> 5 -> 10 -> 15 -> 30 minutes each time the host throttles us again *after* we
|
||||
# already waited a full window out; a host left clear for longer than the top step
|
||||
# starts the ladder over. Keyed by host so every mirror and source shares one view;
|
||||
# in-memory only, so a restart starts clean.
|
||||
_RATE_LIMIT_COOLDOWN_LADDER_SECONDS: tuple[float, ...] = (120.0, 300.0, 600.0, 900.0, 1800.0)
|
||||
# A host that has been clear this long is treated as a fresh episode: the next 429
|
||||
# restarts the ladder at 2 minutes rather than resuming the escalation.
|
||||
_RATE_LIMIT_RESET_AFTER_SECONDS = 1800.0
|
||||
|
||||
|
||||
class _Cooldown(NamedTuple):
|
||||
"""One host's active rate-limit window and how far up the ladder it has climbed."""
|
||||
|
||||
deadline: float # time.monotonic() value at which the wait expires
|
||||
level: int # index into _RATE_LIMIT_COOLDOWN_LADDER_SECONDS
|
||||
|
||||
|
||||
_host_cooldowns: dict[str, _Cooldown] = {}
|
||||
_host_cooldowns_lock = _RLock()
|
||||
|
||||
|
||||
class RateLimitedError(Exception):
|
||||
"""Raised to abandon a request whose host is in a 429 cooldown.
|
||||
|
||||
Not a transport failure - nothing is wrong with the network, the origin is
|
||||
throttling this IP and only time clears it. Callers surface it as a plain failure
|
||||
rather than retrying or handing the URL to the bypasser.
|
||||
"""
|
||||
|
||||
|
||||
def _cooldown_key(url: str) -> str:
|
||||
"""Host a cooldown is keyed by; '' when the URL carries none."""
|
||||
return (urllib.parse.urlparse(url).hostname or "").lower()
|
||||
|
||||
|
||||
def note_rate_limited(url: str) -> float:
|
||||
"""Escalate a host's 429 backoff and (re)arm its cooldown; return the wait applied.
|
||||
|
||||
The step advances only when a fresh 429 arrives *after* the previous window already
|
||||
elapsed - i.e. we waited it out and the host throttled us again. A 429 that lands
|
||||
while the host is still cooling is the same episode: it neither escalates the level
|
||||
nor shortens the wait. See the ladder note above.
|
||||
"""
|
||||
host = _cooldown_key(url)
|
||||
if not host:
|
||||
return 0.0
|
||||
now = time.monotonic()
|
||||
ladder = _RATE_LIMIT_COOLDOWN_LADDER_SECONDS
|
||||
with _host_cooldowns_lock:
|
||||
prev = _host_cooldowns.get(host)
|
||||
if prev is not None and now < prev.deadline:
|
||||
# Still inside the current window - same throttling episode, leave it be.
|
||||
return prev.deadline - now
|
||||
if prev is None or now - prev.deadline > _RATE_LIMIT_RESET_AFTER_SECONDS:
|
||||
level = 0
|
||||
else:
|
||||
level = min(prev.level + 1, len(ladder) - 1)
|
||||
wait = ladder[level]
|
||||
_host_cooldowns[host] = _Cooldown(deadline=now + wait, level=level)
|
||||
logger.info(
|
||||
"Rate limited (429): backing off %s for %.0fs (step %d/%d)",
|
||||
host,
|
||||
wait,
|
||||
level + 1,
|
||||
len(ladder),
|
||||
)
|
||||
return wait
|
||||
|
||||
|
||||
def host_cooldown_remaining(url: str) -> float:
|
||||
"""Seconds left on a host's 429 cooldown; 0.0 when clear or expired.
|
||||
|
||||
Leaves an expired record in place: the ladder level it carries is what a later 429
|
||||
escalates from (or resets, once the clear gap is long enough).
|
||||
"""
|
||||
host = _cooldown_key(url)
|
||||
if not host:
|
||||
return 0.0
|
||||
now = time.monotonic()
|
||||
with _host_cooldowns_lock:
|
||||
rec = _host_cooldowns.get(host)
|
||||
if rec is None or rec.deadline <= now:
|
||||
return 0.0
|
||||
return rec.deadline - now
|
||||
|
||||
|
||||
def is_host_cooling_down(url: str) -> bool:
|
||||
"""True while ``url``'s host is inside its 429 cooldown window."""
|
||||
return host_cooldown_remaining(url) > 0.0
|
||||
|
||||
|
||||
def clear_host_cooldowns() -> None:
|
||||
"""Forget all rate-limit cooldowns (manual reset / tests)."""
|
||||
with _host_cooldowns_lock:
|
||||
_host_cooldowns.clear()
|
||||
|
||||
|
||||
def _ensure_initialized() -> None:
|
||||
"""Lazy guard so runtime setup happens once and late calls still work."""
|
||||
@@ -298,6 +410,24 @@ DNS_PROVIDERS = [
|
||||
("opendns", ["208.67.222.222", "208.67.220.220"], "https://doh.opendns.com/dns-query"),
|
||||
]
|
||||
|
||||
# httpx raises its own hierarchy, which shares no base class with requests', so a
|
||||
# wireformat failure would escape a requests-only except clause.
|
||||
_DOH_REQUEST_ERRORS = (OSError, ValueError, requests.RequestException, httpx.HTTPError)
|
||||
|
||||
|
||||
def _first_proxy(proxies: dict[str, str] | None) -> str | None:
|
||||
"""Pick a single proxy URL from a requests-style mapping, for httpx."""
|
||||
if not proxies:
|
||||
return None
|
||||
return proxies.get("https") or proxies.get("http") or None
|
||||
|
||||
|
||||
# DoH providers that speak RFC 8484 wireformat rather than the (non-standard) JSON API
|
||||
# Cloudflare and Google popularised. Verified against the live services: both reject a
|
||||
# ?name=&type= query outright - Quad9 with 505 (it also mandates HTTP/2 per RFC 8484
|
||||
# section 5.2, which requests cannot speak), OpenDNS with 400 "No valid query received".
|
||||
_DOH_WIREFORMAT_HOSTS = frozenset({"dns.quad9.net", "doh.opendns.com"})
|
||||
|
||||
# Domain patterns that should trigger DNS rotation on failure
|
||||
DNS_ROTATION_DOMAINS = [
|
||||
"annas-archive",
|
||||
@@ -462,8 +592,16 @@ class DoHResolver:
|
||||
# DNS cache: {(hostname, record_type): (ip_list, timestamp)}
|
||||
self._cache: dict[tuple[str, str], tuple[list[str], datetime]] = {}
|
||||
|
||||
# Different headers based on provider
|
||||
if "google" in self.base_url:
|
||||
# RFC 8484 providers get a separate transport: they need wireformat, and Quad9
|
||||
# additionally refuses HTTP/1.1, which requests has no way to upgrade from.
|
||||
self.use_wireformat = urllib.parse.urlparse(self.base_url).hostname in (
|
||||
_DOH_WIREFORMAT_HOSTS
|
||||
)
|
||||
self._http2_client: Any | None = None
|
||||
|
||||
if self.use_wireformat:
|
||||
self.session.headers.update({"Accept": "application/dns-message"})
|
||||
elif "google" in self.base_url:
|
||||
self.session.headers.update(
|
||||
{
|
||||
"Accept": "application/json",
|
||||
@@ -476,6 +614,35 @@ class DoHResolver:
|
||||
}
|
||||
)
|
||||
|
||||
def _get_http2_client(self) -> Any:
|
||||
"""Lazily build the HTTP/2 client used for RFC 8484 providers.
|
||||
|
||||
Built on first use so a resolver pointed at a JSON provider never opens an
|
||||
HTTP/2 connection pool it will not use.
|
||||
"""
|
||||
if self._http2_client is None:
|
||||
self._http2_client = httpx.Client(
|
||||
http2=True,
|
||||
timeout=10,
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
proxy=_first_proxy(get_proxies(self.base_url)),
|
||||
)
|
||||
return self._http2_client
|
||||
|
||||
def _resolve_wireformat(self, hostname: str, record_type: str) -> list[str]:
|
||||
"""Resolve via RFC 8484: base64url query in, DNS message out."""
|
||||
from shelfmark.download import doh_wireformat
|
||||
|
||||
qtype = doh_wireformat.TYPE_AAAA if record_type == "AAAA" else doh_wireformat.TYPE_A
|
||||
param = doh_wireformat.encode_query_param(hostname, qtype)
|
||||
response = self._get_http2_client().get(
|
||||
self.base_url,
|
||||
params={"dns": param},
|
||||
headers={"Accept": "application/dns-message"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return doh_wireformat.decode_answer(response.content, qtype)
|
||||
|
||||
def _get_cached(self, hostname: str, record_type: str) -> list[str] | None:
|
||||
"""Get cached DNS result if still valid."""
|
||||
key = (hostname, record_type)
|
||||
@@ -525,34 +692,37 @@ class DoHResolver:
|
||||
return cached
|
||||
|
||||
try:
|
||||
params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"}
|
||||
if self.use_wireformat:
|
||||
answers = self._resolve_wireformat(hostname, record_type)
|
||||
else:
|
||||
params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"}
|
||||
|
||||
response = self.session.get(
|
||||
self.base_url,
|
||||
params=params,
|
||||
proxies=get_proxies(self.base_url),
|
||||
timeout=10, # Increased from 5s to handle slow network conditions
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
)
|
||||
response.raise_for_status()
|
||||
response = self.session.get(
|
||||
self.base_url,
|
||||
params=params,
|
||||
proxies=get_proxies(self.base_url),
|
||||
timeout=10, # Increased from 5s to handle slow network conditions
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
if "Answer" not in data:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, data)
|
||||
return []
|
||||
data = response.json()
|
||||
if "Answer" not in data:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, data)
|
||||
return []
|
||||
|
||||
# Extract IP addresses from the response
|
||||
answers = [
|
||||
answer["data"]
|
||||
for answer in data["Answer"]
|
||||
if answer.get("type") == (28 if record_type == "AAAA" else 1)
|
||||
]
|
||||
# Extract IP addresses from the response
|
||||
answers = [
|
||||
answer["data"]
|
||||
for answer in data["Answer"]
|
||||
if answer.get("type") == (28 if record_type == "AAAA" else 1)
|
||||
]
|
||||
|
||||
# Cache the result
|
||||
self._set_cached(hostname, record_type, answers)
|
||||
|
||||
# Don't log here - the caller (custom_getaddrinfo) will log the final result
|
||||
except (OSError, ValueError, requests.RequestException) as e:
|
||||
except _DOH_REQUEST_ERRORS as e:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, e)
|
||||
return []
|
||||
else:
|
||||
@@ -616,8 +786,6 @@ def create_custom_getaddrinfo(
|
||||
source: str,
|
||||
provider_label: str,
|
||||
res: Sequence[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]],
|
||||
*,
|
||||
is_bypass: bool = False,
|
||||
) -> None:
|
||||
"""Emit a unified resolver log with the IPs returned.
|
||||
|
||||
@@ -625,7 +793,6 @@ def create_custom_getaddrinfo(
|
||||
source: Description of resolver source
|
||||
provider_label: Label for the DNS provider
|
||||
res: Resolution results
|
||||
is_bypass: If True, log at DEBUG level (for local/IP addresses)
|
||||
|
||||
"""
|
||||
# Skip logging entirely for localhost to reduce noise
|
||||
@@ -641,11 +808,7 @@ def create_custom_getaddrinfo(
|
||||
ip = sockaddr[0]
|
||||
if isinstance(ip, str):
|
||||
ips.append(ip)
|
||||
msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}"
|
||||
if is_bypass:
|
||||
logger.debug(msg)
|
||||
else:
|
||||
logger.info(msg)
|
||||
logger.debug("Resolved %s via %s [%s]: %s", host_str, source, provider_label, ips)
|
||||
|
||||
# Skip custom resolution for IP addresses, local addresses, or if skip check passes
|
||||
if (
|
||||
@@ -655,7 +818,7 @@ def create_custom_getaddrinfo(
|
||||
):
|
||||
# Quietly bypass custom resolution for IP/local targets
|
||||
res = original_getaddrinfo(host, port, family, socket_type, proto, flags)
|
||||
_log_results("system resolver (bypass)", "system", res, is_bypass=True)
|
||||
_log_results("system resolver (bypass)", "system", res)
|
||||
return res
|
||||
|
||||
results: list[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]] = []
|
||||
@@ -1015,14 +1178,18 @@ def rotate_dns_and_reset_aa() -> bool:
|
||||
configured_url = _get_configured_aa_url()
|
||||
|
||||
if configured_url == "auto":
|
||||
# Auto mode always resets to the first mirror to restart the cascade
|
||||
_current_aa_url_index = 0
|
||||
if _aa_urls:
|
||||
_aa_base_url = _aa_urls[0]
|
||||
# Auto mode always resets to the first mirror to restart the cascade. Skip any
|
||||
# quarantined ones: a new DNS provider cannot revive a parked or seized domain.
|
||||
with _dead_aa_urls_lock:
|
||||
restart_urls = [url for url in _aa_urls if url not in _dead_aa_urls] or _aa_urls
|
||||
if restart_urls:
|
||||
_aa_base_url = restart_urls[0]
|
||||
_current_aa_url_index = _aa_urls.index(_aa_base_url)
|
||||
logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url)
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
else:
|
||||
_aa_base_url = ""
|
||||
_current_aa_url_index = 0
|
||||
logger.info("After DNS switch, AA URL remains unconfigured")
|
||||
else:
|
||||
# Keep the user's configured primary mirror (if it exists in the list),
|
||||
@@ -1192,8 +1359,17 @@ def _initialize_aa_state() -> None:
|
||||
global _aa_base_url, _current_aa_url_index, _aa_urls
|
||||
|
||||
# Build URL list from config
|
||||
previous_urls = _aa_urls
|
||||
_aa_urls = _build_aa_urls()
|
||||
|
||||
# Drop quarantine decisions only when the mirror list itself changed - they were
|
||||
# made about a list that no longer applies. This runs on every re-init (settings
|
||||
# sync, DNS rotation, helper subprocess startup), and clearing unconditionally
|
||||
# would resurrect a parked mirror mid-session.
|
||||
if previous_urls != _aa_urls:
|
||||
with _dead_aa_urls_lock:
|
||||
_dead_aa_urls.clear()
|
||||
|
||||
# Get configured base URL from config
|
||||
configured_url = _get_configured_aa_url()
|
||||
|
||||
@@ -1209,26 +1385,34 @@ def _initialize_aa_state() -> None:
|
||||
return
|
||||
|
||||
if configured_url == "auto":
|
||||
if state.get("aa_base_url") and state["aa_base_url"] in _aa_urls:
|
||||
_current_aa_url_index = _aa_urls.index(state["aa_base_url"])
|
||||
_aa_base_url = state["aa_base_url"]
|
||||
# Never restore or probe a mirror quarantined this session: re-init happens
|
||||
# often, and re-electing a parked domain costs a wasted request every time
|
||||
# (its parking page answers 200, so the probe would happily pick it).
|
||||
with _dead_aa_urls_lock:
|
||||
candidates = [url for url in _aa_urls if url not in _dead_aa_urls]
|
||||
restored = state.get("aa_base_url")
|
||||
if restored and restored in candidates:
|
||||
_current_aa_url_index = _aa_urls.index(restored)
|
||||
_aa_base_url = restored
|
||||
else:
|
||||
logger.debug("AA_BASE_URL: auto, checking available urls %s", _aa_urls)
|
||||
for i, url in enumerate(_aa_urls):
|
||||
logger.debug("AA_BASE_URL: auto, checking available urls %s", candidates)
|
||||
for url in candidates:
|
||||
try:
|
||||
response = requests.get(
|
||||
url, proxies=get_proxies(url), timeout=3, verify=get_ssl_verify(url)
|
||||
)
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
_current_aa_url_index = i
|
||||
_current_aa_url_index = _aa_urls.index(url)
|
||||
_aa_base_url = url
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
break
|
||||
except (OSError, requests.RequestException) as exc:
|
||||
logger.debug("Could not reach AA mirror candidate %s: %s", url, exc)
|
||||
if not _aa_base_url or _aa_base_url == "auto":
|
||||
_aa_base_url = _aa_urls[0]
|
||||
_current_aa_url_index = 0
|
||||
# Also covers the case where every probe failed and the previous base is
|
||||
# itself quarantined - keeping it would aim the next search at a dead host.
|
||||
if not _aa_base_url or _aa_base_url == "auto" or _aa_base_url not in candidates:
|
||||
_aa_base_url = (candidates or _aa_urls)[0]
|
||||
_current_aa_url_index = _aa_urls.index(_aa_base_url)
|
||||
elif configured_url not in _aa_urls:
|
||||
logger.info("AA_BASE_URL set to custom value %s; skipping auto-switch", configured_url)
|
||||
_aa_base_url = configured_url
|
||||
@@ -1326,24 +1510,82 @@ def is_aa_auto_mode() -> bool:
|
||||
|
||||
|
||||
def get_available_aa_urls() -> list[str]:
|
||||
"""Get list of configured AA URLs (copy)."""
|
||||
"""Get configured AA URLs (copy), minus any quarantined this process.
|
||||
|
||||
Falls back to the full list when every mirror has been quarantined: a wrong
|
||||
classification must not leave the app with nowhere to search.
|
||||
"""
|
||||
_ensure_initialized()
|
||||
return _aa_urls.copy()
|
||||
with _dead_aa_urls_lock:
|
||||
alive = [url for url in _aa_urls if url not in _dead_aa_urls]
|
||||
if not alive and _aa_urls:
|
||||
logger.warning("All AA mirrors quarantined; retrying the full list")
|
||||
_dead_aa_urls.clear()
|
||||
alive = _aa_urls.copy()
|
||||
# Prefer mirrors that are not serving a 429 cooldown so rotation stops hammering a
|
||||
# throttled host. When every live mirror is cooling, keep the full live list rather
|
||||
# than returning nothing: selection must never be left with nowhere to point, and
|
||||
# the bypasser's fail-fast reports the "all rate-limited" case with a clear error.
|
||||
breathing = [url for url in alive if not is_host_cooling_down(url)]
|
||||
return breathing or alive
|
||||
|
||||
|
||||
def set_aa_url_index(new_index: int) -> bool:
|
||||
"""Set AA base URL by index in available list; returns True if applied."""
|
||||
def _aa_base_for_url(url: str) -> str:
|
||||
"""Return the configured mirror base that ``url`` belongs to, if any."""
|
||||
for base in _aa_urls:
|
||||
if base and url.startswith(base):
|
||||
return base
|
||||
return ""
|
||||
|
||||
|
||||
def mark_aa_url_dead(url: str, reason: str) -> bool:
|
||||
"""Quarantine an AA mirror for the rest of this process.
|
||||
|
||||
Only for hard evidence that the host is not a working AA mirror. Transient
|
||||
failures (403 challenge, 429, 5xx, timeouts) must never come through here -
|
||||
quarantining a live mirror throws away its bypass clearance.
|
||||
"""
|
||||
_ensure_initialized()
|
||||
base = _aa_base_for_url(url) or url
|
||||
with _dead_aa_urls_lock:
|
||||
if base not in _aa_urls or base in _dead_aa_urls:
|
||||
return False
|
||||
# Keep at least one mirror in play, even if it is the failing one.
|
||||
if len([u for u in _aa_urls if u not in _dead_aa_urls]) <= 1:
|
||||
logger.warning("Not quarantining last remaining AA mirror %s (%s)", base, reason)
|
||||
return False
|
||||
_dead_aa_urls.add(base)
|
||||
logger.warning("Quarantined AA mirror %s for this session: %s", base, reason)
|
||||
return True
|
||||
|
||||
|
||||
def get_dead_aa_urls() -> set[str]:
|
||||
"""Return the mirrors quarantined this process (copy)."""
|
||||
with _dead_aa_urls_lock:
|
||||
return set(_dead_aa_urls)
|
||||
|
||||
|
||||
def set_aa_url(url: str) -> bool:
|
||||
"""Set the active AA base URL; returns True if applied."""
|
||||
_ensure_initialized()
|
||||
global _aa_base_url, _current_aa_url_index
|
||||
if new_index < 0 or new_index >= len(_aa_urls):
|
||||
if url not in _aa_urls:
|
||||
return False
|
||||
_current_aa_url_index = new_index
|
||||
_aa_base_url = _aa_urls[_current_aa_url_index]
|
||||
_current_aa_url_index = _aa_urls.index(url)
|
||||
_aa_base_url = url
|
||||
logger.info("Set AA URL to: %s", _aa_base_url)
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
return True
|
||||
|
||||
|
||||
def set_aa_url_index(new_index: int) -> bool:
|
||||
"""Set AA base URL by index in the full configured list; True if applied."""
|
||||
_ensure_initialized()
|
||||
if new_index < 0 or new_index >= len(_aa_urls):
|
||||
return False
|
||||
return set_aa_url(_aa_urls[new_index])
|
||||
|
||||
|
||||
class AAMirrorSelector:
|
||||
"""Keep AA mirror switching consistent across call sites.
|
||||
|
||||
@@ -1352,11 +1594,20 @@ class AAMirrorSelector:
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize mirror state from the current AA configuration."""
|
||||
# Set by html_get_page at each give-up path so a caller that only sees the
|
||||
# returned empty page can still report *why* the fetch produced nothing
|
||||
# (403, 404, redirect loop, bypasser error, mirrors exhausted, ...) instead
|
||||
# of a blanket "network restricted" guess. None means "no failure recorded".
|
||||
self.last_failure: str | None = None
|
||||
self._ensure_fresh_state(reset_attempts=True)
|
||||
|
||||
def _ensure_fresh_state(self, *, reset_attempts: bool = False) -> None:
|
||||
_ensure_initialized()
|
||||
self.aa_urls = get_available_aa_urls()
|
||||
# Rotation walks the live mirrors, but rewriting has to recognise every
|
||||
# configured base: a URL built before a mirror was quarantined still points at
|
||||
# it, and failing to rewrite would send the retry back to the dead host.
|
||||
self.all_aa_urls = _aa_urls.copy()
|
||||
self._index = self._safe_index(get_aa_base_url())
|
||||
self.current_base = self.aa_urls[self._index] if self.aa_urls else ""
|
||||
if reset_attempts:
|
||||
@@ -1369,16 +1620,41 @@ class AAMirrorSelector:
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
"""Replace any known AA base in url with current_base."""
|
||||
for base in self.aa_urls:
|
||||
for base in self.all_aa_urls:
|
||||
if url.startswith(base):
|
||||
return url.replace(base, self.current_base, 1)
|
||||
return url
|
||||
|
||||
def next_mirror_or_rotate_dns(self, *, allow_dns: bool = True) -> tuple[str | None, str]:
|
||||
def quarantine_current(self, reason: str) -> bool:
|
||||
"""Quarantine the mirror this selector is on (hard failures only)."""
|
||||
if not self.current_base:
|
||||
return False
|
||||
dropped = mark_aa_url_dead(self.current_base, reason)
|
||||
if dropped:
|
||||
# Rebuild from the surviving mirrors so the dead one is out of the cycle.
|
||||
self._ensure_fresh_state(reset_attempts=False)
|
||||
return dropped
|
||||
|
||||
def next_mirror_or_rotate_dns(
|
||||
self, *, allow_dns: bool = True, fatal: bool = False, reason: str = ""
|
||||
) -> tuple[str | None, str]:
|
||||
"""Advance to the next mirror or rotate DNS if needed.
|
||||
|
||||
``fatal`` marks the current mirror as not-an-AA-mirror (NXDOMAIN, refused, a
|
||||
200 that isn't AA) and drops it from this process's rotation. Leave it False
|
||||
for anything the mirror can recover from - a challenge or a 5xx means the host
|
||||
is alive, and quarantining it would discard its bypass clearance.
|
||||
|
||||
Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'.
|
||||
"""
|
||||
if fatal and self.quarantine_current(reason or "unusable mirror"):
|
||||
# Quarantining rebuilt the state onto a surviving mirror, so that mirror is
|
||||
# the next one to try - advancing again here would skip straight past it.
|
||||
self.attempts_this_dns += 1
|
||||
if self.current_base and is_aa_auto_mode():
|
||||
set_aa_url(self.current_base)
|
||||
return self.current_base, "mirror"
|
||||
|
||||
self.attempts_this_dns += 1
|
||||
max_attempts = len(self.aa_urls) if is_aa_auto_mode() else 1
|
||||
if self.attempts_this_dns >= max_attempts:
|
||||
@@ -1391,8 +1667,11 @@ class AAMirrorSelector:
|
||||
# Mirror is explicitly configured; do not fail over to other mirrors.
|
||||
return None, "exhausted"
|
||||
|
||||
if not self.aa_urls:
|
||||
return None, "exhausted"
|
||||
|
||||
next_index = (self._index + 1) % len(self.aa_urls)
|
||||
set_aa_url_index(next_index)
|
||||
set_aa_url(self.aa_urls[next_index])
|
||||
self._ensure_fresh_state(reset_attempts=False)
|
||||
return self.current_base, "mirror"
|
||||
|
||||
|
||||
@@ -265,6 +265,8 @@ def queue_release(
|
||||
series_position = release_data.get("series_position") or extra.get("series_position")
|
||||
subtitle = release_data.get("subtitle") or extra.get("subtitle")
|
||||
language = release_data.get("language") or extra.get("language")
|
||||
multi_book = bool(release_data.get("multi_book") or extra.get("multi_book"))
|
||||
book_plan = _normalize_book_plan(release_data.get("book_plan") or extra.get("book_plan"))
|
||||
|
||||
books_output_mode = (
|
||||
str(config.get("BOOKS_OUTPUT_MODE", "folder", user_id=user_id) or "folder")
|
||||
@@ -300,6 +302,8 @@ def queue_release(
|
||||
series_position=series_position,
|
||||
subtitle=subtitle,
|
||||
language=language,
|
||||
multi_book=multi_book or book_plan is not None,
|
||||
book_plan=book_plan,
|
||||
search_mode=search_mode,
|
||||
output_mode=output_mode,
|
||||
output_args=output_args,
|
||||
@@ -408,6 +412,33 @@ def can_retry_download_task(
|
||||
return _has_staged_retry_source(task)
|
||||
|
||||
|
||||
def _normalize_book_plan(value: object) -> list[dict[str, Any]] | None:
|
||||
"""Keep only well-formed pack books: a title plus a non-empty list of file paths."""
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
books: list[dict[str, Any]] = []
|
||||
for entry in value:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
title = normalize_optional_text(entry.get("title"))
|
||||
raw_files = entry.get("files")
|
||||
if title is None or not isinstance(raw_files, list):
|
||||
continue
|
||||
files = [f for f in raw_files if isinstance(f, str) and f.strip()]
|
||||
if not files:
|
||||
continue
|
||||
year = entry.get("year")
|
||||
books.append(
|
||||
{
|
||||
"title": title,
|
||||
"series_position": _optional_number(entry.get("series_position")),
|
||||
"year": year if isinstance(year, int) and not isinstance(year, bool) else None,
|
||||
"files": files,
|
||||
}
|
||||
)
|
||||
return books or None
|
||||
|
||||
|
||||
def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
|
||||
"""Serialize the task state needed for restart-safe retries."""
|
||||
raw_search_mode = getattr(task, "search_mode", None)
|
||||
@@ -437,6 +468,8 @@ def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
|
||||
"subtitle": getattr(task, "subtitle", None),
|
||||
"language": getattr(task, "language", None),
|
||||
"search_mode": search_mode,
|
||||
"multi_book": bool(getattr(task, "multi_book", False)),
|
||||
"book_plan": _normalize_book_plan(getattr(task, "book_plan", None)),
|
||||
"output_mode": getattr(task, "output_mode", None),
|
||||
"output_args": dict(raw_output_args) if isinstance(raw_output_args, dict) else {},
|
||||
"user_id": getattr(task, "user_id", None),
|
||||
@@ -495,6 +528,8 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None:
|
||||
subtitle=normalize_optional_text(payload.get("subtitle")),
|
||||
language=normalize_optional_text(payload.get("language")),
|
||||
search_mode=search_mode,
|
||||
multi_book=bool(payload.get("multi_book", False)),
|
||||
book_plan=_normalize_book_plan(payload.get("book_plan")),
|
||||
output_mode=normalize_optional_text(payload.get("output_mode")),
|
||||
output_args=dict(output_args) if isinstance(output_args, dict) else {},
|
||||
user_id=normalize_positive_int(payload.get("user_id")),
|
||||
|
||||
@@ -105,6 +105,7 @@ def process_folder_output(
|
||||
maybe_run_custom_script,
|
||||
prepare_output_files,
|
||||
record_step,
|
||||
resolve_book_groups,
|
||||
transfer_book_files,
|
||||
)
|
||||
|
||||
@@ -205,6 +206,7 @@ def process_folder_output(
|
||||
is_torrent=is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
organization_mode=plan.organization_mode,
|
||||
source_root=source_path,
|
||||
)
|
||||
|
||||
if error:
|
||||
@@ -259,7 +261,15 @@ def process_folder_output(
|
||||
prepared.cleanup_paths,
|
||||
)
|
||||
|
||||
message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
|
||||
pack_groups = resolve_book_groups(
|
||||
task, prepared.files, organization_mode=plan.organization_mode
|
||||
)
|
||||
if pack_groups is not None:
|
||||
message = f"Complete ({len(pack_groups)} books, {len(final_paths)} files)"
|
||||
elif len(final_paths) == 1:
|
||||
message = "Complete"
|
||||
else:
|
||||
message = f"Complete ({len(final_paths)} files)"
|
||||
status_callback("complete", message)
|
||||
|
||||
return str(final_paths[0])
|
||||
|
||||
@@ -0,0 +1,423 @@
|
||||
"""Multi-book ("pack") release planning.
|
||||
|
||||
A pack is one release that contains several books: a whole-series torrent with one
|
||||
subfolder per book, or a flat folder of `Series 1.0 - Title.m4b` files. The same
|
||||
planning rules serve pre-download inspection (the file list comes from the release
|
||||
source) and post-processing (the file list comes from disk), so what the user
|
||||
approved in the modal is what gets filed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS
|
||||
|
||||
# m4b/m4a hold a whole audiobook in one file; every other audio format (mp3, flac, ...) is
|
||||
# chaptered - many files make up one book. Ebook formats are always one file per book, so
|
||||
# only chaptered *audio* matters here. A flat folder is split one-book-per-file only when
|
||||
# none of its files are chaptered audio: a bare list of `01 - Chapter.mp3` tracks is a
|
||||
# single chaptered audiobook, not a pack of books.
|
||||
_SINGLE_FILE_AUDIO_CONTAINERS = frozenset({"m4b", "m4a"})
|
||||
_CHAPTERED_AUDIO_EXTENSIONS = frozenset(AUDIOBOOK_FORMATS) - _SINGLE_FILE_AUDIO_CONTAINERS
|
||||
|
||||
_YEAR_SUFFIX_RE = re.compile(r"\s*\(\s*(?P<year>\d{4})\s*\)\s*$")
|
||||
_SERIES_MARKER_RE = re.compile(
|
||||
r"""
|
||||
^\s*
|
||||
(?:
|
||||
\[\s*\#?(?P<bracket>\d+(?:\.\d+)?)\s*\] # [03] / [#3]
|
||||
| \#(?P<hash>\d+(?:\.\d+)?) # #3
|
||||
| book\.?\s*(?P<book>\d+(?:\.\d+)?) # Book 3 / Book. 03
|
||||
| (?P<plain>\d+(?:\.\d+)?)(?=[\s\-:.]) # 03 - / 1.0 - / 3.
|
||||
)
|
||||
\s*(?:[-:.]\s*)?
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
_SEPARATOR_CHARS = " \t-_:."
|
||||
# "Gods of Risk 2.5 - Gods of Risk": the title repeated on both sides of the position.
|
||||
_REPEATED_TITLE_RE = re.compile(
|
||||
r"^(?P<left>.+?)\s+(?P<position>\d+(?:\.\d+)?)\s*[-:\u2013]\s*(?P<right>.+)$"
|
||||
)
|
||||
_SERIES_LABEL_WORDS = r"(?:novella|novellas|short\s+story|short|story|novel)"
|
||||
# "Uncrowned Cradle, Book 7" / "Reaper Cradle, Volume 10" / "Wintersteel (Cradle, Book 8)":
|
||||
# an explicit word marks the position at the END of the name. A bare trailing number
|
||||
# is deliberately not matched — "Title - 02" is a chapter, not a series position.
|
||||
_TRAILING_MARKER_RE = re.compile(
|
||||
r"""
|
||||
[\s,\-:\u2013(]*
|
||||
(?:book|volume|vol\.?)\s*\#?(?P<position>\d+(?:\.\d+)?)
|
||||
\s*\)?\s*$
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
# AudiobookBay renders a file inside a folder as "<folder> <file>" with no separator,
|
||||
# so a pack row reads "Author - Title Series, Book 1 Title Series, Book 1".
|
||||
_GLUED_FOLDER_RE = re.compile(
|
||||
r"^(?P<prefix>.+?\s[-\u2013]\s)?(?P<core>.+?)\s+(?P=core)$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackFile:
|
||||
"""One file inside a release, path relative to the release root."""
|
||||
|
||||
path: str
|
||||
size: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackBook:
|
||||
"""One book split out of a pack, files as release-relative paths."""
|
||||
|
||||
title: str
|
||||
series_position: float | None
|
||||
year: int | None
|
||||
files: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackPlan:
|
||||
books: list[PackBook]
|
||||
ignored: list[str]
|
||||
|
||||
@property
|
||||
def is_pack(self) -> bool:
|
||||
return len(self.books) > 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BookGroup:
|
||||
"""One book's on-disk files, ready for transfer."""
|
||||
|
||||
title: str
|
||||
series_position: float | None
|
||||
year: int | None
|
||||
files: list[Path]
|
||||
|
||||
|
||||
def _strip_series_name(name: str, series_name: str | None) -> str:
|
||||
if not series_name:
|
||||
return name
|
||||
prefix = series_name.strip()
|
||||
if not prefix or not name.lower().startswith(prefix.lower()):
|
||||
return name
|
||||
remainder = name[len(prefix) :]
|
||||
if remainder and remainder[0].isalnum():
|
||||
return name
|
||||
return remainder.lstrip(_SEPARATOR_CHARS)
|
||||
|
||||
|
||||
def _strip_series_label(work: str, series_name: str | None) -> str:
|
||||
"""Drop a leading "An <Series> Novella - " style label that some packs prepend."""
|
||||
if not series_name:
|
||||
return work
|
||||
# "The Expanse" is labelled "An Expanse Novella", so match without the article.
|
||||
core = re.sub(r"^(?:the|an?)\s+", "", series_name.strip(), flags=re.IGNORECASE)
|
||||
if not core:
|
||||
return work
|
||||
pattern = re.compile(
|
||||
rf"^(?:an?\s+|the\s+)?{re.escape(core)}\s+{_SERIES_LABEL_WORDS}\s*[-:\u2013]\s*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
return pattern.sub("", work, count=1)
|
||||
|
||||
|
||||
def _collapse_glued_folder(name: str) -> str:
|
||||
match = _GLUED_FOLDER_RE.match(name)
|
||||
if not match:
|
||||
return name
|
||||
prefix = match.group("prefix") or ""
|
||||
core = match.group("core")
|
||||
# "Author - X X" → "Author - X" (the folder carried the author, the file did not).
|
||||
return (prefix + core).strip()
|
||||
|
||||
|
||||
def _strip_author_name(name: str, author_name: str | None) -> str:
|
||||
"""Drop a leading "Author - " (packs are often filed as `Author - Title`)."""
|
||||
if not author_name:
|
||||
return name
|
||||
prefix = author_name.strip()
|
||||
if not prefix or not name.lower().startswith(prefix.lower()):
|
||||
return name
|
||||
remainder = name[len(prefix) :]
|
||||
stripped = remainder.lstrip(_SEPARATOR_CHARS + "\u2013")
|
||||
if stripped == remainder: # no separator after the author: part of the title
|
||||
return name
|
||||
return stripped
|
||||
|
||||
|
||||
def _strip_trailing_series_name(work: str, series_name: str | None) -> str:
|
||||
"""Drop a trailing series name left behind by a trailing position marker."""
|
||||
if not series_name:
|
||||
return work
|
||||
suffix = series_name.strip()
|
||||
if not suffix or not work.lower().endswith(suffix.lower()):
|
||||
return work
|
||||
remainder = work[: -len(suffix)]
|
||||
stripped = remainder.rstrip(_SEPARATOR_CHARS + ",(\u2013")
|
||||
if not stripped or stripped == remainder:
|
||||
return work
|
||||
return stripped
|
||||
|
||||
|
||||
def parse_pack_book_name(
|
||||
name: str, *, series_name: str | None, author_name: str | None = None
|
||||
) -> tuple[str, float | None, int | None]:
|
||||
"""Split a book folder/file-stem name into (title, series position, year).
|
||||
|
||||
Strips a leading series name, a leading position marker (`Book 3 - `, `03 - `,
|
||||
`1.0 - `, `3. `, `[03] `, `#3 `) and a trailing `(YYYY)`. Also understands a
|
||||
trailing marker (`Title Series, Book 3`, `Title (Series, Volume 3)`), a leading
|
||||
`Author - `, and AudiobookBay's glued `<folder> <file>` names. Returns the name
|
||||
unchanged with no position/year when nothing would be left of the title.
|
||||
"""
|
||||
work = _collapse_glued_folder(name.strip())
|
||||
work = _strip_author_name(work, author_name)
|
||||
work = _strip_series_name(work, series_name)
|
||||
|
||||
year: int | None = None
|
||||
year_match = _YEAR_SUFFIX_RE.search(work)
|
||||
if year_match:
|
||||
year = int(year_match.group("year"))
|
||||
work = work[: year_match.start()]
|
||||
|
||||
position: float | None = None
|
||||
repeated = _REPEATED_TITLE_RE.match(work.strip())
|
||||
if (
|
||||
repeated
|
||||
and repeated.group("left").strip().lower() == repeated.group("right").strip().lower()
|
||||
):
|
||||
return repeated.group("right").strip(), float(repeated.group("position")), year
|
||||
|
||||
marker = _SERIES_MARKER_RE.match(work)
|
||||
if marker:
|
||||
raw = (
|
||||
marker.group("bracket")
|
||||
or marker.group("hash")
|
||||
or marker.group("book")
|
||||
or marker.group("plain")
|
||||
)
|
||||
position = float(raw)
|
||||
work = work[marker.end() :]
|
||||
else:
|
||||
trailing = _TRAILING_MARKER_RE.search(work)
|
||||
if trailing and trailing.start() > 0:
|
||||
position = float(trailing.group("position"))
|
||||
work = _strip_trailing_series_name(work[: trailing.start()], series_name)
|
||||
|
||||
work = _strip_series_label(work, series_name)
|
||||
title = work.strip().strip(_SEPARATOR_CHARS).strip()
|
||||
if not title:
|
||||
return name, None, None
|
||||
return title, position, year
|
||||
|
||||
|
||||
def _book_from_name(
|
||||
name: str, files: list[str], series_name: str | None, author_name: str | None = None
|
||||
) -> PackBook:
|
||||
title, position, year = parse_pack_book_name(
|
||||
name, series_name=series_name, author_name=author_name
|
||||
)
|
||||
return PackBook(title=title, series_position=position, year=year, files=files)
|
||||
|
||||
|
||||
def _common_root_parts(paths: list[PurePosixPath]) -> tuple[str, ...]:
|
||||
parents = [p.parent.parts for p in paths]
|
||||
common: list[str] = []
|
||||
for parts in zip(*parents, strict=False):
|
||||
if len(set(parts)) != 1:
|
||||
break
|
||||
common.append(parts[0])
|
||||
return tuple(common)
|
||||
|
||||
|
||||
def plan_pack(
|
||||
files: list[PackFile],
|
||||
*,
|
||||
supported_extensions: set[str],
|
||||
series_name: str | None,
|
||||
author_name: str | None = None,
|
||||
root_depth: int | None = None,
|
||||
) -> PackPlan:
|
||||
"""Group a release's file list into books.
|
||||
|
||||
Files in a subfolder (relative to the common root) group by that subfolder. Files
|
||||
directly in the root split one-book-per-file only when at least two of them carry
|
||||
a series position in their names; otherwise they are one book (a chaptered
|
||||
audiobook, e.g. `01.mp3`, `02.mp3`). `root_depth` fixes how many leading path
|
||||
components form the root instead of deriving it from the files' common parent.
|
||||
"""
|
||||
supported = {ext.lower().lstrip(".") for ext in supported_extensions}
|
||||
book_files: list[PurePosixPath] = []
|
||||
ignored: list[str] = []
|
||||
for pack_file in files:
|
||||
rel = PurePosixPath(pack_file.path.replace("\\", "/").lstrip("./"))
|
||||
if rel.suffix.lower().lstrip(".") in supported:
|
||||
book_files.append(rel)
|
||||
else:
|
||||
ignored.append(pack_file.path)
|
||||
|
||||
if not book_files:
|
||||
return PackPlan(books=[], ignored=ignored)
|
||||
|
||||
root_parts = (
|
||||
_common_root_parts(book_files) if root_depth is None else book_files[0].parts[:root_depth]
|
||||
)
|
||||
depth = len(root_parts)
|
||||
|
||||
root_files: list[PurePosixPath] = []
|
||||
folders: dict[str, list[str]] = {}
|
||||
for rel in book_files:
|
||||
remainder = rel.parts[depth:]
|
||||
if len(remainder) > 1:
|
||||
folders.setdefault(remainder[0], []).append(str(rel))
|
||||
else:
|
||||
root_files.append(rel)
|
||||
|
||||
books: list[PackBook] = []
|
||||
if root_files:
|
||||
parsed = [
|
||||
parse_pack_book_name(f.stem, series_name=series_name, author_name=author_name)
|
||||
for f in root_files
|
||||
]
|
||||
positions = {p[1] for p in parsed if p[1] is not None}
|
||||
titles = {p[0].strip().lower() for p in parsed if p[0]}
|
||||
one_book_per_file = all(
|
||||
rel.suffix.lower().lstrip(".") not in _CHAPTERED_AUDIO_EXTENSIONS for rel in root_files
|
||||
)
|
||||
# Split a flat folder into a book per file only with real evidence of distinct
|
||||
# books: two or more series positions, more than one title, and no chaptered audio
|
||||
# (a bare list of `01 - Chapter.mp3` tracks is one book, not a pack).
|
||||
if len(positions) >= 2 and len(titles) >= 2 and one_book_per_file:
|
||||
books.extend(
|
||||
PackBook(title=title, series_position=position, year=year, files=[str(f)])
|
||||
for f, (title, position, year) in zip(root_files, parsed, strict=True)
|
||||
)
|
||||
elif len(root_files) == 1:
|
||||
books.append(
|
||||
_book_from_name(root_files[0].stem, [str(root_files[0])], series_name, author_name)
|
||||
)
|
||||
else:
|
||||
group_name = root_parts[-1] if root_parts else ""
|
||||
books.append(
|
||||
_book_from_name(group_name, [str(f) for f in root_files], series_name, author_name)
|
||||
)
|
||||
|
||||
books.extend(
|
||||
_book_from_name(folder, paths, series_name, author_name)
|
||||
for folder, paths in folders.items()
|
||||
)
|
||||
return PackPlan(books=books, ignored=ignored)
|
||||
|
||||
|
||||
def _relative_paths(
|
||||
book_files: list[Path], root: Path | None = None
|
||||
) -> tuple[Path, dict[Path, str]]:
|
||||
if root is None:
|
||||
root = Path(os.path.commonpath([str(f.parent) for f in book_files]))
|
||||
return root, {f: f.relative_to(root).as_posix() for f in book_files}
|
||||
|
||||
|
||||
def group_files_into_books(
|
||||
book_files: list[Path],
|
||||
*,
|
||||
series_name: str | None,
|
||||
author_name: str | None = None,
|
||||
root: Path | None = None,
|
||||
) -> list[BookGroup]:
|
||||
"""Heuristically split on-disk files into books (see `plan_pack`).
|
||||
|
||||
`root` pins the release root when grouping a subset of a larger file set.
|
||||
"""
|
||||
if not book_files:
|
||||
return []
|
||||
_root, rel_by_path = _relative_paths(book_files, root)
|
||||
path_by_rel = {rel: path for path, rel in rel_by_path.items()}
|
||||
extensions = {f.suffix.lower().lstrip(".") for f in book_files}
|
||||
plan = plan_pack(
|
||||
[PackFile(rel) for rel in rel_by_path.values()],
|
||||
supported_extensions=extensions,
|
||||
series_name=series_name,
|
||||
author_name=author_name,
|
||||
root_depth=None if root is None else 0,
|
||||
)
|
||||
return [
|
||||
BookGroup(
|
||||
title=book.title,
|
||||
series_position=book.series_position,
|
||||
year=book.year,
|
||||
files=[path_by_rel[rel] for rel in book.files],
|
||||
)
|
||||
for book in plan.books
|
||||
]
|
||||
|
||||
|
||||
def match_plan_to_files(
|
||||
plan: list[PackBook],
|
||||
book_files: list[Path],
|
||||
*,
|
||||
series_name: str | None = None,
|
||||
author_name: str | None = None,
|
||||
) -> list[BookGroup]:
|
||||
"""Apply an approved plan to on-disk files.
|
||||
|
||||
Files match by release-relative path first, then by basename (archive extraction
|
||||
and client save paths can shift the root), then by the on-disk basename being a
|
||||
suffix of the planned name (sources that glue folder and file names together).
|
||||
Book files the plan does not mention fall back to heuristic grouping so nothing
|
||||
is silently dropped.
|
||||
"""
|
||||
if not book_files:
|
||||
return []
|
||||
root, rel_by_path = _relative_paths(book_files)
|
||||
by_rel = {rel: path for path, rel in rel_by_path.items()}
|
||||
by_name: dict[str, list[Path]] = {}
|
||||
for path in book_files:
|
||||
by_name.setdefault(path.name, []).append(path)
|
||||
|
||||
claimed: set[Path] = set()
|
||||
groups: list[BookGroup] = []
|
||||
for book in plan:
|
||||
matched: list[Path] = []
|
||||
for wanted in book.files:
|
||||
wanted_rel = wanted.replace("\\", "/").lstrip("./")
|
||||
candidate = by_rel.get(wanted_rel)
|
||||
if candidate is None:
|
||||
candidates = [
|
||||
p for p in by_name.get(PurePosixPath(wanted_rel).name, []) if p not in claimed
|
||||
]
|
||||
candidate = candidates[0] if candidates else None
|
||||
if candidate is None:
|
||||
wanted_name = PurePosixPath(wanted_rel).name.lower()
|
||||
candidates = [
|
||||
p
|
||||
for p in book_files
|
||||
if p not in claimed and wanted_name.endswith(p.name.lower())
|
||||
]
|
||||
candidate = candidates[0] if len(candidates) == 1 else None
|
||||
if candidate is not None and candidate not in claimed:
|
||||
claimed.add(candidate)
|
||||
matched.append(candidate)
|
||||
if matched:
|
||||
groups.append(
|
||||
BookGroup(
|
||||
title=book.title,
|
||||
series_position=book.series_position,
|
||||
year=book.year,
|
||||
files=matched,
|
||||
)
|
||||
)
|
||||
|
||||
unmatched = [p for p in book_files if p not in claimed]
|
||||
if unmatched:
|
||||
groups.extend(
|
||||
group_files_into_books(
|
||||
unmatched, series_name=series_name, author_name=author_name, root=root
|
||||
)
|
||||
)
|
||||
return groups
|
||||
@@ -40,6 +40,7 @@ from .transfer import (
|
||||
build_metadata_dict,
|
||||
is_torrent_source,
|
||||
process_directory,
|
||||
resolve_book_groups,
|
||||
resolve_hardlink_source,
|
||||
should_hardlink,
|
||||
transfer_book_files,
|
||||
@@ -80,6 +81,7 @@ __all__ = [
|
||||
"process_directory",
|
||||
"record_step",
|
||||
"resolve_custom_script_target",
|
||||
"resolve_book_groups",
|
||||
"resolve_hardlink_source",
|
||||
"run_custom_script",
|
||||
"safe_cleanup_path",
|
||||
|
||||
@@ -52,7 +52,7 @@ def get_file_organization(*, is_audiobook: bool) -> str:
|
||||
"""Get the file organization mode for the content type."""
|
||||
key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
|
||||
mode = _config_text(core_config.config.get(key, "rename")).strip().lower()
|
||||
return mode if mode in ("none", "rename", "organize") else "rename"
|
||||
return mode if mode in ("none", "rename", "rename_and_group", "organize") else "rename"
|
||||
|
||||
|
||||
def get_template(*, is_audiobook: bool, organization_mode: str) -> str:
|
||||
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive
|
||||
from shelfmark.download.fs import run_blocking_io
|
||||
@@ -142,7 +143,7 @@ def scan_directory_tree(
|
||||
|
||||
is_audiobook = check_audiobook(content_type)
|
||||
if is_audiobook:
|
||||
trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"}
|
||||
trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS}
|
||||
else:
|
||||
trackable_exts = {
|
||||
".pdf",
|
||||
@@ -367,7 +368,7 @@ def collect_staged_files(
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
if is_audiobook:
|
||||
trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"}
|
||||
trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS}
|
||||
else:
|
||||
trackable_exts = {
|
||||
".pdf",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -17,6 +18,7 @@ from shelfmark.core.naming import (
|
||||
sanitize_filename,
|
||||
)
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.archive import is_archive
|
||||
from shelfmark.download.fs import (
|
||||
atomic_copy,
|
||||
atomic_hardlink,
|
||||
@@ -25,6 +27,7 @@ from shelfmark.download.fs import (
|
||||
)
|
||||
from shelfmark.download.postprocess.policy import get_file_organization, get_template
|
||||
|
||||
from .packs import BookGroup, PackBook, group_files_into_books, match_plan_to_files
|
||||
from .scan import collect_directory_files, scan_directory_tree
|
||||
from .types import TransferPlan
|
||||
from .workspace import safe_cleanup_path
|
||||
@@ -160,6 +163,24 @@ def _transfer_single_file(
|
||||
return atomic_move(source_path, dest_path, max_attempts=max_attempts), "move"
|
||||
|
||||
|
||||
def _group_folder_name(source_root: Path | None) -> str:
|
||||
"""Name the folder a grouped multi-file audiobook is transferred into.
|
||||
|
||||
A directory names the group directly. A file cannot hold several book files
|
||||
on its own, so a non-directory source that produced more than one means
|
||||
`collect_staged_files` extracted an archive: the stem is the release name and
|
||||
the suffix is packaging, which is why `Book.zip` groups into `Book/` rather
|
||||
than `Book.zip/` or, worse, not at all.
|
||||
"""
|
||||
if source_root is None:
|
||||
return ""
|
||||
if run_blocking_io(source_root.is_dir):
|
||||
return sanitize_filename(source_root.name)
|
||||
if is_archive(source_root):
|
||||
return sanitize_filename(source_root.stem)
|
||||
return ""
|
||||
|
||||
|
||||
def transfer_book_files(
|
||||
book_files: list[Path],
|
||||
destination: Path,
|
||||
@@ -169,6 +190,7 @@ def transfer_book_files(
|
||||
is_torrent: bool,
|
||||
preserve_source: bool = False,
|
||||
organization_mode: str | None = None,
|
||||
source_root: Path | None = None,
|
||||
) -> tuple[list[Path], str | None, dict[str, int]]:
|
||||
"""Transfer discovered book files into their final destination layout."""
|
||||
if not book_files:
|
||||
@@ -176,6 +198,19 @@ def transfer_book_files(
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
organization_mode = organization_mode or get_file_organization(is_audiobook=is_audiobook)
|
||||
|
||||
groups = resolve_book_groups(task, book_files, organization_mode=organization_mode)
|
||||
if groups is not None:
|
||||
return _transfer_book_groups(
|
||||
groups,
|
||||
destination,
|
||||
task,
|
||||
use_hardlink=use_hardlink,
|
||||
is_torrent=is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
organization_mode=organization_mode,
|
||||
)
|
||||
|
||||
max_attempts = _max_attempts_for_batch(len(book_files))
|
||||
|
||||
final_paths: list[Path] = []
|
||||
@@ -238,6 +273,13 @@ def transfer_book_files(
|
||||
|
||||
return final_paths, None, op_counts
|
||||
|
||||
transfer_destination = destination
|
||||
if is_audiobook and len(book_files) > 1 and organization_mode == "rename_and_group":
|
||||
source_folder = _group_folder_name(source_root)
|
||||
if source_folder:
|
||||
transfer_destination = destination / source_folder
|
||||
run_blocking_io(transfer_destination.mkdir, parents=True, exist_ok=True)
|
||||
|
||||
for book_file in book_files:
|
||||
if len(book_files) == 1 and organization_mode != "none":
|
||||
if not task.format:
|
||||
@@ -256,7 +298,7 @@ def transfer_book_files(
|
||||
else:
|
||||
filename = book_file.name
|
||||
|
||||
dest_path = destination / filename
|
||||
dest_path = transfer_destination / filename
|
||||
final_path, op = _transfer_single_file(
|
||||
book_file,
|
||||
dest_path,
|
||||
@@ -272,6 +314,101 @@ def transfer_book_files(
|
||||
return final_paths, None, op_counts
|
||||
|
||||
|
||||
def resolve_book_groups(
|
||||
task: DownloadTask,
|
||||
book_files: list[Path],
|
||||
*,
|
||||
organization_mode: str,
|
||||
) -> list[BookGroup] | None:
|
||||
"""Split a multi-book pack into per-book groups, or None to file as one book.
|
||||
|
||||
An approved `book_plan` wins; a bare `multi_book` flag falls back to heuristic
|
||||
grouping. Organization `none` keeps files as-is, and a split that yields a single
|
||||
group is not a pack at all.
|
||||
"""
|
||||
if organization_mode == "none" or not (task.book_plan or task.multi_book):
|
||||
return None
|
||||
if task.book_plan:
|
||||
plan = [
|
||||
PackBook(
|
||||
title=str(entry.get("title") or ""),
|
||||
series_position=entry.get("series_position"),
|
||||
year=entry.get("year"),
|
||||
files=list(entry.get("files") or []),
|
||||
)
|
||||
for entry in task.book_plan
|
||||
if isinstance(entry, dict)
|
||||
]
|
||||
groups = match_plan_to_files(
|
||||
plan, book_files, series_name=task.series_name, author_name=task.author
|
||||
)
|
||||
else:
|
||||
groups = group_files_into_books(
|
||||
book_files, series_name=task.series_name, author_name=task.author
|
||||
)
|
||||
return groups if len(groups) > 1 else None
|
||||
|
||||
|
||||
def _transfer_book_groups(
|
||||
groups: list[BookGroup],
|
||||
destination: Path,
|
||||
task: DownloadTask,
|
||||
*,
|
||||
use_hardlink: bool,
|
||||
is_torrent: bool,
|
||||
preserve_source: bool,
|
||||
organization_mode: str,
|
||||
) -> tuple[list[Path], str | None, dict[str, int]]:
|
||||
"""Transfer each book of a pack through the normal single-book path.
|
||||
|
||||
Each book gets an isolated task copy (the single-file path mutates `task.format`)
|
||||
carrying its own title, position and year; the searched book's position must not
|
||||
leak onto its siblings, while author and series name apply to all of them.
|
||||
"""
|
||||
all_paths: list[Path] = []
|
||||
totals: dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0}
|
||||
errors: list[str] = []
|
||||
|
||||
for group in groups:
|
||||
book_task = dataclasses.replace(
|
||||
task,
|
||||
title=group.title or task.title,
|
||||
year=str(group.year) if group.year is not None else None,
|
||||
subtitle=None,
|
||||
series_position=group.series_position,
|
||||
multi_book=False,
|
||||
book_plan=None,
|
||||
)
|
||||
paths, error, op_counts = transfer_book_files(
|
||||
group.files,
|
||||
destination,
|
||||
book_task,
|
||||
use_hardlink=use_hardlink,
|
||||
is_torrent=is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
organization_mode=organization_mode,
|
||||
source_root=group.files[0].parent,
|
||||
)
|
||||
for op, count in op_counts.items():
|
||||
totals[op] = totals.get(op, 0) + count
|
||||
if error:
|
||||
errors.append(f"{group.title}: {error}")
|
||||
logger.warning("Task %s: pack book %r failed: %s", task.task_id, group.title, error)
|
||||
continue
|
||||
all_paths.extend(paths)
|
||||
|
||||
if not all_paths:
|
||||
return [], "; ".join(errors) or "No book files found", totals
|
||||
if errors:
|
||||
logger.warning(
|
||||
"Task %s: pack filed with %d failed book(s): %s",
|
||||
task.task_id,
|
||||
len(errors),
|
||||
"; ".join(errors),
|
||||
)
|
||||
return all_paths, None, totals
|
||||
|
||||
|
||||
def process_directory(
|
||||
directory: Path,
|
||||
ingest_dir: Path,
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Boot-time warm-up of the direct-download source.
|
||||
|
||||
The first AA search after a cold start pays for the whole cold path at once: DNS
|
||||
resolution, electing a live mirror, spinning up headless Chrome and solving the
|
||||
DDoS-Guard challenge. That is tens of seconds with the user sat at the search box.
|
||||
|
||||
Running one throwaway search shortly after boot moves that cost off the user's first
|
||||
search. It primes the DNS cache, elects (and quarantines) mirrors, and leaves the
|
||||
clearance cookie in the bypasser's per-domain cache, so the first real search reuses
|
||||
it instead of solving from scratch.
|
||||
|
||||
Runs on a daemon thread and swallows every failure: this is an optimisation, and a
|
||||
source that is down at boot must not affect startup or health.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Delay before the warm-up fires. Long enough that it does not compete with the rest
|
||||
# of startup (and with a container's own health probe) for the first request.
|
||||
_DEFAULT_DELAY_SECONDS = 15.0
|
||||
|
||||
_DEFAULT_QUERY = "The Great Gatsby"
|
||||
|
||||
_warmup_thread: threading.Thread | None = None
|
||||
_warmup_lock = threading.Lock()
|
||||
|
||||
# Set as soon as a real release search starts. The warm-up exists to pay the cold path
|
||||
# *before* the user does; once they have beaten it to the box there is nothing left to
|
||||
# pre-solve, and running anyway is actively harmful - the bypasser serializes on one
|
||||
# browser, so the warm-up's solve goes in front of the search the user is watching. In
|
||||
# the bundle on issue #1276 that cost a full minute of a 2m27s wait, on a container 16
|
||||
# seconds old, for a throwaway "The Great Gatsby" query nobody asked for.
|
||||
_user_search_seen = threading.Event()
|
||||
|
||||
|
||||
def note_user_search() -> None:
|
||||
"""Record that a real search has run, so a pending warm-up stands down."""
|
||||
_user_search_seen.set()
|
||||
|
||||
|
||||
def _as_bool(value: object, *, default: bool) -> bool:
|
||||
"""Coerce a config value that may arrive as a string, bool or None."""
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, str):
|
||||
from shelfmark.config.env import string_to_bool
|
||||
|
||||
return string_to_bool(value)
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _setting(key: str, default: object) -> object:
|
||||
"""Read a warm-up setting, preferring the deployment environment.
|
||||
|
||||
These keys are not in the settings registry, and ``config.get`` only consults the
|
||||
environment for keys it knows about - so reading config alone silently ignored
|
||||
SEARCH_WARMUP_ENABLED and always returned the default. Check os.environ first so
|
||||
the documented switches actually work.
|
||||
"""
|
||||
raw = os.environ.get(key)
|
||||
if raw is not None and raw.strip():
|
||||
return raw
|
||||
return config.get(key, default)
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""Whether the boot-time warm-up search should run."""
|
||||
if not _as_bool(_setting("SEARCH_WARMUP_ENABLED", True), default=True):
|
||||
return False
|
||||
# Nothing to warm if the source is off, and no challenge to pre-solve without
|
||||
# the bypasser - a plain search is fast enough not to need this.
|
||||
if not _as_bool(_setting("DIRECT_DOWNLOAD_ENABLED", True), default=True):
|
||||
logger.debug("Search warm-up skipped: direct download disabled")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def warmup_query() -> str:
|
||||
"""The query used to warm the source."""
|
||||
raw = _setting("SEARCH_WARMUP_QUERY", _DEFAULT_QUERY)
|
||||
query = str(raw).strip() if raw else ""
|
||||
return query or _DEFAULT_QUERY
|
||||
|
||||
|
||||
def run_warmup() -> bool:
|
||||
"""Run one warm-up search. Returns True if it produced results.
|
||||
|
||||
Never raises: every failure mode here is one the next real search would hit
|
||||
anyway, and reporting it is the search path's job, not the warm-up's.
|
||||
"""
|
||||
from shelfmark.core.mirrors import has_aa_mirror_configuration
|
||||
|
||||
# Checked here rather than only at schedule time: the delay is what this races with,
|
||||
# so the user's first search usually lands *during* the wait, not before it.
|
||||
if _user_search_seen.is_set():
|
||||
logger.info("Search warm-up skipped: a real search got there first")
|
||||
return False
|
||||
|
||||
if not has_aa_mirror_configuration():
|
||||
logger.debug("Search warm-up skipped: no Anna's Archive mirrors configured")
|
||||
return False
|
||||
|
||||
query = warmup_query()
|
||||
logger.info("Warming up direct download search (%r)", query)
|
||||
try:
|
||||
from shelfmark.core.models import SearchFilters
|
||||
from shelfmark.release_sources.direct_download import search_books
|
||||
|
||||
results = search_books(query, SearchFilters())
|
||||
except Exception:
|
||||
# Broad by design: a warm-up must never take the app down, and the source
|
||||
# raises everything from network errors to parse failures.
|
||||
logger.warning("Search warm-up did not complete; first user search may be slow")
|
||||
logger.debug("Search warm-up failure detail", exc_info=True)
|
||||
return False
|
||||
|
||||
if results:
|
||||
logger.info("Search warm-up complete: %s results, source is ready", len(results))
|
||||
return True
|
||||
logger.info("Search warm-up returned no results; source reachable but empty")
|
||||
return False
|
||||
|
||||
|
||||
def start(delay_seconds: float = _DEFAULT_DELAY_SECONDS) -> bool:
|
||||
"""Schedule the warm-up on a daemon thread. Safe to call multiple times."""
|
||||
global _warmup_thread
|
||||
|
||||
if not is_enabled():
|
||||
return False
|
||||
|
||||
with _warmup_lock:
|
||||
if _warmup_thread is not None and _warmup_thread.is_alive():
|
||||
logger.debug("Search warm-up already scheduled")
|
||||
return False
|
||||
|
||||
def _run() -> None:
|
||||
run_warmup()
|
||||
|
||||
_warmup_thread = threading.Timer(delay_seconds, _run)
|
||||
_warmup_thread.daemon = True
|
||||
_warmup_thread.name = "SearchWarmup"
|
||||
_warmup_thread.start()
|
||||
|
||||
logger.debug("Search warm-up scheduled in %ss", delay_seconds)
|
||||
return True
|
||||
+49
-24
@@ -38,7 +38,11 @@ from shelfmark.config.env import (
|
||||
string_to_bool,
|
||||
)
|
||||
from shelfmark.config.security import _migrate_security_settings
|
||||
from shelfmark.config.settings import _SUPPORTED_BOOK_LANGUAGE
|
||||
from shelfmark.config.settings import (
|
||||
_SUPPORTED_BOOK_LANGUAGE,
|
||||
migrate_audiobook_format_settings,
|
||||
)
|
||||
from shelfmark.core import search_deadline
|
||||
from shelfmark.core.activity_view_state_service import ActivityViewStateService
|
||||
from shelfmark.core.auth_modes import (
|
||||
get_auth_check_admin_status,
|
||||
@@ -59,6 +63,7 @@ from shelfmark.core.notifications import (
|
||||
notify_user,
|
||||
)
|
||||
from shelfmark.core.prefix_middleware import PrefixMiddleware
|
||||
from shelfmark.core.release_inspect_routes import register_release_inspect_routes
|
||||
from shelfmark.core.request_helpers import (
|
||||
coerce_bool,
|
||||
emit_ws_event,
|
||||
@@ -80,8 +85,9 @@ from shelfmark.core.requests_service import (
|
||||
sync_delivery_states_from_queue_status,
|
||||
)
|
||||
from shelfmark.core.user_db import UserDB
|
||||
from shelfmark.core.utils import normalize_base_path
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS, normalize_base_path
|
||||
from shelfmark.download import orchestrator as backend
|
||||
from shelfmark.download import warmup
|
||||
from shelfmark.release_sources import (
|
||||
BrowseRecord,
|
||||
Release,
|
||||
@@ -168,6 +174,9 @@ except ImportError as e:
|
||||
# Migrate legacy security settings if needed
|
||||
_migrate_security_settings()
|
||||
|
||||
# Widen audiobook formats for installs that still carry the old m4b/mp3-only default
|
||||
migrate_audiobook_format_settings()
|
||||
|
||||
# Initialize user database and register multi-user routes
|
||||
# If CONFIG_DIR doesn't exist or is read-only, multi-user features will be disabled
|
||||
_user_db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db")
|
||||
@@ -200,6 +209,10 @@ except (sqlite3.OperationalError, OSError) as e:
|
||||
# Start download coordinator
|
||||
backend.start()
|
||||
|
||||
# Pre-solve the direct-download source's protection challenge in the background so the
|
||||
# first user search does not pay for a cold Chrome bypass. Never blocks startup.
|
||||
warmup.start()
|
||||
|
||||
# Rate limiting for login attempts
|
||||
# Map usernames to their failed-attempt counters and lockout timestamps.
|
||||
failed_login_attempts: dict[str, dict[str, Any]] = {}
|
||||
@@ -319,19 +332,7 @@ def get_auth_mode() -> str:
|
||||
|
||||
|
||||
_AUDIOBOOK_CATEGORY_RANGE = (3030, 3049)
|
||||
_AUDIOBOOK_FORMAT_HINTS = frozenset(
|
||||
{
|
||||
"m4b",
|
||||
"mp3",
|
||||
"m4a",
|
||||
"flac",
|
||||
"ogg",
|
||||
"wma",
|
||||
"aac",
|
||||
"wav",
|
||||
"opus",
|
||||
}
|
||||
)
|
||||
_AUDIOBOOK_FORMAT_HINTS = frozenset(AUDIOBOOK_FORMATS)
|
||||
|
||||
|
||||
def _contains_audiobook_format_hint(value: Any) -> bool:
|
||||
@@ -1025,6 +1026,9 @@ def _serialize_release(release: Release) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
register_release_inspect_routes(app, login_required)
|
||||
|
||||
|
||||
@app.route("/api/releases/download", methods=["POST"])
|
||||
@login_required
|
||||
def api_download_release() -> Response | tuple[Response, int]:
|
||||
@@ -1151,7 +1155,7 @@ def api_config() -> Response | tuple[Response, int]:
|
||||
"build_version": BUILD_VERSION,
|
||||
"release_version": RELEASE_VERSION,
|
||||
"book_languages": _SUPPORTED_BOOK_LANGUAGE,
|
||||
"default_language": app_config.BOOK_LANGUAGE,
|
||||
"default_language": app_config.get("BOOK_LANGUAGE", ["en"], user_id=db_user_id),
|
||||
"supported_formats": app_config.SUPPORTED_FORMATS,
|
||||
"supported_audiobook_formats": app_config.SUPPORTED_AUDIOBOOK_FORMATS,
|
||||
"search_mode": search_mode,
|
||||
@@ -2841,6 +2845,7 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
manual_query=query_text if source_query_filters is not None else manual_query,
|
||||
indexers=indexers,
|
||||
source_filters=source_query_filters,
|
||||
user_id=db_user_id,
|
||||
)
|
||||
|
||||
if plan.source_filters is not None:
|
||||
@@ -2893,6 +2898,8 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
if languages_param
|
||||
else None
|
||||
)
|
||||
# Without an explicit filter the plan falls back to this user's default languages.
|
||||
db_user_id = get_session_db_user_id(session)
|
||||
# Content type for audiobook vs ebook search
|
||||
content_type = request.args.get("content_type", "ebook").strip()
|
||||
|
||||
@@ -2981,18 +2988,36 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
# Search only enabled sources
|
||||
sources_to_search = [src["name"] for src in list_available_sources() if src["enabled"]]
|
||||
|
||||
# Search each source for releases
|
||||
# Search each source for releases.
|
||||
#
|
||||
# Under a wall-clock budget: this endpoint is synchronous, and the bypass path it
|
||||
# can reach used to be allowed minutes per URL with nothing bounding the request
|
||||
# as a whole. A search that ran into an unsolvable protection challenge therefore
|
||||
# outlived every reverse proxy in front of it and surfaced to the user as
|
||||
# "Server unavailable (504)" - a gateway timeout that blames their proxy for a
|
||||
# challenge failure. The budget is shared across sources, so a stuck first source
|
||||
# cannot spend the whole request on its own. See issue #1276.
|
||||
all_releases = []
|
||||
errors = []
|
||||
source_instances = {} # Keep source instances for column config
|
||||
|
||||
for source_name in sources_to_search:
|
||||
source, releases, error = _search_source_releases(source_name, book)
|
||||
if source is not None:
|
||||
source_instances[source_name] = source
|
||||
all_releases.extend(releases)
|
||||
if error is not None:
|
||||
errors.append(error)
|
||||
# A real search is under way, so a warm-up still sitting on its start-up delay
|
||||
# should stand down rather than queue its throwaway solve in front of this one.
|
||||
warmup.note_user_search()
|
||||
|
||||
with search_deadline.search_deadline():
|
||||
for source_name in sources_to_search:
|
||||
if search_deadline.expired():
|
||||
logger.warning("Release search budget spent; %s not searched", source_name)
|
||||
errors.append(f"{source_name}: {search_deadline.deadline_message()}")
|
||||
continue
|
||||
|
||||
source, releases, error = _search_source_releases(source_name, book)
|
||||
if source is not None:
|
||||
source_instances[source_name] = source
|
||||
all_releases.extend(releases)
|
||||
if error is not None:
|
||||
errors.append(error)
|
||||
|
||||
# Convert Release objects to dicts
|
||||
releases_data = [_serialize_release(release) for release in all_releases]
|
||||
|
||||
@@ -48,6 +48,10 @@ HARDCOVER_PAGE_SIZE = 25 # Hardcover API returns max 25 results per page
|
||||
HARDCOVER_MIN_AUTHOR_PARTS = 2
|
||||
HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH = 2
|
||||
HARDCOVER_MAX_SERIES_OPTIONS = 7
|
||||
# Hardcover hands out short opaque tokens now ("hc_pat_...") instead of the ~500 char
|
||||
# JWTs it used to, so the length floor only applies to keys without that prefix.
|
||||
HARDCOVER_API_KEY_PREFIX = "hc_pat_"
|
||||
HARDCOVER_BEARER_PREFIX_PATTERN = re.compile(r"^bearer\s+", re.IGNORECASE)
|
||||
HARDCOVER_API_KEY_MIN_LENGTH = 100
|
||||
HARDCOVER_LIST_URL_PATTERN = re.compile(
|
||||
r"^/(?:@([\w.-]+)/)?lists?/([\w-]+)/?$",
|
||||
@@ -318,6 +322,7 @@ query SearchFieldOptions(
|
||||
fields: $fields,
|
||||
weights: $weights
|
||||
) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -536,13 +541,19 @@ SORT_MAPPING: dict[SortOrder, str] = {
|
||||
SortOrder.OLDEST: "release_year:asc",
|
||||
}
|
||||
|
||||
# Mapping from abstract search type to Hardcover fields parameter
|
||||
SEARCH_TYPE_FIELDS: dict[SearchType, str] = {
|
||||
SearchType.GENERAL: "title,isbns,series_names,author_names,alternative_titles",
|
||||
SearchType.TITLE: "title,alternative_titles",
|
||||
SearchType.AUTHOR: "author_names",
|
||||
# ISBN is handled separately via search_by_isbn()
|
||||
}
|
||||
# `fields` becomes Typesense's `query_by`, but Hardcover keeps `num_typos` and
|
||||
# `query_by_weights` as fixed-length presets per query_type. Passing a different
|
||||
# number of fields than the preset expects makes Typesense reject the whole search,
|
||||
# complaining that the number of num_typos values does not match the number of
|
||||
# query_by fields. So a Book search may only ever narrow to *these five* names --
|
||||
# a shorter list is rejected outright rather than searched, and any weights sent
|
||||
# alongside must match one-for-one.
|
||||
# Weights only bias ranking: a field weighted 0 still matches, so `fields` can no
|
||||
# longer restrict which fields a Book query looks at.
|
||||
BOOK_SEARCH_FIELDS = "title,alternative_titles,author_names,series_names,isbns"
|
||||
BOOK_SEARCH_FIELD_COUNT = 5
|
||||
BOOK_TITLE_WEIGHTS = "5,1,0,0,0"
|
||||
BOOK_TITLE_AUTHOR_WEIGHTS = "5,1,3,0,0"
|
||||
|
||||
SERIES_SEARCH_FIELDS = "name,books,author_name"
|
||||
SERIES_SEARCH_WEIGHTS = "2,1,1"
|
||||
@@ -550,22 +561,28 @@ SERIES_SEARCH_SORT = "_text_match:desc,readers_count:desc"
|
||||
AUTHOR_SUGGESTION_FIELDS = "name,name_personal,alternate_names"
|
||||
AUTHOR_SUGGESTION_WEIGHTS = "4,3,2"
|
||||
AUTHOR_SUGGESTION_SORT = "_text_match:desc,books_count:desc"
|
||||
TITLE_SUGGESTION_FIELDS = "title,alternative_titles"
|
||||
TITLE_SUGGESTION_WEIGHTS = "5,2"
|
||||
TITLE_SUGGESTION_FIELDS = BOOK_SEARCH_FIELDS
|
||||
TITLE_SUGGESTION_WEIGHTS = "5,2,0,0,0"
|
||||
TITLE_SUGGESTION_SORT = "_text_match:desc,users_count:desc"
|
||||
|
||||
# Hardcover forwards `sort` to Typesense's `sort_by` and rejects the whole search
|
||||
# if it does not like the value -- an unknown field, a bare field name with no
|
||||
# direction, more than three keys. A rejected search comes back as HTTP 200 with
|
||||
# no GraphQL errors and a null `results` body, which is otherwise indistinguishable
|
||||
# from "nothing matched". An empty sort is always accepted, so fall back to it and
|
||||
# keep the fallback sticky for a while rather than paying for a doomed request on
|
||||
# every search.
|
||||
SORT_FALLBACK = ""
|
||||
# from "nothing matched"; the reason only shows up in the sibling `error` field,
|
||||
# so every search asks for it. Dropping `sort` from the request is the one shape
|
||||
# Hardcover always accepts -- an empty string is a value like any other and has
|
||||
# been rejected too -- so retry that way and keep the fallback sticky for a while
|
||||
# rather than paying for a doomed request on every search.
|
||||
SORT_FALLBACK_TTL = 900.0
|
||||
_sort_fallback_until = 0.0
|
||||
|
||||
|
||||
def _without_sort(variables: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop `sort` entirely so Hardcover applies its own default ordering."""
|
||||
return {key: value for key, value in variables.items() if key != "sort"}
|
||||
|
||||
|
||||
def _search_payload_rejected(result: dict[str, Any] | None) -> bool:
|
||||
"""Report whether Hardcover answered a search with a null results body.
|
||||
|
||||
@@ -580,6 +597,17 @@ def _search_payload_rejected(result: dict[str, Any] | None) -> bool:
|
||||
return root["results"] is None
|
||||
|
||||
|
||||
def _search_rejection_reason(result: dict[str, Any] | None) -> str:
|
||||
"""Return Hardcover's explanation for a rejected search, if it sent one."""
|
||||
if not isinstance(result, dict):
|
||||
return ""
|
||||
root = result.get("search", result)
|
||||
if not isinstance(root, dict):
|
||||
return ""
|
||||
error = root.get("error")
|
||||
return error.strip() if isinstance(error, str) else ""
|
||||
|
||||
|
||||
def _combine_headline_description(headline: str | None, description: str | None) -> str | None:
|
||||
"""Combine headline (tagline) and description into a single description."""
|
||||
if headline and description:
|
||||
@@ -646,7 +674,7 @@ def _normalize_series_position(value: Any) -> float | None:
|
||||
def _normalize_hardcover_api_key(value: object) -> str:
|
||||
"""Normalize Hardcover API keys, stripping copied auth-header prefixes."""
|
||||
normalized_value = normalize_optional_text(value) or ""
|
||||
return normalized_value.removeprefix("Bearer ").strip()
|
||||
return HARDCOVER_BEARER_PREFIX_PATTERN.sub("", normalized_value.strip()).strip()
|
||||
|
||||
|
||||
def _normalize_search_text(value: str) -> str:
|
||||
@@ -1012,13 +1040,15 @@ class HardcoverProvider(MetadataProvider):
|
||||
"""Build search query, fields, and weights based on provided values.
|
||||
|
||||
Returns (query, fields, weights) tuple. Fields/weights are None for general search.
|
||||
A narrowed search still sends all of BOOK_SEARCH_FIELDS -- Hardcover rejects a
|
||||
shorter list outright -- and leans on the weights to rank the wanted field first.
|
||||
"""
|
||||
if author and not title and not series:
|
||||
return author, None, None
|
||||
if title and not author and not series:
|
||||
return title, "title,alternative_titles", "5,1"
|
||||
return title, BOOK_SEARCH_FIELDS, BOOK_TITLE_WEIGHTS
|
||||
if author and title and not series:
|
||||
return f"{title} {author}", "title,alternative_titles,author_names", "5,1,3"
|
||||
return f"{title} {author}", BOOK_SEARCH_FIELDS, BOOK_TITLE_AUTHOR_WEIGHTS
|
||||
return default_query, None, None
|
||||
|
||||
def _detect_list_url(self, query: str) -> tuple[str | None, str] | None:
|
||||
@@ -2384,6 +2414,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
graphql_query = """
|
||||
query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String, $fields: String, $weights: String) {
|
||||
search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort, fields: $fields, weights: $weights) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -2392,6 +2423,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
graphql_query = """
|
||||
query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String) {
|
||||
search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -2690,33 +2722,42 @@ class HardcoverProvider(MetadataProvider):
|
||||
|
||||
sort = variables.get("sort")
|
||||
if sort and time.monotonic() < _sort_fallback_until:
|
||||
variables = {**variables, "sort": SORT_FALLBACK}
|
||||
variables = _without_sort(variables)
|
||||
sort = None
|
||||
|
||||
result = self._execute_query(query, variables)
|
||||
if not _search_payload_rejected(result):
|
||||
return result
|
||||
|
||||
reason = _search_rejection_reason(result)
|
||||
if not sort:
|
||||
logger.error(
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s) and returned "
|
||||
"no result body",
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s): %s",
|
||||
variables.get("queryType", "Book"),
|
||||
variables.get("fields"),
|
||||
reason or "no error message",
|
||||
)
|
||||
return None
|
||||
|
||||
retry = self._execute_query(query, _without_sort(variables))
|
||||
if _search_payload_rejected(retry):
|
||||
# The sort was not the culprit, so leave sorting alone for other searches.
|
||||
logger.error(
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s) with and without "
|
||||
"a sort order: %s",
|
||||
variables.get("queryType", "Book"),
|
||||
variables.get("fields"),
|
||||
_search_rejection_reason(retry) or reason or "no error message",
|
||||
)
|
||||
return None
|
||||
|
||||
logger.warning(
|
||||
"Hardcover rejected sort '%s'; retrying searches without a sort order for %ss",
|
||||
"Hardcover rejected sort '%s' (%s); dropping the sort order from searches for %ss",
|
||||
sort,
|
||||
reason or "no error message",
|
||||
int(SORT_FALLBACK_TTL),
|
||||
)
|
||||
_sort_fallback_until = time.monotonic() + SORT_FALLBACK_TTL
|
||||
|
||||
retry = self._execute_query(query, {**variables, "sort": SORT_FALLBACK})
|
||||
if _search_payload_rejected(retry):
|
||||
logger.error("Hardcover rejected this search even without a sort order")
|
||||
return None
|
||||
return retry
|
||||
|
||||
def _parse_search_result(self, item: dict) -> BookMetadata | None:
|
||||
@@ -2982,12 +3023,13 @@ def _test_hardcover_connection(current_values: dict[str, Any] | None = None) ->
|
||||
_save_connected_user(None, None)
|
||||
return {"success": False, "message": "API key is required"}
|
||||
|
||||
if key_len < HARDCOVER_API_KEY_MIN_LENGTH:
|
||||
is_prefixed_key = api_key.startswith(HARDCOVER_API_KEY_PREFIX)
|
||||
if not is_prefixed_key and key_len < HARDCOVER_API_KEY_MIN_LENGTH:
|
||||
return {
|
||||
"success": False,
|
||||
"message": (
|
||||
f"API key seems too short ({key_len} chars). "
|
||||
f"Expected {HARDCOVER_API_KEY_MIN_LENGTH}+ chars."
|
||||
f"API key seems too short ({key_len} chars). Expected a key starting "
|
||||
f"with {HARDCOVER_API_KEY_PREFIX} or {HARDCOVER_API_KEY_MIN_LENGTH}+ chars."
|
||||
),
|
||||
}
|
||||
|
||||
@@ -3094,7 +3136,7 @@ def hardcover_settings() -> list[SettingsField]:
|
||||
PasswordField(
|
||||
key="HARDCOVER_API_KEY",
|
||||
label="API Key",
|
||||
description="Get your API key from hardcover.app/account/api",
|
||||
description="Get your API key from hardcover.app/account/api (starts with hc_pat_)",
|
||||
required=True,
|
||||
),
|
||||
ActionButton(
|
||||
|
||||
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.core.search_plan import ReleaseSearchPlan
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
|
||||
@@ -400,6 +401,14 @@ class DownloadHandler(ABC):
|
||||
"""Return private queue-time fields needed for restart-safe retry."""
|
||||
return {}
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""Return the release's file list without downloading it.
|
||||
|
||||
Lets the UI review a multi-book pack before queueing. Return None when the
|
||||
source cannot know the files ahead of time (magnet links, usenet, ...).
|
||||
"""
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def cancel(self, task_id: str) -> bool:
|
||||
"""Cancel an in-progress download."""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""AudiobookBay download handler - resolves magnet links and uses shared client lifecycle."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.config import config
|
||||
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
DEFAULT_ABB_HOSTNAME = "audiobookbay.lu"
|
||||
@@ -68,6 +69,19 @@ class AudiobookBayHandler(ExternalClientHandler):
|
||||
return task_id
|
||||
return None
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""Read the torrent's file list off the detail page, without downloading."""
|
||||
raw_url = release_data.get("download_url") or release_data.get("source_url")
|
||||
detail_url = raw_url.strip() if isinstance(raw_url, str) else ""
|
||||
hostname = _resolve_allowed_detail_hostname()
|
||||
if not detail_url or not _detail_url_matches_host(detail_url, hostname):
|
||||
logger.debug("Cannot list files for AudiobookBay release without a valid detail URL")
|
||||
return None
|
||||
detail_html = scraper.fetch_detail_html(detail_url, hostname)
|
||||
if not detail_html:
|
||||
return None
|
||||
return scraper.extract_file_list(detail_html)
|
||||
|
||||
def _get_client(self, protocol: str) -> DownloadClient | None:
|
||||
"""Compatibility shim so module-level patching still works in tests."""
|
||||
return get_client(protocol)
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
from threading import Lock
|
||||
from urllib.parse import quote, quote_plus
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
@@ -10,6 +11,8 @@ from bs4 import BeautifulSoup
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.download import http as downloader
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -31,6 +34,13 @@ FIRST_PAGE_SESSION_REFRESH_ATTEMPTS = 2
|
||||
# Legacy search parameter used by older ABB flows
|
||||
LEGACY_CATEGORY_QUERY = "undefined%2Cundefined"
|
||||
|
||||
# Detail pages are fetched once and shared by inspection (file list) and download
|
||||
# (magnet link) so a "review then download" round trip costs ABB a single request.
|
||||
DETAIL_PAGE_CACHE_TTL_SECONDS = 120.0
|
||||
DETAIL_PAGE_CACHE_MAX_ENTRIES = 8
|
||||
_detail_page_cache: dict[str, tuple[float, str]] = {}
|
||||
_detail_page_cache_lock = Lock()
|
||||
|
||||
# Precompiled patterns used while parsing result cards
|
||||
LANGUAGE_PATTERN = re.compile(r"Language:\s*([A-Za-z]+)")
|
||||
POSTED_PATTERN = re.compile(r"Posted:\s*(\d+\s+[A-Za-z]+\s+\d{4})")
|
||||
@@ -38,6 +48,11 @@ FORMAT_PATTERN = re.compile(r"Format:\s*([A-Za-z0-9]+)")
|
||||
BITRATE_PATTERN = re.compile(r"Bitrate:\s*([\d]+\s*[A-Za-z/]+)")
|
||||
SIZE_PATTERN = re.compile(r"File Size:\s*([\d.]+)\s*([A-Za-z]+)")
|
||||
INFO_HASH_LABEL_PATTERN = re.compile(r"Info Hash", re.IGNORECASE)
|
||||
FILE_ROW_SIZE_PATTERN = re.compile(
|
||||
r"^(?P<name>.+?)\s+(?P<size>\d+(?:\.\d+)?)\s*(?P<unit>Bytes?|KBs?|MBs?|GBs?|TBs?)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FILE_SIZE_MULTIPLIERS = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4}
|
||||
|
||||
|
||||
def _coerce_non_negative_float(value: object, default: float) -> float:
|
||||
@@ -98,8 +113,10 @@ def _encode_search_query(query: str, *, exact_phrase: bool) -> str:
|
||||
and not (search_query.startswith('"') and search_query.endswith('"'))
|
||||
):
|
||||
search_query = f'"{search_query}"'
|
||||
# Keep ABB-friendly encoding style (spaces as '+') while percent-encoding quotes.
|
||||
return search_query.replace('"', "%22").replace(" ", "+")
|
||||
# Keep ABB's space-as-'+' style, but percent-encode everything else: a bare
|
||||
# '&' would otherwise start a new query parameter, '%' would open an invalid
|
||||
# escape, and a literal '+' would arrive as a space.
|
||||
return quote_plus(search_query)
|
||||
|
||||
|
||||
def _normalize_result_url(url: str, hostname: str) -> str:
|
||||
@@ -153,6 +170,9 @@ def search_audiobookbay(
|
||||
|
||||
"""
|
||||
results = []
|
||||
# ABB matches the stored, untexturized title, so a curly apostrophe reaching
|
||||
# the search returns nothing at all rather than merely ranking worse.
|
||||
query = normalize_search_punctuation(query)
|
||||
rate_limit_delay = _coerce_non_negative_float(config.get("ABB_RATE_LIMIT_DELAY", 1.0), 1.0)
|
||||
session = requests.Session()
|
||||
|
||||
@@ -342,6 +362,98 @@ def search_audiobookbay(
|
||||
return results
|
||||
|
||||
|
||||
def _get_cached_detail_page(details_url: str) -> str | None:
|
||||
with _detail_page_cache_lock:
|
||||
entry = _detail_page_cache.get(details_url)
|
||||
if entry is None:
|
||||
return None
|
||||
fetched_at, html = entry
|
||||
if time.monotonic() - fetched_at > DETAIL_PAGE_CACHE_TTL_SECONDS:
|
||||
del _detail_page_cache[details_url]
|
||||
return None
|
||||
return html
|
||||
|
||||
|
||||
def _store_cached_detail_page(details_url: str, html: str) -> None:
|
||||
with _detail_page_cache_lock:
|
||||
_detail_page_cache[details_url] = (time.monotonic(), html)
|
||||
while len(_detail_page_cache) > DETAIL_PAGE_CACHE_MAX_ENTRIES:
|
||||
oldest = min(_detail_page_cache, key=lambda key: _detail_page_cache[key][0])
|
||||
del _detail_page_cache[oldest]
|
||||
|
||||
|
||||
def clear_detail_page_cache() -> None:
|
||||
"""Drop cached detail pages (used by tests)."""
|
||||
with _detail_page_cache_lock:
|
||||
_detail_page_cache.clear()
|
||||
|
||||
|
||||
def _fetch_detail_page_once(details_url: str, hostname: str) -> str:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
return _coerce_markup_to_html(
|
||||
downloader.html_get_page(
|
||||
details_url,
|
||||
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fetch_detail_html(details_url: str, hostname: str = "audiobookbay.lu") -> str:
|
||||
"""Fetch a detail page (one retry with a fresh session), cached briefly per URL."""
|
||||
cached = _get_cached_detail_page(details_url)
|
||||
if cached is not None:
|
||||
logger.debug("Reusing recently fetched detail page: %s", details_url)
|
||||
return cached
|
||||
detail_html = _fetch_detail_page_once(details_url, hostname)
|
||||
if not detail_html:
|
||||
detail_html = _fetch_detail_page_once(details_url, hostname)
|
||||
if detail_html:
|
||||
_store_cached_detail_page(details_url, detail_html)
|
||||
return detail_html
|
||||
|
||||
|
||||
def _parse_file_row(text: str) -> PackFile | None:
|
||||
match = FILE_ROW_SIZE_PATTERN.match(text.strip())
|
||||
if not match:
|
||||
return None
|
||||
multiplier = _FILE_SIZE_MULTIPLIERS[match.group("unit")[0].lower()]
|
||||
return PackFile(match.group("name"), int(float(match.group("size")) * multiplier))
|
||||
|
||||
|
||||
def extract_file_list(detail_html: str) -> list[PackFile] | None:
|
||||
"""Read the torrent file rows off a detail page.
|
||||
|
||||
ABB renders the torrent's file table as single-cell rows between the
|
||||
"This is a Multifile Torrent" marker (absent for single-file torrents) and the
|
||||
"Combined File Size" row. Returns None when the page has no such table.
|
||||
"""
|
||||
soup = BeautifulSoup(detail_html, "html.parser")
|
||||
rows: list[PackFile] = []
|
||||
for row in soup.find_all("tr"):
|
||||
cells = row.find_all("td")
|
||||
if not cells:
|
||||
continue
|
||||
label = cells[0].get_text(" ", strip=True)
|
||||
if label.lower().startswith("combined file size"):
|
||||
return rows or None
|
||||
if len(cells) != 1:
|
||||
rows = [] # a two-column metadata row means we're not in the file table yet
|
||||
continue
|
||||
text = cells[0].get_text(" ", strip=True)
|
||||
if "multifile torrent" in text.lower():
|
||||
rows = []
|
||||
continue
|
||||
parsed = _parse_file_row(text)
|
||||
if parsed is not None:
|
||||
rows.append(parsed)
|
||||
return None
|
||||
|
||||
|
||||
def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") -> str | None:
|
||||
"""Extract info hash and trackers from book detail page, then construct magnet link.
|
||||
|
||||
@@ -354,35 +466,7 @@ def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") ->
|
||||
|
||||
"""
|
||||
try:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
|
||||
# Fetch detail page
|
||||
detail_html = _coerce_markup_to_html(
|
||||
downloader.html_get_page(
|
||||
details_url,
|
||||
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
)
|
||||
|
||||
if not detail_html:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
detail_html = _coerce_markup_to_html(
|
||||
downloader.html_get_page(
|
||||
details_url,
|
||||
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
)
|
||||
|
||||
detail_html = fetch_detail_html(details_url, hostname)
|
||||
if not detail_html:
|
||||
logger.warning("Failed to fetch details page")
|
||||
return None
|
||||
|
||||
@@ -23,7 +23,11 @@ from shelfmark.release_sources import (
|
||||
register_source,
|
||||
)
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_hostname, parse_size
|
||||
from shelfmark.release_sources.audiobookbay.utils import (
|
||||
normalize_hostname,
|
||||
normalize_search_punctuation,
|
||||
parse_size,
|
||||
)
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
MIN_RELEVANCE_QUERY_WORD_LENGTH = 2
|
||||
@@ -227,10 +231,12 @@ class AudiobookBaySource(ReleaseSource):
|
||||
deduped_queries[index + 1].lower(),
|
||||
)
|
||||
|
||||
# Extract query words for relevance checking
|
||||
# Extract query words for relevance checking. Both sides of the
|
||||
# comparison are punctuation-normalized: scraped titles carry the
|
||||
# typographic forms WordPress renders, queries carry the ASCII ones.
|
||||
query_words = {
|
||||
word.lower()
|
||||
for word in query_lower.split()
|
||||
for word in normalize_search_punctuation(query_lower).split()
|
||||
if len(word) > MIN_RELEVANCE_QUERY_WORD_LENGTH
|
||||
}
|
||||
|
||||
@@ -239,7 +245,7 @@ class AudiobookBaySource(ReleaseSource):
|
||||
try:
|
||||
raw_title = result["title"]
|
||||
title, author = _split_title_and_author(raw_title)
|
||||
title_for_filter = raw_title.lower()
|
||||
title_for_filter = normalize_search_punctuation(raw_title).lower()
|
||||
|
||||
# Basic relevance check: ensure title contains at least one query word
|
||||
# This filters out homepage "Latest" feed items that may leak through
|
||||
|
||||
@@ -2,6 +2,63 @@
|
||||
|
||||
import re
|
||||
|
||||
# WordPress texturizes punctuation on output only: a post stored as "The
|
||||
# Stranger's Wife" is rendered as "The Stranger’s Wife". ABB's search matches the
|
||||
# stored value, so a query carrying the typographic form matches nothing -- and
|
||||
# because ABB ANDs its search terms, one such term empties the entire result set.
|
||||
# Book metadata and phone keyboards both hand us the typographic forms, so map
|
||||
# them back before they reach a search or a title comparison.
|
||||
_ASCII_PUNCTUATION = str.maketrans(
|
||||
{
|
||||
# Single quotes
|
||||
"‘": "'", # left single quotation mark
|
||||
"’": "'", # right single quotation mark
|
||||
"‚": "'", # single low-9 quotation mark
|
||||
"‛": "'", # single high-reversed-9 quotation mark
|
||||
"′": "'", # prime
|
||||
"´": "'", # acute accent
|
||||
"`": "'", # grave accent
|
||||
# Double quotes
|
||||
"“": '"', # left double quotation mark
|
||||
"”": '"', # right double quotation mark
|
||||
"„": '"', # double low-9 quotation mark
|
||||
"‟": '"', # double high-reversed-9 quotation mark
|
||||
"″": '"', # double prime
|
||||
# Dashes
|
||||
"‐": "-", # hyphen
|
||||
"‑": "-", # non-breaking hyphen
|
||||
"‒": "-", # figure dash
|
||||
"–": "-", # en dash
|
||||
"—": "-", # em dash
|
||||
"―": "-", # horizontal bar
|
||||
"−": "-", # minus sign
|
||||
"﹘": "-", # small em dash
|
||||
"﹣": "-", # small hyphen-minus
|
||||
"-": "-", # fullwidth hyphen-minus
|
||||
# Ellipsis
|
||||
"…": "...", # horizontal ellipsis
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def normalize_search_punctuation(text: str) -> str:
|
||||
"""Replace typographic punctuation with the ASCII forms ABB stores.
|
||||
|
||||
Each character is mapped individually rather than collapsing runs, so an
|
||||
ASCII "--" is left alone: only characters ABB cannot have stored are
|
||||
rewritten.
|
||||
|
||||
Args:
|
||||
text: A search query, or a scraped title being compared against one.
|
||||
|
||||
Returns:
|
||||
The text with curly quotes, dashes and ellipses mapped to ASCII.
|
||||
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
return text.translate(_ASCII_PUNCTUATION)
|
||||
|
||||
|
||||
def normalize_hostname(raw: str | None) -> str:
|
||||
"""Normalize a user-supplied hostname for URL construction.
|
||||
|
||||
@@ -17,6 +17,7 @@ from bs4 import BeautifulSoup, Tag
|
||||
from bs4.element import NavigableString
|
||||
|
||||
from shelfmark.config.env import DEBUG_SKIP_SOURCES, TMP_DIR
|
||||
from shelfmark.core import search_deadline
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.languages import language_alias_map
|
||||
from shelfmark.core.logger import setup_logger
|
||||
@@ -539,6 +540,107 @@ class SearchUnavailableError(SourceUnavailableError):
|
||||
"""Raised when Anna's Archive cannot be reached via any mirror/DNS."""
|
||||
|
||||
|
||||
# Markers that prove a 200 really came from Anna's Archive, and markers that mean we
|
||||
# are looking at a protection interstitial rather than the site. A page with neither
|
||||
# is a domain that answers but is not AA - seized, parked or for sale.
|
||||
#
|
||||
# Deliberately structural rather than the domain name: a parking page's whole job is
|
||||
# to display the domain it is squatting on, so "annas-archive" matches the very pages
|
||||
# this is meant to catch. These paths only exist on the real site.
|
||||
_AA_PAGE_MARKERS = (
|
||||
"/md5/",
|
||||
"aarecord",
|
||||
"anna's archive",
|
||||
"/dyn/",
|
||||
"/datasets",
|
||||
"/fast_download",
|
||||
"/slow_download",
|
||||
)
|
||||
_CHALLENGE_MARKERS = (
|
||||
"ddos-guard",
|
||||
"just a moment",
|
||||
"cloudflare",
|
||||
"checking your browser",
|
||||
"cf-browser-verification",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_aa_page(html: str) -> bool:
|
||||
"""Whether ``html`` is recognisably Anna's Archive itself."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in _AA_PAGE_MARKERS)
|
||||
|
||||
|
||||
def _looks_like_challenge_page(html: str) -> bool:
|
||||
"""Whether ``html`` is a protection interstitial rather than the site behind it."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in _CHALLENGE_MARKERS)
|
||||
|
||||
|
||||
def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[str, Tag | None]:
|
||||
"""Fetch the AA search page, retrying past mirrors that are not actually AA.
|
||||
|
||||
A parked or seized domain answers 200 with a page that has no results table and no
|
||||
"No files found." - indistinguishable from a broken search unless we check whether
|
||||
the response looks like AA at all. Those mirrors are quarantined for the session so
|
||||
later searches skip them instead of paying the timeout again.
|
||||
"""
|
||||
attempt_url = url
|
||||
for _ in range(len(network.get_available_aa_urls()) or 1):
|
||||
# Every mirror shares the protection, so once the search budget is gone another
|
||||
# mirror is another full solve nobody is still waiting for.
|
||||
if search_deadline.expired():
|
||||
raise SearchUnavailableError(search_deadline.deadline_message())
|
||||
|
||||
response = downloader.html_get_page(
|
||||
attempt_url, selector=selector, allow_bypasser_fallback=True
|
||||
)
|
||||
if not response:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients.
|
||||
# html_get_page records the concrete give-up reason on the selector; fall
|
||||
# back to the generic line only if nothing was recorded.
|
||||
detail = getattr(selector, "last_failure", None) or (
|
||||
"Network restricted or mirrors are blocked."
|
||||
)
|
||||
raise SearchUnavailableError(f"Unable to reach download source. {detail}")
|
||||
|
||||
html = _html_response_text(response)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if isinstance(table, Tag):
|
||||
return html, table
|
||||
if table is not None:
|
||||
msg = f"Expected results table tag, got {type(table).__name__}"
|
||||
raise TypeError(msg)
|
||||
if "No files found." in html:
|
||||
# A real, genuinely empty answer from a healthy mirror.
|
||||
return html, None
|
||||
if _looks_like_challenge_page(html):
|
||||
# The bypass did not actually clear the protection - the interstitial is
|
||||
# what came back. Rotating is pointless (every mirror shares the same
|
||||
# protection) and reporting it as an empty result is worse: the user is
|
||||
# told their query found nothing when the search never ran.
|
||||
msg = (
|
||||
"Anna's Archive answered with an unsolved protection challenge. "
|
||||
"Check that the bypasser is reachable and working."
|
||||
)
|
||||
raise SearchUnavailableError(msg)
|
||||
if _looks_like_aa_page(html):
|
||||
# A real AA response in a shape the caller should report as drift.
|
||||
# Not the mirror's fault.
|
||||
return html, None
|
||||
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
fatal=True, reason="responded without an Anna's Archive page"
|
||||
)
|
||||
if action not in ("mirror", "dns") or not new_base:
|
||||
return html, None
|
||||
attempt_url = selector.rewrite(url)
|
||||
logger.info("Retrying search on %s", new_base)
|
||||
|
||||
return "", None
|
||||
|
||||
|
||||
def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
"""Search for books matching the query.
|
||||
|
||||
@@ -601,15 +703,9 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
f"{filters_query}"
|
||||
)
|
||||
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
|
||||
if not html:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
|
||||
soup = BeautifulSoup(_html_response_text(html), "html.parser")
|
||||
tbody = soup.find("table")
|
||||
|
||||
# AA gates /search behind a DDoS-Guard JS challenge, which every mirror shares. Rotating
|
||||
# to another mirror only collects another 403, so let the bypasser solve it.
|
||||
html, tbody = _fetch_search_table(url, selector)
|
||||
if tbody is None:
|
||||
if "No files found." in html:
|
||||
logger.info("No books found for query: %s", query)
|
||||
@@ -657,11 +753,14 @@ def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseR
|
||||
"""
|
||||
url = f"{network.get_aa_base_url()}/md5/{book_id}"
|
||||
selector = network.AAMirrorSelector()
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
|
||||
# Same challenge as search: the detail page is gated on every mirror, so bypass it.
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True)
|
||||
|
||||
if not html:
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
detail = getattr(selector, "last_failure", None) or (
|
||||
"Network restricted or mirrors are blocked."
|
||||
)
|
||||
raise SearchUnavailableError(f"Unable to reach download source. {detail}")
|
||||
|
||||
soup = BeautifulSoup(_html_response_text(html), "html.parser")
|
||||
|
||||
@@ -885,6 +984,9 @@ def _parse_book_info_page(
|
||||
if fetch_download_count:
|
||||
try:
|
||||
summary_url = f"{network.get_aa_base_url()}/dyn/md5/summary/{book_id}"
|
||||
# Unlike search and the detail page above, this one stays off the bypasser: a
|
||||
# download count is decoration on the details modal, not worth holding the
|
||||
# modal open for a browser solve. If it is gated, drop it and move on.
|
||||
summary_response = downloader.html_get_page(
|
||||
summary_url, selector=network.AAMirrorSelector(), allow_bypasser_fallback=False
|
||||
)
|
||||
@@ -1877,6 +1979,12 @@ class DirectDownloadSource(ReleaseSource):
|
||||
query = f"{title} {author}".strip()
|
||||
if not query:
|
||||
continue
|
||||
# `except Exception` below keeps this loop going past a failed variant, which
|
||||
# is right for a parse error and wrong for a spent budget: without this the
|
||||
# variants queue up behind each other and the request outlives the caller.
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent; skipping remaining title variants")
|
||||
break
|
||||
|
||||
logger.debug("Searching direct_download: title_author='%s', langs=%s", query, langs)
|
||||
filters = SearchFilters(lang=langs if langs is not None else [])
|
||||
@@ -1890,7 +1998,11 @@ class DirectDownloadSource(ReleaseSource):
|
||||
except Exception:
|
||||
logger.exception("Search error")
|
||||
|
||||
if not all_results and any(langs for _, langs in searches):
|
||||
if (
|
||||
not all_results
|
||||
and any(langs for _, langs in searches)
|
||||
and not search_deadline.expired()
|
||||
):
|
||||
logger.debug(
|
||||
"No title+author results with language filter, retrying without language filter"
|
||||
)
|
||||
@@ -1898,6 +2010,9 @@ class DirectDownloadSource(ReleaseSource):
|
||||
query = f"{title} {author}".strip()
|
||||
if not query:
|
||||
continue
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent; skipping remaining retries")
|
||||
break
|
||||
|
||||
logger.debug("Searching direct_download: title_author='%s', langs=[]", query)
|
||||
try:
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -18,11 +19,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# All recognized formats for parsing IRC result lines.
|
||||
# This comprehensive list is used to identify file extensions in results.
|
||||
# User-configured formats are used separately for filtering.
|
||||
ALL_RECOGNIZED_FORMATS = {
|
||||
# Ebook formats
|
||||
# Ebook formats recognized in IRC result lines.
|
||||
EBOOK_FORMATS = (
|
||||
"epub",
|
||||
"mobi",
|
||||
"azw3",
|
||||
@@ -41,19 +39,17 @@ ALL_RECOGNIZED_FORMATS = {
|
||||
"cbz",
|
||||
"cdr",
|
||||
"jpg",
|
||||
"rar",
|
||||
"zip",
|
||||
# Audiobook formats
|
||||
"m4b",
|
||||
"mp3",
|
||||
"m4a",
|
||||
"flac",
|
||||
"ogg",
|
||||
"wma",
|
||||
"aac",
|
||||
"wav",
|
||||
"opus",
|
||||
}
|
||||
)
|
||||
|
||||
# All recognized formats for parsing IRC result lines.
|
||||
# This comprehensive list is used to identify file extensions in results.
|
||||
# User-configured formats are used separately for filtering.
|
||||
# Ordered longest-first so that scanning a line matches "azw3" before "azw" and "docx"
|
||||
# before "doc". It used to be a set, which made the winning format for a line naming more
|
||||
# than one extension depend on set iteration order, and therefore vary between restarts.
|
||||
ALL_RECOGNIZED_FORMATS = tuple(
|
||||
sorted({*EBOOK_FORMATS, *ARCHIVE_FORMATS, *AUDIOBOOK_FORMATS}, key=len, reverse=True)
|
||||
)
|
||||
|
||||
|
||||
def _normalize_config_formats(raw_formats: object) -> set[str]:
|
||||
@@ -84,13 +80,22 @@ def _get_supported_formats(content_type: str | None = None) -> set[str]:
|
||||
|
||||
# Regex to parse result lines
|
||||
# Format: !Server Author - Title.format ::INFO:: size
|
||||
#
|
||||
# The extension is matched against the known formats rather than a bare \w+. A bare \w+
|
||||
# happily matched the decimal point in the size, so a line with no file extension parsed
|
||||
# as format="5mb" out of "::INFO:: 620.5MB" - taking the title and size down with it, and
|
||||
# leaving the result to be discarded by every format filter downstream. Restricting the
|
||||
# alternation makes such a line fall through to SIMPLE_RESULT_REGEX and come back as
|
||||
# "unknown", which is what the rest of the parser already expects.
|
||||
_FORMAT_ALTERNATION = "|".join(re.escape(fmt) for fmt in ALL_RECOGNIZED_FORMATS)
|
||||
RESULT_LINE_REGEX = re.compile(
|
||||
r"^!(\S+)\s+" # !ServerName
|
||||
r"(.+?)\s+-\s+" # Author Name -
|
||||
r"(.+?)\.(\w+)" # Title.format
|
||||
rf"(.+?)\.({_FORMAT_ALTERNATION})\b" # Title.format
|
||||
r"(?:\s+::INFO::\s*(.+?))?" # Optional ::INFO:: metadata
|
||||
r"(?:\s+::HASH::\s*(\S+))?" # Optional ::HASH::
|
||||
r"\s*$"
|
||||
r"\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Simpler fallback pattern
|
||||
@@ -187,18 +192,54 @@ def parse_result_line(line: str) -> SearchResult | None:
|
||||
return None
|
||||
|
||||
|
||||
# Words that mark an archive as holding an audiobook rather than an ebook. Multi-file
|
||||
# audiobooks ship as .rar/.zip, so for those the extension says nothing about the content
|
||||
# and the release name is the only evidence there is.
|
||||
_AUDIOBOOK_MARKER_REGEX = re.compile(
|
||||
r"\b(?:audio ?books?|unabridged|abridged|narrat(?:ed|or)|audible|\d+ ?kbps|"
|
||||
+ "|".join(re.escape(fmt) for fmt in AUDIOBOOK_FORMATS)
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_AUDIOBOOK_FORMAT_SET = frozenset(AUDIOBOOK_FORMATS)
|
||||
_EBOOK_FORMAT_SET = frozenset(EBOOK_FORMATS)
|
||||
|
||||
|
||||
def detect_content_type(result: SearchResult) -> str:
|
||||
"""Classify a parsed result as an audiobook or an ebook.
|
||||
|
||||
Extension alone is not enough. It settles the plain cases, but the common audiobook
|
||||
release is a .rar or .zip of MP3s, which is indistinguishable by extension from an
|
||||
ebook archive - so for containers (and for lines with no usable extension) the
|
||||
release name decides.
|
||||
"""
|
||||
if result.format in _AUDIOBOOK_FORMAT_SET:
|
||||
return "audiobook"
|
||||
if result.format in _EBOOK_FORMAT_SET:
|
||||
return "ebook"
|
||||
return "audiobook" if _AUDIOBOOK_MARKER_REGEX.search(result.full_line) else "ebook"
|
||||
|
||||
|
||||
def parse_results_file(content: str, content_type: str | None = None) -> list[SearchResult]:
|
||||
"""Parse a search results file into SearchResult objects."""
|
||||
results = []
|
||||
supported = _get_supported_formats(content_type)
|
||||
requested = "audiobook" if check_audiobook(content_type) else "ebook"
|
||||
|
||||
for line in content.splitlines():
|
||||
result = parse_result_line(line)
|
||||
if result and (result.format in supported or result.format == "unknown"):
|
||||
# Filter to user's configured formats
|
||||
if not result:
|
||||
continue
|
||||
# Classify first, then apply the user's format filter within that bucket. Doing it
|
||||
# the other way round is what lost audiobooks entirely: an audiobook .rar matched
|
||||
# neither the ebook nor the audiobook format list, so it fell out of both.
|
||||
if detect_content_type(result) != requested:
|
||||
continue
|
||||
if result.format in supported or result.format == "unknown":
|
||||
results.append(result)
|
||||
|
||||
logger.info("Parsed %s results from search file", len(results))
|
||||
logger.info("Parsed %s %s results from search file", len(results), requested)
|
||||
return results
|
||||
|
||||
|
||||
|
||||
@@ -102,10 +102,13 @@ def irc_settings() -> list[SettingsField]:
|
||||
key="audiobook_heading",
|
||||
title="Audiobooks",
|
||||
description=(
|
||||
"Some networks index audiobooks in a separate channel from ebooks "
|
||||
"(for example #ebooks for ebooks and #bookz for audiobooks). "
|
||||
"Configure that channel here to search it for audiobook requests. "
|
||||
"Leave these blank to search the main channel above for both."
|
||||
"Most networks index audiobooks in the same channel as ebooks, so leaving "
|
||||
"these blank is the right setting for almost everyone. On irc.irchighway.net "
|
||||
"the audiobooks are in #ebooks and #bookz is effectively inactive — pointing "
|
||||
"this at an empty channel just returns no results. Only fill these in when "
|
||||
"your network really does index audiobooks elsewhere (Undernet's #bookz, for "
|
||||
"example). Audiobooks are usually posted as archives, so keep ZIP and RAR "
|
||||
"enabled under Supported Audiobook Formats or the releases are filtered out."
|
||||
),
|
||||
),
|
||||
TextField(
|
||||
@@ -113,8 +116,9 @@ def irc_settings() -> list[SettingsField]:
|
||||
label="Audiobook channel",
|
||||
placeholder="e.g. bookz",
|
||||
description=(
|
||||
"Optional. Channel name (without the # prefix) to use for audiobook "
|
||||
"searches. Leave blank to use the main channel above for audiobooks too."
|
||||
"Optional. Channel name (without the # prefix) for networks that index "
|
||||
"audiobooks separately, such as Undernet's bookz. Leave blank (the usual "
|
||||
"setting) to search the main channel above for audiobooks too."
|
||||
),
|
||||
required=False,
|
||||
env_supported=True,
|
||||
|
||||
@@ -240,11 +240,11 @@ class IRCReleaseSource(ReleaseSource):
|
||||
nick = _config_text("IRC_NICK")
|
||||
search_bot = _config_text("IRC_SEARCH_BOT")
|
||||
|
||||
# Audiobooks may be indexed in a separate channel from ebooks on some networks
|
||||
# (e.g. #ebooks for ebooks, #bookz for audiobooks). When an audiobook channel is
|
||||
# configured and an audiobook was requested, route the search there (with its own
|
||||
# search bot if set). Otherwise fall back to the main channel/bot, which keeps the
|
||||
# single-channel networks that index both formats working unchanged.
|
||||
# A few networks index audiobooks in a separate channel from ebooks (Undernet's
|
||||
# #bookz, say). When an audiobook channel is configured and an audiobook was
|
||||
# requested, route the search there (with its own search bot if set). Otherwise
|
||||
# fall back to the main channel/bot — that is the common case, since most networks
|
||||
# (irchighway included) serve both formats from the one channel.
|
||||
if is_audiobook(content_type):
|
||||
audiobook_channel = _config_text("IRC_AUDIOBOOK_CHANNEL")
|
||||
if audiobook_channel:
|
||||
@@ -428,14 +428,15 @@ class IRCReleaseSource(ReleaseSource):
|
||||
"m4b": 0,
|
||||
"mp3": 1,
|
||||
"m4a": 2,
|
||||
"flac": 3,
|
||||
"opus": 4,
|
||||
"ogg": 5,
|
||||
"aac": 6,
|
||||
"wav": 7,
|
||||
"wma": 8,
|
||||
"rar": 9,
|
||||
"zip": 10,
|
||||
"mp4": 3,
|
||||
"flac": 4,
|
||||
"opus": 5,
|
||||
"ogg": 6,
|
||||
"aac": 7,
|
||||
"wav": 8,
|
||||
"wma": 9,
|
||||
"rar": 10,
|
||||
"zip": 11,
|
||||
}
|
||||
|
||||
def _convert_to_releases(
|
||||
|
||||
@@ -14,10 +14,6 @@ from shelfmark.release_sources.prowlarr.torznab import parse_torznab_xml
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Newznab standard book category IDs
|
||||
NEWZNAB_BOOKS = 7000
|
||||
NEWZNAB_AUDIOBOOKS = 3030
|
||||
|
||||
|
||||
class NewznabClient:
|
||||
"""Client for any Newznab-compatible indexer API."""
|
||||
|
||||
@@ -8,6 +8,8 @@ from shelfmark.core.settings_registry import (
|
||||
HeadingField,
|
||||
PasswordField,
|
||||
SettingsField,
|
||||
TableField,
|
||||
TagListField,
|
||||
TextField,
|
||||
register_settings,
|
||||
)
|
||||
@@ -15,12 +17,36 @@ from shelfmark.core.utils import normalize_http_url
|
||||
|
||||
|
||||
def _test_newznab_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Test the Newznab connection using current form values."""
|
||||
"""Test all named Newznab connections, or the legacy connection as fallback."""
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.release_sources.newznab.api import NewznabClient
|
||||
from shelfmark.release_sources.newznab.source import _parse_indexer_rows
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
raw_indexers = current_values.get("NEWZNAB_INDEXERS")
|
||||
if raw_indexers is None:
|
||||
raw_indexers = config.get("NEWZNAB_INDEXERS", [])
|
||||
indexers = _parse_indexer_rows(raw_indexers)
|
||||
|
||||
if indexers:
|
||||
details: list[str] = []
|
||||
all_successful = True
|
||||
for name, url, api_key in indexers:
|
||||
try:
|
||||
success, message = NewznabClient(url, api_key).test_connection()
|
||||
except Exception as e: # noqa: BLE001 — surface unexpected errors to the UI
|
||||
success, message = False, f"Connection failed: {e!s}"
|
||||
all_successful = all_successful and success
|
||||
details.append(f"{name}: {message}")
|
||||
|
||||
summary = (
|
||||
f"Connected to all {len(indexers)} indexers"
|
||||
if all_successful
|
||||
else "One or more Newznab indexers failed"
|
||||
)
|
||||
return {"success": all_successful, "message": summary, "details": details}
|
||||
|
||||
raw_url = str(current_values.get("NEWZNAB_URL") or config.get("NEWZNAB_URL", "") or "")
|
||||
api_key = str(current_values.get("NEWZNAB_API_KEY") or config.get("NEWZNAB_API_KEY", "") or "")
|
||||
|
||||
@@ -63,29 +89,88 @@ def newznab_config_settings() -> list[SettingsField]:
|
||||
default=False,
|
||||
description="Enable searching for books via a Newznab-compatible indexer",
|
||||
),
|
||||
TableField(
|
||||
key="NEWZNAB_INDEXERS",
|
||||
label="Named Indexers",
|
||||
description=(
|
||||
"Add each Newznab-compatible indexer separately. The configured name is shown "
|
||||
"beside every result from that indexer."
|
||||
),
|
||||
columns=[
|
||||
{
|
||||
"key": "name",
|
||||
"label": "Name",
|
||||
"type": "text",
|
||||
"placeholder": "NZBGeek",
|
||||
},
|
||||
{
|
||||
"key": "url",
|
||||
"label": "URL",
|
||||
"type": "text",
|
||||
"placeholder": "https://api.nzbgeek.info",
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"type": "password",
|
||||
"placeholder": "Optional",
|
||||
},
|
||||
],
|
||||
default=[],
|
||||
add_label="Add Indexer",
|
||||
empty_message=(
|
||||
"No named indexers configured. The legacy single-indexer fields below are used "
|
||||
"as a fallback."
|
||||
),
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
TextField(
|
||||
key="NEWZNAB_URL",
|
||||
label="Newznab URL",
|
||||
description="Base URL of your Newznab indexer or aggregator",
|
||||
label="Legacy Newznab URL",
|
||||
description="Used only when the named indexer list is empty",
|
||||
placeholder="http://nzbhydra:5076",
|
||||
required=True,
|
||||
required=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
PasswordField(
|
||||
key="NEWZNAB_API_KEY",
|
||||
label="API Key",
|
||||
description="Your Newznab API key (leave blank if not required)",
|
||||
label="Legacy API Key",
|
||||
description="Used only with the legacy Newznab URL",
|
||||
required=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
ActionButton(
|
||||
key="test_newznab",
|
||||
label="Test Connection",
|
||||
description="Verify your Newznab configuration",
|
||||
label="Test Connections",
|
||||
description="Verify every named indexer, or the legacy connection when the list is empty",
|
||||
style="primary",
|
||||
callback=_test_newznab_connection,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
TagListField(
|
||||
key="NEWZNAB_EBOOK_CATEGORIES",
|
||||
label="Ebook Categories",
|
||||
description=(
|
||||
"Newznab category IDs searched for ebooks. Most indexers use the standard 7000, "
|
||||
"but some use custom IDs. Leave empty to use 7000."
|
||||
),
|
||||
placeholder="7000",
|
||||
default=["7000"],
|
||||
normalize_urls=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
TagListField(
|
||||
key="NEWZNAB_AUDIOBOOK_CATEGORIES",
|
||||
label="Audiobook Categories",
|
||||
description=(
|
||||
"Newznab category IDs searched for audiobooks. Most indexers use the standard "
|
||||
"3030, but some use custom IDs. Leave empty to use 3030."
|
||||
),
|
||||
placeholder="3030",
|
||||
default=["3030"],
|
||||
normalize_urls=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
CheckboxField(
|
||||
key="NEWZNAB_AUTO_EXPAND",
|
||||
label="Auto-expand search on no results",
|
||||
|
||||
@@ -2,8 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
from urllib.parse import urlparse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.search_plan import ReleaseSearchPlan
|
||||
@@ -39,15 +43,137 @@ from shelfmark.release_sources.prowlarr.source import (
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Newznab category IDs
|
||||
_AUDIOBOOK_CATS = [3030]
|
||||
_BOOK_CATS = [7000]
|
||||
# Standard Newznab category IDs, used when the indexer's categories aren't configured.
|
||||
_DEFAULT_AUDIOBOOK_CATS = [3030]
|
||||
_DEFAULT_BOOK_CATS = [7000]
|
||||
|
||||
# Reuse the same timeout constant as Prowlarr.
|
||||
NEWZNAB_SEARCH_TIMEOUT_SECONDS = _SEARCH_TIMEOUT
|
||||
|
||||
|
||||
def _newznab_result_to_release(result: dict, content_type: str = "ebook") -> Release:
|
||||
@dataclass(frozen=True)
|
||||
class _NamedClient:
|
||||
"""A configured Newznab connection and its stable cache namespace."""
|
||||
|
||||
name: str
|
||||
connection_id: str
|
||||
client: NewznabClient
|
||||
|
||||
|
||||
def _parse_indexer_rows(raw: object) -> list[tuple[str, str, str]]:
|
||||
"""Normalize structured Newznab indexer settings.
|
||||
|
||||
Invalid/incomplete rows are ignored so one partially edited row cannot disable
|
||||
the other configured indexers.
|
||||
"""
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
|
||||
indexers: list[tuple[str, str, str]] = []
|
||||
seen_connections: set[tuple[str, str]] = set()
|
||||
for row in raw:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
raw_url = str(row.get("url") or "").strip()
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
if raw_url:
|
||||
logger.warning("Newznab: ignoring indexer row with invalid URL '%s'", raw_url)
|
||||
continue
|
||||
|
||||
api_key = str(row.get("api_key") or "").strip()
|
||||
connection_key = (url, api_key)
|
||||
if connection_key in seen_connections:
|
||||
continue
|
||||
seen_connections.add(connection_key)
|
||||
|
||||
configured_name = str(row.get("name") or "").strip()
|
||||
hostname = urlparse(url).hostname or ""
|
||||
name = configured_name or hostname or "Newznab"
|
||||
indexers.append((name, url, api_key))
|
||||
|
||||
return indexers
|
||||
|
||||
|
||||
def _parse_category_ids(raw: object) -> list[int]:
|
||||
"""Parse a configured category setting into Newznab category IDs.
|
||||
|
||||
Accepts a list of values or a comma/whitespace separated string. Entries that
|
||||
aren't positive integers are skipped, and duplicates are dropped.
|
||||
"""
|
||||
if raw is None:
|
||||
return []
|
||||
|
||||
values = list(raw) if isinstance(raw, (list, tuple)) else [raw]
|
||||
|
||||
category_ids: list[int] = []
|
||||
for value in values:
|
||||
for token in re.split(r"[,\s]+", str(value).strip()):
|
||||
if not token:
|
||||
continue
|
||||
try:
|
||||
category_id = int(token)
|
||||
except ValueError:
|
||||
logger.warning("Newznab: ignoring invalid category ID '%s'", token)
|
||||
continue
|
||||
if category_id > 0 and category_id not in category_ids:
|
||||
category_ids.append(category_id)
|
||||
|
||||
return category_ids
|
||||
|
||||
|
||||
def _configured_categories(content_type: str) -> list[int]:
|
||||
"""Return the categories to search for a content type, falling back to defaults."""
|
||||
if content_type == "audiobook":
|
||||
key, defaults = "NEWZNAB_AUDIOBOOK_CATEGORIES", _DEFAULT_AUDIOBOOK_CATS
|
||||
else:
|
||||
key, defaults = "NEWZNAB_EBOOK_CATEGORIES", _DEFAULT_BOOK_CATS
|
||||
|
||||
return _parse_category_ids(config.get(key, None)) or list(defaults)
|
||||
|
||||
|
||||
def _result_category_ids(categories: object) -> set[int]:
|
||||
"""Extract numeric category IDs from a result's categories field."""
|
||||
if not isinstance(categories, (list, tuple)):
|
||||
return set()
|
||||
|
||||
category_ids: set[int] = set()
|
||||
for cat in categories:
|
||||
raw = cat.get("id") if isinstance(cat, dict) else cat
|
||||
try:
|
||||
category_ids.add(int(raw)) # type: ignore[arg-type]
|
||||
except TypeError, ValueError:
|
||||
continue
|
||||
return category_ids
|
||||
|
||||
|
||||
def _resolve_content_type(
|
||||
categories: object,
|
||||
content_type: str,
|
||||
searched_categories: list[int] | None,
|
||||
) -> str:
|
||||
"""Resolve a result's content type, honouring custom indexer categories.
|
||||
|
||||
Indexers using non-standard IDs (e.g. 7100 for ebooks) fall outside the standard
|
||||
ranges, so trust the searched content type when the result carries a category we
|
||||
explicitly asked for.
|
||||
"""
|
||||
category_list = list(categories) if isinstance(categories, (list, tuple)) else []
|
||||
detected = _detect_content_type_from_categories(category_list, content_type)
|
||||
if (
|
||||
detected == "other"
|
||||
and searched_categories
|
||||
and _result_category_ids(category_list) & set(searched_categories)
|
||||
):
|
||||
return "audiobook" if content_type == "audiobook" else "book"
|
||||
return detected
|
||||
|
||||
|
||||
def _newznab_result_to_release(
|
||||
result: dict,
|
||||
content_type: str = "ebook",
|
||||
searched_categories: list[int] | None = None,
|
||||
) -> Release:
|
||||
"""Convert a parsed Newznab XML result dict to a Release object."""
|
||||
raw_title = result.get("title", "Unknown")
|
||||
size_bytes = result.get("size")
|
||||
@@ -67,8 +193,11 @@ def _newznab_result_to_release(result: dict, content_type: str = "ebook") -> Rel
|
||||
else None
|
||||
)
|
||||
|
||||
# Build source_id from GUID
|
||||
source_id = result.get("guid") or f"newznab:{hash(raw_title)}"
|
||||
# Namespace IDs from named connections so identical GUIDs returned by two
|
||||
# indexers cannot overwrite one another in the private release cache.
|
||||
raw_source_id = result.get("guid") or f"newznab:{hash(raw_title)}"
|
||||
connection_id = str(result.get("_newznab_connection_id") or "").strip()
|
||||
source_id = f"newznab:{connection_id}:{raw_source_id}" if connection_id else raw_source_id
|
||||
|
||||
# Cache the raw result for the handler
|
||||
cache_release(source_id, result)
|
||||
@@ -125,7 +254,7 @@ def _newznab_result_to_release(result: dict, content_type: str = "ebook") -> Rel
|
||||
indexer=indexer,
|
||||
seeders=seeders if is_torrent else None,
|
||||
peers=peers_display,
|
||||
content_type=_detect_content_type_from_categories(categories, content_type),
|
||||
content_type=_resolve_content_type(categories, content_type, searched_categories),
|
||||
extra={
|
||||
"publish_date": result.get("publishDate"),
|
||||
"categories": categories,
|
||||
@@ -193,6 +322,7 @@ class NewznabSource(ReleaseSource):
|
||||
)
|
||||
|
||||
def _get_client(self) -> NewznabClient | None:
|
||||
"""Build the legacy single-indexer client."""
|
||||
raw_url = str(config.get("NEWZNAB_URL", "") or "")
|
||||
api_key = str(config.get("NEWZNAB_API_KEY", "") or "")
|
||||
|
||||
@@ -205,6 +335,28 @@ class NewznabSource(ReleaseSource):
|
||||
|
||||
return NewznabClient(url, api_key or "")
|
||||
|
||||
def _get_clients(self) -> list[_NamedClient]:
|
||||
"""Build named clients, falling back to the legacy single connection."""
|
||||
configured = _parse_indexer_rows(config.get("NEWZNAB_INDEXERS", []))
|
||||
if configured:
|
||||
clients: list[_NamedClient] = []
|
||||
for name, url, api_key in configured:
|
||||
digest = sha256(f"{name}\0{url}\0{api_key}".encode()).hexdigest()[:16]
|
||||
clients.append(
|
||||
_NamedClient(
|
||||
name=name,
|
||||
connection_id=digest,
|
||||
client=NewznabClient(url, api_key),
|
||||
)
|
||||
)
|
||||
return clients
|
||||
|
||||
legacy_client = self._get_client()
|
||||
if legacy_client is None:
|
||||
return []
|
||||
legacy_name = str(config.get("NEWZNAB_NAME", "") or "").strip() or "Newznab"
|
||||
return [_NamedClient(name=legacy_name, connection_id="legacy", client=legacy_client)]
|
||||
|
||||
def search(
|
||||
self,
|
||||
book: BookMetadata,
|
||||
@@ -214,8 +366,8 @@ class NewznabSource(ReleaseSource):
|
||||
content_type: str = "ebook",
|
||||
) -> list[Release]:
|
||||
"""Search the Newznab indexer for releases matching the book."""
|
||||
client = self._get_client()
|
||||
if not client:
|
||||
clients = self._get_clients()
|
||||
if not clients:
|
||||
logger.warning("Newznab not configured - skipping search")
|
||||
return []
|
||||
|
||||
@@ -230,12 +382,7 @@ class NewznabSource(ReleaseSource):
|
||||
return []
|
||||
|
||||
# Category selection — omit categories when expanding search
|
||||
if expand_search:
|
||||
categories = None
|
||||
elif content_type == "audiobook":
|
||||
categories = [3030]
|
||||
else:
|
||||
categories = [7000]
|
||||
categories = None if expand_search else _configured_categories(content_type)
|
||||
|
||||
auto_expand = config.get("NEWZNAB_AUTO_EXPAND", False)
|
||||
deadline = time.monotonic() + NEWZNAB_SEARCH_TIMEOUT_SECONDS
|
||||
@@ -250,40 +397,60 @@ class NewznabSource(ReleaseSource):
|
||||
all_results: list[dict] = []
|
||||
|
||||
try:
|
||||
for idx, query in enumerate(queries, start=1):
|
||||
_check_timeout()
|
||||
if len(queries) > 1:
|
||||
logger.debug("Newznab query %d/%d: '%s'", idx, len(queries), query)
|
||||
for connection in clients:
|
||||
try:
|
||||
for idx, query in enumerate(queries, start=1):
|
||||
_check_timeout()
|
||||
if len(queries) > 1:
|
||||
logger.debug(
|
||||
"Newznab [%s] query %d/%d: '%s'",
|
||||
connection.name,
|
||||
idx,
|
||||
len(queries),
|
||||
query,
|
||||
)
|
||||
|
||||
raw = client.search(query=query, categories=categories)
|
||||
raw = connection.client.search(query=query, categories=categories)
|
||||
|
||||
# Auto-expand: retry without category filter if no results
|
||||
if not raw and categories and auto_expand:
|
||||
_check_timeout()
|
||||
logger.info(
|
||||
"Newznab: no results for '%s' with category filter, auto-expanding",
|
||||
query,
|
||||
)
|
||||
raw = client.search(query=query, categories=None)
|
||||
# Auto-expand: retry without category filter if no results
|
||||
if not raw and categories and auto_expand:
|
||||
_check_timeout()
|
||||
logger.info(
|
||||
"Newznab [%s]: no results for '%s' with category filter, "
|
||||
"auto-expanding",
|
||||
connection.name,
|
||||
query,
|
||||
)
|
||||
raw = connection.client.search(query=query, categories=None)
|
||||
|
||||
for r in raw:
|
||||
key = (
|
||||
r.get("guid")
|
||||
or r.get("downloadUrl")
|
||||
or f"{r.get('indexer')}:{r.get('title')}"
|
||||
)
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
for raw_result in raw:
|
||||
r = dict(raw_result)
|
||||
# Aggregators can identify the underlying indexer. Plain feeds
|
||||
# generally cannot, so use the user-configured connection name.
|
||||
r["indexer"] = r.get("indexer") or connection.name
|
||||
r["_newznab_connection_id"] = connection.connection_id
|
||||
key = (
|
||||
connection.connection_id,
|
||||
r.get("guid")
|
||||
or r.get("downloadUrl")
|
||||
or f"{r.get('indexer')}:{r.get('title')}",
|
||||
)
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
except TimeoutError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Newznab search failed for %s", connection.name)
|
||||
|
||||
except TimeoutError as e:
|
||||
logger.warning("Newznab search timed out: %s", e)
|
||||
except Exception:
|
||||
logger.exception("Newznab search failed")
|
||||
return []
|
||||
|
||||
results = [_newznab_result_to_release(r, content_type) for r in all_results]
|
||||
results = [_newznab_result_to_release(r, content_type, categories) for r in all_results]
|
||||
if plan.indexers:
|
||||
selected_indexers = set(plan.indexers)
|
||||
results = [r for r in results if r.indexer in selected_indexers]
|
||||
|
||||
if results:
|
||||
nzb_count = sum(1 for r in results if r.protocol == ReleaseProtocol.NZB)
|
||||
@@ -305,5 +472,7 @@ class NewznabSource(ReleaseSource):
|
||||
def is_available(self) -> bool:
|
||||
if not config.get("NEWZNAB_ENABLED", False):
|
||||
return False
|
||||
if _parse_indexer_rows(config.get("NEWZNAB_INDEXERS", [])):
|
||||
return True
|
||||
url = normalize_http_url(str(config.get("NEWZNAB_URL", "") or ""))
|
||||
return bool(url)
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any, TypedDict
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
@@ -18,6 +19,19 @@ logger = setup_logger(__name__)
|
||||
_HTTP_STATUS_UNAUTHORIZED = HTTPStatus.UNAUTHORIZED
|
||||
_BOOK_CATEGORY_RANGE_START = 7000
|
||||
_BOOK_CATEGORY_RANGE_END = 8000
|
||||
|
||||
# Prowlarr's own JSON endpoints (status, indexer list) read local state and answer
|
||||
# in milliseconds, so they keep a short timeout. A Torznab search is different: it
|
||||
# is Prowlarr proxying a live request to the tracker, which for a Cloudflare-fronted
|
||||
# indexer means waiting on FlareSolverr to solve a challenge. A cold challenge
|
||||
# routinely runs past a minute, so indexer searches get their own, longer budget.
|
||||
DEFAULT_INDEXER_TIMEOUT_SECONDS = 90
|
||||
MIN_INDEXER_TIMEOUT_SECONDS = 5
|
||||
MAX_INDEXER_TIMEOUT_SECONDS = 300
|
||||
|
||||
# Connecting to Prowlarr itself is a LAN hop; only the read is allowed to be slow.
|
||||
_CONNECT_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
_PROWLARR_CLIENT_ERRORS = (
|
||||
requests.exceptions.RequestException,
|
||||
OSError,
|
||||
@@ -27,6 +41,37 @@ _PROWLARR_CLIENT_ERRORS = (
|
||||
)
|
||||
|
||||
|
||||
class ProwlarrSearchError(RuntimeError):
|
||||
"""A Torznab search could not be completed.
|
||||
|
||||
Deliberately distinct from an empty result list. Reporting a failed search as
|
||||
"this indexer has nothing" is what turns a slow FlareSolverr challenge into
|
||||
"No releases found for this book" in the UI (#1249), and it also makes the
|
||||
auto-expand retry fire a second request on top of the one still running.
|
||||
"""
|
||||
|
||||
|
||||
def resolve_indexer_timeout(timeout: object = None) -> int:
|
||||
"""Resolve the per-indexer search timeout, falling back to config.
|
||||
|
||||
Out-of-range and unparsable values are clamped rather than rejected: this
|
||||
feeds an HTTP timeout, and a bad setting should not take searching down.
|
||||
"""
|
||||
if timeout is None:
|
||||
timeout = config.get("PROWLARR_INDEXER_TIMEOUT", DEFAULT_INDEXER_TIMEOUT_SECONDS)
|
||||
|
||||
resolved = coerce_int_like(timeout)
|
||||
if resolved is None:
|
||||
logger.warning(
|
||||
"Invalid PROWLARR_INDEXER_TIMEOUT %r - using %ss",
|
||||
timeout,
|
||||
DEFAULT_INDEXER_TIMEOUT_SECONDS,
|
||||
)
|
||||
return DEFAULT_INDEXER_TIMEOUT_SECONDS
|
||||
|
||||
return max(MIN_INDEXER_TIMEOUT_SECONDS, min(MAX_INDEXER_TIMEOUT_SECONDS, resolved))
|
||||
|
||||
|
||||
class IndexerSeedSettings(TypedDict, total=False):
|
||||
ratio_limit: float
|
||||
seeding_time_limit_minutes: int
|
||||
@@ -77,11 +122,23 @@ def _get_field_value(fields: object, name: str) -> object | None:
|
||||
class ProwlarrClient:
|
||||
"""Client for interacting with the Prowlarr API."""
|
||||
|
||||
def __init__(self, url: str, api_key: str, timeout: int = 30) -> None:
|
||||
"""Initialize the API client with base URL, key, and timeout."""
|
||||
def __init__(
|
||||
self, url: str, api_key: str, timeout: int = 30, indexer_timeout: int | None = None
|
||||
) -> None:
|
||||
"""Initialize the API client with base URL, key, and timeouts.
|
||||
|
||||
Args:
|
||||
url: Prowlarr base URL.
|
||||
api_key: Prowlarr API key.
|
||||
timeout: Timeout for Prowlarr's own JSON endpoints.
|
||||
indexer_timeout: Timeout for Torznab searches, which Prowlarr proxies
|
||||
out to the tracker. Defaults to PROWLARR_INDEXER_TIMEOUT.
|
||||
|
||||
"""
|
||||
self.base_url = normalize_http_url(url)
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
self.indexer_timeout = resolve_indexer_timeout(indexer_timeout)
|
||||
self._session = requests.Session()
|
||||
self._session.headers.update(
|
||||
{
|
||||
@@ -307,6 +364,12 @@ class ProwlarrClient:
|
||||
|
||||
This returns richer fields (e.g., author/booktitle, torznab tags like
|
||||
FreeLeech) than the JSON /api/v1/search endpoint.
|
||||
|
||||
Raises:
|
||||
ProwlarrSearchError: The search could not be completed. An empty list
|
||||
strictly means the indexer answered with no matches, never that
|
||||
the request timed out or errored.
|
||||
|
||||
"""
|
||||
if not query:
|
||||
return []
|
||||
@@ -329,7 +392,7 @@ class ProwlarrClient:
|
||||
response = self._session.get(
|
||||
url=url,
|
||||
params=params,
|
||||
timeout=self.timeout,
|
||||
timeout=(_CONNECT_TIMEOUT_SECONDS, self.indexer_timeout),
|
||||
headers={
|
||||
# Override the session default JSON accept header.
|
||||
"Accept": "application/rss+xml, application/xml;q=0.9, */*;q=0.8"
|
||||
@@ -347,9 +410,20 @@ class ProwlarrClient:
|
||||
for r in results:
|
||||
if r.get("indexerId") is None:
|
||||
r["indexerId"] = int(indexer_id)
|
||||
except Exception:
|
||||
except requests.exceptions.Timeout as e:
|
||||
logger.warning(
|
||||
"Prowlarr Torznab search for indexer %s timed out after %ss. An indexer "
|
||||
"behind FlareSolverr can need far longer than that on a cold Cloudflare "
|
||||
"challenge - raise PROWLARR_INDEXER_TIMEOUT if this keeps happening.",
|
||||
indexer_id,
|
||||
self.indexer_timeout,
|
||||
)
|
||||
msg = f"indexer {indexer_id} did not respond within {self.indexer_timeout}s"
|
||||
raise ProwlarrSearchError(msg) from e
|
||||
except Exception as e:
|
||||
logger.exception("Prowlarr Torznab search failed for indexer %s", indexer_id)
|
||||
return []
|
||||
msg = f"indexer {indexer_id} search failed: {e}"
|
||||
raise ProwlarrSearchError(msg) from e
|
||||
else:
|
||||
return results
|
||||
|
||||
|
||||
@@ -28,6 +28,10 @@ from shelfmark.download.clients.base_handler import (
|
||||
DownloadRequest,
|
||||
ExternalClientHandler,
|
||||
)
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
extract_file_list_from_torrent,
|
||||
extract_torrent_info,
|
||||
)
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
from shelfmark.release_sources import register_handler
|
||||
from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, ProwlarrClient
|
||||
@@ -38,12 +42,14 @@ from shelfmark.release_sources.prowlarr.utils import (
|
||||
coerce_int_like,
|
||||
get_preferred_download_url,
|
||||
get_protocol,
|
||||
sanitize_download_url,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -127,6 +133,24 @@ class ProwlarrHandler(ExternalClientHandler):
|
||||
|
||||
return settings.get(indexer_id)
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""List a cached torrent release's files from its .torrent, without downloading.
|
||||
|
||||
Magnet-only and usenet releases cannot be listed ahead of time.
|
||||
"""
|
||||
source_id = str(release_data.get("source_id") or "")
|
||||
prowlarr_result = get_release(source_id) if source_id else None
|
||||
if not prowlarr_result or get_protocol(prowlarr_result) != "torrent":
|
||||
return None
|
||||
download_url = sanitize_download_url(str(prowlarr_result.get("downloadUrl") or "").strip())
|
||||
if not download_url or download_url.startswith("magnet:"):
|
||||
return None
|
||||
expected_hash = str(prowlarr_result.get("infoHash") or "").strip() or None
|
||||
info = extract_torrent_info(download_url, expected_hash=expected_hash)
|
||||
if not info.torrent_data:
|
||||
return None
|
||||
return extract_file_list_from_torrent(info.torrent_data)
|
||||
|
||||
def _get_client(self, protocol: str) -> DownloadClient | None:
|
||||
"""Compatibility shim so module-level patching still works in tests."""
|
||||
return get_client(protocol)
|
||||
@@ -297,6 +321,8 @@ class ProwlarrHandler(ExternalClientHandler):
|
||||
search_title=title,
|
||||
search_author=task.author,
|
||||
)
|
||||
# No language default here on purpose: this re-finds one exact release by its
|
||||
# guid, and Prowlarr does not filter on plan.languages anyway.
|
||||
plan = build_release_search_plan(
|
||||
book,
|
||||
indexers=[indexer] if indexer is not None else None,
|
||||
|
||||
@@ -10,12 +10,18 @@ from shelfmark.core.settings_registry import (
|
||||
CheckboxField,
|
||||
HeadingField,
|
||||
MultiSelectField,
|
||||
NumberField,
|
||||
PasswordField,
|
||||
SettingsField,
|
||||
TextField,
|
||||
register_settings,
|
||||
)
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.api import (
|
||||
DEFAULT_INDEXER_TIMEOUT_SECONDS,
|
||||
MAX_INDEXER_TIMEOUT_SECONDS,
|
||||
MIN_INDEXER_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
# ==================== Dynamic Options Loaders ====================
|
||||
|
||||
@@ -183,6 +189,20 @@ def prowlarr_config_settings() -> list[SettingsField]:
|
||||
default=[],
|
||||
show_when={"field": "PROWLARR_ENABLED", "value": True},
|
||||
),
|
||||
NumberField(
|
||||
key="PROWLARR_INDEXER_TIMEOUT",
|
||||
label="Indexer Search Timeout (seconds)",
|
||||
description=(
|
||||
"How long to wait for a single indexer to answer a search. Indexers behind "
|
||||
"FlareSolverr can need 90 seconds or more while a cold Cloudflare challenge "
|
||||
"is solved; raise this if searches come back empty and the Prowlarr log "
|
||||
"shows the search still running."
|
||||
),
|
||||
default=DEFAULT_INDEXER_TIMEOUT_SECONDS,
|
||||
min_value=MIN_INDEXER_TIMEOUT_SECONDS,
|
||||
max_value=MAX_INDEXER_TIMEOUT_SECONDS,
|
||||
show_when={"field": "PROWLARR_ENABLED", "value": True},
|
||||
),
|
||||
CheckboxField(
|
||||
key="PROWLARR_AUTO_EXPAND",
|
||||
label="Auto-expand search on no results",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING, ClassVar, NoReturn
|
||||
|
||||
@@ -16,6 +17,7 @@ from shelfmark.core.languages import normalize_language
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.request_helpers import normalize_optional_text
|
||||
from shelfmark.core.search_plan import ReleaseSearchVariant
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS as CORE_AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources import (
|
||||
ColumnAlign,
|
||||
@@ -29,9 +31,14 @@ from shelfmark.release_sources import (
|
||||
ReleaseProtocol,
|
||||
ReleaseSource,
|
||||
SortOption,
|
||||
SourceUnavailableError,
|
||||
register_source,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, ProwlarrClient
|
||||
from shelfmark.release_sources.prowlarr.api import (
|
||||
IndexerSeedSettings,
|
||||
ProwlarrClient,
|
||||
ProwlarrSearchError,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.cache import cache_release
|
||||
from shelfmark.release_sources.prowlarr.utils import (
|
||||
build_source_id,
|
||||
@@ -49,10 +56,10 @@ _PROWLARR_SOURCE_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, Val
|
||||
# Prowlarr indexer priority is 1-50 and lower is preferred; unknown sorts last.
|
||||
_UNRANKED_INDEXER_RANK = 51
|
||||
|
||||
# Errors that can surface from ProwlarrClient.get_indexer_seed_settings(). The
|
||||
# Errors that can surface from a ProwlarrClient call that talks to Prowlarr. The
|
||||
# client raises requests exceptions (subclasses of OSError via IOError lineage
|
||||
# is not guaranteed), so include RequestException explicitly.
|
||||
_PROWLARR_SEED_SETTINGS_ERRORS = (*_PROWLARR_SOURCE_ERRORS, requests.exceptions.RequestException)
|
||||
_PROWLARR_REQUEST_ERRORS = (*_PROWLARR_SOURCE_ERRORS, requests.exceptions.RequestException)
|
||||
|
||||
|
||||
def _raise_timeout_error(message: str) -> NoReturn:
|
||||
@@ -138,6 +145,36 @@ def _build_indexer_priority(indexers: list[dict]) -> dict[int, int]:
|
||||
return priority
|
||||
|
||||
|
||||
def _drop_unknown_indexer_ids(
|
||||
selected_ids: list[int] | None, indexers: list[dict]
|
||||
) -> list[int] | None:
|
||||
"""Keep only selected indexer ids Prowlarr still serves.
|
||||
|
||||
An indexer removed or disabled in Prowlarr stays in the saved selection,
|
||||
where settings can no longer show it - so it cannot be unselected, and every
|
||||
search keeps querying an indexer that is gone (#1283). Dropping it here
|
||||
keeps the saved selection intact for an indexer that comes back.
|
||||
"""
|
||||
if selected_ids is None:
|
||||
return None
|
||||
|
||||
live_ids = {
|
||||
indexer_id
|
||||
for indexer in indexers
|
||||
if (indexer_id := _coerce_indexer_id(indexer.get("id"))) is not None
|
||||
}
|
||||
kept = [indexer_id for indexer_id in selected_ids if indexer_id in live_ids]
|
||||
|
||||
stale = [indexer_id for indexer_id in selected_ids if indexer_id not in live_ids]
|
||||
if stale:
|
||||
logger.warning(
|
||||
"Skipping selected Prowlarr indexers that are no longer enabled in Prowlarr: %s",
|
||||
stale,
|
||||
)
|
||||
|
||||
return kept
|
||||
|
||||
|
||||
def _rank_for_indexer_id(indexer_id: object, priority: dict[int, int]) -> int:
|
||||
"""Preference rank for an indexer id. Lower wins, unknown ranks last."""
|
||||
coerced = _coerce_indexer_id(indexer_id)
|
||||
@@ -222,7 +259,7 @@ EBOOK_FORMATS = [
|
||||
]
|
||||
|
||||
# Common audiobook formats
|
||||
AUDIOBOOK_FORMATS = ["m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus"]
|
||||
AUDIOBOOK_FORMATS = list(CORE_AUDIOBOOK_FORMATS)
|
||||
|
||||
# Combined list for format detection (audiobook formats first for priority)
|
||||
ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS
|
||||
@@ -231,6 +268,35 @@ ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS
|
||||
# Backend safeguard: cap total Prowlarr search time per request.
|
||||
PROWLARR_SEARCH_TIMEOUT_SECONDS = 120.0
|
||||
|
||||
# The overall budget has to leave room for at least a couple of indexers to spend
|
||||
# their full per-indexer timeout, otherwise raising PROWLARR_INDEXER_TIMEOUT for a
|
||||
# Cloudflare-fronted tracker just moves the cutoff here. Capped short of the
|
||||
# gunicorn worker timeout (300s) so the worker is never the thing that gives up.
|
||||
_MAX_SEARCH_BUDGET_SECONDS = 240.0
|
||||
|
||||
|
||||
def _search_budget_seconds(indexer_timeout: int) -> float:
|
||||
"""Total time one Prowlarr search may spend, scaled to the per-indexer timeout."""
|
||||
return min(
|
||||
_MAX_SEARCH_BUDGET_SECONDS,
|
||||
max(PROWLARR_SEARCH_TIMEOUT_SECONDS, indexer_timeout * 2.0),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _IndexerSearchOutcome:
|
||||
"""What one pass over the target indexers produced.
|
||||
|
||||
Separates "every indexer answered, none had this book" from "the indexers
|
||||
never answered", which the caller has to tell apart before it decides to
|
||||
auto-expand or to report the search as failed.
|
||||
"""
|
||||
|
||||
results: list[dict]
|
||||
attempted: int = 0
|
||||
failed: int = 0
|
||||
last_error: str | None = None
|
||||
|
||||
|
||||
def _extract_format(title: str) -> str | None:
|
||||
"""Extract ebook/audiobook format from release title (extension, bracketed, or standalone)."""
|
||||
@@ -281,19 +347,25 @@ def _extract_mam_language(raw_title: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_mam_formats(raw_title: str) -> list[str]:
|
||||
"""Extract a list of formats from MyAnonamouse titles.
|
||||
def _split_mam_formats(raw_title: str) -> tuple[list[str], list[str]]:
|
||||
"""Split the format tokens of a MyAnonamouse title into (recognized, unrecognized).
|
||||
|
||||
Prowlarr's MAM parser appends a structured bracket segment like:
|
||||
[ENG / EPUB MOBI PDF]
|
||||
|
||||
We only trust this structured segment (and do not attempt generic title
|
||||
heuristics for other indexers).
|
||||
|
||||
Tokens after the "/" that Shelfmark does not know as a book or audiobook format
|
||||
(e.g. ``[ENG / AVI]``) are returned separately so the UI can warn that the release
|
||||
will download but cannot be processed, instead of showing a bare content-type icon
|
||||
that looks like an ordinary result.
|
||||
"""
|
||||
if not raw_title:
|
||||
return []
|
||||
return [], []
|
||||
|
||||
format_set = set(ALL_BOOK_FORMATS)
|
||||
first_unrecognized: list[str] | None = None
|
||||
for bracket in re.findall(r"\[([^\]]+)\]", raw_title):
|
||||
if "/" not in bracket:
|
||||
continue
|
||||
@@ -302,15 +374,26 @@ def _extract_mam_formats(raw_title: str) -> list[str]:
|
||||
tokens = re.findall(r"[A-Za-z0-9]+", after_slash)
|
||||
|
||||
formats: list[str] = []
|
||||
unrecognized: list[str] = []
|
||||
for token in tokens:
|
||||
fmt = token.lower()
|
||||
if fmt in format_set and fmt not in formats:
|
||||
formats.append(fmt)
|
||||
if fmt in format_set:
|
||||
if fmt not in formats:
|
||||
formats.append(fmt)
|
||||
elif fmt not in unrecognized:
|
||||
unrecognized.append(fmt)
|
||||
|
||||
if formats:
|
||||
return formats
|
||||
return formats, unrecognized
|
||||
if unrecognized and first_unrecognized is None:
|
||||
first_unrecognized = unrecognized
|
||||
|
||||
return []
|
||||
return [], first_unrecognized or []
|
||||
|
||||
|
||||
def _extract_mam_formats(raw_title: str) -> list[str]:
|
||||
"""Extract the recognized formats from a MyAnonamouse title (see _split_mam_formats)."""
|
||||
return _split_mam_formats(raw_title)[0]
|
||||
|
||||
|
||||
def _formats_display(formats: list[str]) -> str | None:
|
||||
@@ -449,6 +532,7 @@ def _prowlarr_result_to_release(
|
||||
|
||||
format_detected: str | None = None
|
||||
formats: list[str] = []
|
||||
unrecognized_formats: list[str] = []
|
||||
formats_display: str | None = None
|
||||
language_detected: str | None = None
|
||||
if enable_format_detection:
|
||||
@@ -456,7 +540,7 @@ def _prowlarr_result_to_release(
|
||||
if book_title:
|
||||
title = book_title
|
||||
|
||||
formats = _extract_mam_formats(str(raw_title or ""))
|
||||
formats, unrecognized_formats = _split_mam_formats(str(raw_title or ""))
|
||||
format_detected = formats[0] if formats else None
|
||||
formats_display = _formats_display(formats)
|
||||
language_detected = _extract_mam_language(str(raw_title or ""))
|
||||
@@ -518,6 +602,9 @@ def _prowlarr_result_to_release(
|
||||
"info_hash": result.get("infoHash"),
|
||||
"formats": formats or None,
|
||||
"formats_display": formats_display,
|
||||
# Format tokens the indexer declared but Shelfmark can't process (e.g. a MAM
|
||||
# "[ENG / AVI]"). Lets the UI warn instead of showing a bare content icon.
|
||||
"unrecognized_formats": unrecognized_formats or None,
|
||||
# Raw torznab attributes for rich tooltips (enriched indexers)
|
||||
"torznab_attrs": result.get("torznabAttrs"),
|
||||
},
|
||||
@@ -538,7 +625,7 @@ def _fetch_indexer_seed_settings(
|
||||
"""Fetch per-indexer share limits, falling back to last-known-good on failure."""
|
||||
try:
|
||||
fetched = client.get_indexer_seed_settings(restrict_to=indexer_ids)
|
||||
except _PROWLARR_SEED_SETTINGS_ERRORS:
|
||||
except _PROWLARR_REQUEST_ERRORS:
|
||||
with _seed_settings_lock:
|
||||
fallback = dict(_last_known_seed_settings)
|
||||
logger.warning(
|
||||
@@ -894,8 +981,17 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
try:
|
||||
auto_expand_enabled = config.get("PROWLARR_AUTO_EXPAND", False)
|
||||
deadline = time.monotonic() + PROWLARR_SEARCH_TIMEOUT_SECONDS
|
||||
enabled_indexers = client.get_enabled_indexers_detailed()
|
||||
search_budget = _search_budget_seconds(client.indexer_timeout)
|
||||
deadline = time.monotonic() + search_budget
|
||||
try:
|
||||
enabled_indexers = client.get_enabled_indexers_detailed(raise_on_error=True)
|
||||
except _PROWLARR_REQUEST_ERRORS as e:
|
||||
# Prowlarr itself is unreachable. Swallowing this leaves the search
|
||||
# with no indexers to query, which the UI renders as "No releases
|
||||
# found for this book" - the same lie as a swallowed timeout (#1249).
|
||||
msg = f"could not reach Prowlarr: {e}"
|
||||
raise SourceUnavailableError(msg) from e
|
||||
indexer_ids = _drop_unknown_indexer_ids(indexer_ids, enabled_indexers)
|
||||
indexer_priority = _build_indexer_priority(enabled_indexers)
|
||||
# Some indexers benefit from title+author queries and extra format detection.
|
||||
enriched_indexer_ids = client.get_enriched_indexer_ids(
|
||||
@@ -910,18 +1006,16 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
def _check_timeout() -> None:
|
||||
if time.monotonic() > deadline:
|
||||
_raise_timeout_error(
|
||||
f"Prowlarr search timed out after {int(PROWLARR_SEARCH_TIMEOUT_SECONDS)}s"
|
||||
)
|
||||
_raise_timeout_error(f"Prowlarr search timed out after {int(search_budget)}s")
|
||||
|
||||
def search_indexers(
|
||||
query: str, cats: list[int] | None, *, enriched_query: str | None = None
|
||||
) -> list[dict]:
|
||||
) -> _IndexerSearchOutcome:
|
||||
"""Search indexers with given categories via Torznab/Newznab."""
|
||||
results: list[dict] = []
|
||||
outcome = _IndexerSearchOutcome(results=[])
|
||||
target_indexer_ids = self._get_search_indexer_ids(client, indexer_ids, cats)
|
||||
if not target_indexer_ids:
|
||||
return results
|
||||
return outcome
|
||||
|
||||
for indexer_id in target_indexer_ids:
|
||||
_check_timeout()
|
||||
@@ -930,19 +1024,31 @@ class ProwlarrSource(ReleaseSource):
|
||||
if indexer_id in enriched_indexer_ids_set and enriched_query
|
||||
else query
|
||||
)
|
||||
raw = client.torznab_search(
|
||||
indexer_id=indexer_id,
|
||||
query=indexer_query,
|
||||
categories=cats,
|
||||
search_type="book",
|
||||
)
|
||||
outcome.attempted += 1
|
||||
try:
|
||||
raw = client.torznab_search(
|
||||
indexer_id=indexer_id,
|
||||
query=indexer_query,
|
||||
categories=cats,
|
||||
search_type="book",
|
||||
)
|
||||
except ProwlarrSearchError as e:
|
||||
# One unreachable indexer must not sink the others, but it
|
||||
# is not "no results" either - record it so the caller can
|
||||
# report a failed search instead of an empty one.
|
||||
outcome.failed += 1
|
||||
outcome.last_error = str(e)
|
||||
continue
|
||||
if raw:
|
||||
results.extend(raw)
|
||||
outcome.results.extend(raw)
|
||||
|
||||
return results
|
||||
return outcome
|
||||
|
||||
seen_keys: set[tuple[int | None, str]] = set()
|
||||
all_results: list[dict] = []
|
||||
attempted_searches = 0
|
||||
failed_searches = 0
|
||||
last_search_error: str | None = None
|
||||
|
||||
for idx, variant in enumerate(variants, start=1):
|
||||
_check_timeout()
|
||||
@@ -952,23 +1058,39 @@ class ProwlarrSource(ReleaseSource):
|
||||
if len(variants) > 1:
|
||||
logger.debug("Prowlarr query %s/%s: '%s'", idx, len(variants), query)
|
||||
|
||||
raw_results = search_indexers(
|
||||
outcome = search_indexers(
|
||||
query=query, cats=categories, enriched_query=enriched_query
|
||||
)
|
||||
|
||||
# Auto-expand: if no results with categories and auto-expand enabled, retry without
|
||||
if not raw_results and categories and auto_expand_enabled:
|
||||
# Auto-expand: if no results with categories and auto-expand enabled, retry without.
|
||||
# Only when every indexer actually answered: a failed search says nothing about
|
||||
# whether the category filter is what hid the book, and retrying it stacks a second
|
||||
# request on an indexer that is still busy solving a Cloudflare challenge (#1249).
|
||||
if (
|
||||
not outcome.results
|
||||
and not outcome.failed
|
||||
and categories
|
||||
and auto_expand_enabled
|
||||
):
|
||||
_check_timeout()
|
||||
logger.info(
|
||||
"Prowlarr: no results for query '%s' with category filter, auto-expanding search",
|
||||
query,
|
||||
)
|
||||
raw_results = search_indexers(
|
||||
expanded = search_indexers(
|
||||
query=query, cats=None, enriched_query=enriched_query
|
||||
)
|
||||
outcome.results = expanded.results
|
||||
outcome.attempted += expanded.attempted
|
||||
outcome.failed += expanded.failed
|
||||
outcome.last_error = expanded.last_error or outcome.last_error
|
||||
self.last_search_type = "expanded"
|
||||
|
||||
for r in raw_results:
|
||||
attempted_searches += outcome.attempted
|
||||
failed_searches += outcome.failed
|
||||
last_search_error = outcome.last_error or last_search_error
|
||||
|
||||
for r in outcome.results:
|
||||
key = _result_dedup_key(r)
|
||||
if key is not None:
|
||||
if key in seen_keys:
|
||||
@@ -976,6 +1098,14 @@ class ProwlarrSource(ReleaseSource):
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
|
||||
if failed_searches:
|
||||
logger.warning(
|
||||
"Prowlarr: %s of %s indexer searches failed (%s)",
|
||||
failed_searches,
|
||||
attempted_searches,
|
||||
last_search_error,
|
||||
)
|
||||
|
||||
if config.get("PROWLARR_COLLAPSE_DUPLICATES", True):
|
||||
before_collapse = len(all_results)
|
||||
all_results = _collapse_duplicate_indexer_results(all_results, indexer_priority)
|
||||
@@ -1032,6 +1162,10 @@ class ProwlarrSource(ReleaseSource):
|
||||
else:
|
||||
logger.debug("Prowlarr: no results found")
|
||||
|
||||
except SourceUnavailableError:
|
||||
# Already carries its own message for the caller to surface; the blanket
|
||||
# handler below would turn it back into a silent empty result.
|
||||
raise
|
||||
except TimeoutError as e:
|
||||
logger.warning("Prowlarr search timed out: %s", e)
|
||||
raise
|
||||
@@ -1039,6 +1173,15 @@ class ProwlarrSource(ReleaseSource):
|
||||
logger.exception("Prowlarr search failed")
|
||||
return []
|
||||
else:
|
||||
# An empty list is the UI's "No releases found for this book", so it has
|
||||
# to mean the indexers answered and had nothing. When they failed instead,
|
||||
# say so rather than blaming the book (#1249).
|
||||
if not results and failed_searches:
|
||||
msg = (
|
||||
f"{failed_searches} of {attempted_searches} indexer searches failed "
|
||||
f"({last_search_error})"
|
||||
)
|
||||
raise SourceUnavailableError(msg)
|
||||
return results
|
||||
|
||||
def is_available(self) -> bool:
|
||||
|
||||
Generated
+249
-302
@@ -19,9 +19,9 @@
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.4",
|
||||
"@vitejs/plugin-react": "^6.0.5",
|
||||
"knip": "^6.32.0",
|
||||
"oxfmt": "^0.62.0",
|
||||
"oxlint": "^1.77.0",
|
||||
"knip": "^6.32.2",
|
||||
"oxfmt": "^0.63.0",
|
||||
"oxlint": "^1.78.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"typescript": "^7.0.2",
|
||||
@@ -130,9 +130,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-android-arm-eabi": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.142.0.tgz",
|
||||
"integrity": "sha512-ZiRGDutGsv1G6bL/ozy/koC0Sv39T1DqyoC4KD1DOy9ZoACm1O5UWhEK2c02Qdk+4lfLVkvFa/mQ0fm/4h1BtQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.143.0.tgz",
|
||||
"integrity": "sha512-n9uozULWflPqBtdmI8lAabLqGKNgLVNN0ZH8HfgCwpKGNtzRzauB76jTiW/3YLkcA7N1zskpi9GdVnZuu1SAvg==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -147,9 +147,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-android-arm64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.142.0.tgz",
|
||||
"integrity": "sha512-WZkvGRLNQTz8lR9zP5nLjUdlroRCopBu3g9zF1p/laE6DzT1UbQo8Rdz5MWhaJUPYg/6gp+jo7HUgsyKaN1FtQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.143.0.tgz",
|
||||
"integrity": "sha512-9BbdjHETk6O3zH/DDid9IgBtF0GlpLabNKN231uraXpRDSfY+iiZxTP5bk1Z63GBownVdhdINFIeddmMz4MzpQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -164,9 +164,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-darwin-arm64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.142.0.tgz",
|
||||
"integrity": "sha512-l4khS8LQOOVYsGRVARo1gSaCT/aBSceUVXgtovWc2+drnxVuDr082WA3OCHVdVzIz5JIrP/y9CWsSKxBDNmYGg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.143.0.tgz",
|
||||
"integrity": "sha512-gh+6ecoHUy4/sUcolBl/1qPXKBbYNxFY0Pk0ujgQvINTMSftJY7o4yb8gOkDJPeZeB8+a+u7xTe6umoP8N5HFA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -181,9 +181,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-darwin-x64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.142.0.tgz",
|
||||
"integrity": "sha512-QBsNF3nqlXmcH2B1YOPqQYmCJoy4HuIjUxGbBO/k5JAJUl68ghU2psRY2zPk+RyBaWqKP/qfL4oaFgEMCdwskA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.143.0.tgz",
|
||||
"integrity": "sha512-qd1hl2d+lXgHv/VQ/M9qm8TrMC5T4RqDBwtOnl+1D0QMjwcz+8AaB4JSg8STgeag0GP6a6L74XEGAsrTSJWNzQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -198,9 +198,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-freebsd-x64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.142.0.tgz",
|
||||
"integrity": "sha512-b7Q7m4Cqc6XqNhri3R+QhU+GVy646Pn+bkdhrDdWym/Fdi0ZUa+d73H9dm5H91JtbtAQ/z1d8XKMW3oOV8a4tQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.143.0.tgz",
|
||||
"integrity": "sha512-M5XXcNa7aOqLPKTR41msfghKu2yQ4xWvCm11/gwU0JzOzHNk5sgW//rVEjJ+LO48+VDAMzXTSzurUVxIDKwozw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -215,9 +215,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm-gnueabihf": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.142.0.tgz",
|
||||
"integrity": "sha512-3riVS5IhdH3uCZj1Y9ftDQlR0dvLsIlw/edrRqk8JhgNd5K0XSs+UBtgh50N13CAlW9/TXj6sVGXaKNBocd0Yg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.143.0.tgz",
|
||||
"integrity": "sha512-T/GXusuOkPNQhCQCSBbcU/N8j0rAypuDBl1IyFK+lyYT594XsVz80clPC/OtbSSpBGyJxj8uYEfctxVuxVYoww==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -232,9 +232,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm-musleabihf": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.142.0.tgz",
|
||||
"integrity": "sha512-NmXUOpgpTSkhl795TiXmWppTwmSJ92RC1qvD6e4XOF+slgmo3e6Ah+kEu+6AN8s7NAOEwqGmir58MgSQSWmBSA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.143.0.tgz",
|
||||
"integrity": "sha512-oKu4RcBlXSqo3OC62dp6YTnQaZIurNDpCX3BnAM3+bJxt7s8J2TJKMnC0UYer1qhlRaDCg6wkTaTw+2IlsZ12w==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -249,9 +249,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-gc0EXsKtXgerujmU2Bql3u1L1HsSQ2774R83idq/FoNMPVV/RY/1ErFsvnit7KoiP/sLvzQixeUo4Ut0ic0wmw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-WJBbD186AZmMGaSIhlktC+rPl8L3peCTXAh88Ih9uEvK0en2mPojGyCGYiL6mHtV1RPV3JyfJW5t6n5hh0lXhA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -269,9 +269,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm64-musl": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.142.0.tgz",
|
||||
"integrity": "sha512-F2XvmWSE0uWpie+jHKKIFgdVOe9ypGhkEZxKx5DuW215K6cbAC274yYaPkcM7EqY4Df3Weyhpcz3lsURyH2LVg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.143.0.tgz",
|
||||
"integrity": "sha512-t1AcYOwEzgceadT4v5e+vaCCb0AncCA3v5AyzfBAz/tMq11qzVccXKzNHtkWdjBsgvTKwRkaUF3QvT4kot8vcQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -289,9 +289,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-ppc64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-wLMbT21U/QxknQsk+VvNF0b9D2/aGWhcaQQQ+VYlE8FwD5+GoWZIPPXNzyHmkYyhm0KB3itL+TBavjMatqNnYA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-RsnO/NoD8376LMJq8JS8TwI0ieNaFRTuNe2GVJntQg6gwZNMENZsEbknHdVwjpOmxdGLGodcwaGSbAeRr5Bgjw==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -309,9 +309,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-riscv64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-+G8F/4ckwT7FCJV4H2bt09xEzJbjNCfuL4Sp1AYNaFtFMVtgIGMuJlteT82U+K0UIZ/DzAR/LDlMFnEuajG7Kw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-48fSVfR9TZi5CASZFyv0VC6z6BCoeihFsX031mAD/oSH7d9PYsPgIqza7d9mjP7Z2KTEpTFyH6SIu0Ui6R1vdg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -329,9 +329,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-riscv64-musl": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.142.0.tgz",
|
||||
"integrity": "sha512-hTsHtTLxMAfCo+rpF5K3qZJKW2NpPN/CHd4mYB3y7XlSdspHkd2gehDIofP64AacA9nWQw2tY3O7wR6UY8IVOA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.143.0.tgz",
|
||||
"integrity": "sha512-T8CpdD+SfE01DnIOD4HpVxu0ZJOfMJ/VhCvikKfaXAxkZ+9veyLM/D2hpi7Y2hFUyPmVQO3FNZHmYzV/WlVR4g==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -349,9 +349,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-s390x-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-6y7qYY3TCUDYjqswImdTGl92y+KA/80twALegQPN27kfY+bG7Ib1+L3jbmrCZQx6wrVnai9IPsEZp07I0hx7JQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-QLdeMsCcacenPEFsfxnBUDF1y6opyz5+fmOz9bfD5Y7fiGCMupUCuB3KTPQhNwshIG1P9fPqar9MHxuBDd4bwQ==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -369,9 +369,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-x64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-i69kAWU+2LgoH5bR+zWiiu+UzAw7Oxkwv7COeJTeY19pn4e70nKQcr9Pm6cL2Z0Z54d+gl9qADlK/0yyuCPiBA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-659ujfqLy6k7cuH3sbzhd8b+ztSq+i6E2E9pG78Q0BmHjAExfGIdgc8cGgMdwAozDXeZFHkJ+LXYJdWsaGdgyw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -389,9 +389,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-x64-musl": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.142.0.tgz",
|
||||
"integrity": "sha512-4SQs678MmjYVrmhAgCWD4o0vpaFszXw9xLX5p2Z9MMFcltxiLkA88wQjh80YHjPrXtpyZ2CWI5m+1yNKM0m2Pw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.143.0.tgz",
|
||||
"integrity": "sha512-/Mw/9j4TfZcnKphPrzOE6t4MMknXadcAAuVUlDRTF/ETWB5xOgQvOJV2Mh9We/bWxZdoxaGAdc+hy4GuYwQ2yQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -409,9 +409,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-openharmony-arm64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.142.0.tgz",
|
||||
"integrity": "sha512-YHpx9N7Ln3a++Tc8rv+H7mrK1zyJQOAwCFg8LZ3lTs1T5afGWeZrLPhPT9HLnIwSjCyJqPWVMIrMxbjcmBr2oQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.143.0.tgz",
|
||||
"integrity": "sha512-8rIKWR2BFuifbIK/1XB9wTaSdtuJ25dlE7ZQYDnEwj/2xH2vHsxnvIjHT3ZjSVuLLwGGlSslIG/fbOJ8TV8rTw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -425,52 +425,10 @@
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-wasm32-wasi": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.142.0.tgz",
|
||||
"integrity": "sha512-3pLDyY3+oogW73RM5uehNgAiR/Xfb7fvO2Q1Z1gIqZ2+50XDVQmBVlRkHXZTU4gKnQHpwETNsYQVsJ3joVB2iA==",
|
||||
"cpu": [
|
||||
"wasm32"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/core": "1.11.2",
|
||||
"@emnapi/runtime": "1.11.2",
|
||||
"@napi-rs/wasm-runtime": "^1.1.6"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@emnapi/core": {
|
||||
"version": "1.11.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz",
|
||||
"integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.2.2",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@emnapi/runtime": {
|
||||
"version": "1.11.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz",
|
||||
"integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-arm64-msvc": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.142.0.tgz",
|
||||
"integrity": "sha512-Had/VeVY28Oyb0K+Q4FV8KCzoBycIh93oDK6pCbya9lkzdq+ikMHMgBubsdqqlybjJmQRawCQRrnBRHyQwYvcQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.143.0.tgz",
|
||||
"integrity": "sha512-5U9kQYMfRRI6Zq7KDxgbIP0RMnKrfn3gLepRMgJuRkPSUALTiRCk9d/uyhb4lGDjUdzwK7mBkKqhLgzBPCmLpQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -485,9 +443,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-ia32-msvc": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.142.0.tgz",
|
||||
"integrity": "sha512-GGi3+YphVHavvgs6gum2UXoNCqzHAmPt/nXkn8ZQZstV2Q1qZD1Mn8fz/nWrDkefHQtrG/+1/XrbMxsBTo6Svw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.143.0.tgz",
|
||||
"integrity": "sha512-25P7AaHk4R88Yv2XH4gToDVmh0cOu+bEURQU10CRrmvgabfRArSGAP5osmwUKeSUHj0VS50upbpbRWWW/m7mHA==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -502,9 +460,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-x64-msvc": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.142.0.tgz",
|
||||
"integrity": "sha512-Ny/Wv4Us1LGC/ljwNTp+Hx3r/pH15EFfeDF0p+n898gt+TtRd6C9SccHcuUhDiNTb8s5tt7jdeAMDRQZ4Vq6hg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.143.0.tgz",
|
||||
"integrity": "sha512-ORMh3JE1s6V7ySicdRK7vgaDQnn5o+UHg9ct989PlWHbel8O9ARrmWXM6kZjrBMtNucxNayQ8g69G0VfWzhANw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -846,9 +804,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@oxfmt/binding-android-arm-eabi": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.62.0.tgz",
|
||||
"integrity": "sha512-pdsv0C4gPjJ8H1+sd8u0BDx+yLACTL+rgeMIOL1ln4ihSnhw8CWXtYWgvcSkyTfgGBIzFKab+d8rx9Xl4en/Kw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.63.0.tgz",
|
||||
"integrity": "sha512-YmRth4ZPGgEXcgmkhvANbC9uD67dxmSobW7DQuyt5tOBOKvPnIpk5SVHBj88E+7wMNRI2FhqaDbOhQFBix+b8A==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -863,9 +821,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-android-arm64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.62.0.tgz",
|
||||
"integrity": "sha512-WC3YQ7uS/KtDrjmqwBviwFKe9qeoi+eXx8aX1z/ffG23Md75myjrJaQqTuJvdOLPoa4EYTjDWH0dHXfwulCVog==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.63.0.tgz",
|
||||
"integrity": "sha512-icbahX8X2X3sRamOMecvdYeZXWjPDazRDIfvWfy7Ca1nc/ZDT2Y9k5Nt7s46EqFd7NQPdgk+CM3/SgIT5LPCaQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -880,9 +838,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-darwin-arm64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.62.0.tgz",
|
||||
"integrity": "sha512-GM8Yf3LjjaR1I8PD0SfeoIlwhsh9GvSF+cQ8sf624Yxnjsyumn95aFzYfKJVefblfDIiOAnZ7QVm2sa21Er/0Q==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.63.0.tgz",
|
||||
"integrity": "sha512-WV+Ze5v5gI2qoj8jpAovt8KBTW8pjEz/AiMXXjeTQS+Bmf/MmZXTS40S8xNPDszX+W8WDv2Bbk6qKrMTtUGu1A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -897,9 +855,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-darwin-x64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.62.0.tgz",
|
||||
"integrity": "sha512-d5THp7F8bCxLqNogEXDORRsQD6dosf3EyFtnXfBer6v+8tGdcWIjoDX9WaXrrF/26zOmL8qHpPTKCEvpBDmZkQ==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.63.0.tgz",
|
||||
"integrity": "sha512-CJGSBdDxXOWIpoFXHpverimCvz084KA7L483rqJ44c3jDtzv6d4qOSoR/V9ywSHfV+Ks1lwIj2P49BFhunLNAA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -914,9 +872,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-freebsd-x64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.62.0.tgz",
|
||||
"integrity": "sha512-1DnrtXGZooOZ0fHgAXZUaDQzBVh1CM2MNW4oBXyQ2aWKvCHjyljvT9fgBkOM0fEOb96X5eqtcfJ0YUVt9jj66g==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.63.0.tgz",
|
||||
"integrity": "sha512-BDfKY+KhL2078cgswBBFQPAYuxCy93bS/iC5frdSeSbTLcGrR6VC2hsuPTanoJmg84+wSyWl0wWC1eR+uTnkRg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -931,9 +889,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm-gnueabihf": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.62.0.tgz",
|
||||
"integrity": "sha512-4pQDHOYRH+Huqe0StIaWyvk2CVl/aTaqSrbZpA3/pLS2xH24ME7lBgYprhQF2fRkHBzhGGGKliwxFsDdHwx59g==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.63.0.tgz",
|
||||
"integrity": "sha512-Ov1cQEXT4mj7cojAokWSS1eoxkoyvbDfAbxNsGIKY2o36kvdAaFzPxRN6NxFRk9fD72B8oCoTTX/NuYTUWlpsg==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -948,9 +906,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm-musleabihf": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.62.0.tgz",
|
||||
"integrity": "sha512-X0jAaZJFMCVKhB6YyWVTQ/wN2DLsBcZKSMqTS76bF6riT+XZdtg2FPEdjDvdVbunO9cG+tWiVaEs4Zs38lxYog==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.63.0.tgz",
|
||||
"integrity": "sha512-0LE7ro3+6L79jcMANycAZfRaC7zxr9YZ2+vEL5uMD9QlEep+rS/r1kSJsnuLl991NXJZD60euh0PC1GHrR20vw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -965,9 +923,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-682Z8T5s8T5ATArYtsejKvbIfd8LEAXyyDkKkoZVq8HND7Vx8TYLlrDjDSeYfodMeVwHOgkj13lJYR8cj6vUSg==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-izPk+2Z4gjuZK32Fqh5qXoMpT/2NXzLh++ob57HiEiVSQZ1iYXu8EKMzb+K5AvWyIEXhdDIt7ADjGGtFhkT9Bw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -985,9 +943,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm64-musl": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.62.0.tgz",
|
||||
"integrity": "sha512-lk25fAl7KWaLWVJcW0CHEXB7QlQZtx5eDkjpaGMK0hzXTjUe0Wmlu8IKuFHoviSOcEJedRTs4VE/506VqGxGew==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.63.0.tgz",
|
||||
"integrity": "sha512-alPmbOuWXFXiSo+lOtv6X71C7SYMEDW2WVvywOvf9BwKgEhSNGhMTLeFVSjKUMCamcjbbgVdsWF8GN1uy8xshg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1005,9 +963,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-ppc64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-SFyNqHQLwySceWNLhiSldx7wPXRAzP0L0WcW9GegP3uWrpZGJiZlQO85NbHAFPEfxR9PhZ9qSnZryEh7+v+4Gw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-BdzCPvolJc4AWZ+YMzgUDJcDzbQWrFjYuqBHoNHNqP1aCaluQRJNs4k3vNU5IG7vTpjf9zeD73D7MFM1TecZpg==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -1025,9 +983,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-riscv64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-KYj55C1ywJfHo6+aKDuEmUtVEdJALsC5GwayDGsI6FGz2GxFqNr/mA8nxVsNbJzm7sE5MRqTQ9ziImSzhYXysA==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-7sIgfLzqtNKSkMGsGVyRpHwpjNezRg2XONvUOheFZs95TSZpM0JAuPpA8KrQFsWc4wPU95roX2O69JgH8igOgw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1045,9 +1003,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-riscv64-musl": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.62.0.tgz",
|
||||
"integrity": "sha512-BhZDNo5GOU5nC378RhD0/XpvaEBHsH3HLgJp8YZX3A0InC7oivzA63HsRmiXFLtLSHAstEVrDf6fbC7Rs8Jh/A==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.63.0.tgz",
|
||||
"integrity": "sha512-9Tcg0y0WcVa6Mm9AgcgFMseDS+VkFJZpKZ8We9SpDY4gg5jewSwln+0sO04QLcTS1BtfDl9MwR+NfID8L7PUTg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1065,9 +1023,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-s390x-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-UyAFmyHkgSgUJ/wOM4p3U8AC2yAFvRH5PNBs7TnK0fObTT/XSWcdr/lAzPSWaekHaZFaMeFZyk9n93Joq3J93A==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-qWKC1pEOpx1qYhXaugPhHUeXwSfqEOk2wJH2LqVXGPV5iQYfdAZdt+d2XDiX4DTSWA2QDMUcFB+wEORh3Xn/sA==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -1085,9 +1043,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-x64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-1iYMP0leytWazFubD/WnINJuIrzRPuoL1aWEJdlGezEzDbTxcd29R4r8IUzP2oWeKst5V02uMJgR2NILlPlG6w==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-S9wXYOiGSqYGS4Fx/TFsY+xDd/7dE5s+rUgbA4TsHiVF9e8J3ZcKmP7dsP/7iqLI9Wz7Ic7TzEr3mdthRCTdrA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1105,9 +1063,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-x64-musl": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.62.0.tgz",
|
||||
"integrity": "sha512-4rA/URtJSTVNVAQz6Q8wf7SaRvOXVy+TizriT9hs/Y1XhLR/R+92uWKRQG8yFWRAIEBbFHJ6WevQcl/G9SXEfw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.63.0.tgz",
|
||||
"integrity": "sha512-5eGyTJuMZNwBSHCivXt8Yuta6GeTYksOPXRk2MIhajiyFGQx7bjaHIwY+ZusAoFHhT157A9x6sktLjYo9D5oMQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1125,9 +1083,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-openharmony-arm64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.62.0.tgz",
|
||||
"integrity": "sha512-mSZuFHU2ar1KLUjXpI2QBQcJ1VsOB3mOCgQXuXCpKs19dgh4u+OaovNfrWDfiJb+ihJ2+f7YFcaO9bS2dlTCXA==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.63.0.tgz",
|
||||
"integrity": "sha512-Rz7hx+Dv3DoW/S6pwVAyjfFXp7/trdQ1zg+vNmsdsdDNlUccugp4XNqambSuEAeP0DaG9k72AtNyfDXCEg0AGw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1142,9 +1100,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-win32-arm64-msvc": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.62.0.tgz",
|
||||
"integrity": "sha512-OfwuhkcjDlqC4EgDojtiV9mzpLqeB9KqTOWPOjLEYBVdDCVSxqW3qzp/xcIxsbtI0UgGCnKvAqYKyY25kf5JZw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.63.0.tgz",
|
||||
"integrity": "sha512-T/IuizKN9mr4Xw6YYnptkXRNdLkyIlUZ7c8zfTOBpoytZyJ1BAsMUvsMDEx0X4YvSMpaivm+DR8112rQfzC25g==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1159,9 +1117,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-win32-ia32-msvc": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.62.0.tgz",
|
||||
"integrity": "sha512-P9uDDNFRzghO3X8QAzhkjKhK7JvtABsVn8UYtFX7uor12IAnwNt8nNIctvfWj1JkQU/kE+fmLRPiw7XlrIHsZw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.63.0.tgz",
|
||||
"integrity": "sha512-XjrO5FJ5Wl9vsAxtCP1G/eaeT6y1K2s9CICUHGE42cEjou32/J6S+B1KnrOAboj6E7uhJnwPbRSvznWcxNdA0g==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -1176,9 +1134,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-win32-x64-msvc": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.62.0.tgz",
|
||||
"integrity": "sha512-dlI5SY7XYQCiCBafntWagCR6HcAJB/NpsLtdlPx8x08+Osz8Ok1HHz1GZuusegCe/VoJ6pAnF5a4pd5OZAq7qQ==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.63.0.tgz",
|
||||
"integrity": "sha512-sgsHCQy432OTQH4Ikk3tZptp3GqwnhwUDuY0loBH41zyHWfMZY9v8Dy78wsnSofHejvFozZGgJgBB1A0LQRwMQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1277,9 +1235,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@oxlint/binding-android-arm-eabi": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.77.0.tgz",
|
||||
"integrity": "sha512-E06sKWS6PiI6HRxS1wyQg22HvApt01hI7fV+T3wUk3OSbaaP4a3hYGY/MIQDmASqCiRjBdpRQYkgMkqH82cWmQ==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.78.0.tgz",
|
||||
"integrity": "sha512-Bu819lmAfZMUHErrpe0cEWj3iaefuUODHSU8+UbXy67V/r7/7f4K3FL0NmbD85E+wiFLDYuhP8Zlv0XnVeXshw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1294,9 +1252,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-android-arm64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.77.0.tgz",
|
||||
"integrity": "sha512-NvsKz0KZxTp9cYWPLf+FXaSZwB3oO3peAjtukpOMBgse2vhQSoIIVqeO1yR0lEo/UcdZIDL18uq+kL0LzQ0ytA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.78.0.tgz",
|
||||
"integrity": "sha512-CDfxZgB61B7buRdY2FJoAYYPPXCZ1EoC1LKscnC5dg3kjobdxiconvAvvN1BmHyW4PyFT3jRLDag/BY/roSNBQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1311,9 +1269,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-darwin-arm64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.77.0.tgz",
|
||||
"integrity": "sha512-bgjTn6nW4bQCFBvSvuHCpDD+sONvmpo4lGI4PxzMt1quBA+xYxhczk6RiCn3GZ9gY8uhaBbwhj9MdKGfu6T9DA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.78.0.tgz",
|
||||
"integrity": "sha512-2Y2U9Ahrz+OO0Ej88f9SJYq51/jUBp1Mc7iZu0ukrbeeZ3gpRGfzIFnoqfHDY96xr0GEfNrPUBFEy0nN5aD7HA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1328,9 +1286,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-darwin-x64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.77.0.tgz",
|
||||
"integrity": "sha512-aotaIttH1R6j1Rwhx0M0htgeZyGtVQqYNTVEYMN/UcgHPquGA6kmk9OyuDc3a2GKUQBC+3C3GVQCcrRPMYqAFA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.78.0.tgz",
|
||||
"integrity": "sha512-rpych6eJq6m9jDRypTEaPD1xysaEW5h9+xuxhGK/QhOg+/xaqPZrCrTNoIl/f3nEjuJeCEmstNDlrE9rJi/3/g==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1345,9 +1303,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-freebsd-x64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.77.0.tgz",
|
||||
"integrity": "sha512-nNx/wta7ksRAdYvq+l4AWjXkLxEXHALhENxjj2cYbQAIR4ybaA5L+hCbE63HOmft5czQ6ks+hb8vmEAnn7YGPg==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.78.0.tgz",
|
||||
"integrity": "sha512-IcMGrQT3QizkOESUJd5et+rOhVqSkNDfNik1cvrKDqIbzqx9KMtRswpFgkCuNTSwylCFLKhGUu8KmqY1ZnC0Dg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1362,9 +1320,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm-gnueabihf": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.77.0.tgz",
|
||||
"integrity": "sha512-tMLLjM7xXtzXisVCzkOTXNCy9bZVId2wteNwjohlFDR/jY6WagpEDA1c1wu4xRc20Hojaxj+V6DSR7gbKxijWA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.78.0.tgz",
|
||||
"integrity": "sha512-/uLdoJ0IXE6vo/0f0LKjinQAp+re+VMaCWaNT8ENIv2EOCkSsc8SGaflXAuW0Jua2dq5+GLVWm1NQK7P3UFSNQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1379,9 +1337,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm-musleabihf": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.77.0.tgz",
|
||||
"integrity": "sha512-MiAFDFaqR0tmHTAyo0YDcZ5hyLREdYw/RQhc2R3cbT+8O3tB+zqPM2th9TTQ+Uo3jn/embS+DO+HyX9ztCPkOQ==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.78.0.tgz",
|
||||
"integrity": "sha512-7xi4Wb/O8NRJhLoUXmDJMUVpNYvB5kefdhFU1Jb8rtae4QoXlTiLwI14X4YvAXVZLNZChP8m5qO9SQAlWQTbkQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1396,9 +1354,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-/xqQ3B16i1T4cyt/9Mn+4CpzhUXoBXp7kVpIwzOXNFLj5JmK1bIjsbSnX296Gg8A/o7oDtKWikFgBx0SLwztkw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-4hFW0+fVXa3OIh1Y4A5SPkmvI4wuuBSrCVKzOyE7PTjhc7yEqZ1pmvEEeS5Lj/MaqvegFxXyF33N+6jkehxdyg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1416,9 +1374,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm64-musl": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.77.0.tgz",
|
||||
"integrity": "sha512-LSbwuRKiNCenPDcbARqAZ5RfBy7gmj7vOvfJRLeCDU3gFtSxWbhv/+VTlaUqzUhNj1gFLHB8h7ALnxa/Az6z6g==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.78.0.tgz",
|
||||
"integrity": "sha512-oC0mvsgBJjlMijSDEhx9KuvR9zYeHXceA9MjbuXB1F8NSR78Yj2unOBrstEvTVaq+pko+kuue6DajC00eqvTdg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1436,9 +1394,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-ppc64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-QWdcH31mXEUe5Nq1s0CfCpceaKjIo9uZtwDjAuL681g1axf+5x8xrg/eXWaw//4NCxYZ4V4e5Hu5tvdR+pTBlg==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-XAllT5SUZS+ohjuZ3/5S0cwe0r7eboiuigeStCZ5DXRYx/2KVM2UvQXvAfyzXEimtQjAB7cDQ2YxDe2Zl2WNQQ==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -1456,9 +1414,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-riscv64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-GnOfYgJxbcElOiPZaDFDl406ONddwvOWk2jvAAAEjwAl4GofNoHF+/HHUIBYa6bFCArlcGPi0XjC4cU1pkgF/Q==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-trucMER/0QtecoXvc1y/UVqE3kwJipDwrx4oHfj+nNm3dq2zjP44WT0CfHNDPM3G1DXIkx/gY6lAD21NSCZVhA==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1476,9 +1434,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-riscv64-musl": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.77.0.tgz",
|
||||
"integrity": "sha512-AyEMTUCf0xY+hHF+IxqXFQIX0yQOIR8ykpY0lJNOw9xYqOzUX8dyZfRvlG0RfXwuQn2eonf/8NrMmDSZJjdqsA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.78.0.tgz",
|
||||
"integrity": "sha512-cm3O4F/HQbdzOUX5mKHqG5KDL6E5w0pnlZ+fbBy2rmLryPOowkuLagFHTopQsEIpjcaZoPOrL+BmmAytAG9HFg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1496,9 +1454,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-s390x-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-sPLzEcNvxd/oyVQ5oZo92CiHkFkpBeRop13E/P3TPY+hZfXHKCOWKI70TE2RYwMKFJDc20EMjH16L7NZICtKTw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-33wRf6HqGNsybJ3qX4cGaQN2ODPxNmc1rMa0mrTmx3eFq1VzOnvQooi9bIGVYakW8a/wmqVx1mgsUm8R2xfTiw==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -1516,9 +1474,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-x64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-1Oh2ssH2L7lwyvkdSqaMUfsGfwU2Wfvew+obBUYjRVqhpBcUpwnsPSEr1IzVi9XqkuY10geiLsNKecqaZC34Dw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-rRdISSYegj6VganMZ9tjRjijowfHJ09IZU01i0toBAqr6n5LEtwHq2IeS4FjW2RoskOHlb6efB26H5izYb3GEQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1536,9 +1494,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-x64-musl": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.77.0.tgz",
|
||||
"integrity": "sha512-0j/2wRgNGO+Qj/M1uu/p57h/hFTTWWcfie0ufkbabeus2s5+/QqkCflnMOwLLN5m2GsNeWp4xdl4cPa4n7QCOQ==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.78.0.tgz",
|
||||
"integrity": "sha512-GmsP4rW0xTL6u5CVdcDsaN5Fbc7hBc382Wmar1kttbnwSEviM+rSINKOMQ+UQ6iH+AGwC+8gaAiwu134Tgh6Lg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1556,9 +1514,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-openharmony-arm64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.77.0.tgz",
|
||||
"integrity": "sha512-BJ/j54qS0usEnyDkLYURMj2iiD9h5Cyy+ppzeMSXBGRXaGRNWnj1Mw14NqWMR5E/PzdgB30OOCCzLzbRoduafw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.78.0.tgz",
|
||||
"integrity": "sha512-sy9yeYuADc8a+n4TLBayzMCZiHPW78DcIFVpOXTmdKHWQeM9xe5uzkqIIZmi326D5hY9XVwacipEB1p7tQjPAg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1573,9 +1531,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-win32-arm64-msvc": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.77.0.tgz",
|
||||
"integrity": "sha512-Yh8w+g2Lpx7StrvtYkoz9JJvXjB9wxgFChFNb85nrXm/wj/XTwGWS1hve9+900HL7llrntYB3YP+y32E3tRqzA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.78.0.tgz",
|
||||
"integrity": "sha512-rjc2hF1KfMi8fZj1X/m3AmnHbdsF3rL0v6KQg0Uc880Yb2khjz+3U14sfdZ7jWTpRnN1m1NQa/TT7uU9lJWPrA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1590,9 +1548,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-win32-ia32-msvc": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.77.0.tgz",
|
||||
"integrity": "sha512-zja5b7+6a7UsRFgAQSrnax5vrzliEyNPLCjfXONu/vTWswaIVZGFajJZptaeRvPE4LghtFdAzVFlexTm7MVTGA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.78.0.tgz",
|
||||
"integrity": "sha512-zcuXFVrEFHIafRfkCQT8w/Xe41o07ozl/vwHq7p94vB29xVzsB0sZGYORU1jhcYKv3Lr0J3HbJ2T4fHH5rWmvA==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -1607,9 +1565,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-win32-x64-msvc": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.77.0.tgz",
|
||||
"integrity": "sha512-+teyvPDZ2RjUvo+SuCqS/UhaJl1QtdW5fWT5NJTV61V5MIuIS90Db9LixmtEGvXixyttiK62P96MSu3UlpviBw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.78.0.tgz",
|
||||
"integrity": "sha512-Sb5ocmLSuYeOuXd+CFOToGKp/gjXUEWDnvIGwhnh8aq8wY4TMmEnKnvbogSW7RdMZv77JSARduS7/gv+khYEjA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2979,9 +2937,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/knip": {
|
||||
"version": "6.32.0",
|
||||
"resolved": "https://registry.npmjs.org/knip/-/knip-6.32.0.tgz",
|
||||
"integrity": "sha512-KDX9OmmOFmlvmxTkrx6Z0GHISMut+pXMSKR8eg84bovaxJKx2NdQD4JYCXveSbvieRe107W6vCD2xCpmz0qBYA==",
|
||||
"version": "6.32.2",
|
||||
"resolved": "https://registry.npmjs.org/knip/-/knip-6.32.2.tgz",
|
||||
"integrity": "sha512-WXTXbmocrw7gqm1A1TQvFN0OgJ7hUSU6E1g6SPRIzzHFogUBhXByc7cYeOFVtJ2uODg7DP4VbESYBYnfbtBYsg==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -2999,13 +2957,13 @@
|
||||
"formatly": "^0.3.0",
|
||||
"get-tsconfig": "4.14.1",
|
||||
"jiti": "^2.7.0",
|
||||
"oxc-parser": "^0.142.0",
|
||||
"oxc-parser": "^0.143.0",
|
||||
"oxc-resolver": "11.24.2",
|
||||
"picomatch": "^4.0.5",
|
||||
"smol-toml": "^1.7.1",
|
||||
"strip-json-comments": "5.0.3",
|
||||
"tinyglobby": "^0.2.17",
|
||||
"unbash": "^4.0.4",
|
||||
"unbash": "^4.0.9",
|
||||
"yaml": "^2.9.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
@@ -3311,13 +3269,13 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/oxc-parser": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.142.0.tgz",
|
||||
"integrity": "sha512-kKR+jPiRJYJDexVoziIg/FVGvr1fT1FZSSJOk6tVoMKKSlsf1Cso+cgGCJkOEDWOP174vRntCPFKg+AS7InWvw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.143.0.tgz",
|
||||
"integrity": "sha512-ov0NzaDCOInknS7mP1cwKdJERt3utPW8ldjtdUXQ8Ty0GEFD08wk422vCUN0d7pST6kqtV7dxoI9w1Zi0l/9TA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@oxc-project/types": "^0.142.0"
|
||||
"@oxc-project/types": "^0.143.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
@@ -3326,36 +3284,25 @@
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oxc-parser/binding-android-arm-eabi": "0.142.0",
|
||||
"@oxc-parser/binding-android-arm64": "0.142.0",
|
||||
"@oxc-parser/binding-darwin-arm64": "0.142.0",
|
||||
"@oxc-parser/binding-darwin-x64": "0.142.0",
|
||||
"@oxc-parser/binding-freebsd-x64": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm-gnueabihf": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm-musleabihf": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm64-musl": "0.142.0",
|
||||
"@oxc-parser/binding-linux-ppc64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-riscv64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-riscv64-musl": "0.142.0",
|
||||
"@oxc-parser/binding-linux-s390x-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-x64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-x64-musl": "0.142.0",
|
||||
"@oxc-parser/binding-openharmony-arm64": "0.142.0",
|
||||
"@oxc-parser/binding-wasm32-wasi": "0.142.0",
|
||||
"@oxc-parser/binding-win32-arm64-msvc": "0.142.0",
|
||||
"@oxc-parser/binding-win32-ia32-msvc": "0.142.0",
|
||||
"@oxc-parser/binding-win32-x64-msvc": "0.142.0"
|
||||
}
|
||||
},
|
||||
"node_modules/oxc-parser/node_modules/@oxc-project/types": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz",
|
||||
"integrity": "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
"@oxc-parser/binding-android-arm-eabi": "0.143.0",
|
||||
"@oxc-parser/binding-android-arm64": "0.143.0",
|
||||
"@oxc-parser/binding-darwin-arm64": "0.143.0",
|
||||
"@oxc-parser/binding-darwin-x64": "0.143.0",
|
||||
"@oxc-parser/binding-freebsd-x64": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm-gnueabihf": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm-musleabihf": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm64-musl": "0.143.0",
|
||||
"@oxc-parser/binding-linux-ppc64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-riscv64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-riscv64-musl": "0.143.0",
|
||||
"@oxc-parser/binding-linux-s390x-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-x64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-x64-musl": "0.143.0",
|
||||
"@oxc-parser/binding-openharmony-arm64": "0.143.0",
|
||||
"@oxc-parser/binding-win32-arm64-msvc": "0.143.0",
|
||||
"@oxc-parser/binding-win32-ia32-msvc": "0.143.0",
|
||||
"@oxc-parser/binding-win32-x64-msvc": "0.143.0"
|
||||
}
|
||||
},
|
||||
"node_modules/oxc-resolver": {
|
||||
@@ -3390,9 +3337,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/oxfmt": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.62.0.tgz",
|
||||
"integrity": "sha512-vxgGHTmnDU9j4CX7dDBLzxgmHxfda/yPcgJkGCMUSCwRmz+euo/V08xXLNgXTeqAB9Fhf3Pe2nO1RNKLCVgphQ==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.63.0.tgz",
|
||||
"integrity": "sha512-kgdDwv35wvVf6554U2Ab8Jnd0zTM+TsEQWwaB70RAjK3gICFAFGO+2Hd3Be27GMoXj3XRL9IKSNRVl7KBQL6iw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -3408,25 +3355,25 @@
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oxfmt/binding-android-arm-eabi": "0.62.0",
|
||||
"@oxfmt/binding-android-arm64": "0.62.0",
|
||||
"@oxfmt/binding-darwin-arm64": "0.62.0",
|
||||
"@oxfmt/binding-darwin-x64": "0.62.0",
|
||||
"@oxfmt/binding-freebsd-x64": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm-gnueabihf": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm-musleabihf": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm64-musl": "0.62.0",
|
||||
"@oxfmt/binding-linux-ppc64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-riscv64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-riscv64-musl": "0.62.0",
|
||||
"@oxfmt/binding-linux-s390x-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-x64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-x64-musl": "0.62.0",
|
||||
"@oxfmt/binding-openharmony-arm64": "0.62.0",
|
||||
"@oxfmt/binding-win32-arm64-msvc": "0.62.0",
|
||||
"@oxfmt/binding-win32-ia32-msvc": "0.62.0",
|
||||
"@oxfmt/binding-win32-x64-msvc": "0.62.0"
|
||||
"@oxfmt/binding-android-arm-eabi": "0.63.0",
|
||||
"@oxfmt/binding-android-arm64": "0.63.0",
|
||||
"@oxfmt/binding-darwin-arm64": "0.63.0",
|
||||
"@oxfmt/binding-darwin-x64": "0.63.0",
|
||||
"@oxfmt/binding-freebsd-x64": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm-gnueabihf": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm-musleabihf": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm64-musl": "0.63.0",
|
||||
"@oxfmt/binding-linux-ppc64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-riscv64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-riscv64-musl": "0.63.0",
|
||||
"@oxfmt/binding-linux-s390x-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-x64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-x64-musl": "0.63.0",
|
||||
"@oxfmt/binding-openharmony-arm64": "0.63.0",
|
||||
"@oxfmt/binding-win32-arm64-msvc": "0.63.0",
|
||||
"@oxfmt/binding-win32-ia32-msvc": "0.63.0",
|
||||
"@oxfmt/binding-win32-x64-msvc": "0.63.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"svelte": "^5.0.0",
|
||||
@@ -3442,9 +3389,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/oxlint": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.77.0.tgz",
|
||||
"integrity": "sha512-qnGh8XJHaQ0dprrDXNQZgS0FgjI6v+V3+X8DwmaV++5Aamy6jGKfDdQ1TUvhUxtmKFAbEf4/WeO5QZX+5WSngg==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.78.0.tgz",
|
||||
"integrity": "sha512-QgQePuxIqKOzo1KSjG2EnITEeWvWnKAm77eq8nrMtf6AGoA+zyGc4PFYtDNJSD25g/ibOwfQ851hZ4/SPkMVoA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
@@ -3457,25 +3404,25 @@
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oxlint/binding-android-arm-eabi": "1.77.0",
|
||||
"@oxlint/binding-android-arm64": "1.77.0",
|
||||
"@oxlint/binding-darwin-arm64": "1.77.0",
|
||||
"@oxlint/binding-darwin-x64": "1.77.0",
|
||||
"@oxlint/binding-freebsd-x64": "1.77.0",
|
||||
"@oxlint/binding-linux-arm-gnueabihf": "1.77.0",
|
||||
"@oxlint/binding-linux-arm-musleabihf": "1.77.0",
|
||||
"@oxlint/binding-linux-arm64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-arm64-musl": "1.77.0",
|
||||
"@oxlint/binding-linux-ppc64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-riscv64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-riscv64-musl": "1.77.0",
|
||||
"@oxlint/binding-linux-s390x-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-x64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-x64-musl": "1.77.0",
|
||||
"@oxlint/binding-openharmony-arm64": "1.77.0",
|
||||
"@oxlint/binding-win32-arm64-msvc": "1.77.0",
|
||||
"@oxlint/binding-win32-ia32-msvc": "1.77.0",
|
||||
"@oxlint/binding-win32-x64-msvc": "1.77.0"
|
||||
"@oxlint/binding-android-arm-eabi": "1.78.0",
|
||||
"@oxlint/binding-android-arm64": "1.78.0",
|
||||
"@oxlint/binding-darwin-arm64": "1.78.0",
|
||||
"@oxlint/binding-darwin-x64": "1.78.0",
|
||||
"@oxlint/binding-freebsd-x64": "1.78.0",
|
||||
"@oxlint/binding-linux-arm-gnueabihf": "1.78.0",
|
||||
"@oxlint/binding-linux-arm-musleabihf": "1.78.0",
|
||||
"@oxlint/binding-linux-arm64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-arm64-musl": "1.78.0",
|
||||
"@oxlint/binding-linux-ppc64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-riscv64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-riscv64-musl": "1.78.0",
|
||||
"@oxlint/binding-linux-s390x-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-x64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-x64-musl": "1.78.0",
|
||||
"@oxlint/binding-openharmony-arm64": "1.78.0",
|
||||
"@oxlint/binding-win32-arm64-msvc": "1.78.0",
|
||||
"@oxlint/binding-win32-ia32-msvc": "1.78.0",
|
||||
"@oxlint/binding-win32-x64-msvc": "1.78.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"oxlint-tsgolint": ">=7.0.2001",
|
||||
@@ -3874,9 +3821,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/unbash": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/unbash/-/unbash-4.0.4.tgz",
|
||||
"integrity": "sha512-60m9IVGbavD6jholbxt0jVBXZkEB/HsMZq7Tyaghseve2/Sf0zQRAIfWsD34sde+DKP2tBxJS2wP88ZM0D1FhA==",
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/unbash/-/unbash-4.0.10.tgz",
|
||||
"integrity": "sha512-b7zoBQvpWp0vuN5q2vK2RRBR2SvuruQAs50DApdDveBSn3eSYd84IaHodFqQIMlvY9K2VnyBUEXgwOBuGU9GBg==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
|
||||
@@ -28,9 +28,9 @@
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.4",
|
||||
"@vitejs/plugin-react": "^6.0.5",
|
||||
"knip": "^6.32.0",
|
||||
"oxfmt": "^0.62.0",
|
||||
"oxlint": "^1.77.0",
|
||||
"knip": "^6.32.2",
|
||||
"oxfmt": "^0.63.0",
|
||||
"oxlint": "^1.78.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"typescript": "^7.0.2",
|
||||
|
||||
+10
-43
@@ -58,7 +58,6 @@ import {
|
||||
isApiResponseError,
|
||||
updateSelfUser,
|
||||
setBookTargetState,
|
||||
type DownloadReleasePayload,
|
||||
} from './services/api';
|
||||
import type {
|
||||
Book,
|
||||
@@ -87,11 +86,13 @@ import { bookSupportsTargets } from './utils/bookTargetLoader';
|
||||
import { buildSearchQuery } from './utils/buildSearchQuery';
|
||||
import { wasDownloadQueuedAfterResponseError } from './utils/downloadRecovery';
|
||||
import { getDynamicOptionGroup } from './utils/dynamicFieldOptions';
|
||||
import { resolveDefaultLanguageCodes } from './utils/languageFilters';
|
||||
import { getConfiguredMetadataProviderForContentType } from './utils/metadataProviders';
|
||||
import { getEffectiveMetadataSort } from './utils/metadataSort';
|
||||
import { isRecord } from './utils/objectHelpers';
|
||||
import { policyTrace } from './utils/policyTrace';
|
||||
import { buildQueryTargets, getDefaultQueryTargetKey } from './utils/queryTargets';
|
||||
import { buildReleaseDownloadPayload, type ReleaseDownloadOptions } from './utils/releasePayload';
|
||||
import { applyRequestNoteToPayload } from './utils/requestConfirmation';
|
||||
import { bookFromRequestData } from './utils/requestFulfil';
|
||||
import {
|
||||
@@ -218,6 +219,7 @@ type PendingOnBehalfDownload =
|
||||
release: Release;
|
||||
releaseContentType: ContentType;
|
||||
actingAsUser: ActingAsUserSelection;
|
||||
options?: ReleaseDownloadOptions;
|
||||
}
|
||||
| {
|
||||
type: 'combined';
|
||||
@@ -1071,41 +1073,6 @@ function App() {
|
||||
[],
|
||||
);
|
||||
|
||||
const buildReleaseDownloadPayload = useCallback(
|
||||
(book: Book, release: Release, releaseContentType: ContentType): DownloadReleasePayload => {
|
||||
const isManual = book.provider === 'manual';
|
||||
const releasePreview =
|
||||
typeof release.extra?.preview === 'string' ? release.extra.preview : undefined;
|
||||
const releaseAuthor =
|
||||
typeof release.extra?.author === 'string' ? release.extra.author : undefined;
|
||||
|
||||
return {
|
||||
source: release.source,
|
||||
source_id: release.source_id,
|
||||
title: isManual ? release.title : book.title,
|
||||
author: isManual ? releaseAuthor || '' : book.author,
|
||||
year: book.year,
|
||||
format: release.format,
|
||||
size: release.size,
|
||||
size_bytes: release.size_bytes,
|
||||
download_url: release.download_url,
|
||||
protocol: release.protocol,
|
||||
indexer: release.indexer,
|
||||
seeders: release.seeders,
|
||||
extra: release.extra,
|
||||
preview: isManual ? releasePreview || undefined : book.preview,
|
||||
content_type: releaseContentType,
|
||||
series_name: book.series_name,
|
||||
series_position: book.series_position,
|
||||
subtitle: book.subtitle,
|
||||
// From the release, never the book: book.language is the provider's
|
||||
// canonical edition, which would mislabel a translated release.
|
||||
language: release.language ?? undefined,
|
||||
};
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
// When downloading a book while browsing a Hardcover list the user owns,
|
||||
// automatically remove it from that list (fire-and-forget).
|
||||
const searchFieldLabelsRef = useRef(searchFieldLabels);
|
||||
@@ -1213,12 +1180,13 @@ function App() {
|
||||
release: Release,
|
||||
releaseContentType: ContentType,
|
||||
onBehalfOfUserId?: number,
|
||||
options?: ReleaseDownloadOptions,
|
||||
): Promise<void> => {
|
||||
const requestStartedAtSeconds = Date.now() / 1000;
|
||||
try {
|
||||
trackRelease(book.id, release.source_id);
|
||||
await downloadRelease(
|
||||
buildReleaseDownloadPayload(book, release, releaseContentType),
|
||||
buildReleaseDownloadPayload(book, release, releaseContentType, options),
|
||||
onBehalfOfUserId,
|
||||
);
|
||||
await fetchStatus();
|
||||
@@ -1300,7 +1268,6 @@ function App() {
|
||||
}
|
||||
},
|
||||
[
|
||||
buildReleaseDownloadPayload,
|
||||
fetchStatus,
|
||||
openRequestConfirmation,
|
||||
refreshRequestPolicy,
|
||||
@@ -1415,6 +1382,7 @@ function App() {
|
||||
effectivePendingOnBehalfDownload.release,
|
||||
effectivePendingOnBehalfDownload.releaseContentType,
|
||||
onBehalfOfUserId,
|
||||
effectivePendingOnBehalfDownload.options,
|
||||
);
|
||||
}
|
||||
setPendingOnBehalfDownload(null);
|
||||
@@ -1639,6 +1607,7 @@ function App() {
|
||||
book: Book,
|
||||
release: Release,
|
||||
releaseContentType: ContentType,
|
||||
options?: ReleaseDownloadOptions,
|
||||
) => {
|
||||
policyTrace('release.action:start', {
|
||||
bookId: book.id,
|
||||
@@ -1654,11 +1623,12 @@ function App() {
|
||||
release,
|
||||
releaseContentType,
|
||||
actingAsUser: effectiveActingAsUser,
|
||||
options,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await executeReleaseDownload(book, release, releaseContentType);
|
||||
await executeReleaseDownload(book, release, releaseContentType, undefined, options);
|
||||
};
|
||||
|
||||
const handleReleaseRequest = useCallback(
|
||||
@@ -1927,10 +1897,7 @@ function App() {
|
||||
);
|
||||
const supportedFormats = config?.supported_formats || DEFAULT_SUPPORTED_FORMATS;
|
||||
const defaultLanguageCodes = useMemo(
|
||||
() =>
|
||||
config?.default_language && config.default_language.length > 0
|
||||
? config.default_language
|
||||
: [bookLanguages[0]?.code || 'en'],
|
||||
() => resolveDefaultLanguageCodes(config?.default_language, bookLanguages),
|
||||
[config?.default_language, bookLanguages],
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { useState } from 'react';
|
||||
|
||||
import type { PackBook, PackPlan, Release } from '../types';
|
||||
import {
|
||||
describePackPlan,
|
||||
parseSeriesPositionInput,
|
||||
toBookPlanPayload,
|
||||
updateReviewBook,
|
||||
} from '../utils/packReview';
|
||||
import { ToggleSwitch } from './shared/ToggleSwitch';
|
||||
|
||||
interface PackReviewPanelProps {
|
||||
release: Release;
|
||||
plan: PackPlan;
|
||||
books: PackBook[];
|
||||
onChange: (books: PackBook[]) => void;
|
||||
onBack: () => void;
|
||||
/** `null` means "treat the whole release as one book". */
|
||||
onConfirm: (books: PackBook[] | null) => Promise<void>;
|
||||
isSubmitting: boolean;
|
||||
}
|
||||
|
||||
const inputClassName =
|
||||
'w-full rounded-md border border-(--border-muted) bg-(--bg) px-2 py-1 text-sm text-(--text) focus:border-emerald-500 focus:outline-none';
|
||||
|
||||
export const PackReviewPanel = ({
|
||||
release,
|
||||
plan,
|
||||
books,
|
||||
onChange,
|
||||
onBack,
|
||||
onConfirm,
|
||||
isSubmitting,
|
||||
}: PackReviewPanelProps) => {
|
||||
const [singleBook, setSingleBook] = useState(false);
|
||||
const [expandedFiles, setExpandedFiles] = useState<number | null>(null);
|
||||
const [showIgnored, setShowIgnored] = useState(false);
|
||||
|
||||
const payloadBooks = toBookPlanPayload(books);
|
||||
const canConfirm = !isSubmitting && (singleBook || payloadBooks.length > 0);
|
||||
const confirmLabel = singleBook
|
||||
? 'Download as one book'
|
||||
: `Download ${payloadBooks.length} ${payloadBooks.length === 1 ? 'book' : 'books'}`;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4 px-5 py-4" data-testid="pack-review-panel">
|
||||
<div>
|
||||
<h3 className="text-base font-semibold text-(--text)">
|
||||
This release contains several books
|
||||
</h3>
|
||||
<p className="mt-1 text-sm text-zinc-500 dark:text-zinc-400">
|
||||
<span className="font-medium text-(--text)">{release.title}</span> ·{' '}
|
||||
{describePackPlan(books, plan.ignored)}
|
||||
</p>
|
||||
<p className="mt-1 text-xs text-zinc-500 dark:text-zinc-400">
|
||||
Each book below is filed separately with its own title. Fix any titles before downloading
|
||||
— the author and series come from the book you searched.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between rounded-lg border border-(--border-muted) px-3 py-2">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-(--text)">Treat as a single book</p>
|
||||
<p className="text-xs text-zinc-500 dark:text-zinc-400">
|
||||
Use this if the split is wrong and the files are really one audiobook.
|
||||
</p>
|
||||
</div>
|
||||
<ToggleSwitch
|
||||
checked={singleBook}
|
||||
onChange={setSingleBook}
|
||||
color="emerald"
|
||||
ariaLabel="Treat as a single book"
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div
|
||||
className={`flex flex-col divide-y divide-zinc-200/60 dark:divide-zinc-800/60 ${
|
||||
singleBook ? 'pointer-events-none opacity-40' : ''
|
||||
}`}
|
||||
>
|
||||
<div className="grid grid-cols-[minmax(0,1fr)_72px_72px_80px] gap-2 pb-1 text-xs font-medium tracking-wide text-zinc-500 uppercase dark:text-zinc-400">
|
||||
<span>Title</span>
|
||||
<span>Series #</span>
|
||||
<span>Year</span>
|
||||
<span className="text-right">Files</span>
|
||||
</div>
|
||||
{books.map((book, index) => (
|
||||
<div key={book.files[0] ?? index} className="py-2">
|
||||
<div className="grid grid-cols-[minmax(0,1fr)_72px_72px_80px] items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={book.title}
|
||||
onChange={(e) =>
|
||||
onChange(updateReviewBook(books, index, { title: e.target.value }))
|
||||
}
|
||||
aria-label={`Title for book ${index + 1}`}
|
||||
className={inputClassName}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
inputMode="decimal"
|
||||
value={book.series_position ?? ''}
|
||||
onChange={(e) =>
|
||||
onChange(
|
||||
updateReviewBook(books, index, {
|
||||
series_position: parseSeriesPositionInput(e.target.value),
|
||||
}),
|
||||
)
|
||||
}
|
||||
aria-label={`Series position for book ${index + 1}`}
|
||||
className={inputClassName}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={book.year ?? ''}
|
||||
onChange={(e) => {
|
||||
const parsed = parseSeriesPositionInput(e.target.value);
|
||||
onChange(
|
||||
updateReviewBook(books, index, {
|
||||
year: parsed === null ? null : Math.trunc(parsed),
|
||||
}),
|
||||
);
|
||||
}}
|
||||
aria-label={`Year for book ${index + 1}`}
|
||||
className={inputClassName}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setExpandedFiles(expandedFiles === index ? null : index)}
|
||||
className="hover-surface rounded-md px-2 py-1 text-right text-sm text-zinc-500 transition-colors dark:text-zinc-400"
|
||||
aria-expanded={expandedFiles === index}
|
||||
>
|
||||
{book.files.length} {book.files.length === 1 ? 'file' : 'files'}
|
||||
</button>
|
||||
</div>
|
||||
{expandedFiles === index && (
|
||||
<ul className="mt-2 max-h-40 overflow-y-auto rounded-md bg-(--bg-soft) px-3 py-2 font-mono text-xs break-all text-zinc-600 dark:text-zinc-300">
|
||||
{book.files.map((file) => (
|
||||
<li key={file}>{file}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{plan.ignored.length > 0 && (
|
||||
<div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowIgnored(!showIgnored)}
|
||||
className="text-xs text-zinc-500 underline-offset-2 hover:underline dark:text-zinc-400"
|
||||
aria-expanded={showIgnored}
|
||||
>
|
||||
{plan.ignored.length} {plan.ignored.length === 1 ? 'file' : 'files'} ignored (not a book
|
||||
format)
|
||||
</button>
|
||||
{showIgnored && (
|
||||
<ul className="mt-2 max-h-32 overflow-y-auto rounded-md bg-(--bg-soft) px-3 py-2 font-mono text-xs break-all text-zinc-600 dark:text-zinc-300">
|
||||
{plan.ignored.map((file) => (
|
||||
<li key={file}>{file}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-center justify-end gap-3 border-t border-(--border-muted) pt-4">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onBack}
|
||||
disabled={isSubmitting}
|
||||
className="hover-surface rounded-lg px-3 py-1.5 text-sm font-medium text-(--text) transition-colors disabled:opacity-50"
|
||||
>
|
||||
← Back
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void onConfirm(singleBook ? null : payloadBooks)}
|
||||
disabled={!canConfirm}
|
||||
className="rounded-lg bg-emerald-600 px-4 py-1.5 text-sm font-medium text-white transition-colors hover:bg-emerald-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{isSubmitting ? 'Queuing…' : confirmLabel}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
toStringArray,
|
||||
toStringValue,
|
||||
} from '../utils/objectHelpers';
|
||||
import { getUnrecognizedReleaseFormats } from '../utils/releaseFormats';
|
||||
import { Tooltip } from './shared/Tooltip';
|
||||
|
||||
interface ReleaseCellProps {
|
||||
@@ -424,6 +425,38 @@ export const ReleaseCell = ({
|
||||
const primaryFormat = formats?.[0] || null;
|
||||
const additionalFormats = formats?.slice(1) || [];
|
||||
|
||||
// The indexer named a format Shelfmark can't process (e.g. MAM "[ENG / AVI]").
|
||||
// Downloading it would only fail post-processing, so warn instead of showing the
|
||||
// bare content-type icon that makes it look like any other result.
|
||||
const unrecognizedFormats = primaryFormat ? [] : getUnrecognizedReleaseFormats(release);
|
||||
if (unrecognizedFormats.length > 0) {
|
||||
const unsupportedLabel = unrecognizedFormats.map((fmt) => fmt.toUpperCase()).join(', ');
|
||||
const unsupportedTitle = `Unsupported format (${unsupportedLabel}) - Shelfmark cannot process this release`;
|
||||
if (compact) {
|
||||
return (
|
||||
<span
|
||||
className="font-semibold text-amber-600 dark:text-amber-400"
|
||||
title={unsupportedTitle}
|
||||
>
|
||||
{unrecognizedFormats[0].toUpperCase()}
|
||||
{unrecognizedFormats.length > 1 && ` +${unrecognizedFormats.length - 1}`}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="flex items-center justify-start" title={unsupportedTitle}>
|
||||
<span className="inline-flex items-center gap-1">
|
||||
<span className="w-13 rounded-lg bg-amber-500/20 py-0.5 text-center text-[10px] font-semibold tracking-wide whitespace-nowrap text-amber-700 sm:text-[11px] dark:text-amber-400">
|
||||
{unrecognizedFormats[0].toUpperCase()}
|
||||
</span>
|
||||
<span className="text-[10px] font-medium whitespace-nowrap text-amber-700 sm:text-[11px] dark:text-amber-400">
|
||||
Unsupported
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Use blue for book, violet for audiobook when no format specified
|
||||
const noFormatStyle = isAudiobook
|
||||
? { bg: 'bg-violet-500/20', text: 'text-violet-600 dark:text-violet-400' }
|
||||
|
||||
@@ -7,6 +7,7 @@ import { useReleaseSearchSession } from '../hooks/releaseModal/useReleaseSearchS
|
||||
import { useTabIndicator } from '../hooks/ui/useTabIndicator';
|
||||
import { useBodyScrollLock } from '../hooks/useBodyScrollLock';
|
||||
import { useEscapeKey } from '../hooks/useEscapeKey';
|
||||
import { inspectRelease } from '../services/api';
|
||||
import type {
|
||||
Book,
|
||||
Release,
|
||||
@@ -18,6 +19,8 @@ import type {
|
||||
LeadingCellConfig,
|
||||
ContentType,
|
||||
RequestPolicyMode,
|
||||
PackBook,
|
||||
PackPlan,
|
||||
} from '../types';
|
||||
import { isMetadataBook } from '../types';
|
||||
import { bookSupportsTargets } from '../utils/bookTargetLoader';
|
||||
@@ -29,7 +32,9 @@ import {
|
||||
buildLanguageNormalizer,
|
||||
} from '../utils/languageFilters';
|
||||
import { getNestedValue, toComparableText, toStringValue } from '../utils/objectHelpers';
|
||||
import { toBookPlanPayload } from '../utils/packReview';
|
||||
import { getReleaseFormats } from '../utils/releaseFormats';
|
||||
import { buildReleaseDownloadPayload, type ReleaseDownloadOptions } from '../utils/releasePayload';
|
||||
import {
|
||||
getBookTitleCandidates,
|
||||
getBookAuthorCandidates,
|
||||
@@ -50,6 +55,7 @@ import { BookTargetDropdown } from './BookTargetDropdown';
|
||||
import { Dropdown } from './Dropdown';
|
||||
import { DropdownList } from './DropdownList';
|
||||
import { LanguageMultiSelect } from './LanguageMultiSelect';
|
||||
import { PackReviewPanel } from './PackReviewPanel';
|
||||
import { ReleaseCell } from './ReleaseCell';
|
||||
|
||||
// Combined mode configuration for the ReleaseModal
|
||||
@@ -140,7 +146,12 @@ const DEFAULT_COLUMN_CONFIG: ReleaseColumnConfig = {
|
||||
interface ReleaseModalProps {
|
||||
book: Book | null;
|
||||
onClose: () => void;
|
||||
onDownload: (book: Book, release: Release, contentType: ContentType) => Promise<void>;
|
||||
onDownload: (
|
||||
book: Book,
|
||||
release: Release,
|
||||
contentType: ContentType,
|
||||
options?: ReleaseDownloadOptions,
|
||||
) => Promise<void>;
|
||||
onRequestRelease?: (book: Book, release: Release, contentType: ContentType) => Promise<void>;
|
||||
onRequestBook?: (book: Book, contentType: ContentType) => Promise<void>;
|
||||
getPolicyModeForSource?: (source: string, contentType: ContentType) => RequestPolicyMode;
|
||||
@@ -762,6 +773,15 @@ const ReleaseModalSession = ({
|
||||
: supportedFormats;
|
||||
const [isRequestingBook, setIsRequestingBook] = useState(false);
|
||||
const [selectedRelease, setSelectedRelease] = useState<Release | null>(null);
|
||||
// Multi-book packs: `multiBook` is the manual header toggle (heuristic split for
|
||||
// releases we can't inspect); `packReview` holds an inspected pack awaiting approval.
|
||||
const [multiBook, setMultiBook] = useState(false);
|
||||
const [packReview, setPackReview] = useState<{
|
||||
release: Release;
|
||||
plan: PackPlan;
|
||||
books: PackBook[];
|
||||
} | null>(null);
|
||||
const [packSubmitting, setPackSubmitting] = useState(false);
|
||||
const isCombinedMode = combinedMode != null;
|
||||
const combinedPhase = combinedMode?.phase ?? null;
|
||||
const combinedStepLabel = combinedMode?.stepLabel ?? '';
|
||||
@@ -1196,7 +1216,36 @@ const ReleaseModalSession = ({
|
||||
|
||||
const mode = getReleaseActionMode(release);
|
||||
if (mode === 'download') {
|
||||
await onDownload(book, release, contentType);
|
||||
// Look at the release's files before queueing so a whole-series pack can be
|
||||
// reviewed and filed as separate books instead of one mangled item.
|
||||
let inspected = false;
|
||||
let plan: PackPlan | null = null;
|
||||
let reason: string | null = null;
|
||||
try {
|
||||
const inspection = await inspectRelease(
|
||||
buildReleaseDownloadPayload(book, release, contentType),
|
||||
);
|
||||
inspected = inspection.inspected;
|
||||
plan = inspection.plan;
|
||||
reason = inspection.reason;
|
||||
} catch (error) {
|
||||
console.error('Release inspection failed:', error);
|
||||
}
|
||||
if (inspected && plan?.is_pack) {
|
||||
setPackReview({ release, plan, books: plan.books });
|
||||
return;
|
||||
}
|
||||
// Not a pack (or couldn't be inspected): queue exactly as before. A release we
|
||||
// couldn't inspect might still be an unnoticed pack, so leave a console breadcrumb
|
||||
// rather than interrupting the user; the multi-book toggle forces the split.
|
||||
if (!inspected && !multiBook) {
|
||||
console.warn(
|
||||
`Could not inspect release "${release.title}" before download${
|
||||
reason ? `: ${reason}` : ''
|
||||
}. If it contains several books, enable the multi-book pack toggle.`,
|
||||
);
|
||||
}
|
||||
await onDownload(book, release, contentType, multiBook ? { multiBook: true } : {});
|
||||
handleClose();
|
||||
return;
|
||||
}
|
||||
@@ -1215,9 +1264,31 @@ const ReleaseModalSession = ({
|
||||
onRequestRelease,
|
||||
contentType,
|
||||
handleClose,
|
||||
multiBook,
|
||||
],
|
||||
);
|
||||
|
||||
const handlePackConfirm = useCallback(
|
||||
async (books: PackBook[] | null): Promise<void> => {
|
||||
if (!book || !packReview) {
|
||||
return;
|
||||
}
|
||||
setPackSubmitting(true);
|
||||
try {
|
||||
await onDownload(
|
||||
book,
|
||||
packReview.release,
|
||||
contentType,
|
||||
books ? { multiBook: true, bookPlan: toBookPlanPayload(books) } : {},
|
||||
);
|
||||
handleClose();
|
||||
} finally {
|
||||
setPackSubmitting(false);
|
||||
}
|
||||
},
|
||||
[book, packReview, onDownload, contentType, handleClose],
|
||||
);
|
||||
|
||||
const titleId = `release-modal-title-${book.id}`;
|
||||
const providerDisplay =
|
||||
book.provider_display_name ||
|
||||
@@ -1720,6 +1791,37 @@ const ReleaseModalSession = ({
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3 pr-1 pl-2">
|
||||
{/* Multi-book pack toggle (fallback for releases that can't be inspected) */}
|
||||
{!isCombinedMode && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMultiBook((prev) => !prev)}
|
||||
className={`hover-surface relative rounded-full p-2.5 text-zinc-500 transition-colors dark:text-zinc-400 ${
|
||||
multiBook ? 'text-emerald-600 dark:text-emerald-400' : ''
|
||||
}`}
|
||||
aria-label="Multi-book pack"
|
||||
aria-pressed={multiBook}
|
||||
title="Multi-book pack: file each subfolder (or each file) as a separate book. Only needed when a release can't be inspected before download."
|
||||
>
|
||||
<svg
|
||||
className="h-4 w-4"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth={1.5}
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M6.429 9.75 2.25 12l4.179 2.25m0-4.5 5.571 3 5.571-3m-11.142 0L2.25 7.5 12 2.25l9.75 5.25-4.179 2.25m0 0L21.75 12l-4.179 2.25m0 0 4.179 2.25L12 21.75 2.25 16.5l4.179-2.25m11.142 0-5.571 3-5.571-3"
|
||||
/>
|
||||
</svg>
|
||||
{multiBook && (
|
||||
<span className="absolute top-1 right-1 h-2 w-2 rounded-full bg-emerald-500" />
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
|
||||
{/* Manual query button */}
|
||||
<button
|
||||
type="button"
|
||||
@@ -2120,6 +2222,19 @@ const ReleaseModalSession = ({
|
||||
{/* Release list content */}
|
||||
<div className="min-h-[200px]">
|
||||
{(() => {
|
||||
if (packReview) {
|
||||
return (
|
||||
<PackReviewPanel
|
||||
release={packReview.release}
|
||||
plan={packReview.plan}
|
||||
books={packReview.books}
|
||||
onChange={(books) => setPackReview({ ...packReview, books })}
|
||||
onBack={() => setPackReview(null)}
|
||||
onConfirm={handlePackConfirm}
|
||||
isSubmitting={packSubmitting}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (sourcesLoading) {
|
||||
return <ReleaseSkeleton />;
|
||||
}
|
||||
|
||||
@@ -223,6 +223,14 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
);
|
||||
const showActiveTargetLabel = queryTargets.length > 0 && activeTarget.source !== 'general';
|
||||
|
||||
// Manual search browses release sources directly, one media type at a time — the
|
||||
// combined ("both") flow doesn't apply. Present a plain, switchable Books/Audiobooks
|
||||
// choice for it, even when combined search is forced on for metadata targets.
|
||||
const isManualTarget = activeTarget?.source === 'manual';
|
||||
const combinedSelectionActive = combinedMode && !isManualTarget;
|
||||
const combinedSelectorLocked = combinedModeLocked && !isManualTarget;
|
||||
const combinedToggleAvailable = !!onCombinedModeChange && !isManualTarget;
|
||||
|
||||
useDismiss(isSelectorOpen, [selectorRef], () => setIsSelectorOpen(false));
|
||||
useDismiss(isSelectOpen, [selectPanelRef, selectTriggerRef], () => setIsSelectOpen(false));
|
||||
useDismiss(isAutocompleteOpen, [autocompletePanelRef, inputRef], () =>
|
||||
@@ -356,7 +364,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
contentType,
|
||||
activeTarget,
|
||||
placeholder,
|
||||
combinedMode,
|
||||
combinedSelectionActive,
|
||||
);
|
||||
const effectiveInputAriaLabel = activeTarget
|
||||
? `${inputAriaLabel}: ${activeTarget.label}`
|
||||
@@ -537,7 +545,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
|
||||
let selectorContentTypeLabel = 'audiobooks';
|
||||
let selectorIcon = <AudiobookIcon />;
|
||||
if (combinedMode) {
|
||||
if (combinedSelectionActive) {
|
||||
selectorContentTypeLabel = 'books and audiobooks';
|
||||
selectorIcon = <BothIcon />;
|
||||
} else if (contentType === 'ebook') {
|
||||
@@ -665,7 +673,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
<div className="max-h-[min(24rem,calc(100vh-8rem))] overflow-y-auto p-3">
|
||||
{showContentTypeSelector && (
|
||||
<div
|
||||
className={`border-b ${onCombinedModeChange ? 'pb-0' : 'pb-3'}`}
|
||||
className={`border-b ${combinedToggleAvailable ? 'pb-0' : 'pb-3'}`}
|
||||
style={{ borderColor: 'var(--border-muted)' }}
|
||||
>
|
||||
<div className="flex items-center justify-between px-1 pb-2">
|
||||
@@ -712,34 +720,38 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('ebook')}
|
||||
className={`flex items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'ebook' || combinedMode
|
||||
contentType === 'ebook' || combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'ebook' || combinedMode
|
||||
contentType === 'ebook' || combinedSelectionActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{contentType === 'ebook' || combinedMode ? <CheckIcon /> : <BookIcon />}
|
||||
{contentType === 'ebook' || combinedSelectionActive ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<BookIcon />
|
||||
)}
|
||||
<span>Books</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('audiobook')}
|
||||
className={`flex items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'audiobook' || combinedMode
|
||||
contentType === 'audiobook' || combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'audiobook' || combinedMode
|
||||
contentType === 'audiobook' || combinedSelectionActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{contentType === 'audiobook' || combinedMode ? (
|
||||
{contentType === 'audiobook' || combinedSelectionActive ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<AudiobookIcon />
|
||||
@@ -747,9 +759,9 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
<span>Audiobooks</span>
|
||||
</button>
|
||||
</div>
|
||||
{onCombinedModeChange &&
|
||||
{combinedToggleAvailable &&
|
||||
(() => {
|
||||
const lineColor = combinedMode
|
||||
const lineColor = combinedSelectionActive
|
||||
? 'bg-emerald-500'
|
||||
: 'bg-(--border-muted) group-hover:bg-zinc-400 dark:group-hover:bg-zinc-500';
|
||||
return (
|
||||
@@ -787,7 +799,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
{/* Chain icon centered at bottom */}
|
||||
<div
|
||||
className={`relative z-10 mx-auto rounded-full p-1 transition-colors ${
|
||||
combinedMode
|
||||
combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'bg-(--bg) text-zinc-400 group-hover:bg-zinc-200 group-hover:text-zinc-600 dark:text-zinc-500 dark:group-hover:bg-zinc-700 dark:group-hover:text-zinc-300'
|
||||
}`}
|
||||
@@ -800,7 +812,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
{combinedModeLocked ? (
|
||||
{combinedSelectorLocked ? (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { Dispatch, SetStateAction } from 'react';
|
||||
import { useMountEffect } from '@/hooks/useMountEffect';
|
||||
import type { AppConfig, AdvancedFilterState, ContentType, SearchMode, SortOption } from '@/types';
|
||||
import { buildSearchQuery } from '@/utils/buildSearchQuery';
|
||||
import { resolveDefaultLanguageCodes } from '@/utils/languageFilters';
|
||||
import { getEffectiveMetadataSort } from '@/utils/metadataSort';
|
||||
import type { ParsedUrlSearch } from '@/utils/parseUrlSearchParams';
|
||||
|
||||
@@ -73,10 +74,10 @@ export const UrlSearchBootstrapMount = ({
|
||||
}
|
||||
|
||||
const bookLanguages = config.book_languages || [];
|
||||
const defaultLanguageCodes =
|
||||
config.default_language && config.default_language.length > 0
|
||||
? config.default_language
|
||||
: [bookLanguages[0]?.code || 'en'];
|
||||
const defaultLanguageCodes = resolveDefaultLanguageCodes(
|
||||
config.default_language,
|
||||
bookLanguages,
|
||||
);
|
||||
|
||||
if (parsedParams.searchInput) {
|
||||
setSearchInput(parsedParams.searchInput);
|
||||
|
||||
@@ -353,12 +353,12 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
|
||||
);
|
||||
}
|
||||
|
||||
// text/path
|
||||
// text/password/path
|
||||
return (
|
||||
<div key={col.key} className="flex min-w-0 flex-col gap-1">
|
||||
{mobileLabel}
|
||||
<input
|
||||
type="text"
|
||||
type={col.type === 'password' ? 'password' : 'text'}
|
||||
value={toPrimitiveString(cellValue)}
|
||||
onChange={(e) => updateCell(rowIndex, col.key, e.target.value)}
|
||||
placeholder={col.placeholder}
|
||||
|
||||
@@ -7,7 +7,12 @@ import type {
|
||||
} from '../../../types/settings';
|
||||
import { HeadingField, MultiSelectField, SelectField, TextField } from '../fields';
|
||||
import { FieldWrapper } from '../shared';
|
||||
import { getFieldByKey, toNormalizedLowercaseTextValue, toTextValue } from './fieldHelpers';
|
||||
import {
|
||||
getFieldByKey,
|
||||
resolveListOverride,
|
||||
toNormalizedLowercaseTextValue,
|
||||
toTextValue,
|
||||
} from './fieldHelpers';
|
||||
import type { PerUserSettings } from './types';
|
||||
|
||||
interface UserOverridesSectionProps {
|
||||
@@ -175,16 +180,6 @@ export const UserOverridesSection = ({
|
||||
label: 'Email Recipient',
|
||||
description: 'Email address used for this user in Email output mode.',
|
||||
};
|
||||
const browserDownloadGlobalValue = Array.isArray(globalValues.DOWNLOAD_TO_BROWSER_CONTENT_TYPES)
|
||||
? globalValues.DOWNLOAD_TO_BROWSER_CONTENT_TYPES.map((entry) => String(entry).trim()).filter(
|
||||
(entry) => entry.length > 0,
|
||||
)
|
||||
: [];
|
||||
const browserDownloadUserValue = Array.isArray(userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES)
|
||||
? userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES.map((entry) => entry.trim()).filter(
|
||||
(entry) => entry.length > 0,
|
||||
)
|
||||
: [];
|
||||
|
||||
const isOverridden = (key: DeliverySettingKey): boolean => {
|
||||
if (
|
||||
@@ -200,10 +195,12 @@ export const UserOverridesSection = ({
|
||||
return userValue !== globalValue;
|
||||
};
|
||||
|
||||
const isBrowserDownloadOverridden =
|
||||
Object.prototype.hasOwnProperty.call(userSettings, 'DOWNLOAD_TO_BROWSER_CONTENT_TYPES') &&
|
||||
userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES !== null &&
|
||||
JSON.stringify(browserDownloadUserValue) !== JSON.stringify(browserDownloadGlobalValue);
|
||||
const { value: browserDownloadContentTypes, isOverridden: isBrowserDownloadOverridden } =
|
||||
resolveListOverride(
|
||||
userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES,
|
||||
globalValues.DOWNLOAD_TO_BROWSER_CONTENT_TYPES,
|
||||
Object.prototype.hasOwnProperty.call(userSettings, 'DOWNLOAD_TO_BROWSER_CONTENT_TYPES'),
|
||||
);
|
||||
|
||||
const resetKeys = (keys: DeliverySettingKey[]) => {
|
||||
setUserSettings((prev) => {
|
||||
@@ -232,9 +229,6 @@ export const UserOverridesSection = ({
|
||||
const outputModeValue = readValue('BOOKS_OUTPUT_MODE', 'folder');
|
||||
const effectiveOutputMode = normalizeMode(outputModeValue);
|
||||
|
||||
const browserDownloadContentTypes = isBrowserDownloadOverridden
|
||||
? browserDownloadUserValue
|
||||
: browserDownloadGlobalValue;
|
||||
const destinationValue = readValue('DESTINATION');
|
||||
const destinationAudiobookValue = readValue('DESTINATION_AUDIOBOOK');
|
||||
const libraryValue = readValue('BOOKLORE_LIBRARY_ID');
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
import type { DeliveryPreferencesResponse } from '../../../services/api';
|
||||
import type { HeadingFieldConfig, SelectFieldConfig } from '../../../types/settings';
|
||||
import { HeadingField, SelectField } from '../fields';
|
||||
import type {
|
||||
HeadingFieldConfig,
|
||||
MultiSelectFieldConfig,
|
||||
SelectFieldConfig,
|
||||
} from '../../../types/settings';
|
||||
import { HeadingField, MultiSelectField, SelectField } from '../fields';
|
||||
import { FieldWrapper } from '../shared';
|
||||
import { getFieldByKey, toNormalizedLowercaseTextValue, toTextValue } from './fieldHelpers';
|
||||
import {
|
||||
getFieldByKey,
|
||||
resolveListOverride,
|
||||
toNormalizedLowercaseTextValue,
|
||||
toTextValue,
|
||||
} from './fieldHelpers';
|
||||
import type { PerUserSettings } from './types';
|
||||
|
||||
interface UserSearchPreferencesSectionProps {
|
||||
@@ -14,6 +23,7 @@ interface UserSearchPreferencesSectionProps {
|
||||
|
||||
type SearchSettingKey =
|
||||
| 'SEARCH_MODE'
|
||||
| 'BOOK_LANGUAGE'
|
||||
| 'METADATA_PROVIDER'
|
||||
| 'METADATA_PROVIDER_AUDIOBOOK'
|
||||
| 'DEFAULT_RELEASE_SOURCE'
|
||||
@@ -68,6 +78,15 @@ const fallbackDefaultAudiobookReleaseSourceField: SelectFieldConfig = {
|
||||
options: [{ value: '', label: 'Use book release source' }],
|
||||
};
|
||||
|
||||
const fallbackBookLanguageField: MultiSelectFieldConfig = {
|
||||
type: 'MultiSelectField',
|
||||
key: 'BOOK_LANGUAGE',
|
||||
label: 'Default Book Languages',
|
||||
description: 'Default language filter for searches.',
|
||||
value: [],
|
||||
options: [],
|
||||
};
|
||||
|
||||
const searchHeading: HeadingFieldConfig = {
|
||||
type: 'HeadingField',
|
||||
key: 'search_preferences_heading',
|
||||
@@ -120,6 +139,13 @@ export const UserSearchPreferencesSection = ({
|
||||
'DEFAULT_RELEASE_SOURCE_AUDIOBOOK',
|
||||
fallbackDefaultAudiobookReleaseSourceField,
|
||||
);
|
||||
const bookLanguageField = getFieldByKey(fields, 'BOOK_LANGUAGE', fallbackBookLanguageField);
|
||||
|
||||
const { value: bookLanguageValue, isOverridden: isBookLanguageOverridden } = resolveListOverride(
|
||||
userSettings.BOOK_LANGUAGE,
|
||||
globalValues.BOOK_LANGUAGE,
|
||||
Object.prototype.hasOwnProperty.call(userSettings, 'BOOK_LANGUAGE'),
|
||||
);
|
||||
|
||||
const isOverridden = (key: SearchSettingKey): boolean => {
|
||||
if (
|
||||
@@ -172,9 +198,12 @@ export const UserSearchPreferencesSection = ({
|
||||
const canOverrideDefaultAudiobookReleaseSource =
|
||||
isUserOverridable('DEFAULT_RELEASE_SOURCE_AUDIOBOOK') &&
|
||||
preferenceKeySet.has('DEFAULT_RELEASE_SOURCE_AUDIOBOOK');
|
||||
const canOverrideBookLanguage =
|
||||
isUserOverridable('BOOK_LANGUAGE') && preferenceKeySet.has('BOOK_LANGUAGE');
|
||||
|
||||
if (
|
||||
!canOverrideSearchMode &&
|
||||
!canOverrideBookLanguage &&
|
||||
!canOverrideMetadataProvider &&
|
||||
!canOverrideAudiobookMetadataProvider &&
|
||||
!canOverrideDefaultReleaseSource &&
|
||||
@@ -208,6 +237,27 @@ export const UserSearchPreferencesSection = ({
|
||||
</FieldWrapper>
|
||||
)}
|
||||
|
||||
{canOverrideBookLanguage && (
|
||||
<FieldWrapper
|
||||
field={bookLanguageField}
|
||||
resetAction={
|
||||
isBookLanguageOverridden
|
||||
? {
|
||||
disabled: Boolean(bookLanguageField.fromEnv),
|
||||
onClick: () => resetKeys(['BOOK_LANGUAGE']),
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
<MultiSelectField
|
||||
field={bookLanguageField}
|
||||
value={bookLanguageValue}
|
||||
onChange={(value) => setUserSettings((prev) => ({ ...prev, BOOK_LANGUAGE: value }))}
|
||||
disabled={Boolean(bookLanguageField.fromEnv)}
|
||||
/>
|
||||
</FieldWrapper>
|
||||
)}
|
||||
|
||||
{effectiveSearchMode === 'universal' && canOverrideMetadataProvider && (
|
||||
<FieldWrapper
|
||||
field={metadataProviderField}
|
||||
|
||||
@@ -31,6 +31,33 @@ export const toNormalizedLowercaseTextValue = (value: unknown): string => {
|
||||
return toTrimmedTextValue(value).toLowerCase();
|
||||
};
|
||||
|
||||
const toStringListValue = (value: unknown): string[] => {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.map((entry) => toTrimmedTextValue(entry)).filter((entry) => entry.length > 0);
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve a list-valued per-user override against its global value.
|
||||
*
|
||||
* A key absent from userSettings, or set to null, is not an override. A stored list
|
||||
* that matches the global one is treated as inherited, matching how
|
||||
* buildUserSettingsPayload clears it on save.
|
||||
*/
|
||||
export const resolveListOverride = (
|
||||
userValue: unknown,
|
||||
globalValue: unknown,
|
||||
hasUserKey: boolean,
|
||||
): { value: string[]; isOverridden: boolean } => {
|
||||
const globalList = toStringListValue(globalValue);
|
||||
const userList = toStringListValue(userValue);
|
||||
const isOverridden =
|
||||
hasUserKey && userValue !== null && JSON.stringify(userList) !== JSON.stringify(globalList);
|
||||
|
||||
return { value: isOverridden ? userList : globalList, isOverridden };
|
||||
};
|
||||
|
||||
export const toComparableValue = (value: unknown): string => {
|
||||
if (value === null || value === undefined) {
|
||||
return '';
|
||||
|
||||
@@ -10,6 +10,7 @@ export interface PerUserSettings {
|
||||
EMAIL_RECIPIENT?: string;
|
||||
DOWNLOAD_TO_BROWSER_CONTENT_TYPES?: string[];
|
||||
SEARCH_MODE?: string;
|
||||
BOOK_LANGUAGE?: string[];
|
||||
METADATA_PROVIDER?: string;
|
||||
METADATA_PROVIDER_AUDIOBOOK?: string;
|
||||
DEFAULT_RELEASE_SOURCE?: string;
|
||||
|
||||
@@ -12,6 +12,8 @@ import type {
|
||||
RequestSubmissionResult,
|
||||
MetadataProvidersResponse,
|
||||
MetadataSearchConfig,
|
||||
PackBook,
|
||||
InspectReleaseResponse,
|
||||
} from '../types';
|
||||
import type {
|
||||
ActionResult,
|
||||
@@ -144,6 +146,11 @@ const mapApiErrorToActionResult = (error: unknown): ActionResult | null => {
|
||||
// Default request timeout in milliseconds (30 seconds)
|
||||
const DEFAULT_TIMEOUT_MS = 30000;
|
||||
|
||||
// Release searches can be long-running: a source behind Cloudflare/DDoS-Guard has
|
||||
// to spin up the bypasser and solve the challenge before any results come back,
|
||||
// which routinely takes well over the default timeout.
|
||||
const SEARCH_TIMEOUT_MS = 180000;
|
||||
|
||||
// Utility function for JSON fetch with credentials and timeout
|
||||
async function fetchJSON<T>(
|
||||
url: string,
|
||||
@@ -235,6 +242,8 @@ export const searchBooks = async (query: string): Promise<Book[]> => {
|
||||
if (!query) return [];
|
||||
const response = await fetchJSON<ReleasesResponse>(
|
||||
`${API_BASE}/releases?source=direct_download&${query}`,
|
||||
{},
|
||||
SEARCH_TIMEOUT_MS,
|
||||
);
|
||||
return response.releases.map(transformReleaseToDirectBook);
|
||||
};
|
||||
@@ -503,6 +512,18 @@ export type DownloadReleasePayload = {
|
||||
language?: string; // Release language code, for the {Language} naming variable
|
||||
search_author?: string;
|
||||
search_mode?: 'direct' | 'universal';
|
||||
multi_book?: boolean; // Split a multi-book pack into one book per subfolder/file
|
||||
book_plan?: PackBook[]; // The split the user approved before download
|
||||
};
|
||||
|
||||
/** Inspect a release's file list before download (same body as downloadRelease). */
|
||||
export const inspectRelease = async (
|
||||
release: DownloadReleasePayload,
|
||||
): Promise<InspectReleaseResponse> => {
|
||||
return fetchJSON<InspectReleaseResponse>(`${API_BASE}/releases/inspect`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(release),
|
||||
});
|
||||
};
|
||||
|
||||
export const downloadRelease = async (
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
buildLanguageNormalizer,
|
||||
getReleaseSearchLanguageParams,
|
||||
releaseLanguageMatchesFilter,
|
||||
resolveDefaultLanguageCodes,
|
||||
} from '../utils/languageFilters';
|
||||
|
||||
const supportedLanguages: Language[] = [
|
||||
@@ -67,3 +68,36 @@ describe('releaseLanguageMatchesFilter', () => {
|
||||
expect(visibleLanguages).toHaveLength(48);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveDefaultLanguageCodes', () => {
|
||||
it('keeps an explicitly empty default as "no default filter"', () => {
|
||||
// The backend stores [] to mean "do not filter"; substituting the first
|
||||
// supported language here would make the UI filter where the server does not.
|
||||
expect(resolveDefaultLanguageCodes([], supportedLanguages)).toEqual([]);
|
||||
});
|
||||
|
||||
it('leaves a configured default untouched', () => {
|
||||
expect(resolveDefaultLanguageCodes(['de', 'hu'], supportedLanguages)).toEqual(['de', 'hu']);
|
||||
});
|
||||
|
||||
it('falls back to the first supported language only when nothing is configured', () => {
|
||||
expect(resolveDefaultLanguageCodes(undefined, supportedLanguages)).toEqual(['en']);
|
||||
expect(resolveDefaultLanguageCodes(null, [])).toEqual(['en']);
|
||||
});
|
||||
|
||||
it('sends no language filter when an empty default is the whole selection', () => {
|
||||
const defaults = resolveDefaultLanguageCodes([], supportedLanguages);
|
||||
|
||||
expect(
|
||||
getReleaseSearchLanguageParams([LANGUAGE_OPTION_DEFAULT], supportedLanguages, defaults),
|
||||
).toBe(undefined);
|
||||
});
|
||||
|
||||
it('does not smuggle the first language into a Default+German selection', () => {
|
||||
const defaults = resolveDefaultLanguageCodes([], supportedLanguages);
|
||||
|
||||
expect(
|
||||
getReleaseSearchLanguageParams([LANGUAGE_OPTION_DEFAULT, 'de'], supportedLanguages, defaults),
|
||||
).toEqual(['de']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import type { PackBook } from '../types';
|
||||
import {
|
||||
describePackPlan,
|
||||
parseSeriesPositionInput,
|
||||
toBookPlanPayload,
|
||||
updateReviewBook,
|
||||
} from '../utils/packReview';
|
||||
|
||||
const books: PackBook[] = [
|
||||
{ title: 'Leviathan Wakes', series_position: 1, year: 2011, files: ['a.m4b'] },
|
||||
{ title: 'Caliban’s War', series_position: 2, year: 2012, files: ['b.m4b', 'b2.m4b'] },
|
||||
];
|
||||
|
||||
describe('packReview.updateReviewBook', () => {
|
||||
it('replaces one book without touching the others', () => {
|
||||
const next = updateReviewBook(books, 1, { title: 'Caliban’s War (Unabridged)' });
|
||||
expect(next[0]).toBe(books[0]);
|
||||
expect(next[1]).toEqual({ ...books[1], title: 'Caliban’s War (Unabridged)' });
|
||||
expect(books[1].title).toBe('Caliban’s War');
|
||||
});
|
||||
});
|
||||
|
||||
describe('packReview.parseSeriesPositionInput', () => {
|
||||
it('accepts whole and fractional positions', () => {
|
||||
expect(parseSeriesPositionInput('3')).toBe(3);
|
||||
expect(parseSeriesPositionInput('2.5')).toBe(2.5);
|
||||
});
|
||||
|
||||
it('treats blank or junk as no position', () => {
|
||||
expect(parseSeriesPositionInput('')).toBeNull();
|
||||
expect(parseSeriesPositionInput('abc')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('packReview.toBookPlanPayload', () => {
|
||||
it('trims titles, drops books without a title, and keeps file lists', () => {
|
||||
const edited = updateReviewBook(books, 0, { title: ' ' });
|
||||
expect(toBookPlanPayload(edited)).toEqual([
|
||||
{ title: 'Caliban’s War', series_position: 2, year: 2012, files: ['b.m4b', 'b2.m4b'] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('packReview.describePackPlan', () => {
|
||||
it('summarises books, files and ignored sidecars', () => {
|
||||
expect(describePackPlan(books, ['a.txt', 'cover.jpg'])).toBe(
|
||||
'2 books · 3 files · 2 files ignored',
|
||||
);
|
||||
expect(describePackPlan([books[0]], [])).toBe('1 book · 1 file');
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import type { Release } from '../types';
|
||||
import { getReleaseFormats } from '../utils/releaseFormats';
|
||||
import { getReleaseFormats, getUnrecognizedReleaseFormats } from '../utils/releaseFormats';
|
||||
|
||||
function buildRelease(overrides: Partial<Release>): Release {
|
||||
return {
|
||||
@@ -38,3 +38,26 @@ describe('releaseFormats.getReleaseFormats', () => {
|
||||
expect(getReleaseFormats(release)).toEqual(['pdf']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('releaseFormats.getUnrecognizedReleaseFormats', () => {
|
||||
it('returns normalized, deduplicated unrecognized formats from extra', () => {
|
||||
const release = buildRelease({
|
||||
extra: { unrecognized_formats: ['AVI', ' avi ', 'WEBM'] },
|
||||
});
|
||||
|
||||
expect(getUnrecognizedReleaseFormats(release)).toEqual(['avi', 'webm']);
|
||||
});
|
||||
|
||||
it('accepts a single string value', () => {
|
||||
const release = buildRelease({ extra: { unrecognized_formats: 'AVI' } });
|
||||
|
||||
expect(getUnrecognizedReleaseFormats(release)).toEqual(['avi']);
|
||||
});
|
||||
|
||||
it('returns an empty list when nothing was flagged', () => {
|
||||
expect(getUnrecognizedReleaseFormats(buildRelease({}))).toEqual([]);
|
||||
expect(
|
||||
getUnrecognizedReleaseFormats(buildRelease({ extra: { unrecognized_formats: null } })),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import type { Book, Release } from '../types';
|
||||
import { buildReleaseDownloadPayload } from '../utils/releasePayload';
|
||||
|
||||
const book: Book = {
|
||||
id: 'hc-1',
|
||||
title: 'Drive',
|
||||
author: 'James S. A. Corey',
|
||||
year: '2012',
|
||||
preview: 'https://img/drive.jpg',
|
||||
series_name: 'The Expanse',
|
||||
series_position: 2.6,
|
||||
subtitle: 'An Expanse Short Story',
|
||||
provider: 'hardcover',
|
||||
provider_id: 'hc-1',
|
||||
source: 'direct_download',
|
||||
};
|
||||
|
||||
const release: Release = {
|
||||
source: 'audiobookbay',
|
||||
source_id: 'abb-1',
|
||||
title: 'James S. A. Corey - The Expanse Complete 2.0',
|
||||
format: 'm4b',
|
||||
language: 'en',
|
||||
download_url: 'https://audiobookbay.lu/abss/expanse/',
|
||||
};
|
||||
|
||||
describe('buildReleaseDownloadPayload', () => {
|
||||
it('describes the searched book and the chosen release', () => {
|
||||
const payload = buildReleaseDownloadPayload(book, release, 'audiobook');
|
||||
expect(payload).toMatchObject({
|
||||
source: 'audiobookbay',
|
||||
source_id: 'abb-1',
|
||||
title: 'Drive',
|
||||
author: 'James S. A. Corey',
|
||||
series_name: 'The Expanse',
|
||||
series_position: 2.6,
|
||||
language: 'en',
|
||||
content_type: 'audiobook',
|
||||
});
|
||||
expect(payload.multi_book).toBeUndefined();
|
||||
expect(payload.book_plan).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses the release title and author for manual books', () => {
|
||||
const manual: Book = { ...book, provider: 'manual', title: 'ignored' };
|
||||
const withAuthor = { ...release, extra: { author: 'Release Author' } };
|
||||
const payload = buildReleaseDownloadPayload(manual, withAuthor, 'audiobook');
|
||||
expect(payload.title).toBe(release.title);
|
||||
expect(payload.author).toBe('Release Author');
|
||||
});
|
||||
|
||||
it('flags a manual multi-book pack', () => {
|
||||
const payload = buildReleaseDownloadPayload(book, release, 'audiobook', { multiBook: true });
|
||||
expect(payload.multi_book).toBe(true);
|
||||
expect(payload.book_plan).toBeUndefined();
|
||||
});
|
||||
|
||||
it('attaches the approved book plan', () => {
|
||||
const plan = [{ title: 'Leviathan Wakes', series_position: 1, year: 2011, files: ['a.m4b'] }];
|
||||
const payload = buildReleaseDownloadPayload(book, release, 'audiobook', {
|
||||
multiBook: true,
|
||||
bookPlan: plan,
|
||||
});
|
||||
expect(payload.multi_book).toBe(true);
|
||||
expect(payload.book_plan).toEqual(plan);
|
||||
});
|
||||
});
|
||||
@@ -460,6 +460,27 @@ export interface SourceSearchInfo {
|
||||
}
|
||||
|
||||
// Response from /api/releases endpoint
|
||||
/** One book split out of a multi-book pack release, files as release-relative paths. */
|
||||
export interface PackBook {
|
||||
title: string;
|
||||
series_position: number | null;
|
||||
year: number | null;
|
||||
files: string[];
|
||||
}
|
||||
|
||||
export interface PackPlan {
|
||||
is_pack: boolean;
|
||||
books: PackBook[];
|
||||
ignored: string[];
|
||||
}
|
||||
|
||||
export interface InspectReleaseResponse {
|
||||
inspected: boolean;
|
||||
reason: string | null;
|
||||
files: { path: string; size: number | null }[];
|
||||
plan: PackPlan | null;
|
||||
}
|
||||
|
||||
export interface ReleasesResponse {
|
||||
releases: Release[];
|
||||
book: {
|
||||
|
||||
@@ -146,7 +146,13 @@ export interface TableFieldColumnOption {
|
||||
childOf?: string;
|
||||
}
|
||||
|
||||
export type TableFieldColumnType = 'text' | 'select' | 'multiselect' | 'checkbox' | 'path';
|
||||
export type TableFieldColumnType =
|
||||
| 'text'
|
||||
| 'password'
|
||||
| 'select'
|
||||
| 'multiselect'
|
||||
| 'checkbox'
|
||||
| 'path';
|
||||
|
||||
export interface TableFieldColumn {
|
||||
key: string;
|
||||
|
||||
@@ -27,6 +27,24 @@ export const normalizeLanguageSelection = (selected: string[]): string[] => {
|
||||
return unique.length ? unique : [LANGUAGE_OPTION_DEFAULT];
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve the language codes the "Default" filter option stands for.
|
||||
*
|
||||
* An explicitly empty list is a deliberate "no default filter" and is returned as-is;
|
||||
* only a missing value falls back to the first supported language. Substituting a
|
||||
* language for the empty list would make the UI filter by a language the backend
|
||||
* does not apply.
|
||||
*/
|
||||
export const resolveDefaultLanguageCodes = (
|
||||
configuredDefault: string[] | null | undefined,
|
||||
supportedLanguages: Language[],
|
||||
): string[] => {
|
||||
if (Array.isArray(configuredDefault)) {
|
||||
return configuredDefault;
|
||||
}
|
||||
return [supportedLanguages[0]?.code || 'en'];
|
||||
};
|
||||
|
||||
export const getLanguageFilterValues = (
|
||||
selection: string[],
|
||||
supportedLanguages: Language[],
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { PackBook } from '../types';
|
||||
|
||||
/** Return a copy of `books` with one entry patched; the input is not mutated. */
|
||||
export function updateReviewBook(
|
||||
books: PackBook[],
|
||||
index: number,
|
||||
patch: Partial<PackBook>,
|
||||
): PackBook[] {
|
||||
return books.map((book, i) => (i === index ? { ...book, ...patch } : book));
|
||||
}
|
||||
|
||||
/** Parse a series-position text field: "3" → 3, "2.5" → 2.5, blank/junk → null. */
|
||||
export function parseSeriesPositionInput(value: string): number | null {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return null;
|
||||
const parsed = Number(trimmed);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
/** The plan sent with the download: trimmed titles, untitled books dropped. */
|
||||
export function toBookPlanPayload(books: PackBook[]): PackBook[] {
|
||||
return books
|
||||
.map((book) => ({ ...book, title: book.title.trim() }))
|
||||
.filter((book) => book.title.length > 0 && book.files.length > 0);
|
||||
}
|
||||
|
||||
function plural(count: number, noun: string): string {
|
||||
return `${count} ${noun}${count === 1 ? '' : 's'}`;
|
||||
}
|
||||
|
||||
/** "2 books · 3 files · 2 files ignored" */
|
||||
export function describePackPlan(books: PackBook[], ignored: string[]): string {
|
||||
const fileCount = books.reduce((sum, book) => sum + book.files.length, 0);
|
||||
const parts = [plural(books.length, 'book'), plural(fileCount, 'file')];
|
||||
if (ignored.length > 0) {
|
||||
parts.push(`${plural(ignored.length, 'file')} ignored`);
|
||||
}
|
||||
return parts.join(' · ');
|
||||
}
|
||||
@@ -33,3 +33,27 @@ export function getReleaseFormats(release: Release): string[] {
|
||||
|
||||
return formats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format tokens the indexer declared but the backend could not map to a known
|
||||
* book/audiobook format (e.g. MyAnonamouse "[ENG / AVI]"). Such a release will
|
||||
* download but fail post-processing, so the UI warns instead of showing a bare
|
||||
* content-type icon.
|
||||
*/
|
||||
export function getUnrecognizedReleaseFormats(release: Release): string[] {
|
||||
const raw = release.extra?.unrecognized_formats;
|
||||
const values = Array.isArray(raw) ? raw : [raw];
|
||||
const formats: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
values.forEach((value) => {
|
||||
const normalized = normalizeFormatValue(value);
|
||||
if (!normalized || seen.has(normalized)) {
|
||||
return;
|
||||
}
|
||||
seen.add(normalized);
|
||||
formats.push(normalized);
|
||||
});
|
||||
|
||||
return formats;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { DownloadReleasePayload } from '../services/api';
|
||||
import type { Book, ContentType, PackBook, Release } from '../types';
|
||||
|
||||
export interface ReleaseDownloadOptions {
|
||||
/** Ask post-processing to split the release into one book per subfolder/file. */
|
||||
multiBook?: boolean;
|
||||
/** The split the user approved in the pack review panel. */
|
||||
bookPlan?: PackBook[];
|
||||
}
|
||||
|
||||
/** Build the body for /api/releases/download (and /api/releases/inspect). */
|
||||
export function buildReleaseDownloadPayload(
|
||||
book: Book,
|
||||
release: Release,
|
||||
releaseContentType: ContentType,
|
||||
options: ReleaseDownloadOptions = {},
|
||||
): DownloadReleasePayload {
|
||||
const isManual = book.provider === 'manual';
|
||||
const releasePreview =
|
||||
typeof release.extra?.preview === 'string' ? release.extra.preview : undefined;
|
||||
const releaseAuthor =
|
||||
typeof release.extra?.author === 'string' ? release.extra.author : undefined;
|
||||
|
||||
const payload: DownloadReleasePayload = {
|
||||
source: release.source,
|
||||
source_id: release.source_id,
|
||||
title: isManual ? release.title : book.title,
|
||||
author: isManual ? releaseAuthor || '' : book.author,
|
||||
year: book.year,
|
||||
format: release.format,
|
||||
size: release.size,
|
||||
size_bytes: release.size_bytes,
|
||||
download_url: release.download_url,
|
||||
protocol: release.protocol,
|
||||
indexer: release.indexer,
|
||||
seeders: release.seeders,
|
||||
extra: release.extra,
|
||||
preview: isManual ? releasePreview || undefined : book.preview,
|
||||
content_type: releaseContentType,
|
||||
series_name: book.series_name,
|
||||
series_position: book.series_position,
|
||||
subtitle: book.subtitle,
|
||||
// From the release, never the book: book.language is the provider's
|
||||
// canonical edition, which would mislabel a translated release.
|
||||
language: release.language ?? undefined,
|
||||
};
|
||||
|
||||
if (options.multiBook || options.bookPlan) {
|
||||
payload.multi_book = true;
|
||||
}
|
||||
if (options.bookPlan) {
|
||||
payload.book_plan = options.bookPlan;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
"""AudiobookBay test fixtures."""
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_detail_page_cache():
|
||||
"""Detail pages are cached briefly in production; tests must not share them."""
|
||||
scraper.clear_detail_page_cache()
|
||||
yield
|
||||
scraper.clear_detail_page_cache()
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Tests for reading the torrent file list off an AudiobookBay detail page."""
|
||||
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
|
||||
# Trimmed from a real detail page (2026-08): the file rows sit between the
|
||||
# "Multifile Torrent" marker and the "Combined File Size" row.
|
||||
MULTIFILE_DETAIL_HTML = """
|
||||
<table>
|
||||
<tr><td>Tracker:</td><td>udp://tracker.torrent.eu.org:451/announce</td></tr>
|
||||
<tr><td>Creation Date:</td><td>Sun, 29 Mar 2026 21:09:39 +0200</td></tr>
|
||||
<tr><td colspan='2'>This is a Multifile Torrent</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 9.0 - Leviathan Falls (2021).m4b 1.05 GBs</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 0.1 - An Expanse Novella - Drive (2012).txt 340 Bytes</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 0.2 - An Expanse Novella - The Churn (2014).m4b 125.72 MBs</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 2.0 - Caliban’s War (2012).m4b 578.97 MBs</td></tr>
|
||||
<tr><td>Combined File Size:</td><td><span style='color:#00f;'>7.87</span> GBs</td></tr>
|
||||
<tr><td>Info Hash:</td><td>e4a5538e26987ee58a43aa629ec2c4f2b2d46526</td></tr>
|
||||
</table>
|
||||
"""
|
||||
|
||||
SINGLE_FILE_DETAIL_HTML = """
|
||||
<table>
|
||||
<tr><td>Creation Date:</td><td>Sun, 29 Mar 2026 21:09:39 +0200</td></tr>
|
||||
<tr><td colspan='2'>Drive.m4b 41.55 MBs</td></tr>
|
||||
<tr><td>Combined File Size:</td><td><span style='color:#00f;'>41.55</span> MBs</td></tr>
|
||||
<tr><td>Info Hash:</td><td>e4a5538e26987ee58a43aa629ec2c4f2b2d46526</td></tr>
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
def test_extracts_multifile_rows_with_byte_sizes():
|
||||
files = scraper.extract_file_list(MULTIFILE_DETAIL_HTML)
|
||||
assert files == [
|
||||
PackFile("The Expanse 9.0 - Leviathan Falls (2021).m4b", int(1.05 * 1024**3)),
|
||||
PackFile("The Expanse 0.1 - An Expanse Novella - Drive (2012).txt", 340),
|
||||
PackFile(
|
||||
"The Expanse 0.2 - An Expanse Novella - The Churn (2014).m4b", int(125.72 * 1024**2)
|
||||
),
|
||||
PackFile("The Expanse 2.0 - Caliban’s War (2012).m4b", int(578.97 * 1024**2)),
|
||||
]
|
||||
|
||||
|
||||
def test_single_file_torrent_lists_the_row_before_combined_size():
|
||||
assert scraper.extract_file_list(SINGLE_FILE_DETAIL_HTML) == [
|
||||
PackFile("Drive.m4b", int(41.55 * 1024**2))
|
||||
]
|
||||
|
||||
|
||||
def test_page_without_file_table_returns_none():
|
||||
assert scraper.extract_file_list("<html><body><p>nothing here</p></body></html>") is None
|
||||
|
||||
|
||||
class TestHandlerListFiles:
|
||||
def test_lists_files_from_detail_page(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
from shelfmark.release_sources.audiobookbay.handler import AudiobookBayHandler
|
||||
|
||||
with patch(
|
||||
"shelfmark.release_sources.audiobookbay.handler.scraper.fetch_detail_html",
|
||||
return_value=SINGLE_FILE_DETAIL_HTML,
|
||||
) as fetch:
|
||||
files = AudiobookBayHandler().list_files(
|
||||
{"source_id": "abc", "download_url": "https://audiobookbay.lu/abss/drive/"}
|
||||
)
|
||||
assert files == [PackFile("Drive.m4b", int(41.55 * 1024**2))]
|
||||
fetch.assert_called_once_with("https://audiobookbay.lu/abss/drive/", "audiobookbay.lu")
|
||||
|
||||
def test_rejects_detail_url_on_other_host(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
from shelfmark.release_sources.audiobookbay.handler import AudiobookBayHandler
|
||||
|
||||
with patch(
|
||||
"shelfmark.release_sources.audiobookbay.handler.scraper.fetch_detail_html"
|
||||
) as fetch:
|
||||
files = AudiobookBayHandler().list_files(
|
||||
{"source_id": "abc", "download_url": "https://evil.example/abss/drive/"}
|
||||
)
|
||||
assert files is None
|
||||
fetch.assert_not_called()
|
||||
|
||||
|
||||
def test_extract_magnet_link_and_file_list_share_one_page_fetch():
|
||||
from unittest.mock import patch
|
||||
|
||||
page = MULTIFILE_DETAIL_HTML
|
||||
with (
|
||||
patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
|
||||
return_value=page,
|
||||
) as get_page,
|
||||
patch("shelfmark.release_sources.audiobookbay.scraper._bootstrap_abb_session"),
|
||||
):
|
||||
scraper.clear_detail_page_cache()
|
||||
url = "https://audiobookbay.lu/abss/expanse/"
|
||||
assert scraper.fetch_detail_html(url, "audiobookbay.lu") == page
|
||||
magnet = scraper.extract_magnet_link(url, "audiobookbay.lu")
|
||||
assert magnet is not None
|
||||
assert "e4a5538e26987ee58a43aa629ec2c4f2b2d46526".upper() in magnet
|
||||
assert get_page.call_count == 1
|
||||
@@ -317,6 +317,44 @@ class TestSearchAudiobookbay:
|
||||
assert "s=%22test+query%22" in requested_url
|
||||
assert "cat=undefined%2Cundefined" in requested_url
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_normalizes_curly_apostrophe(self, mock_config_get, mock_html_get):
|
||||
"""Test curly apostrophes are searched as the ASCII form ABB stores."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
|
||||
|
||||
scraper.search_audiobookbay(
|
||||
"the stranger’s wife",
|
||||
max_pages=1,
|
||||
hostname="audiobookbay.lu",
|
||||
)
|
||||
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert "s=the+stranger%27s+wife" in requested_url
|
||||
assert "%E2%80%99" not in requested_url
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_percent_encodes_reserved_characters(
|
||||
self, mock_config_get, mock_html_get
|
||||
):
|
||||
"""Test reserved characters cannot break out of the search parameter."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
|
||||
|
||||
scraper.search_audiobookbay(
|
||||
"sense & sensibility 100% c++",
|
||||
max_pages=1,
|
||||
hostname="audiobookbay.lu",
|
||||
)
|
||||
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert "s=sense+%26+sensibility+100%25+c%2B%2B" in requested_url
|
||||
# The only surviving '&' introduces the legacy category parameter.
|
||||
assert requested_url.count("&") == 1
|
||||
assert requested_url.endswith("&cat=undefined%2Cundefined")
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_always_uses_legacy_category_query(
|
||||
|
||||
@@ -306,6 +306,42 @@ class TestAudiobookBaySource:
|
||||
assert len(results) == 1
|
||||
assert results[0].title == "Test Book by Test Author"
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
|
||||
def test_search_relevance_filtering_spans_typographic_punctuation(self, mock_search):
|
||||
"""Test an ASCII query still matches the typographic title ABB renders."""
|
||||
mock_search.return_value = [
|
||||
{
|
||||
"title": "The Stranger’s Wife — Anna‑Lou Weatherley",
|
||||
"link": "https://audiobookbay.lu/abss/the-strangers-wife/",
|
||||
"format": "M4B",
|
||||
"size": "259 MB",
|
||||
"language": "English",
|
||||
},
|
||||
]
|
||||
|
||||
source = AudiobookBaySource()
|
||||
book = BookMetadata(
|
||||
provider="test",
|
||||
provider_id="123",
|
||||
title="Stranger's",
|
||||
authors=["Anna-Lou Weatherley"],
|
||||
)
|
||||
# Every query word carries punctuation, so the result survives only when
|
||||
# both sides of the comparison are normalized.
|
||||
plan = ReleaseSearchPlan(
|
||||
languages=["en"],
|
||||
isbn_candidates=[],
|
||||
author="",
|
||||
title_variants=[ReleaseSearchVariant(title="Stranger's", author="")],
|
||||
grouped_title_variants=[],
|
||||
)
|
||||
|
||||
results = source.search(book, plan, content_type="audiobook")
|
||||
|
||||
assert len(results) == 1
|
||||
# The release keeps the title as ABB rendered it; only matching normalizes.
|
||||
assert results[0].title == "The Stranger’s Wife — Anna‑Lou Weatherley"
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
|
||||
def test_search_result_mapping(self, mock_search):
|
||||
"""Test conversion of scraper results to Release objects."""
|
||||
|
||||
@@ -2,7 +2,52 @@
|
||||
Tests for AudiobookBay utility functions.
|
||||
"""
|
||||
|
||||
from shelfmark.release_sources.audiobookbay.utils import parse_size
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation, parse_size
|
||||
|
||||
|
||||
class TestNormalizeSearchPunctuation:
|
||||
"""Tests for the normalize_search_punctuation function."""
|
||||
|
||||
def test_curly_apostrophe_becomes_ascii(self):
|
||||
"""ABB matches the stored ASCII apostrophe, not the rendered curly one."""
|
||||
assert normalize_search_punctuation("The Stranger’s Wife") == "The Stranger's Wife"
|
||||
|
||||
def test_all_single_quote_variants(self):
|
||||
"""Every single-quote lookalike collapses to the ASCII apostrophe."""
|
||||
for variant in ("‘", "’", "‚", "‛", "′", "´", "`"):
|
||||
assert normalize_search_punctuation(f"don{variant}t") == "don't"
|
||||
|
||||
def test_all_double_quote_variants(self):
|
||||
"""Every double-quote lookalike collapses to the ASCII double quote."""
|
||||
for variant in ("“", "”", "„", "‟", "″"):
|
||||
assert normalize_search_punctuation(f"{variant}quoted{variant}") == '"quoted"'
|
||||
|
||||
def test_all_dash_variants(self):
|
||||
"""Every dash lookalike collapses to the ASCII hyphen."""
|
||||
for variant in ("‐", "‑", "‒", "–", "—", "―", "−", "﹘", "﹣", "-"):
|
||||
assert normalize_search_punctuation(f"anna{variant}lou") == "anna-lou"
|
||||
|
||||
def test_ellipsis_expands_to_three_dots(self):
|
||||
"""WordPress renders '...' as a single ellipsis character."""
|
||||
assert normalize_search_punctuation("And Then…") == "And Then..."
|
||||
|
||||
def test_ascii_query_is_unchanged(self):
|
||||
"""A query that is already ASCII passes through untouched."""
|
||||
assert normalize_search_punctuation("The Stranger's Wife") == "The Stranger's Wife"
|
||||
|
||||
def test_ascii_dash_runs_are_not_collapsed(self):
|
||||
"""Only characters ABB cannot have stored are rewritten."""
|
||||
assert normalize_search_punctuation("Book -- Subtitle") == "Book -- Subtitle"
|
||||
|
||||
def test_other_punctuation_is_preserved(self):
|
||||
"""Colons and commas carry search signal and are left alone."""
|
||||
assert normalize_search_punctuation("Weatherley: Book 3, Part 1") == (
|
||||
"Weatherley: Book 3, Part 1"
|
||||
)
|
||||
|
||||
def test_empty_query(self):
|
||||
"""An empty query is returned as-is."""
|
||||
assert normalize_search_punctuation("") == ""
|
||||
|
||||
|
||||
class TestParseSize:
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
"""How long the internal bypasser is allowed to spend, and on what.
|
||||
|
||||
Issue #1276: MAX_RETRY drove *both* the outer page-load loop and the per-page method
|
||||
loop, so the default of 10 meant ~40 solve attempts on one browser. That overran the
|
||||
worker deadline, and the failure reached the user as `RuntimeError: TimeoutError` - a
|
||||
message that says nothing about a protection challenge and sent people looking at their
|
||||
reverse proxy instead.
|
||||
|
||||
Also covered: the undisturbed window a passive challenge gets before anything touches the
|
||||
page. Anna's Archive's DDoS-Guard check has no click target and clears itself; going
|
||||
straight to the click/reload methods meant the one thing that solves it was never tried.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bypass(monkeypatch):
|
||||
"""internal_bypasser with sleeps and jitter removed."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
async def _no_sleep(_seconds) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.asyncio, "sleep", _no_sleep)
|
||||
monkeypatch.setattr(internal_bypasser._RNG, "uniform", lambda _a, _b: 0)
|
||||
return internal_bypasser
|
||||
|
||||
|
||||
def _recording_methods(calls: list[str], count: int = 4):
|
||||
def _make(name: str):
|
||||
async def _method(_page) -> bool:
|
||||
calls.append(name)
|
||||
return False
|
||||
|
||||
_method.__name__ = name
|
||||
return _method
|
||||
|
||||
return [_make(f"m{i}") for i in range(count)]
|
||||
|
||||
|
||||
def _stub_page_state(monkeypatch, bypass, *, bypassed=False, challenge="ddos_guard"):
|
||||
async def _is_bypassed(*_args, **_kwargs) -> bool:
|
||||
return bypassed
|
||||
|
||||
async def _detect(*_args, **_kwargs) -> str:
|
||||
return challenge
|
||||
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _is_bypassed)
|
||||
monkeypatch.setattr(bypass, "_detect_challenge_type", _detect)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The method loop must not read MAX_RETRY
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_method_loop_budget_is_independent_of_max_retry(monkeypatch, bypass):
|
||||
"""MAX_RETRY is the outer page-load retry; reading it here squared the budget.
|
||||
|
||||
Exercised against a challenge whose *type* keeps changing, because that is the case
|
||||
where max_retries is what bounds the loop: the stuck-challenge guard only fires on a
|
||||
run of the same type, so with a stable challenge it hid the real budget entirely.
|
||||
"""
|
||||
monkeypatch.setattr(type(bypass.app_config), "MAX_RETRY", 50, raising=False)
|
||||
|
||||
types = iter(["ddos_guard", "cloudflare"] * 100)
|
||||
|
||||
async def _alternating(*_args, **_kwargs) -> str:
|
||||
return next(types)
|
||||
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _never_passes)
|
||||
_stub_page_state(monkeypatch, bypass)
|
||||
monkeypatch.setattr(bypass, "_detect_challenge_type", _alternating)
|
||||
|
||||
assert asyncio.run(bypass._bypass(object())) is False
|
||||
assert len(calls) == bypass._BYPASS_METHOD_ATTEMPTS
|
||||
assert len(calls) < 50, "MAX_RETRY must not reach the method loop"
|
||||
|
||||
|
||||
def test_method_attempt_budget_is_reachable(bypass):
|
||||
"""The number reported as `attempt N/X` must be a number the loop can reach.
|
||||
|
||||
It used to be MAX_RETRY (10) while the stuck-challenge guard capped the loop at 5,
|
||||
so logs showed `4/10` and stopped, which reads like six lost attempts.
|
||||
"""
|
||||
assert bypass._BYPASS_METHOD_ATTEMPTS == len(bypass.BYPASS_METHODS) + 1
|
||||
assert bypass._BYPASS_METHOD_ATTEMPTS >= (
|
||||
max(bypass.MAX_CONSECUTIVE_SAME_CHALLENGE, len(bypass.BYPASS_METHODS) + 1)
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# A passive challenge gets an undisturbed window first
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def _never_passes(*_args, **_kwargs) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def test_passive_challenge_is_given_time_before_any_method_runs(monkeypatch, bypass):
|
||||
"""DDoS-Guard's JS check clears itself; nothing should click or reload first."""
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
||||
_stub_page_state(monkeypatch, bypass)
|
||||
|
||||
async def _passes(*_args, **_kwargs) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _passes)
|
||||
|
||||
assert asyncio.run(bypass._bypass(object())) is True
|
||||
assert calls == [], "the page must not be touched while the check can still pass"
|
||||
|
||||
|
||||
def test_passive_wait_happens_once_not_before_every_method(monkeypatch, bypass):
|
||||
"""It is a settling window, not a delay bolted onto each attempt."""
|
||||
waits: list[int] = []
|
||||
calls: list[str] = []
|
||||
|
||||
async def _count_wait(*_args, **_kwargs) -> bool:
|
||||
waits.append(1)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _count_wait)
|
||||
_stub_page_state(monkeypatch, bypass)
|
||||
|
||||
asyncio.run(bypass._bypass(object()))
|
||||
|
||||
assert len(waits) == 1
|
||||
assert calls == ["m0", "m1", "m2", "m3"]
|
||||
|
||||
|
||||
def test_no_passive_wait_when_no_challenge_is_detected(monkeypatch, bypass):
|
||||
"""The 'none' branch has its own settle-and-refresh handling."""
|
||||
waits: list[int] = []
|
||||
|
||||
async def _count_wait(*_args, **_kwargs) -> bool:
|
||||
waits.append(1)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _count_wait)
|
||||
_stub_page_state(monkeypatch, bypass, challenge="none")
|
||||
|
||||
class _Page:
|
||||
async def reload(self, **_kwargs) -> None:
|
||||
return None
|
||||
|
||||
asyncio.run(bypass._bypass(_Page(), max_retries=1))
|
||||
|
||||
assert waits == []
|
||||
|
||||
|
||||
def test_wait_for_passive_solve_returns_as_soon_as_the_page_clears(monkeypatch, bypass):
|
||||
polls = {"n": 0}
|
||||
|
||||
async def _is_bypassed(*_args, **_kwargs) -> bool:
|
||||
polls["n"] += 1
|
||||
return polls["n"] >= 3
|
||||
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _is_bypassed)
|
||||
|
||||
assert asyncio.run(bypass._wait_for_passive_solve(object())) is True
|
||||
assert polls["n"] == 3
|
||||
|
||||
|
||||
def test_wait_for_passive_solve_gives_up_at_the_window(monkeypatch, bypass):
|
||||
"""It must not poll forever - the methods still need their share of the budget."""
|
||||
clock = {"now": 0.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
|
||||
async def _tick(*_args, **_kwargs) -> bool:
|
||||
clock["now"] += 1.0
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _tick)
|
||||
|
||||
assert asyncio.run(bypass._wait_for_passive_solve(object())) is False
|
||||
assert clock["now"] >= bypass._PASSIVE_SOLVE_SECONDS
|
||||
|
||||
|
||||
def test_wait_for_passive_solve_honours_cancellation(monkeypatch, bypass):
|
||||
import threading
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
|
||||
cancel = threading.Event()
|
||||
cancel.set()
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _never_passes)
|
||||
|
||||
with pytest.raises(BypassCancelledError):
|
||||
asyncio.run(bypass._wait_for_passive_solve(object(), cancel))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The page-load loop stops while there is still time to report a real failure
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_page_load_loop_stops_before_the_worker_deadline(monkeypatch, bypass):
|
||||
"""A stubborn challenge must produce "bypass failed", not a cancelled coroutine."""
|
||||
clock = {"now": 0.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
monkeypatch.delenv(bypass._BYPASS_CHILD_ENV, raising=False)
|
||||
|
||||
attempts = {"n": 0}
|
||||
|
||||
async def _create(_url):
|
||||
return object()
|
||||
|
||||
async def _get(_url, _driver, _cancel=None) -> str:
|
||||
attempts["n"] += 1
|
||||
# Each pass eats a realistic slice of the budget.
|
||||
clock["now"] += 120.0
|
||||
return ""
|
||||
|
||||
async def _close(_driver) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(bypass, "_create_cdp_browser", _create)
|
||||
monkeypatch.setattr(bypass, "_get", _get)
|
||||
monkeypatch.setattr(bypass, "_close_cdp_driver", _close)
|
||||
|
||||
class _RealWorker:
|
||||
def run(self, coro, timeout=None):
|
||||
return asyncio.run(coro)
|
||||
|
||||
monkeypatch.setattr(bypass, "_CDP_WORKER", _RealWorker())
|
||||
|
||||
result = bypass._run_bypass_in_current_process("https://example.com", 10)
|
||||
|
||||
assert result == ""
|
||||
# Well short of the 10 it was asked for, and short of the deadline it had.
|
||||
assert attempts["n"] < 10
|
||||
budget = bypass._IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
assert clock["now"] < budget, "the loop must leave room to report the failure"
|
||||
|
||||
|
||||
def test_page_load_loop_still_makes_one_attempt_on_a_spent_budget(monkeypatch, bypass):
|
||||
"""The deadline check must never skip the request entirely."""
|
||||
clock = {"now": 10_000.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
monkeypatch.delenv(bypass._BYPASS_CHILD_ENV, raising=False)
|
||||
|
||||
attempts = {"n": 0}
|
||||
|
||||
async def _create(_url):
|
||||
return object()
|
||||
|
||||
async def _get(_url, _driver, _cancel=None) -> str:
|
||||
attempts["n"] += 1
|
||||
return "<html>solved</html>"
|
||||
|
||||
async def _close(_driver) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(bypass, "_create_cdp_browser", _create)
|
||||
monkeypatch.setattr(bypass, "_get", _get)
|
||||
monkeypatch.setattr(bypass, "_close_cdp_driver", _close)
|
||||
|
||||
class _RealWorker:
|
||||
def run(self, coro, timeout=None):
|
||||
return asyncio.run(coro)
|
||||
|
||||
monkeypatch.setattr(bypass, "_CDP_WORKER", _RealWorker())
|
||||
|
||||
assert bypass._run_bypass_in_current_process("https://example.com", 10) == "<html>solved</html>"
|
||||
assert attempts["n"] == 1
|
||||
@@ -0,0 +1,306 @@
|
||||
"""DDoS-Guard cookie reuse between requests.
|
||||
|
||||
Anna's Archive issues nine cookies after a solve, and they are not equivalent:
|
||||
|
||||
__ddg1_/__ddg2_/__ddgid_ ~1 year clearance
|
||||
__ddgmark_ ~1 day
|
||||
__ddg5_ session
|
||||
__ddg8_/__ddg9_/__ddg10_ ~40 min one check: token, CLIENT IP, TIMESTAMP
|
||||
|
||||
Replaying the last three is what produces the ?check=1 redirect loop. They describe a
|
||||
single check, so once the timestamp ages out - or the egress IP changes, routine
|
||||
behind a VPN - DDoS-Guard stops recognising the caller and re-arms the challenge on
|
||||
every request. Storing an expired cookie and sending it forever has the same effect.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.cookie_store as cs
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_cookie_store(monkeypatch):
|
||||
monkeypatch.setattr(cs, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cs, "_cf_user_agents", {})
|
||||
|
||||
|
||||
class _Cookie:
|
||||
"""Stand-in for the CDP cookie objects the bypasser extracts."""
|
||||
|
||||
def __init__(self, name, value="v", expires=None, domain="annas-archive.gl"):
|
||||
self.name = name
|
||||
self.value = value
|
||||
self.expires = expires
|
||||
self.domain = domain
|
||||
self.path = "/"
|
||||
self.secure = True
|
||||
|
||||
|
||||
def _store(cookies, url="https://annas-archive.gl/search"):
|
||||
cs.store_extracted_cookies(url=url, cookies=cookies, user_agent="UA/1.0")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cookie_store_logs():
|
||||
"""Collect cookie-store log messages.
|
||||
|
||||
The store's logger is built outside the standard hierarchy, so its records never
|
||||
reach the root handler caplog installs.
|
||||
"""
|
||||
import logging
|
||||
|
||||
messages: list[str] = []
|
||||
|
||||
class _Capture(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
messages.append(record.getMessage())
|
||||
|
||||
handler = _Capture()
|
||||
cs.logger.addHandler(handler)
|
||||
previous = cs.logger.level
|
||||
cs.logger.setLevel(logging.DEBUG)
|
||||
# setup_logger builds its loggers with CustomLogger(name) rather than getLogger, so
|
||||
# they are not in the manager's hierarchy - and Logger.setLevel only invalidates the
|
||||
# is-enabled cache *through* the manager. Without this the logger keeps answering
|
||||
# "DEBUG is off" from a cache entry made while it was at INFO.
|
||||
cs.logger._cache.clear()
|
||||
try:
|
||||
yield messages
|
||||
finally:
|
||||
cs.logger.removeHandler(handler)
|
||||
cs.logger.setLevel(previous)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Per-check cookies must not be persisted for replay
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_per_check_cookies_are_not_stored():
|
||||
"""The IP/timestamp trio describes one check and must not outlive it."""
|
||||
_store(
|
||||
[
|
||||
_Cookie("__ddg1_", "clearance"),
|
||||
_Cookie("__ddg2_", "clearance2"),
|
||||
_Cookie("__ddg8_", "opaque"),
|
||||
_Cookie("__ddg9_", "203.0.113.7"),
|
||||
_Cookie("__ddg10_", "1786826304"),
|
||||
_Cookie("ddg_last_challenge", "1786826304"),
|
||||
]
|
||||
)
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(stored) == {"__ddg1_", "__ddg2_"}
|
||||
for ephemeral in ("__ddg8_", "__ddg9_", "__ddg10_", "ddg_last_challenge"):
|
||||
assert ephemeral not in stored
|
||||
|
||||
|
||||
def test_clearance_cookies_survive():
|
||||
_store([_Cookie("__ddg1_", "a"), _Cookie("__ddg2_", "b"), _Cookie("__ddgid_", "c")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"__ddg1_": "a", "__ddg2_": "b", "__ddgid_": "c"}
|
||||
|
||||
|
||||
def test_cloudflare_cookies_are_unaffected():
|
||||
_store([_Cookie("cf_clearance", "token"), _Cookie("__cf_bm", "bm")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"cf_clearance": "token", "__cf_bm": "bm"}
|
||||
|
||||
|
||||
def test_per_check_cookies_are_excluded_even_for_full_session_domains(monkeypatch):
|
||||
"""extract_all exists for Z-Library sessions; it must not resurrect the trio."""
|
||||
monkeypatch.setattr(cs, "_get_full_cookie_domains", lambda: {"annas-archive.gl"})
|
||||
_store([_Cookie("sessionid", "s"), _Cookie("__ddg9_", "203.0.113.7")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert "sessionid" in stored
|
||||
assert "__ddg9_" not in stored
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Expiry must be honoured for every cookie, not only cf_clearance
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_expired_ddg_cookies_are_dropped():
|
||||
"""The old code only expiry-checked cf_clearance, so DDoS-Guard domains - which
|
||||
have none - replayed dead cookies forever."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "live"), _Cookie("__ddgmark_", "dead", expires=past)])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_all_cookies_expired_returns_empty_so_caller_re_solves():
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "dead", expires=past)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
assert cs.has_valid_cf_cookies("annas-archive.gl") is False
|
||||
|
||||
|
||||
def test_unexpired_cookies_are_kept():
|
||||
future = int(time.time()) + 3600
|
||||
_store([_Cookie("__ddg1_", "live", expires=future)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_session_cookies_never_expire():
|
||||
"""expires<=0 means a session cookie, not an already-expired one."""
|
||||
_store([_Cookie("__ddg5_", "s", expires=0), _Cookie("__ddg1_", "a", expires=None)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg5_": "s", "__ddg1_": "a"}
|
||||
|
||||
|
||||
def test_expired_cf_clearance_still_drops_the_whole_domain():
|
||||
"""Pre-existing Cloudflare behaviour must not regress."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("cf_clearance", "dead", expires=past), _Cookie("__cf_bm", "bm")])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_expired_cookies_are_pruned_from_the_store():
|
||||
"""A dropped cookie must not linger and be re-evaluated on every request."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "live"), _Cookie("__ddgmark_", "dead", expires=past)])
|
||||
|
||||
ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(cs._cf_cookies["annas-archive.gl"]) == {"__ddg1_"}
|
||||
|
||||
|
||||
def test_solve_that_yields_only_per_check_cookies_stores_nothing():
|
||||
"""No clearance means no reuse - the caller must go back to the bypasser rather
|
||||
than believe it holds a valid session."""
|
||||
_store([_Cookie("__ddg9_", "203.0.113.7"), _Cookie("__ddg10_", "1786826304")])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Rejected cookies are discarded, never retried forever
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _Resp:
|
||||
def __init__(self, status_code, text="page"):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
|
||||
def _seed(monkeypatch):
|
||||
_store([_Cookie("__ddg1_", "clearance"), _Cookie("__ddg2_", "c2")])
|
||||
monkeypatch.setattr(ib, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(ib, "get_ssl_verify", lambda _url: True)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
|
||||
def test_rejected_cached_cookies_are_discarded(monkeypatch):
|
||||
"""A 403 while presenting cookies proves they are dead - keep them and every
|
||||
later request re-presents a known-rejected cookie."""
|
||||
_seed(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
assert (
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl") is None
|
||||
)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_redirect_loop_on_cached_cookies_discards_them(monkeypatch):
|
||||
"""DDoS-Guard answers dead clearance with an endless ?check=1 bounce, which
|
||||
surfaces as an exception rather than a status code."""
|
||||
_seed(monkeypatch)
|
||||
|
||||
def boom(*_a, **_k):
|
||||
raise ib.requests.exceptions.TooManyRedirects("Exceeded 30 redirects")
|
||||
|
||||
monkeypatch.setattr(ib.requests, "get", boom)
|
||||
|
||||
assert (
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl") is None
|
||||
)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_working_cookies_are_kept(monkeypatch):
|
||||
_seed(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(200, "the page"))
|
||||
|
||||
result = ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl")
|
||||
|
||||
assert result == "the page"
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") != {}
|
||||
|
||||
|
||||
def test_failure_only_clears_the_failing_host(monkeypatch):
|
||||
"""clear_cf_cookies('') means every host - a blank hostname must not wipe
|
||||
clearance for sites that are working fine."""
|
||||
_seed(monkeypatch)
|
||||
_store([_Cookie("__ddg1_", "other")], url="https://other-site.test/x")
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl")
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
assert ib.get_cf_cookies_for_domain("other-site.test") == {"__ddg1_": "other"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Settling what DDoS-Guard actually treats as clearance (issue #1276)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_per_check_cookies_can_be_kept_for_a_field_test(monkeypatch):
|
||||
"""Which __ddg* cookies are clearance is not settled, so it has to be testable.
|
||||
|
||||
The store's premise - that __ddg8_/__ddg9_/__ddg10_ describe one check and must not
|
||||
be replayed - is contradicted by the field reports on #1276, where every request
|
||||
after a successful solve was challenged again. This env-only switch is how that gets
|
||||
answered against a live host without building a branch.
|
||||
"""
|
||||
from shelfmark.config import env
|
||||
|
||||
monkeypatch.setattr(env, "DDG_REPLAY_PER_CHECK_COOKIES", True)
|
||||
_store(
|
||||
[
|
||||
_Cookie("__ddg1_", "clearance"),
|
||||
_Cookie("__ddg8_", "opaque"),
|
||||
_Cookie("__ddg9_", "203.0.113.7"),
|
||||
_Cookie("__ddg10_", "1786826304"),
|
||||
]
|
||||
)
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(stored) == {"__ddg1_", "__ddg8_", "__ddg9_", "__ddg10_"}
|
||||
|
||||
|
||||
def test_dropping_per_check_cookies_is_the_default(monkeypatch):
|
||||
"""The switch is for reproducing the question, not a behaviour change."""
|
||||
from shelfmark.config import env
|
||||
|
||||
assert env.DDG_REPLAY_PER_CHECK_COOKIES is False
|
||||
_store([_Cookie("__ddg1_", "clearance"), _Cookie("__ddg9_", "203.0.113.7")])
|
||||
|
||||
assert set(ib.get_cf_cookies_for_domain("annas-archive.gl")) == {"__ddg1_"}
|
||||
|
||||
|
||||
def test_a_solve_logs_which_cookies_it_won_and_which_were_held_back(cookie_store_logs):
|
||||
"""Without this, a debug log shows a solve succeed and the next request challenged,
|
||||
with nothing in between to explain why."""
|
||||
_store([_Cookie("__ddg1_", "clearance"), _Cookie("__ddg9_", "203.0.113.7")])
|
||||
|
||||
messages = cookie_store_logs
|
||||
line = next((m for m in messages if "won" in m and "dropping" in m), None)
|
||||
assert line is not None, messages
|
||||
assert "__ddg1_" in line
|
||||
assert "__ddg9_" in line
|
||||
# Names only - a clearance cookie's value is a credential.
|
||||
assert "clearance" not in line
|
||||
assert "203.0.113.7" not in line
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Tests for the external bypasser flow."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
@@ -55,6 +57,165 @@ def test_fetch_via_bypasser_posts_expected_payload_and_uses_ssl_verify(monkeypat
|
||||
]
|
||||
|
||||
|
||||
def _stub_solution(monkeypatch, external_bypasser, solution: dict) -> None:
|
||||
"""Answer one bypass with `solution`, with config and SSL stubbed out."""
|
||||
|
||||
def fake_get(key, default=""):
|
||||
values = {
|
||||
"EXT_BYPASSER_URL": "https://bypass.example",
|
||||
"EXT_BYPASSER_PATH": "/v1",
|
||||
"EXT_BYPASSER_TIMEOUT": 60000,
|
||||
}
|
||||
return values.get(key, default)
|
||||
|
||||
monkeypatch.setattr(external_bypasser.config, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
external_bypasser.requests,
|
||||
"post",
|
||||
lambda *_a, **_k: _FakeResponse({"status": "ok", "solution": solution}),
|
||||
)
|
||||
monkeypatch.setattr(external_bypasser, "get_ssl_verify", lambda _url: False)
|
||||
|
||||
|
||||
def test_solved_clearance_is_stored_for_reuse(monkeypatch):
|
||||
"""A solve costs tens of seconds of real browser; its clearance must be kept.
|
||||
|
||||
Without this every request paid a 403 plus a full solve, and the file download -
|
||||
which the solver cannot proxy - presented no clearance at all.
|
||||
"""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"userAgent": "Mozilla/5.0 (solver)",
|
||||
"cookies": [
|
||||
{"name": "__ddg1_", "value": "clearance", "domain": ".annas-archive.gl"},
|
||||
{"name": "__ddg2_", "value": "c2", "domain": ".annas-archive.gl"},
|
||||
# Per-check cookies: kept out of the store, same as the internal path.
|
||||
{"name": "__ddg9_", "value": "203.0.113.7", "domain": ".annas-archive.gl"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {
|
||||
"__ddg1_": "clearance",
|
||||
"__ddg2_": "c2",
|
||||
}
|
||||
# Cloudflare ties clearance to the solving UA, so replaying one without the other fails.
|
||||
assert cookie_store.get_cf_user_agent_for_domain("annas-archive.gl") == "Mozilla/5.0 (solver)"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "shape"),
|
||||
[
|
||||
# Byparr drives Playwright/camoufox, whose cookies spell it "expires".
|
||||
("expires", "playwright"),
|
||||
# FlareSolverr assigns driver.get_cookies() - the WebDriver cookie object,
|
||||
# which spells it "expiry". Reading only "expires" made every FlareSolverr
|
||||
# cookie immortal, so dead clearance was replayed forever.
|
||||
("expiry", "webdriver"),
|
||||
],
|
||||
)
|
||||
def test_expired_solution_cookie_is_not_replayed(monkeypatch, field, shape):
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"cookies": [{"name": "__ddg1_", "value": "dead", field: int(time.time()) - 60}],
|
||||
},
|
||||
)
|
||||
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}, (
|
||||
f"a dead {shape} cookie was kept for replay"
|
||||
)
|
||||
|
||||
|
||||
def test_solution_cookie_expiry_is_coerced_not_trusted(monkeypatch):
|
||||
"""The solver is not ours; a stringified expiry must be read, not raised on."""
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"cookies": [
|
||||
{"name": "__ddg1_", "value": "live", "expires": str(int(time.time()) + 3600)},
|
||||
{"name": "__ddg2_", "value": "dead", "expires": str(int(time.time()) - 60)},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_storing_clearance_can_never_discard_the_solved_page(monkeypatch):
|
||||
"""A solve costs ~30s; a surprise in the cookie shape must not throw it away.
|
||||
|
||||
The store call sits inside the request try/except, whose handler returns None -
|
||||
so without its own guard a raising store turned a good page into a failed fetch
|
||||
and sent the caller round for up to MAX_RETRY more solves.
|
||||
"""
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{"response": "<html>ok</html>", "cookies": [{"name": "__ddg1_", "value": "v"}]},
|
||||
)
|
||||
|
||||
def boom(*_args, **_kwargs):
|
||||
raise TypeError("unexpected cookie shape")
|
||||
|
||||
monkeypatch.setattr(external_bypasser, "store_extracted_cookies", boom)
|
||||
|
||||
assert (
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
== "<html>ok</html>"
|
||||
)
|
||||
|
||||
|
||||
def test_solution_without_cookies_is_still_returned(monkeypatch):
|
||||
"""A solver that returns no cookie list must not break the page fetch."""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(monkeypatch, external_bypasser, {"response": "<html>ok</html>"})
|
||||
|
||||
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_get_bypassed_page_retries_and_rotates_selector_between_attempts(monkeypatch):
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""A recording that never happened must say why.
|
||||
|
||||
Issue #1276: the debug bundle's recording/ directory was empty, and the only trace was
|
||||
three "FFmpeg already stopped" debug lines - one per bypass, each logged 20-56s after
|
||||
the recorder was started, meaning ffmpeg had exited almost immediately every time. It ran
|
||||
with `-loglevel 0` and no stderr capture, so nothing anywhere recorded the reason. The
|
||||
screen recording is the single most useful artifact for diagnosing a bypass failure.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_display():
|
||||
before = dict(ib.DISPLAY)
|
||||
ib.DISPLAY["ffmpeg"] = None
|
||||
ib.DISPLAY["ffmpeg_output"] = None
|
||||
ib.DISPLAY["ffmpeg_error_log"] = None
|
||||
yield
|
||||
ib.DISPLAY.update(before)
|
||||
|
||||
|
||||
class _Proc:
|
||||
def __init__(self, returncode):
|
||||
self.returncode = returncode
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
|
||||
def test_ffmpeg_errors_are_captured_to_a_file_beside_the_recording(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(ib, "RECORDING_DIR", tmp_path)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_popen(cmd, **kwargs):
|
||||
captured["cmd"] = cmd
|
||||
captured["stderr"] = kwargs.get("stderr")
|
||||
return _Proc(None)
|
||||
|
||||
monkeypatch.setattr(ib.subprocess, "Popen", fake_popen)
|
||||
|
||||
ib._start_ffmpeg_recording(display=":99")
|
||||
|
||||
cmd = captured["cmd"]
|
||||
# Errors must not be thrown away any more.
|
||||
assert "-loglevel" in cmd
|
||||
assert cmd[cmd.index("-loglevel") + 1] == "error"
|
||||
# stderr goes to a real file, not a pipe nothing would drain.
|
||||
assert captured["stderr"] is not None
|
||||
assert captured["stderr"] is not subprocess.PIPE
|
||||
|
||||
error_log = ib.DISPLAY["ffmpeg_error_log"]
|
||||
assert error_log is not None
|
||||
assert error_log.parent == tmp_path
|
||||
# It sits beside the mp4, so it travels in the debug bundle.
|
||||
assert error_log.name.startswith("screen_recording_")
|
||||
|
||||
|
||||
def test_an_early_exit_is_reported_with_ffmpegs_own_reason(monkeypatch, tmp_path, caplog):
|
||||
reason = "[x11grab @ 0x1] Cannot open display :99, error 1."
|
||||
error_log = tmp_path / "screen_recording_x.ffmpeg.log"
|
||||
error_log.write_text(reason, encoding="utf-8")
|
||||
|
||||
ib.DISPLAY["ffmpeg"] = _Proc(1)
|
||||
ib.DISPLAY["ffmpeg_output"] = tmp_path / "screen_recording_x.mp4"
|
||||
ib.DISPLAY["ffmpeg_error_log"] = error_log
|
||||
|
||||
messages: list[str] = []
|
||||
|
||||
class _Capture:
|
||||
def emit(self, record):
|
||||
messages.append(record.getMessage())
|
||||
|
||||
import logging
|
||||
|
||||
handler = logging.Handler()
|
||||
handler.emit = _Capture().emit # type: ignore[method-assign]
|
||||
ib.logger.addHandler(handler)
|
||||
previous = ib.logger.level
|
||||
ib.logger.setLevel(logging.DEBUG)
|
||||
ib.logger._cache.clear()
|
||||
try:
|
||||
ib._stop_ffmpeg_recording()
|
||||
finally:
|
||||
ib.logger.removeHandler(handler)
|
||||
ib.logger.setLevel(previous)
|
||||
|
||||
line = next((m for m in messages if "exited early" in m), None)
|
||||
assert line is not None, messages
|
||||
assert "code 1" in line
|
||||
assert "Cannot open display" in line
|
||||
assert ib.DISPLAY["ffmpeg"] is None
|
||||
|
||||
|
||||
def test_summary_is_explicit_when_ffmpeg_logged_nothing(tmp_path):
|
||||
empty = tmp_path / "screen_recording_y.ffmpeg.log"
|
||||
empty.write_text("", encoding="utf-8")
|
||||
ib.DISPLAY["ffmpeg_error_log"] = empty
|
||||
|
||||
assert "logged nothing" in ib._ffmpeg_error_summary()
|
||||
|
||||
|
||||
def test_summary_survives_a_missing_log():
|
||||
ib.DISPLAY["ffmpeg_error_log"] = None
|
||||
|
||||
assert "No FFmpeg error log" in ib._ffmpeg_error_summary()
|
||||
@@ -1,5 +1,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -113,7 +115,9 @@ def test_extract_cookies_from_cdp_keeps_full_session_cookies_for_configured_zlib
|
||||
async def evaluate(self, _expr):
|
||||
return "TestUA/1.0"
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "_get_full_cookie_domains", lambda: {"z-lib.fm"})
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_get_full_cookie_domains", lambda: {"z-lib.fm"})
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
asyncio.run(
|
||||
@@ -165,12 +169,14 @@ def test_extract_cookies_from_cdp_normalizes_session_expiry():
|
||||
)
|
||||
)
|
||||
|
||||
stored = internal_bypasser._cf_cookies.get("example.com", {})
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
stored = cookie_store._cf_cookies.get("example.com", {})
|
||||
assert stored["cf_clearance"]["expiry"] is None
|
||||
assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {"cf_clearance": "abc"}
|
||||
|
||||
# Verify fallback to "expires" key for expiry checks
|
||||
internal_bypasser._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10
|
||||
cookie_store._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10
|
||||
assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {}
|
||||
|
||||
|
||||
@@ -374,7 +380,9 @@ def test_try_with_cached_cookies_returns_none_on_request_exception(monkeypatch):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
internal_bypasser._cf_cookies["example.com"] = {
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
cookie_store._cf_cookies["example.com"] = {
|
||||
"cf_clearance": {
|
||||
"value": "abc",
|
||||
"domain": "example.com",
|
||||
@@ -482,3 +490,239 @@ def test_run_bypass_in_current_process_bounds_its_wait(monkeypatch):
|
||||
|
||||
assert result == "html"
|
||||
assert observed["timeout"] == internal_bypasser._IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def _write_fake_proc_entry(proc_root, pid: int, pgid: int, argv: list[str]) -> None:
|
||||
"""Create a /proc-shaped entry for a fake process."""
|
||||
entry = proc_root / str(pid)
|
||||
entry.mkdir()
|
||||
(entry / "cmdline").write_bytes(b"\0".join(arg.encode() for arg in argv) + b"\0")
|
||||
# pid (comm) state ppid pgrp ... - comm is parenthesised and may contain spaces.
|
||||
(entry / "stat").write_text(f"{pid} (some (odd) name) S 1 {pgid} {pgid} 0 -1 4194304 0 0")
|
||||
|
||||
|
||||
def test_cleanup_only_kills_own_and_abandoned_browser_sessions(monkeypatch, tmp_path):
|
||||
"""Regression test for issue #1231: the sweep used a container-wide `pkill -f chrome`,
|
||||
so every worker that started a bypass killed the browsers the other workers were
|
||||
still driving. Only our own process group and groups whose leader is gone are ours."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
proc_root = tmp_path / "proc"
|
||||
proc_root.mkdir()
|
||||
_write_fake_proc_entry(proc_root, 1000, 1000, ["python", "-m", "shelfmark.bypass"])
|
||||
_write_fake_proc_entry(proc_root, 1001, 1000, ["/usr/bin/chromium", "--headless"])
|
||||
_write_fake_proc_entry(proc_root, 1002, 1000, ["Xvfb", ":99"])
|
||||
# Live sibling session: another worker is solving a challenge with these right now.
|
||||
_write_fake_proc_entry(proc_root, 2000, 2000, ["python", "-m", "shelfmark.bypass"])
|
||||
_write_fake_proc_entry(proc_root, 2001, 2000, ["/usr/bin/chromium", "--headless"])
|
||||
# Abandoned session: its leader (pid 3000) is gone, so its browser really is an orphan.
|
||||
_write_fake_proc_entry(proc_root, 3001, 3000, ["/usr/bin/chromium", "--headless"])
|
||||
|
||||
killed: list[int] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.env, "DOCKERMODE", True)
|
||||
monkeypatch.setattr(internal_bypasser, "_PROC_ROOT", proc_root)
|
||||
monkeypatch.setattr(internal_bypasser.os, "getpid", lambda: 1000)
|
||||
monkeypatch.setattr(internal_bypasser.os, "getpgrp", lambda: 1000)
|
||||
monkeypatch.setattr(internal_bypasser.os, "kill", lambda pid, _sig: killed.append(pid))
|
||||
monkeypatch.setattr(internal_bypasser.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert internal_bypasser._cleanup_orphan_processes() == 3
|
||||
assert sorted(killed) == [1001, 1002, 3001]
|
||||
|
||||
|
||||
def test_cleanup_is_skipped_without_proc(monkeypatch, tmp_path):
|
||||
"""Without /proc there is no way to tell sessions apart, so kill nothing."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.env, "DOCKERMODE", True)
|
||||
monkeypatch.setattr(internal_bypasser, "_PROC_ROOT", tmp_path / "missing")
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.os, "kill", lambda *_args: pytest.fail("must not kill anything")
|
||||
)
|
||||
|
||||
assert internal_bypasser._cleanup_orphan_processes() == 0
|
||||
|
||||
|
||||
class _FakeHelperStdin:
|
||||
"""The request pipe: a write is how the helper receives one request."""
|
||||
|
||||
def __init__(self, process):
|
||||
self._process = process
|
||||
self.closed = False
|
||||
|
||||
def write(self, data):
|
||||
self._process.serve(data)
|
||||
|
||||
def flush(self):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeHelperProcess:
|
||||
"""Stand-in for the bypass helper subprocess.
|
||||
|
||||
The helper serves one request per line of stdin and answers by writing the result file
|
||||
the request named, so that is what this fakes: a write produces an answer.
|
||||
"""
|
||||
|
||||
def __init__(self, *_args, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.pid = 4242
|
||||
self.returncode = None
|
||||
self.answers = True
|
||||
self.killed = False
|
||||
self.waited = False
|
||||
self.stdin = _FakeHelperStdin(self)
|
||||
self.requests: list[dict] = []
|
||||
|
||||
def serve(self, payload):
|
||||
request = json.loads(payload)
|
||||
self.requests.append(request)
|
||||
if not self.answers:
|
||||
return
|
||||
result = {"ok": True, "html": "<html>solved</html>", "cookies": {}, "user_agents": {}}
|
||||
Path(request["result_path"]).write_text(json.dumps(result), encoding="utf-8")
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def kill(self):
|
||||
self.killed = True
|
||||
self.returncode = -9
|
||||
|
||||
def wait(self, timeout=None):
|
||||
self.waited = True
|
||||
if self.returncode is None:
|
||||
self.returncode = 0
|
||||
return self.returncode
|
||||
|
||||
|
||||
def _patch_helper_subprocess(monkeypatch, internal_bypasser, process, killed_groups):
|
||||
monkeypatch.setattr(internal_bypasser.subprocess, "Popen", lambda *a, **kw: process(*a, **kw))
|
||||
monkeypatch.setattr(internal_bypasser.network, "get_dns_config", dict)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.os, "killpg", lambda pgid, _sig: killed_groups.append(pgid)
|
||||
)
|
||||
# A fresh helper per test: the module-level one is shared, and a process parked by one
|
||||
# test would be handed to the next.
|
||||
helper = internal_bypasser._BypassHelper()
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 0.0)
|
||||
monkeypatch.setattr(internal_bypasser, "_BYPASS_HELPER", helper)
|
||||
return helper
|
||||
|
||||
|
||||
def test_helper_runs_in_its_own_session_and_is_torn_down(monkeypatch):
|
||||
"""Regression test for issue #1231: the helper's Chrome and Xvfb must belong to the
|
||||
helper's own process group, and the whole group must die with it - otherwise the
|
||||
leftovers break the next worker's browser and can only be cleared by a sweep broad
|
||||
enough to kill a concurrent worker's browser too.
|
||||
|
||||
The helper outlives a single request, so the teardown happens when it is dropped rather
|
||||
than after every solve. Each bypass still closes its own browser, so what survives in
|
||||
between is the process, not a Chrome.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
helper = _patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
|
||||
assert internal_bypasser._get_via_subprocess("https://example.com", 1) == "<html>solved</html>"
|
||||
assert processes[0].kwargs["start_new_session"] is True
|
||||
assert killed_groups == [], "the helper was torn down after a single request"
|
||||
|
||||
helper._discard()
|
||||
|
||||
assert killed_groups == [processes[0].pid]
|
||||
|
||||
|
||||
def test_helper_serves_a_second_request_without_respawning(monkeypatch):
|
||||
"""The interpreter start and imports are paid once, not per protected request."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
_patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
|
||||
internal_bypasser._get_via_subprocess("https://example.com/one", 1)
|
||||
internal_bypasser._get_via_subprocess("https://example.com/two", 1)
|
||||
|
||||
assert len(processes) == 1
|
||||
assert [request["url"] for request in processes[0].requests] == [
|
||||
"https://example.com/one",
|
||||
"https://example.com/two",
|
||||
]
|
||||
|
||||
|
||||
def test_helper_timeout_kills_the_whole_session(monkeypatch):
|
||||
"""A timed-out solve must not leave a live browser behind for the next worker."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
process.answers = False # accepts the request, never writes a result
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
_patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
monkeypatch.setattr(internal_bypasser, "_BYPASS_SUBPROCESS_TIMEOUT_SECONDS", 0.1)
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
internal_bypasser._get_via_subprocess("https://example.com", 1)
|
||||
|
||||
assert killed_groups == [processes[0].pid]
|
||||
assert processes[0].killed is True
|
||||
|
||||
|
||||
def test_helper_takes_the_browser_down_when_its_parent_dies(monkeypatch):
|
||||
"""Cleanup only reclaims process groups whose leader is gone (#1231), so an orphaned
|
||||
helper must not sit there holding a browser no later bypass is allowed to touch."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
terminated: list[str] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.os, "getppid", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser, "_terminate_own_session", lambda: terminated.append("terminated")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.time, "sleep", lambda _seconds: pytest.fail("should not wait")
|
||||
)
|
||||
|
||||
internal_bypasser._watch_parent_process(999, interval=0.0)
|
||||
|
||||
assert terminated == ["terminated"]
|
||||
|
||||
|
||||
def test_helper_watchdog_waits_while_its_parent_is_alive(monkeypatch):
|
||||
"""The watchdog must only fire on a changed ppid, not on every poll."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
ppids = iter([999, 999, 1])
|
||||
sleeps: list[float] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.os, "getppid", lambda: next(ppids))
|
||||
monkeypatch.setattr(internal_bypasser, "_terminate_own_session", lambda: None)
|
||||
monkeypatch.setattr(internal_bypasser.time, "sleep", sleeps.append)
|
||||
|
||||
internal_bypasser._watch_parent_process(999, interval=0.5)
|
||||
|
||||
assert sleeps == [0.5, 0.5]
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""A 429 is throttling, not a dead clearance cookie.
|
||||
|
||||
Issue #1276: every rejection of the cached cookies took the same exit, which cleared the
|
||||
host's clearance. That is right for a 403 and for the ?check=1 redirect loop - being
|
||||
challenged while presenting a cookie proves the cookie is dead - and wrong for a 429,
|
||||
where the origin is rate-limiting the IP and would answer a real browser holding the very
|
||||
same cookies identically.
|
||||
|
||||
The cost in the reported bundle: a solve completed at 13:41:23 and stored five cookies;
|
||||
six seconds later a 429 threw them away, and the next query bought its own 56-second
|
||||
browser solve. Reuse rate across the whole log was 0 of 2.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.cookie_store as cs
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
URL = "https://annas-archive.gl/search?q=dune"
|
||||
HOST = "annas-archive.gl"
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code, text="page"):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean(monkeypatch):
|
||||
monkeypatch.setattr(cs, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cs, "_cf_user_agents", {})
|
||||
monkeypatch.setattr(cs, "_get_full_cookie_domains", set)
|
||||
monkeypatch.setattr(ib, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(ib, "get_ssl_verify", lambda _url: True)
|
||||
cs._cf_cookies[HOST] = {
|
||||
"__ddg1_": {"value": "clearance", "expiry": None},
|
||||
"__ddg2_": {"value": "c2", "expiry": None},
|
||||
}
|
||||
|
||||
|
||||
def _cooldowns(monkeypatch):
|
||||
"""Record note_rate_limited calls without arming the real per-host ladder."""
|
||||
armed: list[str] = []
|
||||
monkeypatch.setattr(ib.network, "note_rate_limited", lambda url: armed.append(url) or 120.0)
|
||||
return armed
|
||||
|
||||
|
||||
def test_429_keeps_the_clearance(monkeypatch):
|
||||
armed = _cooldowns(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(429))
|
||||
|
||||
assert ib._try_with_cached_cookies(URL, HOST) is None
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {"__ddg1_": "clearance", "__ddg2_": "c2"}
|
||||
assert armed == [URL], "the backoff must still be armed"
|
||||
|
||||
|
||||
def test_403_still_discards_the_clearance(monkeypatch):
|
||||
"""The pre-existing behaviour for a genuine rejection must not regress."""
|
||||
_cooldowns(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
assert ib._try_with_cached_cookies(URL, HOST) is None
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {}
|
||||
|
||||
|
||||
def test_redirect_loop_still_discards_the_clearance(monkeypatch):
|
||||
_cooldowns(monkeypatch)
|
||||
|
||||
def boom(*_a, **_k):
|
||||
raise ib.requests.exceptions.TooManyRedirects("Exceeded 30 redirects")
|
||||
|
||||
monkeypatch.setattr(ib.requests, "get", boom)
|
||||
|
||||
assert ib._try_with_cached_cookies(URL, HOST) is None
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {}
|
||||
|
||||
|
||||
def test_a_throttled_host_is_not_handed_a_browser_solve(monkeypatch):
|
||||
"""A solve cannot clear a throttle, and is itself more traffic at a host asking for
|
||||
less. get_bypassed_page checks the cooldown before the queue; get() has to re-check
|
||||
after it, because a request can hold for LOCKED while another collects the 429."""
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(429))
|
||||
monkeypatch.setattr(ib.network, "note_rate_limited", lambda _url: 120.0)
|
||||
monkeypatch.setattr(ib.network, "host_cooldown_remaining", lambda _url: 118.0)
|
||||
|
||||
solved: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
ib, "_run_bypass_in_current_process", lambda url, *a, **k: solved.append(url) or "html"
|
||||
)
|
||||
monkeypatch.setattr(ib.env, "DOCKERMODE", False)
|
||||
|
||||
with pytest.raises(ib.network.RateLimitedError) as excinfo:
|
||||
ib.get(URL, retry=1)
|
||||
|
||||
assert solved == [], "no browser should have been started"
|
||||
assert "rate-limited" in str(excinfo.value)
|
||||
# And the clearance survives, ready for when the cooldown clears.
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {"__ddg1_": "clearance", "__ddg2_": "c2"}
|
||||
|
||||
|
||||
def test_a_host_that_is_not_throttled_still_solves(monkeypatch):
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
monkeypatch.setattr(ib.network, "host_cooldown_remaining", lambda _url: 0.0)
|
||||
|
||||
solved: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
ib, "_run_bypass_in_current_process", lambda url, *a, **k: solved.append(url) or "html"
|
||||
)
|
||||
monkeypatch.setattr(ib.env, "DOCKERMODE", False)
|
||||
|
||||
assert ib.get(URL, retry=1) == "html"
|
||||
assert solved == [URL]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user