mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 22:00:21 +01:00
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9452ebc70d | ||
|
|
d3f4ccd79a | ||
|
|
cb690b45b8 | ||
|
|
3d7ea40088 | ||
|
|
633004ecf0 | ||
|
|
c06b8ce8ef | ||
|
|
69ff0d6a78 | ||
|
|
3937ae119b | ||
|
|
d7fe28595c | ||
|
|
68c0e83330 | ||
|
|
faaa119884 | ||
|
|
be41a92436 | ||
|
|
7de9319c7a | ||
|
|
97e289ae13 | ||
|
|
c95ee72ad5 | ||
|
|
b25acdb2ad | ||
|
|
7569aaecc5 | ||
|
|
f441b85da2 | ||
|
|
02b7e9d958 | ||
|
|
ff06a1a581 | ||
|
|
463ef49ac3 | ||
|
|
a5595cf9f1 | ||
|
|
9bcf595111 | ||
|
|
65e2e3be20 | ||
|
|
ddc26f01b6 | ||
|
|
89104ae80f |
@@ -25,14 +25,14 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
uses: github/codeql-action/autobuild@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v3
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
||||
@@ -236,6 +236,7 @@ pyrightconfig.json
|
||||
*.local.*
|
||||
AGENTS.md
|
||||
.claude/
|
||||
CLAUDE.md
|
||||
.nvmrc
|
||||
.playwright-mcp/
|
||||
frontend-dist/
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ RUN npm run build
|
||||
FROM ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1 AS uv
|
||||
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14.7-slim@sha256:ce40764625a4ff50df3548277632e7f96c4e77fe75fa848aae9885476e7df5a4 AS base
|
||||
FROM python:3.14.7-slim@sha256:cae66f2ef0ec51a9891263eeee7f987dacf0a9879e8aa9353d5606e0530619a5 AS base
|
||||
|
||||
# Add build argument for version
|
||||
ARG BUILD_VERSION
|
||||
|
||||
@@ -276,6 +276,27 @@ class DownloadHandler(ABC):
|
||||
pass
|
||||
```
|
||||
|
||||
### Optional: Listing Files Before Download
|
||||
|
||||
Some releases bundle several books (a whole-series torrent). Shelfmark inspects a
|
||||
release before queueing it so the user can review how it will be split into books.
|
||||
Override `list_files` when your source can enumerate a release's files without
|
||||
downloading it; the default returns `None`, which the UI reports as "can't inspect":
|
||||
|
||||
```python
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""Return the release's files (release-relative paths + sizes), or None."""
|
||||
torrent_bytes = ... # e.g. fetch the .torrent, or scrape the indexer's detail page
|
||||
return extract_file_list_from_torrent(torrent_bytes) # from download.clients.torrent_utils
|
||||
```
|
||||
|
||||
`release_data` is the same payload the frontend sends to `/api/releases/download`
|
||||
(`source_id`, `download_url`, `content_type`, `series_name`, ...). Built-in examples:
|
||||
Prowlarr parses the `.torrent` it already fetches (magnet-only releases return
|
||||
`None`), and AudiobookBay reads the file table off its detail page.
|
||||
|
||||
### Download Method Parameters
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|
||||
@@ -247,7 +247,7 @@ Seconds since the last WireGuard handshake before the healthcheck bounces the tu
|
||||
| `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). | string | _none_ |
|
||||
| `AUDIOBOOK_LIBRARY_URL` | Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text. | string | _none_ |
|
||||
| `SUPPORTED_FORMATS` | Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found. | string (comma-separated) | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
|
||||
| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar` |
|
||||
| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3,m4a,mp4,flac,ogg,wma,aac,wav,opus,zip,rar` |
|
||||
| `BOOK_LANGUAGE` | Default language filter for searches. | string (comma-separated) | `en` |
|
||||
|
||||
<details>
|
||||
@@ -296,16 +296,7 @@ Book formats to include in search results. ZIP/RAR archives are extracted automa
|
||||
Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar`
|
||||
|
||||
#### `BOOK_LANGUAGE`
|
||||
|
||||
**Default Book Languages**
|
||||
|
||||
Default language filter for searches.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `en`
|
||||
- **Default:** `m4b,mp3,m4a,mp4,flac,ogg,wma,aac,wav,opus,zip,rar`
|
||||
|
||||
</details>
|
||||
|
||||
@@ -314,6 +305,7 @@ Default language filter for searches.
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `SEARCH_MODE` | How you want to search for and download books. | string (choice) | `universal` |
|
||||
| `BOOK_LANGUAGE` | Default language filter for searches. Users can override this for their own account. | string (comma-separated) | `en` |
|
||||
| `AA_DEFAULT_SORT` | Default sort order for search results. | string (choice) | `relevance` |
|
||||
| `SHOW_RELEASE_SOURCE_LINKS` | Show clickable release-source links in release and details modals. Metadata provider links stay enabled. | boolean | `true` |
|
||||
| `SHOW_COMBINED_SELECTOR` | Show the option to search for and download both a book and audiobook together. | boolean | `true` |
|
||||
@@ -337,6 +329,15 @@ How you want to search for and download books.
|
||||
- **Default:** `universal`
|
||||
- **Options:** `direct` (Direct), `universal` (Universal)
|
||||
|
||||
#### `BOOK_LANGUAGE`
|
||||
|
||||
**Default Book Languages**
|
||||
|
||||
Default language filter for searches. Users can override this for their own account.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `en`
|
||||
|
||||
#### `AA_DEFAULT_SORT`
|
||||
|
||||
**Default Sort Order**
|
||||
@@ -749,6 +750,7 @@ Automatically open the downloads sidebar when a new download is queued.
|
||||
Automatically download completed files to your browser for the selected content types.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
|
||||
- **Default:** _empty list_
|
||||
|
||||
#### `MAX_CONCURRENT_DOWNLOADS`
|
||||
@@ -1313,8 +1315,9 @@ Apply per-indexer seed time and ratio preferences from Prowlarr when sending tor
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `NEWZNAB_ENABLED` | Enable searching for books via a Newznab-compatible indexer | boolean | `false` |
|
||||
| `NEWZNAB_URL` | Base URL of your Newznab indexer or aggregator | string | _none_ |
|
||||
| `NEWZNAB_API_KEY` | Your Newznab API key (leave blank if not required) | string (secret) | _none_ |
|
||||
| `NEWZNAB_INDEXERS` | Named Newznab connections. Each row accepts `name`, `url`, and `api_key`. | JSON array | `[]` |
|
||||
| `NEWZNAB_URL` | Legacy single-indexer URL, used when `NEWZNAB_INDEXERS` is empty | string | _none_ |
|
||||
| `NEWZNAB_API_KEY` | Legacy single-indexer API key | string (secret) | _none_ |
|
||||
| `NEWZNAB_EBOOK_CATEGORIES` | Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000. | string (comma-separated) | `7000` |
|
||||
| `NEWZNAB_AUDIOBOOK_CATEGORIES` | Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030. | string (comma-separated) | `3030` |
|
||||
| `NEWZNAB_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
|
||||
@@ -1331,21 +1334,36 @@ Enable searching for books via a Newznab-compatible indexer
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `NEWZNAB_INDEXERS`
|
||||
|
||||
**Named Indexers**
|
||||
|
||||
Configure multiple named Newznab-compatible indexers. The name is shown beside each search result. For environment-based configuration, provide a JSON array:
|
||||
|
||||
```json
|
||||
[
|
||||
{"name":"NZBGeek","url":"https://api.nzbgeek.info","api_key":"..."},
|
||||
{"name":"DrunkenSlug","url":"https://drunkenslug.com","api_key":"..."}
|
||||
]
|
||||
```
|
||||
|
||||
- **Type:** JSON array
|
||||
- **Default:** `[]`
|
||||
|
||||
#### `NEWZNAB_URL`
|
||||
|
||||
**Newznab URL**
|
||||
**Legacy Newznab URL**
|
||||
|
||||
Base URL of your Newznab indexer or aggregator
|
||||
Single-indexer fallback used only when `NEWZNAB_INDEXERS` is empty.
|
||||
|
||||
- **Type:** string
|
||||
- **Default:** _none_
|
||||
- **Required:** Yes
|
||||
|
||||
#### `NEWZNAB_API_KEY`
|
||||
|
||||
**API Key**
|
||||
**Legacy API Key**
|
||||
|
||||
Your Newznab API key (leave blank if not required)
|
||||
API key for the legacy Newznab URL.
|
||||
|
||||
- **Type:** string (secret)
|
||||
- **Default:** _none_
|
||||
@@ -2161,6 +2179,7 @@ Enable Moly.hu as a metadata provider for book searches
|
||||
| `SOURCE_PRIORITY` | Fallback sources, may have waiting. Requires bypasser. Drag to reorder. | JSON array | _see UI for defaults_ |
|
||||
| `MAX_RETRY` | Maximum retry attempts for failed downloads. | number | `10` |
|
||||
| `DEFAULT_SLEEP` | Wait time between download retry attempts. | number | `5` |
|
||||
| `RELEASE_SEARCH_TIMEOUT` | How long one release search may run before it gives up and reports why. A first search on a cold start pays for a browser solve, so leave room for one. If you use a reverse proxy, its read timeout should be at least this high or it will cut the search off with a 504 first. | number | `300` |
|
||||
| `AA_CONTENT_TYPE_ROUTING` | Override destination based on content type metadata. | boolean | `false` |
|
||||
| `AA_CONTENT_TYPE_DIR_FICTION` | Fiction Books | string | _none_ |
|
||||
| `AA_CONTENT_TYPE_DIR_NON_FICTION` | Non-Fiction Books | string | _none_ |
|
||||
@@ -2239,6 +2258,16 @@ Wait time between download retry attempts.
|
||||
- **Default:** `5`
|
||||
- **Constraints:** min: 1, max: 60
|
||||
|
||||
#### `RELEASE_SEARCH_TIMEOUT`
|
||||
|
||||
**Release Search Timeout (seconds)**
|
||||
|
||||
How long one release search may run before it gives up and reports why. A first search on a cold start pays for a browser solve, so leave room for one. If you use a reverse proxy, its read timeout should be at least this high or it will cut the search off with a 504 first.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `300`
|
||||
- **Constraints:** min: 30, max: 1800
|
||||
|
||||
#### `AA_CONTENT_TYPE_ROUTING`
|
||||
|
||||
**Enable Content-Type Routing**
|
||||
@@ -2315,6 +2344,7 @@ Override destination based on content type metadata.
|
||||
| `EXT_BYPASSER_URL` | URL of the external bypasser service (e.g., FlareSolverr). | string | `http://flaresolverr:8191` |
|
||||
| `EXT_BYPASSER_PATH` | API path for the external bypasser. | string | `/v1` |
|
||||
| `EXT_BYPASSER_TIMEOUT` | Timeout for external bypasser requests in milliseconds. | number | `60000` |
|
||||
| `BYPASS_PAGE_SOURCE_TIMEOUT` | How long to wait for a solved page to produce its content before the bypass is retried. Raise it if solves succeed but searches still fail. | number | `20` |
|
||||
| `BYPASS_BROWSER_IDLE_TIMEOUT` | How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner. | number | `180` |
|
||||
|
||||
<details>
|
||||
@@ -2371,6 +2401,16 @@ Timeout for external bypasser requests in milliseconds.
|
||||
- **Requires restart:** Yes
|
||||
- **Constraints:** min: 10000, max: 300000
|
||||
|
||||
#### `BYPASS_PAGE_SOURCE_TIMEOUT`
|
||||
|
||||
**Page Read Timeout (seconds)**
|
||||
|
||||
How long to wait for a solved page to produce its content before the bypass is retried. Raise it if solves succeed but searches still fail.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `20`
|
||||
- **Constraints:** min: 1, max: 120
|
||||
|
||||
#### `BYPASS_BROWSER_IDLE_TIMEOUT`
|
||||
|
||||
**Bypasser Idle Timeout (seconds)**
|
||||
|
||||
@@ -30,7 +30,7 @@ Requires mounting your Calibre-Web `app.db` to `/auth/app.db`.
|
||||
|
||||
Admins can configure per-user settings by editing a user in the user management panel. Non-admin users can also edit their own settings through **My Account** (accessible from the user menu). Admins control which sections are visible in My Account via the **Visible Self-Settings Sections** option.
|
||||
|
||||
There are three categories of per-user settings:
|
||||
There are four categories of per-user settings:
|
||||
|
||||
### Delivery Preferences
|
||||
|
||||
@@ -42,6 +42,15 @@ Override where a user's downloads are sent. Options depend on the global output
|
||||
- **BookLore library/path** — Per-user BookLore target (when using BookLore output mode)
|
||||
- **Email recipient** — Per-user email address (when using Email output mode)
|
||||
|
||||
### Search Preferences
|
||||
|
||||
Override how a user searches, on top of the global search defaults:
|
||||
|
||||
- **Search mode** — Direct or Universal for this user
|
||||
- **Default book languages** — The languages a user's searches fall back to when they don't pick one themselves. Useful for a shared instance where readers want different languages.
|
||||
- **Metadata providers** — Book, audiobook, and combined-mode provider for this user
|
||||
- **Default release sources** — The release tab opened first for books and audiobooks
|
||||
|
||||
### Notifications
|
||||
|
||||
Users can configure personal notification routes, separate from the global notification settings. Each route targets a URL (e.g. an Apprise-compatible endpoint) and can be scoped to specific event types or all events.
|
||||
|
||||
+2
-2
@@ -32,7 +32,7 @@ dependencies = [
|
||||
browser = [
|
||||
"pyvirtualdisplay",
|
||||
"pyautogui",
|
||||
"seleniumbase==4.52.1",
|
||||
"seleniumbase==4.53.5",
|
||||
"python-xlib",
|
||||
]
|
||||
|
||||
@@ -43,7 +43,7 @@ dev = [
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pytest-xdist>=3.8.0",
|
||||
"ruff==0.16.4",
|
||||
"ruff==0.16.5",
|
||||
"vulture>=2.14",
|
||||
]
|
||||
|
||||
|
||||
@@ -147,6 +147,7 @@ See the full [Environment Variables Reference](docs/environment-variables.md) fo
|
||||
Some of the additional options available in Settings:
|
||||
- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
|
||||
- **Additional audiobook sources** - Configure additional sources for audiobook discovery
|
||||
- **Direct Download mirrors** - Supply your own Anna's Archive mirror URLs; Auto mode tries them in the order listed. The `annas-archive.is` domain does not currently work as a source — use `annas-archive.gl` instead (checked August 2026; mirror availability changes)
|
||||
- **IRC** - Add details for IRC book sources and download directly from the UI. Most networks serve audiobooks from the same channel as ebooks (on `irc.irchighway.net` that's `#ebooks`, while `#bookz` is effectively inactive), so leave the separate audiobook channel blank unless your network actually indexes one. IRC audiobooks usually arrive as ZIP/RAR archives — keep those enabled under Supported Audiobook Formats or the releases are filtered out of results
|
||||
- **Library Link** - Add a link to your Calibre-Web or Grimmory instance in the UI header
|
||||
- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
|
||||
|
||||
@@ -3,3 +3,14 @@
|
||||
|
||||
class BypassCancelledError(Exception):
|
||||
"""Raised when a bypass operation is cancelled."""
|
||||
|
||||
|
||||
class ChallengeNotSolvedError(Exception):
|
||||
"""Raised when a bypasser ran but the site still answered with a challenge.
|
||||
|
||||
Distinct from a bypasser that is broken or unreachable, which is what every
|
||||
"the bypass failed" message used to say. A solver can do its job perfectly and
|
||||
still be handed something it cannot clear - DDoS-Guard's manual CAPTCHA page is
|
||||
the case from #1292 - and telling the user to go check that FlareSolverr is
|
||||
reachable sends them to fix a service that is working.
|
||||
"""
|
||||
|
||||
@@ -71,11 +71,18 @@ def _get_full_cookie_domains() -> set[str]:
|
||||
return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()}
|
||||
|
||||
|
||||
def _replay_per_check_cookies() -> bool:
|
||||
"""Whether the per-check trio is kept rather than dropped (see env.py)."""
|
||||
from shelfmark.config import env
|
||||
|
||||
return env.DDG_REPLAY_PER_CHECK_COOKIES
|
||||
|
||||
|
||||
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
|
||||
"""Determine if a cookie should be extracted based on its name."""
|
||||
# Checked before extract_all: a per-check token is wrong to replay for every
|
||||
# domain, including the full-session ones.
|
||||
if name in DDG_EPHEMERAL_COOKIE_NAMES:
|
||||
if name in DDG_EPHEMERAL_COOKIE_NAMES and not _replay_per_check_cookies():
|
||||
return False
|
||||
if extract_all:
|
||||
return True
|
||||
@@ -138,9 +145,11 @@ def store_extracted_cookies(
|
||||
extract_all = base_domain in _get_full_cookie_domains()
|
||||
|
||||
cookies_found: dict[str, dict[str, Any]] = {}
|
||||
dropped: list[str] = []
|
||||
for cookie in cookies:
|
||||
name = _cookie_field(cookie, "name") or ""
|
||||
if not _should_extract_cookie(name, extract_all=extract_all):
|
||||
dropped.append(name)
|
||||
continue
|
||||
secure = _cookie_field(cookie, "secure")
|
||||
cookies_found[name] = {
|
||||
@@ -152,6 +161,18 @@ def store_extracted_cookies(
|
||||
"httpOnly": True,
|
||||
}
|
||||
|
||||
# Names only, never values. Which cookies a solve won, and which of them were held
|
||||
# back, is the evidence needed to settle what DDoS-Guard actually treats as clearance
|
||||
# (issue #1276) - and without it a debug log shows a solve succeeding and the next
|
||||
# request being challenged with nothing in between to explain why.
|
||||
logger.debug(
|
||||
"Solve on %s won %s; keeping %s; dropping %s",
|
||||
base_domain,
|
||||
sorted({_cookie_field(c, "name") or "" for c in cookies}),
|
||||
sorted(cookies_found),
|
||||
sorted(set(dropped)) or "nothing",
|
||||
)
|
||||
|
||||
if not cookies_found:
|
||||
return
|
||||
|
||||
|
||||
@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass import BypassCancelledError, ChallengeNotSolvedError
|
||||
from shelfmark.bypass.challenge import challenge_marker
|
||||
from shelfmark.bypass.cookie_store import store_extracted_cookies
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
@@ -91,7 +92,13 @@ def _store_solution_clearance(target_url: str, solution: Mapping[str, Any]) -> N
|
||||
|
||||
|
||||
def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
"""Make a single request to the external bypasser service. Returns HTML or None."""
|
||||
"""Make a single request to the external bypasser service. Returns HTML or None.
|
||||
|
||||
Raises:
|
||||
ChallengeNotSolvedError: the service answered with a page that is still a
|
||||
challenge, whatever verdict it reported on itself.
|
||||
|
||||
"""
|
||||
raw_bypasser_url = _coerce_config_str(
|
||||
config.get("EXT_BYPASSER_URL", "http://flaresolverr:8191"),
|
||||
"http://flaresolverr:8191",
|
||||
@@ -143,6 +150,32 @@ def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
logger.warning("External bypasser returned empty response for '%s'", target_url)
|
||||
return None
|
||||
|
||||
# "Challenge solved!" is the solver's verdict on its own work, and #1289 showed
|
||||
# it can be reported alongside a page the caller then rejects. Say what actually
|
||||
# came back, so a later report does not have to infer it from downstream errors.
|
||||
marker = challenge_marker(html)
|
||||
logger.debug(
|
||||
"External bypasser page for '%s': %d bytes, challenge_marker=%r",
|
||||
target_url,
|
||||
len(html),
|
||||
marker,
|
||||
)
|
||||
if marker:
|
||||
# The solver's verdict is not evidence; the page is. Returning this one as a
|
||||
# success is what made #1292 unrecoverable: the retry-and-rotate loop that
|
||||
# could still have saved the search - the next mirror is a different
|
||||
# DDoS-Guard host, in its own state - was never entered, and the challenge
|
||||
# page's own __ddg cookies were filed as this host's clearance and replayed
|
||||
# on every later request.
|
||||
logger.warning(
|
||||
"External bypasser reported success but returned a challenge page for "
|
||||
"'%s' (%d bytes, marker=%r) - the solve did not clear the protection",
|
||||
target_url,
|
||||
len(html),
|
||||
marker,
|
||||
)
|
||||
raise ChallengeNotSolvedError(marker)
|
||||
|
||||
try:
|
||||
_store_solution_clearance(target_url, solution)
|
||||
except AttributeError, KeyError, TypeError, ValueError:
|
||||
@@ -192,16 +225,33 @@ def get_bypassed_page(
|
||||
selector: network.AAMirrorSelector | None = None,
|
||||
cancel_flag: Event | None = None,
|
||||
) -> str | None:
|
||||
"""Fetch HTML via external bypasser with retries and mirror rotation."""
|
||||
"""Fetch HTML via external bypasser with retries and mirror rotation.
|
||||
|
||||
Raises:
|
||||
ChallengeNotSolvedError: every attempt came back still carrying a challenge.
|
||||
Reported apart from returning None because the two ask the user for
|
||||
opposite things: None means go and check the bypasser, this means the
|
||||
bypasser is fine and the host is the one refusing.
|
||||
BypassCancelledError: the caller's cancel flag was set.
|
||||
|
||||
"""
|
||||
from shelfmark.download import network as network_module
|
||||
|
||||
sel = selector or network_module.AAMirrorSelector()
|
||||
unsolved_marker: str | None = None
|
||||
|
||||
for attempt in range(1, MAX_RETRY + 1):
|
||||
_check_cancelled(cancel_flag, "by user")
|
||||
|
||||
attempt_url = sel.rewrite(url)
|
||||
result = _fetch_via_bypasser(attempt_url)
|
||||
try:
|
||||
result = _fetch_via_bypasser(attempt_url)
|
||||
except ChallengeNotSolvedError as e:
|
||||
# Worth the remaining attempts rather than an immediate give-up: the retry
|
||||
# rotates onto the next mirror, and that is a different DDoS-Guard host with
|
||||
# its own idea of whether this caller needs a CAPTCHA.
|
||||
unsolved_marker = str(e) or unsolved_marker
|
||||
result = None
|
||||
if result:
|
||||
return result
|
||||
|
||||
@@ -222,4 +272,11 @@ def get_bypassed_page(
|
||||
if action in ("mirror", "dns") and new_base:
|
||||
logger.info("Rotated %s for retry", action)
|
||||
|
||||
if unsolved_marker:
|
||||
msg = (
|
||||
"The bypasser ran, but the site kept answering with a protection challenge "
|
||||
f"(marker={unsolved_marker!r}). That is usually a manual CAPTCHA, which no "
|
||||
"bypasser can answer - the bypasser itself is working. Try again shortly."
|
||||
)
|
||||
raise ChallengeNotSolvedError(msg)
|
||||
return None
|
||||
|
||||
@@ -82,12 +82,18 @@ _HELPER_RESULT_POLL_SECONDS = 0.05
|
||||
# what it is doing and exit before its session is killed instead.
|
||||
_HELPER_SHUTDOWN_GRACE_SECONDS = 15.0
|
||||
_HELPER_IDLE_TIMEOUT_DEFAULT = 180.0
|
||||
# How long to wait for a solved page to produce its document before the attempt is
|
||||
# abandoned. SeleniumBase's own get_page_source() allows one second; see _read_page_source.
|
||||
_PAGE_SOURCE_TIMEOUT_DEFAULT = 20.0
|
||||
_PARENT_WATCHDOG_INTERVAL_SECONDS = 5.0
|
||||
# How much of ffmpeg's stderr to quote when reporting that it died.
|
||||
_FFMPEG_ERROR_TAIL_CHARS = 500
|
||||
|
||||
|
||||
class _DisplayState(TypedDict):
|
||||
ffmpeg: subprocess.Popen[bytes] | None
|
||||
ffmpeg_output: Path | None
|
||||
ffmpeg_error_log: Path | None
|
||||
|
||||
|
||||
class _PageWithWindowRect(Protocol):
|
||||
@@ -101,6 +107,7 @@ class _BrowserWithWindowRectPage(Protocol):
|
||||
DISPLAY: _DisplayState = {
|
||||
"ffmpeg": None,
|
||||
"ffmpeg_output": None,
|
||||
"ffmpeg_error_log": None,
|
||||
}
|
||||
LOCKED = threading.Lock()
|
||||
_PROC_ROOT = Path("/proc")
|
||||
@@ -518,18 +525,6 @@ async def _bypass_method_humanlike(page: Any) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def _bypass_method_cdp_solve(page: Any) -> bool:
|
||||
"""CDP Mode with solve_captcha() - auto-detects challenge type."""
|
||||
try:
|
||||
logger.debug("Attempting bypass: CDP solve_captcha")
|
||||
await page.solve_captcha()
|
||||
await asyncio.sleep(_RNG.uniform(3, 5))
|
||||
return await _is_bypassed(page)
|
||||
except _CDP_OPERATION_ERRORS as e:
|
||||
logger.debug("CDP solve_captcha failed: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
CDP_CLICK_SELECTORS = [
|
||||
"#turnstile-widget div", # Cloudflare Turnstile
|
||||
"#cf-turnstile div", # Alternative CF Turnstile
|
||||
@@ -609,8 +604,13 @@ async def _bypass_method_cdp_gui_click(page: Any) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
# Ordered cheapest-first, and deliberately without a bare `solve_captcha()` entry:
|
||||
# _bypass_method_cdp_gui_click opens by doing exactly that and returns the moment it
|
||||
# works, so a separate method ahead of it could only ever repeat the half that had
|
||||
# already failed - one wasted round trip plus the backoff before the next attempt, on
|
||||
# every solve that gets this far. Measured at ~5.5s of the ~26s each solve cost, and
|
||||
# 0/19 successes for the standalone method against DDoS-Guard. See issue #1285.
|
||||
BYPASS_METHODS = [
|
||||
_bypass_method_cdp_solve,
|
||||
_bypass_method_cdp_gui_click,
|
||||
_bypass_method_cdp_click,
|
||||
_bypass_method_humanlike,
|
||||
@@ -618,6 +618,25 @@ BYPASS_METHODS = [
|
||||
|
||||
MAX_CONSECUTIVE_SAME_CHALLENGE = 3
|
||||
|
||||
# How many method attempts one _bypass() pass may make. Deliberately *not* MAX_RETRY:
|
||||
# that value is already the outer page-load retry in _run_bypass_in_current_process, and
|
||||
# reading it here too squared the budget - the default 10 meant 10 page loads x 4 methods
|
||||
# = 40 solve attempts on one browser, which overruns the worker deadline and reports
|
||||
# `TimeoutError` instead of a plain "bypass failed". One full pass through the methods
|
||||
# plus a spare is all this loop can use anyway: the stuck-challenge guard below aborts at
|
||||
# len(BYPASS_METHODS) + 1, so a larger number here only ever showed up in the logs.
|
||||
_BYPASS_METHOD_ATTEMPTS = len(BYPASS_METHODS) + 1
|
||||
|
||||
# The undisturbed window a passive challenge gets before any method runs. Sized off the
|
||||
# real thing: a desktop browser clears Anna's Archive's DDoS-Guard JS check in under 10s.
|
||||
_PASSIVE_SOLVE_SECONDS = 15.0
|
||||
_PASSIVE_SOLVE_POLL_SECONDS = 1.0
|
||||
|
||||
# Head-room the retry loop leaves itself so it can return a real failure rather than be
|
||||
# cancelled at the worker deadline. Enough for the pass in flight to unwind and the
|
||||
# browser to close.
|
||||
_RESERVE_FOR_CLEAN_FAILURE_SECONDS = 60.0
|
||||
|
||||
|
||||
def _check_cancellation(cancel_flag: Event | None, message: str) -> None:
|
||||
"""Check if cancellation was requested and raise if so."""
|
||||
@@ -627,13 +646,26 @@ def _check_cancellation(cancel_flag: Event | None, message: str) -> None:
|
||||
raise BypassCancelledError(msg)
|
||||
|
||||
|
||||
async def _wait_for_passive_solve(page: Any, cancel_flag: Event | None = None) -> bool:
|
||||
"""Poll for a challenge that clears itself, without touching the page.
|
||||
|
||||
Returns True as soon as the page looks bypassed, False once the window is spent.
|
||||
"""
|
||||
logger.info("Waiting up to %.0fs for the challenge to clear itself...", _PASSIVE_SOLVE_SECONDS)
|
||||
deadline = time.monotonic() + _PASSIVE_SOLVE_SECONDS
|
||||
while time.monotonic() < deadline:
|
||||
_check_cancellation(cancel_flag, "Bypass cancelled while waiting for a passive solve")
|
||||
await asyncio.sleep(_PASSIVE_SOLVE_POLL_SECONDS)
|
||||
if await _is_bypassed(page):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def _bypass(
|
||||
page: Any, max_retries: int | None = None, cancel_flag: Event | None = None
|
||||
) -> bool:
|
||||
"""Attempt to bypass Cloudflare/DDOS-Guard protection using multiple methods."""
|
||||
max_retries = (
|
||||
max_retries if max_retries is not None else _coerce_positive_int(app_config.MAX_RETRY, 10)
|
||||
)
|
||||
max_retries = max_retries if max_retries is not None else _BYPASS_METHOD_ATTEMPTS
|
||||
|
||||
last_challenge_type = None
|
||||
consecutive_same_challenge = 0
|
||||
@@ -651,6 +683,20 @@ async def _bypass(
|
||||
challenge_type = await _detect_challenge_type(page)
|
||||
logger.debug("Challenge detected: %s", challenge_type)
|
||||
|
||||
# Give a passive check the undisturbed window it needs before touching the page.
|
||||
# DDoS-Guard's JS check on Anna's Archive has no click target: it runs, then
|
||||
# navigates on its own - a desktop browser clears it in well under 15s. Every
|
||||
# method below either clicks a selector that is not there or reloads, and a reload
|
||||
# restarts an in-flight check (which DDoS-Guard also throttles), so going straight
|
||||
# to them meant the one thing that actually solves this challenge was the one
|
||||
# thing never tried. Costs one 15s window per solve against a minutes-long budget,
|
||||
# and a challenge that needs interaction simply falls through to the methods.
|
||||
if try_count == 0 and challenge_type != "none":
|
||||
if await _wait_for_passive_solve(page, cancel_flag):
|
||||
logger.info("Bypass successful: %s challenge cleared itself", challenge_type)
|
||||
return True
|
||||
logger.debug("Challenge did not clear on its own; trying bypass methods")
|
||||
|
||||
# No challenge detected but page doesn't look bypassed - wait and retry
|
||||
if challenge_type == "none":
|
||||
logger.info("No challenge detected, waiting for page to settle...")
|
||||
@@ -769,6 +815,22 @@ def _build_host_resolver_rules() -> list[str]:
|
||||
DRIVER_RESET_ERRORS = {"ProtocolException", "RuntimeError", "TimeoutError"}
|
||||
|
||||
|
||||
async def _read_page_source(page: Any) -> str:
|
||||
"""Read a solved page's HTML, waiting for the document to arrive.
|
||||
|
||||
`get_page_source()` waits one second for the `html` element. A page released from a
|
||||
challenge is often still navigating to the real content, so the read times out even
|
||||
though the solve succeeded: the whole attempt is retried, and the repeated requests
|
||||
are what earn a 429 from a host that was about to serve us.
|
||||
"""
|
||||
timeout = _coerce_non_negative_float(
|
||||
app_config.get("BYPASS_PAGE_SOURCE_TIMEOUT", _PAGE_SOURCE_TIMEOUT_DEFAULT),
|
||||
_PAGE_SOURCE_TIMEOUT_DEFAULT,
|
||||
)
|
||||
element = await page.find("html", timeout=timeout)
|
||||
return await element.get_html_async()
|
||||
|
||||
|
||||
async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str:
|
||||
"""Fetch URL with Cloudflare bypass using a CDP browser."""
|
||||
_check_cancellation(cancel_flag, "Bypass cancelled before starting")
|
||||
@@ -792,7 +854,7 @@ async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str:
|
||||
logger.debug("Starting bypass process...")
|
||||
if await _bypass(page, cancel_flag=cancel_flag):
|
||||
await _extract_cookies_from_cdp(driver, page, url)
|
||||
return await page.get_page_source()
|
||||
return await _read_page_source(page)
|
||||
|
||||
logger.warning("Bypass completed but page still shows protection")
|
||||
try:
|
||||
@@ -810,14 +872,33 @@ async def _get(url: str, driver: Any, cancel_flag: Event | None = None) -> str:
|
||||
|
||||
def _run_bypass_in_current_process(url: str, retry: int, cancel_flag: Event | None = None) -> str:
|
||||
"""Run the CDP bypass in the current process."""
|
||||
timeout = (
|
||||
_CHILD_BYPASS_TIMEOUT_SECONDS
|
||||
if os.environ.get(_BYPASS_CHILD_ENV) == "1"
|
||||
else _IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
)
|
||||
|
||||
async def _run_bypass() -> str:
|
||||
driver = None
|
||||
# Stop retrying while there is still time to say so. A challenge nothing can solve
|
||||
# would otherwise spend every one of `retry` passes and be cut off mid-pass by the
|
||||
# worker deadline, which surfaces to the caller as `RuntimeError: TimeoutError` -
|
||||
# a message that says nothing about protection and sent users looking at their
|
||||
# reverse proxy. Giving up a pass early returns the real "bypass failed" instead.
|
||||
deadline = time.monotonic() + timeout - _RESERVE_FOR_CLEAN_FAILURE_SECONDS
|
||||
try:
|
||||
driver = await _create_cdp_browser(url)
|
||||
|
||||
for attempt in range(retry):
|
||||
_check_cancellation(cancel_flag, "Bypass cancelled before attempt")
|
||||
if attempt > 0 and time.monotonic() >= deadline:
|
||||
logger.warning(
|
||||
"Bypass budget spent after %s/%s attempts; giving up on %s",
|
||||
attempt,
|
||||
retry,
|
||||
url,
|
||||
)
|
||||
break
|
||||
|
||||
try:
|
||||
result = await _get(url, driver, cancel_flag)
|
||||
@@ -838,7 +919,7 @@ def _run_bypass_in_current_process(url: str, retry: int, cancel_flag: Event | No
|
||||
await _close_cdp_driver(driver)
|
||||
driver = await _create_cdp_browser(url)
|
||||
|
||||
logger.error("Bypass failed after %s attempts", retry)
|
||||
logger.error("Bypass failed for %s", url)
|
||||
return ""
|
||||
finally:
|
||||
if driver:
|
||||
@@ -852,12 +933,9 @@ def _run_bypass_in_current_process(url: str, retry: int, cancel_flag: Event | No
|
||||
# one call and closes it on the way out, so a helper serving many requests would build
|
||||
# and tear down a loop per bypass and would carry no deadline of its own. The worker's
|
||||
# loop lives in a thread, outlives any single bypass, and cancels the coroutine when the
|
||||
# deadline passes.
|
||||
timeout = (
|
||||
_CHILD_BYPASS_TIMEOUT_SECONDS
|
||||
if os.environ.get(_BYPASS_CHILD_ENV) == "1"
|
||||
else _IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
)
|
||||
# deadline passes. `_run_bypass` aims to finish inside this same budget of its own
|
||||
# accord, so reaching this deadline now means a wedged session rather than a stubborn
|
||||
# challenge - which is the only case worth reporting as a timeout.
|
||||
return _CDP_WORKER.run(_run_bypass(), timeout=timeout)
|
||||
|
||||
|
||||
@@ -1155,6 +1233,20 @@ def get(url: str, retry: int | None = None, cancel_flag: Event | None = None) ->
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
# Re-checked after the cached attempt, not just in get_bypassed_page: that check
|
||||
# ran before the queue, and this call may have spent minutes holding for LOCKED
|
||||
# while another request collected a 429 (or collected one itself, just above).
|
||||
# A solve cannot clear a throttle - the challenge renders, the solve "succeeds",
|
||||
# and the cleared request is refused again while the backoff is renewed.
|
||||
remaining = network.host_cooldown_remaining(url)
|
||||
if remaining > 0:
|
||||
hostname = urlparse(url).hostname or url
|
||||
msg = (
|
||||
f"{hostname} is rate-limited (429); skipping bypass for ~{remaining:.0f}s "
|
||||
"until the cooldown clears."
|
||||
)
|
||||
raise network.RateLimitedError(msg)
|
||||
|
||||
if env.DOCKERMODE and os.environ.get(_BYPASS_CHILD_ENV) != "1":
|
||||
return _get_via_subprocess(url, retry, cancel_flag)
|
||||
return _run_bypass_in_current_process(url, retry, cancel_flag)
|
||||
@@ -1339,13 +1431,48 @@ def _start_ffmpeg_recording(display: str) -> None:
|
||||
"-an",
|
||||
output_file.as_posix(),
|
||||
"-nostats",
|
||||
# Was "0", which discards everything including the reason it could not start.
|
||||
# Recordings have been arriving empty with no explanation anywhere: on issue
|
||||
# #1276 all three of a session's recordings were gone and the log said only
|
||||
# "FFmpeg already stopped", because ffmpeg exits before creating the file when
|
||||
# it cannot open the X display. Errors only - this is a debug-mode recorder, not
|
||||
# something to make chatty.
|
||||
"-loglevel",
|
||||
"0",
|
||||
"error",
|
||||
]
|
||||
logger.debug("Starting FFmpeg recording to %s", output_file)
|
||||
logger.debug_trace(f"FFmpeg command: {' '.join(ffmpeg_cmd)}")
|
||||
DISPLAY["ffmpeg"] = subprocess.Popen(ffmpeg_cmd)
|
||||
# Kept beside the recording so it travels in the debug bundle, which is the only
|
||||
# place anyone will look for it. A file rather than a pipe: nothing here would drain
|
||||
# a pipe, and a full one would wedge ffmpeg partway through a capture.
|
||||
error_log = output_file.with_suffix(".ffmpeg.log")
|
||||
try:
|
||||
stderr_handle = error_log.open("wb")
|
||||
except OSError as exc:
|
||||
logger.debug("Could not open FFmpeg error log %s: %s", error_log, exc)
|
||||
stderr_handle = None
|
||||
DISPLAY["ffmpeg"] = subprocess.Popen(
|
||||
ffmpeg_cmd, stderr=stderr_handle, stdout=subprocess.DEVNULL
|
||||
)
|
||||
if stderr_handle is not None:
|
||||
# The child holds its own descriptor; this one has done its job.
|
||||
stderr_handle.close()
|
||||
DISPLAY["ffmpeg_output"] = output_file
|
||||
DISPLAY["ffmpeg_error_log"] = error_log
|
||||
|
||||
|
||||
def _ffmpeg_error_summary() -> str:
|
||||
"""What ffmpeg wrote to stderr, for the log line that reports it died."""
|
||||
error_log = DISPLAY.get("ffmpeg_error_log")
|
||||
if not error_log:
|
||||
return "No FFmpeg error log was captured."
|
||||
try:
|
||||
text = Path(error_log).read_text(encoding="utf-8", errors="replace").strip()
|
||||
except OSError as exc:
|
||||
return f"FFmpeg error log unreadable ({exc})."
|
||||
if not text:
|
||||
return f"FFmpeg logged nothing to {error_log}."
|
||||
return f"FFmpeg said: {text[-_FFMPEG_ERROR_TAIL_CHARS:]}"
|
||||
|
||||
|
||||
def _stop_ffmpeg_recording() -> None:
|
||||
@@ -1357,9 +1484,17 @@ def _stop_ffmpeg_recording() -> None:
|
||||
if not proc:
|
||||
return
|
||||
if proc.poll() is not None:
|
||||
logger.debug("FFmpeg already stopped")
|
||||
# Not "already stopped" - ffmpeg was asked to record until now and is gone, so
|
||||
# the recording for this bypass does not exist. Say so, with the reason, rather
|
||||
# than leaving an empty recording/ directory to be discovered later.
|
||||
logger.warning(
|
||||
"FFmpeg exited early (code %s); no recording for this bypass. %s",
|
||||
proc.returncode,
|
||||
_ffmpeg_error_summary(),
|
||||
)
|
||||
DISPLAY["ffmpeg"] = None
|
||||
DISPLAY["ffmpeg_output"] = None
|
||||
DISPLAY["ffmpeg_error_log"] = None
|
||||
return
|
||||
try:
|
||||
proc.send_signal(signal.SIGINT)
|
||||
@@ -1374,6 +1509,7 @@ def _stop_ffmpeg_recording() -> None:
|
||||
proc.kill()
|
||||
DISPLAY["ffmpeg"] = None
|
||||
DISPLAY["ffmpeg_output"] = None
|
||||
DISPLAY["ffmpeg_error_log"] = None
|
||||
|
||||
|
||||
def _try_with_cached_cookies(url: str, hostname: str) -> str | None:
|
||||
@@ -1400,6 +1536,20 @@ def _try_with_cached_cookies(url: str, hostname: str) -> str | None:
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
logger.debug("Cached cookies worked, skipped Chrome bypass")
|
||||
return response.text
|
||||
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
|
||||
# Throttled, not challenged. The clearance is still good - the origin is
|
||||
# rate-limiting this IP and would answer 429 to a browser holding the very
|
||||
# same cookies. Discarding it here (as every other rejection does) meant a
|
||||
# solve won seconds earlier was thrown away and the next query bought its
|
||||
# own 20-60s browser solve, which is itself more traffic at a host that has
|
||||
# just asked for less. Keep it, arm the backoff, and let the caller wait.
|
||||
wait = network.note_rate_limited(url)
|
||||
logger.debug(
|
||||
"Cached cookies hit a 429 for %s; keeping them and backing off ~%.0fs",
|
||||
url,
|
||||
wait,
|
||||
)
|
||||
return None
|
||||
logger.debug(
|
||||
"Cached cookies rejected (%s) for %s; discarding them",
|
||||
response.status_code,
|
||||
@@ -1438,6 +1588,18 @@ def get_bypassed_page(
|
||||
attempt_url = sel.rewrite(url)
|
||||
hostname = urlparse(attempt_url).hostname or ""
|
||||
|
||||
# A 429 means the origin is throttling this IP; the challenge still renders, so a
|
||||
# solve "succeeds" but the cleared request is rejected again and the throttle is only
|
||||
# renewed. Never spend a minutes-long Chrome solve on a cooling-down host - fail fast
|
||||
# so the caller waits the backoff out instead of looping the solve.
|
||||
remaining = network.host_cooldown_remaining(attempt_url)
|
||||
if remaining > 0:
|
||||
msg = (
|
||||
f"{hostname} is rate-limited (429); skipping bypass for ~{remaining:.0f}s "
|
||||
"until the cooldown clears."
|
||||
)
|
||||
raise network.RateLimitedError(msg)
|
||||
|
||||
cached_result = _try_with_cached_cookies(attempt_url, hostname)
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
@@ -203,6 +203,21 @@ ONBOARDING = string_to_bool(os.getenv("ONBOARDING", "true"))
|
||||
_DEBUG_SKIP_SOURCES_RAW = os.getenv("DEBUG_SKIP_SOURCES", "").strip().lower()
|
||||
DEBUG_SKIP_SOURCES = {s.strip() for s in _DEBUG_SKIP_SOURCES_RAW.split(",") if s.strip()}
|
||||
|
||||
# Debug: keep DDoS-Guard's __ddg8_/__ddg9_/__ddg10_ in the clearance store instead of
|
||||
# dropping them after a solve.
|
||||
#
|
||||
# Which of DDoS-Guard's cookies actually *are* clearance is not settled. The store treats
|
||||
# the trio as describing one check (client IP, timestamp, token) and drops them, on the
|
||||
# reasoning that replaying a stale IP/timestamp is what re-arms the ?check=1 loop - see
|
||||
# shelfmark.bypass.cookie_store. Field reports on issue #1276 point the other way: every
|
||||
# request after a successful solve was challenged again, which is only consistent with
|
||||
# what the store keeps not being sufficient clearance on its own.
|
||||
#
|
||||
# Deliberately env-only and off by default: this is a knob for reproducing the question
|
||||
# against a live host, not a setting to offer users. Set it to true, solve once, and watch
|
||||
# whether the next search still logs "Redirect loop detected".
|
||||
DDG_REPLAY_PER_CHECK_COOKIES = string_to_bool(os.getenv("DDG_REPLAY_PER_CHECK_COOKIES", "false"))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Legacy migration support - will be removed in future version
|
||||
|
||||
@@ -430,13 +430,6 @@ def general_settings() -> list[SettingsField]:
|
||||
options=_AUDIOBOOK_FORMAT_OPTIONS,
|
||||
default=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS],
|
||||
),
|
||||
MultiSelectField(
|
||||
key="BOOK_LANGUAGE",
|
||||
label="Default Book Languages",
|
||||
description="Default language filter for searches.",
|
||||
options=_LANGUAGE_OPTIONS,
|
||||
default=["en"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -474,6 +467,17 @@ def search_mode_settings() -> list[SettingsField]:
|
||||
default="universal",
|
||||
user_overridable=True,
|
||||
),
|
||||
MultiSelectField(
|
||||
key="BOOK_LANGUAGE",
|
||||
label="Default Book Languages",
|
||||
description=(
|
||||
"Default language filter for searches. Users can override this for their "
|
||||
"own account."
|
||||
),
|
||||
options=_LANGUAGE_OPTIONS,
|
||||
default=["en"],
|
||||
user_overridable=True,
|
||||
),
|
||||
SelectField(
|
||||
key="AA_DEFAULT_SORT",
|
||||
label="Default Sort Order",
|
||||
@@ -1556,6 +1560,19 @@ def download_source_settings() -> list[SettingsField]:
|
||||
min_value=1,
|
||||
max_value=60,
|
||||
),
|
||||
NumberField(
|
||||
key="RELEASE_SEARCH_TIMEOUT",
|
||||
label="Release Search Timeout (seconds)",
|
||||
description=(
|
||||
"How long one release search may run before it gives up and reports why. "
|
||||
"A first search on a cold start pays for a browser solve, so leave room "
|
||||
"for one. If you use a reverse proxy, its read timeout should be at least "
|
||||
"this high or it will cut the search off with a 504 first."
|
||||
),
|
||||
default=300,
|
||||
min_value=30,
|
||||
max_value=1800,
|
||||
),
|
||||
HeadingField(
|
||||
key="content_type_routing_heading",
|
||||
title="Content-Type Routing",
|
||||
@@ -1666,6 +1683,18 @@ def cloudflare_bypass_settings() -> list[SettingsField]:
|
||||
requires_restart=True,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
|
||||
),
|
||||
NumberField(
|
||||
key="BYPASS_PAGE_SOURCE_TIMEOUT",
|
||||
label="Page Read Timeout (seconds)",
|
||||
description=(
|
||||
"How long to wait for a solved page to produce its content before the "
|
||||
"bypass is retried. Raise it if solves succeed but searches still fail."
|
||||
),
|
||||
default=20,
|
||||
min_value=1,
|
||||
max_value=120,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": False},
|
||||
),
|
||||
NumberField(
|
||||
key="BYPASS_BROWSER_IDLE_TIMEOUT",
|
||||
label="Bypasser Idle Timeout (seconds)",
|
||||
|
||||
@@ -7,6 +7,7 @@ that talks to /api/admin/users endpoints.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from shelfmark.core.languages import normalize_language
|
||||
from shelfmark.core.request_policy import (
|
||||
get_source_content_type_capabilities,
|
||||
parse_policy_mode,
|
||||
@@ -61,7 +62,7 @@ _SELF_SETTINGS_SECTION_OPTIONS = [
|
||||
{
|
||||
"value": "search",
|
||||
"label": "Search Preferences",
|
||||
"description": "Show personal search mode and provider settings.",
|
||||
"description": "Show personal search mode, language, and provider settings.",
|
||||
},
|
||||
{
|
||||
"value": "notifications",
|
||||
@@ -77,8 +78,9 @@ _SEARCH_PREFERENCE_PROVIDER_KEYS = {
|
||||
"METADATA_PROVIDER_AUDIOBOOK",
|
||||
"METADATA_PROVIDER_COMBINED",
|
||||
}
|
||||
_SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
|
||||
SEARCH_PREFERENCE_VALIDATABLE_KEYS = {
|
||||
"SEARCH_MODE",
|
||||
"BOOK_LANGUAGE",
|
||||
"DEFAULT_RELEASE_SOURCE",
|
||||
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK",
|
||||
"SHOW_COMBINED_SELECTOR",
|
||||
@@ -178,14 +180,43 @@ def _get_request_policy_rule_columns() -> list[dict[str, object]]:
|
||||
]
|
||||
|
||||
|
||||
def _validate_book_languages(value: Any) -> tuple[Any, str | None]:
|
||||
"""Validate a per-user default language list against the known languages.
|
||||
|
||||
Accepts the list the settings UI sends as well as a comma-separated string, so an
|
||||
API client can spell the value the way the env var does. Blank entries are skipped
|
||||
rather than rejected, which makes "" and "en," mean the same as [] and ["en"]. An
|
||||
empty result is a deliberate override meaning "no default language filter", so it
|
||||
is kept as-is; ``None`` clears the override further up the chain.
|
||||
"""
|
||||
entries = value.split(",") if isinstance(value, str) else value
|
||||
if not isinstance(entries, (list, tuple)):
|
||||
return value, "BOOK_LANGUAGE must be a list of language codes"
|
||||
|
||||
normalized: list[str] = []
|
||||
for entry in entries:
|
||||
if entry is None or (isinstance(entry, str) and not entry.strip()):
|
||||
continue
|
||||
code = normalize_language(entry)
|
||||
if code is None:
|
||||
return value, f"BOOK_LANGUAGE contains an unsupported language: {entry}"
|
||||
if code not in normalized:
|
||||
normalized.append(code)
|
||||
|
||||
return normalized, None
|
||||
|
||||
|
||||
def validate_search_preference_value(key: str, value: Any) -> tuple[Any, str | None]:
|
||||
"""Validate and normalize a search preference value for user overrides."""
|
||||
if key not in _SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
if key not in SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
return value, None
|
||||
|
||||
if value is None:
|
||||
return None, None
|
||||
|
||||
if key == "BOOK_LANGUAGE":
|
||||
return _validate_book_languages(value)
|
||||
|
||||
normalized_value = str(value).strip()
|
||||
|
||||
if key == "SEARCH_MODE":
|
||||
@@ -298,7 +329,7 @@ def _on_save_users(values: dict[str, object]) -> dict[str, object]:
|
||||
}
|
||||
values["REQUEST_POLICY_RULES"] = normalized_rules
|
||||
|
||||
for key in _SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
for key in SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
if key not in values:
|
||||
continue
|
||||
normalized_value, validation_error = validate_search_preference_value(key, values[key])
|
||||
|
||||
@@ -11,7 +11,10 @@ from shelfmark.config.notifications_settings import (
|
||||
is_valid_notification_url,
|
||||
normalize_notification_routes,
|
||||
)
|
||||
from shelfmark.config.users_settings import validate_search_preference_value
|
||||
from shelfmark.config.users_settings import (
|
||||
SEARCH_PREFERENCE_VALIDATABLE_KEYS,
|
||||
validate_search_preference_value,
|
||||
)
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.request_policy import parse_policy_mode, validate_policy_rules
|
||||
from shelfmark.core.settings_registry import load_config_file
|
||||
@@ -91,13 +94,9 @@ def validate_user_settings(
|
||||
if search_validation_error:
|
||||
errors.append(search_validation_error)
|
||||
continue
|
||||
if key in {
|
||||
"SEARCH_MODE",
|
||||
"METADATA_PROVIDER",
|
||||
"METADATA_PROVIDER_AUDIOBOOK",
|
||||
"DEFAULT_RELEASE_SOURCE",
|
||||
"DEFAULT_RELEASE_SOURCE_AUDIOBOOK",
|
||||
}:
|
||||
# Every key the search validator recognises keeps its normalized value;
|
||||
# a hand-maintained subset here silently dropped normalization for the rest.
|
||||
if key in SEARCH_PREFERENCE_VALIDATABLE_KEYS:
|
||||
valid[key] = normalized_search_value
|
||||
continue
|
||||
|
||||
|
||||
@@ -136,6 +136,12 @@ class DownloadTask:
|
||||
default_factory=dict
|
||||
) # Per-output parameters (e.g. email recipient)
|
||||
|
||||
# Multi-book packs: one release holding several books. `book_plan` is the split the
|
||||
# user approved before download (list of {title, series_position, year, files});
|
||||
# `multi_book` asks post-processing to split heuristically when no plan exists.
|
||||
multi_book: bool = False
|
||||
book_plan: list[dict[str, Any]] | None = None
|
||||
|
||||
# User association (multi-user support)
|
||||
user_id: int | None = None # DB user ID who queued this download
|
||||
username: str | None = None # Username for {User} template variable
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Pre-download release inspection: list a release's files and plan a multi-book split."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import is_audiobook
|
||||
from shelfmark.download.postprocess.packs import PackFile, PackPlan, plan_pack
|
||||
from shelfmark.download.postprocess.policy import (
|
||||
get_supported_audiobook_formats,
|
||||
get_supported_formats,
|
||||
)
|
||||
from shelfmark.release_sources import get_handler
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from flask import Flask, Response
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
_INSPECT_ERRORS = (OSError, RuntimeError, ValueError, TypeError, KeyError, AttributeError)
|
||||
NOT_INSPECTABLE_REASON = "This source cannot list the release's files before downloading"
|
||||
|
||||
|
||||
def _serialize_plan(plan: PackPlan) -> dict[str, Any]:
|
||||
return {
|
||||
"is_pack": plan.is_pack,
|
||||
"ignored": plan.ignored,
|
||||
"books": [
|
||||
{
|
||||
"title": book.title,
|
||||
"series_position": book.series_position,
|
||||
"year": book.year,
|
||||
"files": book.files,
|
||||
}
|
||||
for book in plan.books
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def inspect_release(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Build the inspect response for a release payload (same shape as a download)."""
|
||||
source = str(data["source"])
|
||||
handler = get_handler(source)
|
||||
try:
|
||||
files: list[PackFile] | None = handler.list_files(data)
|
||||
except _INSPECT_ERRORS as exc:
|
||||
logger.warning(
|
||||
"Could not list files for %s release %s: %s", source, data.get("source_id"), exc
|
||||
)
|
||||
return {"inspected": False, "reason": str(exc), "files": [], "plan": None}
|
||||
|
||||
if files is None:
|
||||
return {"inspected": False, "reason": NOT_INSPECTABLE_REASON, "files": [], "plan": None}
|
||||
|
||||
content_type = data.get("content_type")
|
||||
supported = (
|
||||
get_supported_audiobook_formats()
|
||||
if is_audiobook(content_type if isinstance(content_type, str) else None)
|
||||
else get_supported_formats()
|
||||
)
|
||||
series_name = data.get("series_name")
|
||||
author_name = data.get("author")
|
||||
plan = plan_pack(
|
||||
files,
|
||||
supported_extensions=set(supported),
|
||||
series_name=series_name if isinstance(series_name, str) else None,
|
||||
author_name=author_name if isinstance(author_name, str) else None,
|
||||
)
|
||||
return {
|
||||
"inspected": True,
|
||||
"reason": None,
|
||||
"files": [{"path": f.path, "size": f.size} for f in files],
|
||||
"plan": _serialize_plan(plan),
|
||||
}
|
||||
|
||||
|
||||
def register_release_inspect_routes(
|
||||
app: Flask,
|
||||
login_required: Callable[..., Any],
|
||||
) -> None:
|
||||
"""Register POST /api/releases/inspect."""
|
||||
|
||||
@app.route("/api/releases/inspect", methods=["POST"])
|
||||
@login_required
|
||||
def api_inspect_release() -> Response | tuple[Response, int]:
|
||||
data = request.get_json(silent=True)
|
||||
if not isinstance(data, dict):
|
||||
return jsonify({"error": "No data provided"}), 400
|
||||
if not data.get("source_id"):
|
||||
return jsonify({"error": "source_id is required"}), 400
|
||||
if not data.get("source"):
|
||||
return jsonify({"error": "source is required"}), 400
|
||||
try:
|
||||
get_handler(str(data["source"]))
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify(inspect_release(data))
|
||||
@@ -0,0 +1,134 @@
|
||||
"""A wall-clock budget for one release search, enforced through the existing cancel flag.
|
||||
|
||||
`/api/releases` is synchronous: the browser waits on it while the search runs. Nothing
|
||||
bounded that wait, and the bypasser's own worst case is minutes long
|
||||
(`internal_bypasser.max_duration_seconds()`), so a search that ran into an unsolvable
|
||||
protection challenge outlived every reverse proxy in front of it. The user then saw
|
||||
"Server unavailable (504)" - a gateway timeout that says nothing about what went wrong
|
||||
and points the blame at their proxy config. See issue #1276.
|
||||
|
||||
The budget is expressed as the cancel flag the download path already understands: an
|
||||
Event armed by a timer. `html_get_page`, the bypassers and the helper subprocess all poll
|
||||
it, so an expired budget stops a solve already in flight rather than only refusing the
|
||||
next one. When it trips, the search fails with a message that names the real cause.
|
||||
|
||||
Scoped to a context variable so it applies to the request that set it and to nothing else
|
||||
- a queued download must keep its own, much longer, budget.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# What one search may spend. A first search on a cold start legitimately pays for a
|
||||
# browser solve - jfmlima measured 60-120s for a successful one on Anna's Archive - so
|
||||
# this cannot be as tight as a proxy's default read timeout without breaking working
|
||||
# setups. It is instead well below the ~840s the bypass path could previously reach,
|
||||
# which is what turned a failing challenge into a gateway timeout.
|
||||
DEFAULT_SEARCH_BUDGET_SECONDS = 300.0
|
||||
|
||||
_MIN_SEARCH_BUDGET_SECONDS = 30.0
|
||||
_MAX_SEARCH_BUDGET_SECONDS = 1800.0
|
||||
|
||||
# Raised to the caller when the budget runs out, so the API can say so plainly.
|
||||
SEARCH_DEADLINE_MESSAGE = (
|
||||
"The release search ran out of time (%.0fs). Anna's Archive is behind a protection "
|
||||
"challenge the bypasser could not solve in that window. Raise the release search "
|
||||
"timeout if your setup is simply slow."
|
||||
)
|
||||
|
||||
|
||||
class SearchDeadline:
|
||||
"""A budget with an Event that trips when it expires."""
|
||||
|
||||
def __init__(self, budget_seconds: float) -> None:
|
||||
self.budget_seconds = budget_seconds
|
||||
self.expires_at = time.monotonic() + budget_seconds
|
||||
# A plain threading.Event on purpose: this is handed on as a cancel flag, and
|
||||
# that is the type the download path, the CDP worker thread and the bypass helper
|
||||
# already poll.
|
||||
self.event = threading.Event()
|
||||
self._timer = threading.Timer(budget_seconds, self.event.set)
|
||||
self._timer.daemon = True
|
||||
|
||||
def start(self) -> None:
|
||||
self._timer.start()
|
||||
|
||||
def cancel(self) -> None:
|
||||
self._timer.cancel()
|
||||
|
||||
@property
|
||||
def remaining(self) -> float:
|
||||
return max(0.0, self.expires_at - time.monotonic())
|
||||
|
||||
@property
|
||||
def expired(self) -> bool:
|
||||
return self.event.is_set() or self.remaining <= 0
|
||||
|
||||
|
||||
_current: ContextVar[SearchDeadline | None] = ContextVar("search_deadline", default=None)
|
||||
|
||||
|
||||
def budget_seconds() -> float:
|
||||
"""The configured budget for one release search."""
|
||||
from shelfmark.core.config import config as app_config
|
||||
|
||||
raw = app_config.get("RELEASE_SEARCH_TIMEOUT", DEFAULT_SEARCH_BUDGET_SECONDS)
|
||||
if isinstance(raw, bool) or not isinstance(raw, int | float | str):
|
||||
return DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
try:
|
||||
value = float(raw)
|
||||
except TypeError, ValueError:
|
||||
return DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
if value <= 0:
|
||||
return DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
return min(max(value, _MIN_SEARCH_BUDGET_SECONDS), _MAX_SEARCH_BUDGET_SECONDS)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def search_deadline(budget: float | None = None) -> Iterator[SearchDeadline]:
|
||||
"""Apply a budget to everything the calling context does."""
|
||||
deadline = SearchDeadline(budget if budget is not None else budget_seconds())
|
||||
token = _current.set(deadline)
|
||||
deadline.start()
|
||||
logger.debug("Release search budget: %.0fs", deadline.budget_seconds)
|
||||
try:
|
||||
yield deadline
|
||||
finally:
|
||||
deadline.cancel()
|
||||
_current.reset(token)
|
||||
|
||||
|
||||
def current() -> SearchDeadline | None:
|
||||
"""The budget in force, or None outside a search."""
|
||||
return _current.get()
|
||||
|
||||
|
||||
def expired() -> bool:
|
||||
"""Whether the budget in force has run out. False when there is no budget."""
|
||||
deadline = _current.get()
|
||||
return deadline is not None and deadline.expired
|
||||
|
||||
|
||||
def cancel_event() -> threading.Event | None:
|
||||
"""The Event that trips when the budget runs out, for use as a cancel flag."""
|
||||
deadline = _current.get()
|
||||
return deadline.event if deadline is not None else None
|
||||
|
||||
|
||||
def deadline_message() -> str:
|
||||
"""The failure to report when the budget has run out."""
|
||||
deadline = _current.get()
|
||||
budget = deadline.budget_seconds if deadline else DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
return SEARCH_DEADLINE_MESSAGE % budget
|
||||
+103
-28
@@ -7,6 +7,7 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.metadata_providers import (
|
||||
BookMetadata,
|
||||
build_localized_search_titles,
|
||||
@@ -16,6 +17,8 @@ from shelfmark.metadata_providers import (
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.models import SearchFilters
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
MANUAL_QUERY_MAX_LEN = 256
|
||||
|
||||
|
||||
@@ -52,44 +55,110 @@ class ReleaseSearchPlan:
|
||||
return self.title_variants[0].query if self.title_variants else ""
|
||||
|
||||
|
||||
def _normalize_languages(languages: list[str] | None) -> list[str] | None:
|
||||
def _to_language_codes(values: Iterable[object], *, source: str) -> list[str] | None:
|
||||
"""Resolve any spelling of a language to the ISO code the sources expect.
|
||||
|
||||
Anna's Archive matches `lang=` against ISO codes: `lang=english` is not a loose
|
||||
spelling of `lang=en`, it is a facet value AA does not have, and it filters every
|
||||
search down to nothing. Only the *per-user* override was normalised
|
||||
(config.users_settings.validate), so a global BOOK_LANGUAGE=english - the spelling
|
||||
the old docs used - reached the query verbatim and silently emptied every search
|
||||
with no error anywhere. See issue #1276.
|
||||
|
||||
An entry that resolves to nothing is dropped with a warning rather than passed
|
||||
through: searching unfiltered and saying so beats reporting "no results" for a book
|
||||
the source is full of.
|
||||
"""
|
||||
from shelfmark.core.languages import normalize_language
|
||||
|
||||
codes: list[str] = []
|
||||
unresolved: list[str] = []
|
||||
for value in values:
|
||||
text = str(value).strip() if value is not None else ""
|
||||
if not text:
|
||||
continue
|
||||
if text.lower() == "all":
|
||||
# An explicit "search every language", not a language.
|
||||
return None
|
||||
code = normalize_language(text)
|
||||
if code is None:
|
||||
unresolved.append(text)
|
||||
continue
|
||||
if code not in codes:
|
||||
codes.append(code)
|
||||
|
||||
if unresolved:
|
||||
logger.warning(
|
||||
"Ignoring unrecognised language(s) in %s: %s. Use an ISO code such as 'en', "
|
||||
"a three-letter code, or an English name like 'English'.",
|
||||
source,
|
||||
", ".join(unresolved),
|
||||
)
|
||||
|
||||
return codes or None
|
||||
|
||||
|
||||
def _normalize_languages(languages: list[str] | None, user_id: int | None) -> list[str] | None:
|
||||
if not languages:
|
||||
default = getattr(config, "BOOK_LANGUAGE", None)
|
||||
default = config.get("BOOK_LANGUAGE", None, user_id=user_id)
|
||||
if isinstance(default, str):
|
||||
default_values: list[object] = [default]
|
||||
elif isinstance(default, Iterable) and not isinstance(default, (bytes, bytearray, dict)):
|
||||
default_values = list(default)
|
||||
else:
|
||||
return None
|
||||
return [str(lang).strip() for lang in default_values if str(lang).strip()]
|
||||
return _to_language_codes(default_values, source="BOOK_LANGUAGE")
|
||||
|
||||
normalized: list[str] = []
|
||||
for lang in languages:
|
||||
if not lang:
|
||||
continue
|
||||
s = str(lang).strip()
|
||||
if not s:
|
||||
continue
|
||||
normalized.append(s)
|
||||
|
||||
if any(lang.lower() == "all" for lang in normalized):
|
||||
return None
|
||||
|
||||
return normalized or None
|
||||
return _to_language_codes(languages, source="the search request")
|
||||
|
||||
|
||||
def _pick_search_author(book: BookMetadata) -> str:
|
||||
def first_author(value: str) -> str:
|
||||
"""The first name in a possibly comma-joined author string.
|
||||
|
||||
Both ends of the app hand us every contributor in one string. The frontend joins
|
||||
`authors` with ", " for display (`bookTransformers.ts`) and that display string comes
|
||||
straight back as the `author` request parameter, while several providers set
|
||||
`search_author` from the same joined text. Searching a release source for
|
||||
"Blindness Jose Saramago, Giovanni Pontiero, ..." - the author plus two translators -
|
||||
matches nothing, and the user is told the book has no releases at all.
|
||||
|
||||
A "Last, First" author collapses to the surname, which is still a usable search term
|
||||
and is what the authors[] fallback has always done with the same input. See #1252.
|
||||
"""
|
||||
first, _, _ = value.partition(",")
|
||||
return first.strip()
|
||||
|
||||
|
||||
def pick_search_author(book: BookMetadata) -> str:
|
||||
"""The one author a release query should carry, from whichever field holds one.
|
||||
|
||||
Every release source that builds its own query wants exactly this, so it lives here
|
||||
rather than being re-derived per source - the two branches below drifted apart once
|
||||
already (#1252) and the IRC source carried a third copy of the same preference.
|
||||
|
||||
#1290 fixed the same report by merging the two branches and trimming whichever one
|
||||
won; this keeps that outcome ("Blindness Jose Saramago" from either field, measured
|
||||
there at 0 releases before and 49 after) and adds the empty-narrowing fallback, so a
|
||||
credit list that merely starts with a blank entry does not fall out to title-only.
|
||||
"""
|
||||
# Narrowing can come back empty - the joined string starts with a comma because the
|
||||
# first contributor was blank, and `authors.join(', ')` does not drop the empty entry.
|
||||
# Falling through to authors[] then still finds a usable name; returning "" would
|
||||
# search by title alone and lose the author we were holding all along.
|
||||
if book.search_author:
|
||||
return book.search_author
|
||||
narrowed = first_author(book.search_author)
|
||||
if narrowed:
|
||||
return narrowed
|
||||
|
||||
if not book.authors:
|
||||
return ""
|
||||
# A bare string here would otherwise be iterated one character at a time; the IRC
|
||||
# source guarded against exactly that before it shared this helper.
|
||||
authors = book.authors if isinstance(book.authors, list) else [book.authors or ""]
|
||||
for author in authors:
|
||||
narrowed = first_author(author or "")
|
||||
if narrowed:
|
||||
return narrowed
|
||||
|
||||
first = book.authors[0]
|
||||
if "," in first:
|
||||
first = first.split(",")[0].strip()
|
||||
|
||||
return first
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_search_title(book: BookMetadata) -> str:
|
||||
@@ -102,15 +171,21 @@ def build_release_search_plan(
|
||||
manual_query: str | None = None,
|
||||
indexers: list[str] | None = None,
|
||||
source_filters: SearchFilters | None = None,
|
||||
user_id: int | None = None,
|
||||
) -> ReleaseSearchPlan:
|
||||
"""Build normalized search variants shared across release sources."""
|
||||
resolved_languages = _normalize_languages(languages)
|
||||
"""Build normalized search variants shared across release sources.
|
||||
|
||||
``user_id`` picks up that user's default languages when the caller does not
|
||||
filter explicitly, so a search started without a language filter uses the
|
||||
reader's own default rather than the instance-wide one.
|
||||
"""
|
||||
resolved_languages = _normalize_languages(languages, user_id)
|
||||
|
||||
resolved_manual_query = None
|
||||
if manual_query:
|
||||
resolved_manual_query = manual_query.strip()[:MANUAL_QUERY_MAX_LEN] or None
|
||||
|
||||
author = _pick_search_author(book)
|
||||
author = pick_search_author(book)
|
||||
base_title = _pick_search_title(book)
|
||||
|
||||
if resolved_manual_query:
|
||||
|
||||
@@ -122,7 +122,12 @@ def is_audiobook(content_type: str | None) -> bool:
|
||||
# had drifted apart: the settings UI only offered m4b/mp3/m4a, which meant a FLAC
|
||||
# audiobook could never be enabled, was silently dropped from every search result, and
|
||||
# was rejected after download as "format not supported".
|
||||
AUDIOBOOK_FORMATS = ("m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus")
|
||||
#
|
||||
# "mp4" is here because some trackers (MyAnonamouse in particular) ship AAC audiobooks
|
||||
# as per-chapter .mp4 files - the same ISO-BMFF container as .m4a/.m4b, just with the
|
||||
# generic extension. Without it those releases downloaded fine and then failed
|
||||
# post-processing with "No book files found in download".
|
||||
AUDIOBOOK_FORMATS = ("m4b", "mp3", "m4a", "mp4", "flac", "ogg", "wma", "aac", "wav", "opus")
|
||||
|
||||
# Multi-file audiobooks are almost always distributed as an archive. These are containers
|
||||
# rather than formats: they are what a *release* looks like, and the formats above are
|
||||
|
||||
@@ -76,6 +76,7 @@ _BOOK_EXTENSIONS = (
|
||||
".m4b",
|
||||
".mobi",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".ogg",
|
||||
".opus",
|
||||
".pdf",
|
||||
|
||||
@@ -45,6 +45,10 @@ _HASH_LENGTH_ED2K = 32
|
||||
_HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN
|
||||
_HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND
|
||||
_METADATA_DOWNLOAD_STATES = {"forcedMetaDL", "metaDL"}
|
||||
# How long add_download waits for magnet metadata before falling back to the info
|
||||
# hash it already knows, rather than holding the download queue on a thin swarm.
|
||||
_METADATA_WAIT_POLLS = 20
|
||||
_METADATA_WAIT_INTERVAL_SECONDS = 0.5
|
||||
_ONE_WEEK_IN_SECONDS = 604800
|
||||
|
||||
|
||||
@@ -221,6 +225,9 @@ class QBittorrentClient(DownloadClient):
|
||||
self._category = config_text(config.get("QBITTORRENT_CATEGORY", "books"))
|
||||
self._download_dir = config_text(config.get("QBITTORRENT_DOWNLOAD_DIR", ""))
|
||||
self._tags = _normalize_tags(config.get("QBITTORRENT_TAG", []))
|
||||
# download_id -> qBittorrent's current primary hash, for identities that no
|
||||
# longer match it directly. See _resolve_torrent().
|
||||
self._primary_hashes: dict[str, str] = {}
|
||||
|
||||
@property
|
||||
def _can_reauthenticate(self) -> bool:
|
||||
@@ -307,13 +314,31 @@ class QBittorrentClient(DownloadClient):
|
||||
params = {"category": category} if category else {}
|
||||
return self._request_torrent_info_records(params)
|
||||
|
||||
def _remember_primary_hash(self, download_id: str, torrent: SimpleNamespace) -> None:
|
||||
"""Note the primary hash a listing scan found, so later lookups skip the scan."""
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
self._primary_hashes[download_id.lower()] = torrent_hash.lower()
|
||||
|
||||
def _resolve_torrent(
|
||||
self, download_id: str, category: str | None = None
|
||||
) -> tuple[SimpleNamespace | None, str | None]:
|
||||
"""Resolve any known torrent identity to its current qBittorrent record."""
|
||||
torrent, error = self._get_torrent_info(download_id)
|
||||
if error or torrent:
|
||||
return torrent, error
|
||||
"""Resolve any known torrent identity to its current qBittorrent record.
|
||||
|
||||
A hybrid torrent's primary hash switches from the v1 hash to the truncated v2
|
||||
hash once metadata resolves, so a download tracked by its v1 hash misses the
|
||||
`hashes=` lookup and falls through to a full listing. Since `get_status()`
|
||||
polls every couple of seconds for the life of the download, remember the
|
||||
primary hash a scan finds and try it first.
|
||||
"""
|
||||
cached = self._primary_hashes.get(download_id.lower())
|
||||
for candidate in (item for item in dict.fromkeys((cached, download_id)) if item):
|
||||
torrent, error = self._get_torrent_info(candidate)
|
||||
if error:
|
||||
return None, error
|
||||
if torrent:
|
||||
self._remember_primary_hash(download_id, torrent)
|
||||
return torrent, None
|
||||
|
||||
categories = [candidate for candidate in (category, self._category) if candidate]
|
||||
for candidate in dict.fromkeys(categories):
|
||||
@@ -325,18 +350,41 @@ class QBittorrentClient(DownloadClient):
|
||||
None,
|
||||
)
|
||||
if torrent:
|
||||
self._remember_primary_hash(download_id, torrent)
|
||||
return torrent, None
|
||||
|
||||
torrents, error = self._list_torrents_by_category(None)
|
||||
if error:
|
||||
return None, error
|
||||
return (
|
||||
next(
|
||||
(item for item in torrents if _torrent_matches_download_id(item, download_id)),
|
||||
None,
|
||||
),
|
||||
torrent = next(
|
||||
(item for item in torrents if _torrent_matches_download_id(item, download_id)),
|
||||
None,
|
||||
)
|
||||
if torrent:
|
||||
self._remember_primary_hash(download_id, torrent)
|
||||
else:
|
||||
# The torrent is gone; drop the note so a re-add is not looked up by a
|
||||
# hash that no longer exists.
|
||||
self._primary_hashes.pop(download_id.lower(), None)
|
||||
return torrent, None
|
||||
|
||||
def _current_hash(self, download_id: str) -> str:
|
||||
"""qBittorrent's current primary hash for any identity we know the torrent by.
|
||||
|
||||
Falls back to the given ID when the torrent cannot be found, so callers
|
||||
still address the hash they were handed and surface the client's error.
|
||||
"""
|
||||
try:
|
||||
torrent, error = self._resolve_torrent(download_id)
|
||||
except _QBITTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("Could not resolve current hash for %s: %s", download_id, e)
|
||||
return download_id
|
||||
if error or not torrent:
|
||||
return download_id
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
return torrent_hash
|
||||
return download_id
|
||||
|
||||
def _list_category_hashes(self, category: str | None) -> set[str] | None:
|
||||
"""Snapshot the hashes qBittorrent currently reports for a category."""
|
||||
@@ -495,9 +543,13 @@ class QBittorrentClient(DownloadClient):
|
||||
message = f"{message} (torrent file fetch failed: {torrent_info.fetch_error})"
|
||||
_raise_runtime_error(message)
|
||||
|
||||
# Wait until qBittorrent has resolved magnet metadata so the returned
|
||||
# hash is its stable primary torrent ID, which may differ from the v1 hash.
|
||||
for _ in range(20):
|
||||
# Prefer qBittorrent's primary torrent ID, which for hybrid torrents
|
||||
# switches from the v1 hash to the truncated v2 hash once metadata
|
||||
# resolves. A magnet with few peers can take minutes to fetch metadata,
|
||||
# and the torrent is worth keeping in the meantime: every lookup goes
|
||||
# through `_resolve_torrent`, which still matches the v1 hash against
|
||||
# `infohash_v1` after the primary ID has changed.
|
||||
for _ in range(_METADATA_WAIT_POLLS):
|
||||
torrent, error = self._resolve_torrent(expected_hash, category)
|
||||
if error:
|
||||
logger.debug("qBittorrent add_download: %s", error)
|
||||
@@ -506,17 +558,18 @@ class QBittorrentClient(DownloadClient):
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
logger.info("Added torrent: %s", torrent_hash)
|
||||
return torrent_hash.lower()
|
||||
time.sleep(0.5)
|
||||
time.sleep(_METADATA_WAIT_INTERVAL_SECONDS)
|
||||
|
||||
_raise_runtime_error(
|
||||
"Torrent metadata resolution was not confirmed within the visibility grace period "
|
||||
f"(response={result_text})"
|
||||
logger.info(
|
||||
"Added torrent %s; metadata still pending after %.0fs, tracking it by info hash",
|
||||
expected_hash,
|
||||
_METADATA_WAIT_POLLS * _METADATA_WAIT_INTERVAL_SECONDS,
|
||||
)
|
||||
except _QBITTORRENT_CLIENT_ERRORS:
|
||||
logger.exception("qBittorrent add failed")
|
||||
raise
|
||||
else:
|
||||
return expected_hash
|
||||
return expected_hash.lower()
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""Get torrent status by hash.
|
||||
@@ -529,7 +582,7 @@ class QBittorrentClient(DownloadClient):
|
||||
|
||||
"""
|
||||
try:
|
||||
torrent, error = self._get_torrent_info(download_id)
|
||||
torrent, error = self._resolve_torrent(download_id)
|
||||
if error:
|
||||
return DownloadStatus.error(error)
|
||||
if not torrent:
|
||||
@@ -613,7 +666,9 @@ class QBittorrentClient(DownloadClient):
|
||||
|
||||
"""
|
||||
try:
|
||||
self._client.torrents_delete(torrent_hashes=download_id, delete_files=delete_files)
|
||||
torrent_hash = self._current_hash(download_id)
|
||||
self._client.torrents_delete(torrent_hashes=torrent_hash, delete_files=delete_files)
|
||||
self._primary_hashes.pop(download_id.lower(), None)
|
||||
logger.info(
|
||||
"Removed torrent from qBittorrent: %s%s",
|
||||
download_id,
|
||||
@@ -635,7 +690,7 @@ class QBittorrentClient(DownloadClient):
|
||||
logger.debug("Could not create category '%s': %s", category, e)
|
||||
|
||||
self._client.torrents_set_category(
|
||||
torrent_hashes=download_id,
|
||||
torrent_hashes=self._current_hash(download_id),
|
||||
category=category,
|
||||
)
|
||||
logger.info("Set qBittorrent category for %s to '%s'", download_id, category)
|
||||
@@ -657,7 +712,7 @@ class QBittorrentClient(DownloadClient):
|
||||
- join `save_path` with the torrent's top-level directory
|
||||
"""
|
||||
try:
|
||||
torrent, error = self._get_torrent_info(download_id)
|
||||
torrent, error = self._resolve_torrent(download_id)
|
||||
if error:
|
||||
logger.debug("qBittorrent get_download_path: %s", error)
|
||||
return None
|
||||
@@ -758,6 +813,33 @@ class QBittorrentClient(DownloadClient):
|
||||
)
|
||||
return None
|
||||
|
||||
def _await_existing_torrent(
|
||||
self, info_hash: str, category: str | None
|
||||
) -> tuple[str, DownloadStatus] | None:
|
||||
"""Report a torrent already in qBittorrent, waiting out magnet metadata first."""
|
||||
for _ in range(_METADATA_WAIT_POLLS):
|
||||
torrent, error = self._resolve_torrent(info_hash, category)
|
||||
if error:
|
||||
logger.debug("qBittorrent find_existing: %s", error)
|
||||
return None
|
||||
if not torrent:
|
||||
return None
|
||||
if getattr(torrent, "state", None) not in _METADATA_DOWNLOAD_STATES:
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
torrent_hash = torrent_hash.lower()
|
||||
return (torrent_hash, self.get_status(torrent_hash))
|
||||
time.sleep(_METADATA_WAIT_INTERVAL_SECONDS)
|
||||
|
||||
# Metadata is still pending, but the torrent is here and `add_download` keeps
|
||||
# one in this state rather than giving up. Report it by info hash so the
|
||||
# caller joins the download in progress instead of adding a duplicate.
|
||||
logger.info(
|
||||
"Existing torrent %s is still fetching metadata; joining it by info hash",
|
||||
info_hash,
|
||||
)
|
||||
return (info_hash.lower(), self.get_status(info_hash))
|
||||
|
||||
def find_existing(
|
||||
self, url: str, category: str | None = None
|
||||
) -> tuple[str, DownloadStatus] | None:
|
||||
@@ -767,21 +849,9 @@ class QBittorrentClient(DownloadClient):
|
||||
if not torrent_info.info_hash:
|
||||
return None
|
||||
|
||||
for _ in range(20):
|
||||
torrent, error = self._resolve_torrent(torrent_info.info_hash, category)
|
||||
if error:
|
||||
logger.debug("qBittorrent find_existing: %s", error)
|
||||
return None
|
||||
if not torrent:
|
||||
return None
|
||||
if getattr(torrent, "state", None) not in _METADATA_DOWNLOAD_STATES:
|
||||
torrent_hash = getattr(torrent, "hash", None)
|
||||
if isinstance(torrent_hash, str) and torrent_hash:
|
||||
torrent_hash = torrent_hash.lower()
|
||||
return (torrent_hash, self.get_status(torrent_hash))
|
||||
time.sleep(0.5)
|
||||
existing = self._await_existing_torrent(torrent_info.info_hash, category)
|
||||
except _QBITTORRENT_CLIENT_ERRORS as e:
|
||||
logger.debug("Error checking for existing torrent: %s", e)
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
return existing
|
||||
|
||||
@@ -80,6 +80,7 @@ _BOOK_EXTENSIONS = (
|
||||
".m4b",
|
||||
".mobi",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".ogg",
|
||||
".opus",
|
||||
".pdf",
|
||||
|
||||
@@ -248,6 +248,15 @@ class SABnzbdClient(DownloadClient):
|
||||
if trusted_url and _url_origin(trusted_url) == target_origin:
|
||||
return True
|
||||
|
||||
named_indexers = config.get("NEWZNAB_INDEXERS", [])
|
||||
if isinstance(named_indexers, list):
|
||||
for row in named_indexers:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
trusted_url = normalize_http_config_url(row.get("url"))
|
||||
if trusted_url and _url_origin(trusted_url) == target_origin:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _get_prowlarr_headers(self, url: str) -> dict:
|
||||
|
||||
@@ -17,6 +17,7 @@ from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -433,6 +434,56 @@ def extract_info_hash_from_torrent(torrent_data: bytes) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _decode_torrent_text(value: object) -> str | None:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def extract_file_list_from_torrent(torrent_data: bytes) -> list[PackFile] | None:
|
||||
"""List the files a .torrent describes, release-relative, without downloading it.
|
||||
|
||||
Multi-file torrents nest every path under the torrent name (which becomes the
|
||||
client's save folder); single-file torrents are just the named file.
|
||||
"""
|
||||
try:
|
||||
decoded, _ = bencode_decode(torrent_data)
|
||||
except _TORRENT_PARSE_ERRORS as e:
|
||||
logger.debug("Failed to parse torrent file list: %s", e)
|
||||
return None
|
||||
if not isinstance(decoded, dict):
|
||||
return None
|
||||
info = decoded.get(b"info")
|
||||
if not isinstance(info, dict):
|
||||
return None
|
||||
|
||||
name = _decode_torrent_text(info.get(b"name")) or ""
|
||||
raw_files = info.get(b"files")
|
||||
if not isinstance(raw_files, list):
|
||||
length = info.get(b"length")
|
||||
if not name:
|
||||
return None
|
||||
return [PackFile(name, length if isinstance(length, int) else None)]
|
||||
|
||||
files: list[PackFile] = []
|
||||
for entry in raw_files:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
raw_path = entry.get(b"path")
|
||||
if not isinstance(raw_path, list):
|
||||
continue
|
||||
segments = [seg for seg in (_decode_torrent_text(part) for part in raw_path) if seg]
|
||||
if not segments:
|
||||
continue
|
||||
if name:
|
||||
segments.insert(0, name)
|
||||
length = entry.get(b"length")
|
||||
files.append(PackFile("/".join(segments), length if isinstance(length, int) else None))
|
||||
return files
|
||||
|
||||
|
||||
def extract_hash_from_magnet(magnet_url: str) -> str | None:
|
||||
"""Extract info_hash from a magnet URL."""
|
||||
if not magnet_url.startswith("magnet:"):
|
||||
|
||||
+100
-4
@@ -5,13 +5,14 @@ import time
|
||||
from http import HTTPStatus
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING, NoReturn
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
|
||||
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError, cookie_store
|
||||
from shelfmark.bypass import BypassCancelledError, ChallengeNotSolvedError, cookie_store
|
||||
from shelfmark.bypass.challenge import challenge_marker
|
||||
from shelfmark.core import search_deadline
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int
|
||||
@@ -28,6 +29,10 @@ logger = setup_logger(__name__)
|
||||
_RNG = random.SystemRandom()
|
||||
|
||||
_MAX_REDIRECTS = 5
|
||||
# DDoS-Guard's re-check probe. Its 302 to `?check=1` is one hop of a handshake rather
|
||||
# than a page: the parameter asserts the caller already holds the cookies that hop
|
||||
# issued.
|
||||
_DDG_CHECK_PARAM = "check"
|
||||
# Z-Library answers the first hit with a 503 whose only real payload is a Set-Cookie; echoing
|
||||
# that cookie back returns the 302 to the real page. Two attempts cover the handshake without
|
||||
# letting a server that keeps re-issuing cookies hold us in the loop.
|
||||
@@ -47,11 +52,13 @@ _BYPASS_GRACE_SLACK_SECONDS = 30.0
|
||||
_BYPASSER_ERRORS = (
|
||||
AttributeError,
|
||||
BypassCancelledError,
|
||||
ChallengeNotSolvedError,
|
||||
KeyError,
|
||||
OSError,
|
||||
RuntimeError,
|
||||
TypeError,
|
||||
ValueError,
|
||||
network.RateLimitedError,
|
||||
requests.exceptions.RequestException,
|
||||
)
|
||||
|
||||
@@ -250,6 +257,33 @@ def _response_challenge_marker(response: requests.Response) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _solvable_url(url: str) -> str:
|
||||
"""The URL a solver should open, given one we may be mid-handshake on.
|
||||
|
||||
The manual AA redirect follower in `html_get_page` walks DDoS-Guard's handshake by
|
||||
reassigning `current_url`, so by the time a 403, a 503 challenge or a redirect loop
|
||||
hands that URL to a bypasser it is often the `?check=1` probe rather than the page
|
||||
we actually wanted. A solver opens it in a fresh browser holding none of the cookies
|
||||
the probe exists to collect, so DDoS-Guard cannot verify it automatically and answers
|
||||
with the manual CAPTCHA page that nothing can solve - the failure in #1292, where
|
||||
FlareSolverr reported "Challenge solved!" over a 4.7 KB DDOS-GUARD interstitial.
|
||||
|
||||
Handing over the pre-probe URL instead lets the solver's browser run the whole
|
||||
handshake itself, which is what a real browser does and what the solver is for.
|
||||
|
||||
Scoped to the hosts whose redirects we follow manually: everywhere else `check` is
|
||||
an ordinary query parameter and none of our business.
|
||||
"""
|
||||
if not network.should_rotate_dns_for_url(url):
|
||||
return url
|
||||
parsed = urlparse(url)
|
||||
params = parse_qsl(parsed.query, keep_blank_values=True)
|
||||
kept = [(key, value) for key, value in params if key != _DDG_CHECK_PARAM]
|
||||
if len(kept) == len(params):
|
||||
return url
|
||||
return urlunparse(parsed._replace(query=urlencode(kept)))
|
||||
|
||||
|
||||
def _fatal_mirror_reason(e: Exception) -> str | None:
|
||||
"""Return why ``e`` proves the mirror is unusable, or None if it may recover.
|
||||
|
||||
@@ -337,6 +371,14 @@ def html_get_page(
|
||||
# so it must be a concrete selector, not the Optional parameter.
|
||||
selector = selector or network.AAMirrorSelector()
|
||||
|
||||
# A release search runs under a wall-clock budget (see shelfmark.core.search_deadline).
|
||||
# Adopting it as the cancel flag is what makes the budget bite on a solve already in
|
||||
# flight: the bypassers and the helper subprocess poll this flag but know nothing about
|
||||
# deadlines. Only when the caller has no flag of its own - a queued download brings one
|
||||
# and must keep it, and runs outside any search context anyway.
|
||||
if cancel_flag is None:
|
||||
cancel_flag = search_deadline.cancel_event()
|
||||
|
||||
def _result(html: str, response_url: str) -> str | tuple[str, str]:
|
||||
if include_response_url:
|
||||
return html, response_url
|
||||
@@ -361,6 +403,16 @@ def html_get_page(
|
||||
retry-loop branch above with `continue`, and with MAX_RETRY=1 there is no
|
||||
later attempt for that branch to run on either.
|
||||
"""
|
||||
# Every handoff reaches the solver through here, so this is the one place the
|
||||
# mid-handshake `?check=1` URL has to be unwound. See _solvable_url.
|
||||
bypass_url = _solvable_url(bypass_url)
|
||||
# Never start a minutes-long browser solve on a budget that has already run out:
|
||||
# nothing downstream would get to report the real reason before the caller's
|
||||
# deadline (or its reverse proxy) cut the request off.
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent; not starting a bypass for %s", bypass_url)
|
||||
return _fail(search_deadline.deadline_message(), bypass_url)
|
||||
|
||||
if status_callback:
|
||||
status_callback("resolving", "Bypassing protection...")
|
||||
try:
|
||||
@@ -377,6 +429,29 @@ def html_get_page(
|
||||
"not solved. Check that FlareSolverr/the CF bypasser is reachable.",
|
||||
bypass_url,
|
||||
)
|
||||
except network.RateLimitedError as e:
|
||||
# Not a bypasser malfunction: the host is throttling this IP and a solve
|
||||
# cannot help. Surface the wait as a plain failure so the search ends cleanly
|
||||
# instead of looping another minutes-long solve against a 429.
|
||||
logger.info("Skipping bypass (rate-limited): %s", e)
|
||||
if status_callback:
|
||||
try:
|
||||
status_callback("resolving", "Rate limited, try again shortly")
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Rate-limit status callback failed", exc_info=True)
|
||||
return _fail(str(e), bypass_url)
|
||||
except ChallengeNotSolvedError as e:
|
||||
# Not a bypasser malfunction: it ran, and the host answered with something it
|
||||
# cannot clear - DDoS-Guard's manual CAPTCHA, typically. Must precede the
|
||||
# generic handler below, whose "the protection bypasser failed" is what sent
|
||||
# #1292 off to fix a FlareSolverr that was working perfectly.
|
||||
logger.info("Bypass ran but did not clear the protection: %s", e)
|
||||
if status_callback:
|
||||
try:
|
||||
status_callback("error", str(e))
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Unsolved-challenge status callback failed", exc_info=True)
|
||||
return _fail(str(e), bypass_url)
|
||||
except _BYPASSER_ERRORS as e:
|
||||
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
|
||||
# Surface the real reason. Without this the caller only sees an empty
|
||||
@@ -388,6 +463,10 @@ def html_get_page(
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Bypass error status callback failed", exc_info=True)
|
||||
if isinstance(e, BypassCancelledError):
|
||||
# The budget trips the same cancel flag a user's cancel does, so tell them
|
||||
# apart here - "cancelled" is a confusing thing to read when nobody did.
|
||||
if search_deadline.expired():
|
||||
return _fail(search_deadline.deadline_message(), bypass_url)
|
||||
return _fail("The protection bypass was cancelled.", bypass_url)
|
||||
return _fail(f"The protection bypasser failed: {type(e).__name__}: {e}", bypass_url)
|
||||
finally:
|
||||
@@ -444,6 +523,9 @@ def html_get_page(
|
||||
for attempt in range(1, retry_limit + 1):
|
||||
# Check for cancellation before each attempt
|
||||
if cancel_flag and cancel_flag.is_set():
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent before attempt %s", attempt)
|
||||
return _fail(search_deadline.deadline_message(), current_url)
|
||||
logger.info("html_get_page cancelled before attempt %s", attempt)
|
||||
return _fail("The request was cancelled.", current_url)
|
||||
|
||||
@@ -472,8 +554,15 @@ def html_get_page(
|
||||
current_url,
|
||||
proxies=get_proxies(current_url),
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
# Bypasser-derived cookies win: they came from a real solved challenge.
|
||||
cookies={**handshake_cookies, **cookies},
|
||||
# Handshake cookies win. They were issued by *this* exchange, so by
|
||||
# definition they are fresher than anything the store holds, and the
|
||||
# server is waiting to see them echoed back on the very next hop.
|
||||
# Letting the store overwrite them meant a stored cookie of the same
|
||||
# name (DDoS-Guard reuses __ddg1_/__ddg2_ for both) was replayed on
|
||||
# every hop and the freshly issued value never left this process - the
|
||||
# ?check=1 probe could then never terminate, so every request ended in
|
||||
# the redirect-loop handoff and paid for a full browser solve.
|
||||
cookies={**cookies, **handshake_cookies},
|
||||
headers=headers,
|
||||
allow_redirects=allow_redirects,
|
||||
verify=get_ssl_verify(current_url),
|
||||
@@ -698,6 +787,12 @@ def html_get_page(
|
||||
f"Anna's Archive returned 404 Not Found for {current_url}.", current_url
|
||||
)
|
||||
|
||||
# 429 = origin throttling this IP. Arm the per-host backoff so selection and
|
||||
# the bypasser stop hammering it, then fall through to normal rotation onto a
|
||||
# mirror that is not (yet) rate-limited.
|
||||
if status == _HTTP_STATUS_RATE_LIMITED:
|
||||
network.note_rate_limited(current_url)
|
||||
|
||||
# Try mirror/DNS rotation on retryable errors. A failure that proves the
|
||||
# mirror is unusable also drops it from this process's rotation, so the
|
||||
# next search does not pay for it again.
|
||||
@@ -851,6 +946,7 @@ def download_url(
|
||||
# Rate limited - skip to next source immediately
|
||||
# (waiting doesn't help with concurrent downloads hitting the same server)
|
||||
if status == _HTTP_STATUS_RATE_LIMITED:
|
||||
network.note_rate_limited(current_url)
|
||||
logger.info("Rate limited (429) - trying next source")
|
||||
if status_callback:
|
||||
status_callback("resolving", "Server busy, trying next")
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
import fnmatch
|
||||
import ipaddress
|
||||
import socket
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
from socket import AddressFamily, SocketKind
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
||||
|
||||
import dns.resolver
|
||||
import httpx
|
||||
@@ -287,6 +288,108 @@ _dead_aa_urls: set[str] = set()
|
||||
_dead_aa_urls_lock = _RLock()
|
||||
|
||||
|
||||
# Per-host rate-limit backoff. A 429 is the origin throttling *this IP*, not a challenge:
|
||||
# a DDoS-Guard/Cloudflare solve still renders, so the bypass "succeeds" yet the cleared
|
||||
# request is rejected again and the throttle is only renewed. The single answer is to
|
||||
# wait, so a 429 sidelines the host for a growing window - mirror selection and the
|
||||
# bypasser both skip a cooling-down host until its deadline passes. The wait escalates
|
||||
# 2 -> 5 -> 10 -> 15 -> 30 minutes each time the host throttles us again *after* we
|
||||
# already waited a full window out; a host left clear for longer than the top step
|
||||
# starts the ladder over. Keyed by host so every mirror and source shares one view;
|
||||
# in-memory only, so a restart starts clean.
|
||||
_RATE_LIMIT_COOLDOWN_LADDER_SECONDS: tuple[float, ...] = (120.0, 300.0, 600.0, 900.0, 1800.0)
|
||||
# A host that has been clear this long is treated as a fresh episode: the next 429
|
||||
# restarts the ladder at 2 minutes rather than resuming the escalation.
|
||||
_RATE_LIMIT_RESET_AFTER_SECONDS = 1800.0
|
||||
|
||||
|
||||
class _Cooldown(NamedTuple):
|
||||
"""One host's active rate-limit window and how far up the ladder it has climbed."""
|
||||
|
||||
deadline: float # time.monotonic() value at which the wait expires
|
||||
level: int # index into _RATE_LIMIT_COOLDOWN_LADDER_SECONDS
|
||||
|
||||
|
||||
_host_cooldowns: dict[str, _Cooldown] = {}
|
||||
_host_cooldowns_lock = _RLock()
|
||||
|
||||
|
||||
class RateLimitedError(Exception):
|
||||
"""Raised to abandon a request whose host is in a 429 cooldown.
|
||||
|
||||
Not a transport failure - nothing is wrong with the network, the origin is
|
||||
throttling this IP and only time clears it. Callers surface it as a plain failure
|
||||
rather than retrying or handing the URL to the bypasser.
|
||||
"""
|
||||
|
||||
|
||||
def _cooldown_key(url: str) -> str:
|
||||
"""Host a cooldown is keyed by; '' when the URL carries none."""
|
||||
return (urllib.parse.urlparse(url).hostname or "").lower()
|
||||
|
||||
|
||||
def note_rate_limited(url: str) -> float:
|
||||
"""Escalate a host's 429 backoff and (re)arm its cooldown; return the wait applied.
|
||||
|
||||
The step advances only when a fresh 429 arrives *after* the previous window already
|
||||
elapsed - i.e. we waited it out and the host throttled us again. A 429 that lands
|
||||
while the host is still cooling is the same episode: it neither escalates the level
|
||||
nor shortens the wait. See the ladder note above.
|
||||
"""
|
||||
host = _cooldown_key(url)
|
||||
if not host:
|
||||
return 0.0
|
||||
now = time.monotonic()
|
||||
ladder = _RATE_LIMIT_COOLDOWN_LADDER_SECONDS
|
||||
with _host_cooldowns_lock:
|
||||
prev = _host_cooldowns.get(host)
|
||||
if prev is not None and now < prev.deadline:
|
||||
# Still inside the current window - same throttling episode, leave it be.
|
||||
return prev.deadline - now
|
||||
if prev is None or now - prev.deadline > _RATE_LIMIT_RESET_AFTER_SECONDS:
|
||||
level = 0
|
||||
else:
|
||||
level = min(prev.level + 1, len(ladder) - 1)
|
||||
wait = ladder[level]
|
||||
_host_cooldowns[host] = _Cooldown(deadline=now + wait, level=level)
|
||||
logger.info(
|
||||
"Rate limited (429): backing off %s for %.0fs (step %d/%d)",
|
||||
host,
|
||||
wait,
|
||||
level + 1,
|
||||
len(ladder),
|
||||
)
|
||||
return wait
|
||||
|
||||
|
||||
def host_cooldown_remaining(url: str) -> float:
|
||||
"""Seconds left on a host's 429 cooldown; 0.0 when clear or expired.
|
||||
|
||||
Leaves an expired record in place: the ladder level it carries is what a later 429
|
||||
escalates from (or resets, once the clear gap is long enough).
|
||||
"""
|
||||
host = _cooldown_key(url)
|
||||
if not host:
|
||||
return 0.0
|
||||
now = time.monotonic()
|
||||
with _host_cooldowns_lock:
|
||||
rec = _host_cooldowns.get(host)
|
||||
if rec is None or rec.deadline <= now:
|
||||
return 0.0
|
||||
return rec.deadline - now
|
||||
|
||||
|
||||
def is_host_cooling_down(url: str) -> bool:
|
||||
"""True while ``url``'s host is inside its 429 cooldown window."""
|
||||
return host_cooldown_remaining(url) > 0.0
|
||||
|
||||
|
||||
def clear_host_cooldowns() -> None:
|
||||
"""Forget all rate-limit cooldowns (manual reset / tests)."""
|
||||
with _host_cooldowns_lock:
|
||||
_host_cooldowns.clear()
|
||||
|
||||
|
||||
def _ensure_initialized() -> None:
|
||||
"""Lazy guard so runtime setup happens once and late calls still work."""
|
||||
global _initialized
|
||||
@@ -1418,8 +1521,13 @@ def get_available_aa_urls() -> list[str]:
|
||||
if not alive and _aa_urls:
|
||||
logger.warning("All AA mirrors quarantined; retrying the full list")
|
||||
_dead_aa_urls.clear()
|
||||
return _aa_urls.copy()
|
||||
return alive
|
||||
alive = _aa_urls.copy()
|
||||
# Prefer mirrors that are not serving a 429 cooldown so rotation stops hammering a
|
||||
# throttled host. When every live mirror is cooling, keep the full live list rather
|
||||
# than returning nothing: selection must never be left with nowhere to point, and
|
||||
# the bypasser's fail-fast reports the "all rate-limited" case with a clear error.
|
||||
breathing = [url for url in alive if not is_host_cooling_down(url)]
|
||||
return breathing or alive
|
||||
|
||||
|
||||
def _aa_base_for_url(url: str) -> str:
|
||||
|
||||
@@ -265,6 +265,8 @@ def queue_release(
|
||||
series_position = release_data.get("series_position") or extra.get("series_position")
|
||||
subtitle = release_data.get("subtitle") or extra.get("subtitle")
|
||||
language = release_data.get("language") or extra.get("language")
|
||||
multi_book = bool(release_data.get("multi_book") or extra.get("multi_book"))
|
||||
book_plan = _normalize_book_plan(release_data.get("book_plan") or extra.get("book_plan"))
|
||||
|
||||
books_output_mode = (
|
||||
str(config.get("BOOKS_OUTPUT_MODE", "folder", user_id=user_id) or "folder")
|
||||
@@ -300,6 +302,8 @@ def queue_release(
|
||||
series_position=series_position,
|
||||
subtitle=subtitle,
|
||||
language=language,
|
||||
multi_book=multi_book or book_plan is not None,
|
||||
book_plan=book_plan,
|
||||
search_mode=search_mode,
|
||||
output_mode=output_mode,
|
||||
output_args=output_args,
|
||||
@@ -408,6 +412,33 @@ def can_retry_download_task(
|
||||
return _has_staged_retry_source(task)
|
||||
|
||||
|
||||
def _normalize_book_plan(value: object) -> list[dict[str, Any]] | None:
|
||||
"""Keep only well-formed pack books: a title plus a non-empty list of file paths."""
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
books: list[dict[str, Any]] = []
|
||||
for entry in value:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
title = normalize_optional_text(entry.get("title"))
|
||||
raw_files = entry.get("files")
|
||||
if title is None or not isinstance(raw_files, list):
|
||||
continue
|
||||
files = [f for f in raw_files if isinstance(f, str) and f.strip()]
|
||||
if not files:
|
||||
continue
|
||||
year = entry.get("year")
|
||||
books.append(
|
||||
{
|
||||
"title": title,
|
||||
"series_position": _optional_number(entry.get("series_position")),
|
||||
"year": year if isinstance(year, int) and not isinstance(year, bool) else None,
|
||||
"files": files,
|
||||
}
|
||||
)
|
||||
return books or None
|
||||
|
||||
|
||||
def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
|
||||
"""Serialize the task state needed for restart-safe retries."""
|
||||
raw_search_mode = getattr(task, "search_mode", None)
|
||||
@@ -437,6 +468,8 @@ def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]:
|
||||
"subtitle": getattr(task, "subtitle", None),
|
||||
"language": getattr(task, "language", None),
|
||||
"search_mode": search_mode,
|
||||
"multi_book": bool(getattr(task, "multi_book", False)),
|
||||
"book_plan": _normalize_book_plan(getattr(task, "book_plan", None)),
|
||||
"output_mode": getattr(task, "output_mode", None),
|
||||
"output_args": dict(raw_output_args) if isinstance(raw_output_args, dict) else {},
|
||||
"user_id": getattr(task, "user_id", None),
|
||||
@@ -495,6 +528,8 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None:
|
||||
subtitle=normalize_optional_text(payload.get("subtitle")),
|
||||
language=normalize_optional_text(payload.get("language")),
|
||||
search_mode=search_mode,
|
||||
multi_book=bool(payload.get("multi_book", False)),
|
||||
book_plan=_normalize_book_plan(payload.get("book_plan")),
|
||||
output_mode=normalize_optional_text(payload.get("output_mode")),
|
||||
output_args=dict(output_args) if isinstance(output_args, dict) else {},
|
||||
user_id=normalize_positive_int(payload.get("user_id")),
|
||||
|
||||
@@ -105,6 +105,7 @@ def process_folder_output(
|
||||
maybe_run_custom_script,
|
||||
prepare_output_files,
|
||||
record_step,
|
||||
resolve_book_groups,
|
||||
transfer_book_files,
|
||||
)
|
||||
|
||||
@@ -260,7 +261,15 @@ def process_folder_output(
|
||||
prepared.cleanup_paths,
|
||||
)
|
||||
|
||||
message = "Complete" if len(final_paths) == 1 else f"Complete ({len(final_paths)} files)"
|
||||
pack_groups = resolve_book_groups(
|
||||
task, prepared.files, organization_mode=plan.organization_mode
|
||||
)
|
||||
if pack_groups is not None:
|
||||
message = f"Complete ({len(pack_groups)} books, {len(final_paths)} files)"
|
||||
elif len(final_paths) == 1:
|
||||
message = "Complete"
|
||||
else:
|
||||
message = f"Complete ({len(final_paths)} files)"
|
||||
status_callback("complete", message)
|
||||
|
||||
return str(final_paths[0])
|
||||
|
||||
@@ -0,0 +1,423 @@
|
||||
"""Multi-book ("pack") release planning.
|
||||
|
||||
A pack is one release that contains several books: a whole-series torrent with one
|
||||
subfolder per book, or a flat folder of `Series 1.0 - Title.m4b` files. The same
|
||||
planning rules serve pre-download inspection (the file list comes from the release
|
||||
source) and post-processing (the file list comes from disk), so what the user
|
||||
approved in the modal is what gets filed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS
|
||||
|
||||
# m4b/m4a hold a whole audiobook in one file; every other audio format (mp3, flac, ...) is
|
||||
# chaptered - many files make up one book. Ebook formats are always one file per book, so
|
||||
# only chaptered *audio* matters here. A flat folder is split one-book-per-file only when
|
||||
# none of its files are chaptered audio: a bare list of `01 - Chapter.mp3` tracks is a
|
||||
# single chaptered audiobook, not a pack of books.
|
||||
_SINGLE_FILE_AUDIO_CONTAINERS = frozenset({"m4b", "m4a"})
|
||||
_CHAPTERED_AUDIO_EXTENSIONS = frozenset(AUDIOBOOK_FORMATS) - _SINGLE_FILE_AUDIO_CONTAINERS
|
||||
|
||||
_YEAR_SUFFIX_RE = re.compile(r"\s*\(\s*(?P<year>\d{4})\s*\)\s*$")
|
||||
_SERIES_MARKER_RE = re.compile(
|
||||
r"""
|
||||
^\s*
|
||||
(?:
|
||||
\[\s*\#?(?P<bracket>\d+(?:\.\d+)?)\s*\] # [03] / [#3]
|
||||
| \#(?P<hash>\d+(?:\.\d+)?) # #3
|
||||
| book\.?\s*(?P<book>\d+(?:\.\d+)?) # Book 3 / Book. 03
|
||||
| (?P<plain>\d+(?:\.\d+)?)(?=[\s\-:.]) # 03 - / 1.0 - / 3.
|
||||
)
|
||||
\s*(?:[-:.]\s*)?
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
_SEPARATOR_CHARS = " \t-_:."
|
||||
# "Gods of Risk 2.5 - Gods of Risk": the title repeated on both sides of the position.
|
||||
_REPEATED_TITLE_RE = re.compile(
|
||||
r"^(?P<left>.+?)\s+(?P<position>\d+(?:\.\d+)?)\s*[-:\u2013]\s*(?P<right>.+)$"
|
||||
)
|
||||
_SERIES_LABEL_WORDS = r"(?:novella|novellas|short\s+story|short|story|novel)"
|
||||
# "Uncrowned Cradle, Book 7" / "Reaper Cradle, Volume 10" / "Wintersteel (Cradle, Book 8)":
|
||||
# an explicit word marks the position at the END of the name. A bare trailing number
|
||||
# is deliberately not matched — "Title - 02" is a chapter, not a series position.
|
||||
_TRAILING_MARKER_RE = re.compile(
|
||||
r"""
|
||||
[\s,\-:\u2013(]*
|
||||
(?:book|volume|vol\.?)\s*\#?(?P<position>\d+(?:\.\d+)?)
|
||||
\s*\)?\s*$
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
# AudiobookBay renders a file inside a folder as "<folder> <file>" with no separator,
|
||||
# so a pack row reads "Author - Title Series, Book 1 Title Series, Book 1".
|
||||
_GLUED_FOLDER_RE = re.compile(
|
||||
r"^(?P<prefix>.+?\s[-\u2013]\s)?(?P<core>.+?)\s+(?P=core)$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackFile:
|
||||
"""One file inside a release, path relative to the release root."""
|
||||
|
||||
path: str
|
||||
size: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackBook:
|
||||
"""One book split out of a pack, files as release-relative paths."""
|
||||
|
||||
title: str
|
||||
series_position: float | None
|
||||
year: int | None
|
||||
files: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackPlan:
|
||||
books: list[PackBook]
|
||||
ignored: list[str]
|
||||
|
||||
@property
|
||||
def is_pack(self) -> bool:
|
||||
return len(self.books) > 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BookGroup:
|
||||
"""One book's on-disk files, ready for transfer."""
|
||||
|
||||
title: str
|
||||
series_position: float | None
|
||||
year: int | None
|
||||
files: list[Path]
|
||||
|
||||
|
||||
def _strip_series_name(name: str, series_name: str | None) -> str:
|
||||
if not series_name:
|
||||
return name
|
||||
prefix = series_name.strip()
|
||||
if not prefix or not name.lower().startswith(prefix.lower()):
|
||||
return name
|
||||
remainder = name[len(prefix) :]
|
||||
if remainder and remainder[0].isalnum():
|
||||
return name
|
||||
return remainder.lstrip(_SEPARATOR_CHARS)
|
||||
|
||||
|
||||
def _strip_series_label(work: str, series_name: str | None) -> str:
|
||||
"""Drop a leading "An <Series> Novella - " style label that some packs prepend."""
|
||||
if not series_name:
|
||||
return work
|
||||
# "The Expanse" is labelled "An Expanse Novella", so match without the article.
|
||||
core = re.sub(r"^(?:the|an?)\s+", "", series_name.strip(), flags=re.IGNORECASE)
|
||||
if not core:
|
||||
return work
|
||||
pattern = re.compile(
|
||||
rf"^(?:an?\s+|the\s+)?{re.escape(core)}\s+{_SERIES_LABEL_WORDS}\s*[-:\u2013]\s*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
return pattern.sub("", work, count=1)
|
||||
|
||||
|
||||
def _collapse_glued_folder(name: str) -> str:
|
||||
match = _GLUED_FOLDER_RE.match(name)
|
||||
if not match:
|
||||
return name
|
||||
prefix = match.group("prefix") or ""
|
||||
core = match.group("core")
|
||||
# "Author - X X" → "Author - X" (the folder carried the author, the file did not).
|
||||
return (prefix + core).strip()
|
||||
|
||||
|
||||
def _strip_author_name(name: str, author_name: str | None) -> str:
|
||||
"""Drop a leading "Author - " (packs are often filed as `Author - Title`)."""
|
||||
if not author_name:
|
||||
return name
|
||||
prefix = author_name.strip()
|
||||
if not prefix or not name.lower().startswith(prefix.lower()):
|
||||
return name
|
||||
remainder = name[len(prefix) :]
|
||||
stripped = remainder.lstrip(_SEPARATOR_CHARS + "\u2013")
|
||||
if stripped == remainder: # no separator after the author: part of the title
|
||||
return name
|
||||
return stripped
|
||||
|
||||
|
||||
def _strip_trailing_series_name(work: str, series_name: str | None) -> str:
|
||||
"""Drop a trailing series name left behind by a trailing position marker."""
|
||||
if not series_name:
|
||||
return work
|
||||
suffix = series_name.strip()
|
||||
if not suffix or not work.lower().endswith(suffix.lower()):
|
||||
return work
|
||||
remainder = work[: -len(suffix)]
|
||||
stripped = remainder.rstrip(_SEPARATOR_CHARS + ",(\u2013")
|
||||
if not stripped or stripped == remainder:
|
||||
return work
|
||||
return stripped
|
||||
|
||||
|
||||
def parse_pack_book_name(
|
||||
name: str, *, series_name: str | None, author_name: str | None = None
|
||||
) -> tuple[str, float | None, int | None]:
|
||||
"""Split a book folder/file-stem name into (title, series position, year).
|
||||
|
||||
Strips a leading series name, a leading position marker (`Book 3 - `, `03 - `,
|
||||
`1.0 - `, `3. `, `[03] `, `#3 `) and a trailing `(YYYY)`. Also understands a
|
||||
trailing marker (`Title Series, Book 3`, `Title (Series, Volume 3)`), a leading
|
||||
`Author - `, and AudiobookBay's glued `<folder> <file>` names. Returns the name
|
||||
unchanged with no position/year when nothing would be left of the title.
|
||||
"""
|
||||
work = _collapse_glued_folder(name.strip())
|
||||
work = _strip_author_name(work, author_name)
|
||||
work = _strip_series_name(work, series_name)
|
||||
|
||||
year: int | None = None
|
||||
year_match = _YEAR_SUFFIX_RE.search(work)
|
||||
if year_match:
|
||||
year = int(year_match.group("year"))
|
||||
work = work[: year_match.start()]
|
||||
|
||||
position: float | None = None
|
||||
repeated = _REPEATED_TITLE_RE.match(work.strip())
|
||||
if (
|
||||
repeated
|
||||
and repeated.group("left").strip().lower() == repeated.group("right").strip().lower()
|
||||
):
|
||||
return repeated.group("right").strip(), float(repeated.group("position")), year
|
||||
|
||||
marker = _SERIES_MARKER_RE.match(work)
|
||||
if marker:
|
||||
raw = (
|
||||
marker.group("bracket")
|
||||
or marker.group("hash")
|
||||
or marker.group("book")
|
||||
or marker.group("plain")
|
||||
)
|
||||
position = float(raw)
|
||||
work = work[marker.end() :]
|
||||
else:
|
||||
trailing = _TRAILING_MARKER_RE.search(work)
|
||||
if trailing and trailing.start() > 0:
|
||||
position = float(trailing.group("position"))
|
||||
work = _strip_trailing_series_name(work[: trailing.start()], series_name)
|
||||
|
||||
work = _strip_series_label(work, series_name)
|
||||
title = work.strip().strip(_SEPARATOR_CHARS).strip()
|
||||
if not title:
|
||||
return name, None, None
|
||||
return title, position, year
|
||||
|
||||
|
||||
def _book_from_name(
|
||||
name: str, files: list[str], series_name: str | None, author_name: str | None = None
|
||||
) -> PackBook:
|
||||
title, position, year = parse_pack_book_name(
|
||||
name, series_name=series_name, author_name=author_name
|
||||
)
|
||||
return PackBook(title=title, series_position=position, year=year, files=files)
|
||||
|
||||
|
||||
def _common_root_parts(paths: list[PurePosixPath]) -> tuple[str, ...]:
|
||||
parents = [p.parent.parts for p in paths]
|
||||
common: list[str] = []
|
||||
for parts in zip(*parents, strict=False):
|
||||
if len(set(parts)) != 1:
|
||||
break
|
||||
common.append(parts[0])
|
||||
return tuple(common)
|
||||
|
||||
|
||||
def plan_pack(
|
||||
files: list[PackFile],
|
||||
*,
|
||||
supported_extensions: set[str],
|
||||
series_name: str | None,
|
||||
author_name: str | None = None,
|
||||
root_depth: int | None = None,
|
||||
) -> PackPlan:
|
||||
"""Group a release's file list into books.
|
||||
|
||||
Files in a subfolder (relative to the common root) group by that subfolder. Files
|
||||
directly in the root split one-book-per-file only when at least two of them carry
|
||||
a series position in their names; otherwise they are one book (a chaptered
|
||||
audiobook, e.g. `01.mp3`, `02.mp3`). `root_depth` fixes how many leading path
|
||||
components form the root instead of deriving it from the files' common parent.
|
||||
"""
|
||||
supported = {ext.lower().lstrip(".") for ext in supported_extensions}
|
||||
book_files: list[PurePosixPath] = []
|
||||
ignored: list[str] = []
|
||||
for pack_file in files:
|
||||
rel = PurePosixPath(pack_file.path.replace("\\", "/").lstrip("./"))
|
||||
if rel.suffix.lower().lstrip(".") in supported:
|
||||
book_files.append(rel)
|
||||
else:
|
||||
ignored.append(pack_file.path)
|
||||
|
||||
if not book_files:
|
||||
return PackPlan(books=[], ignored=ignored)
|
||||
|
||||
root_parts = (
|
||||
_common_root_parts(book_files) if root_depth is None else book_files[0].parts[:root_depth]
|
||||
)
|
||||
depth = len(root_parts)
|
||||
|
||||
root_files: list[PurePosixPath] = []
|
||||
folders: dict[str, list[str]] = {}
|
||||
for rel in book_files:
|
||||
remainder = rel.parts[depth:]
|
||||
if len(remainder) > 1:
|
||||
folders.setdefault(remainder[0], []).append(str(rel))
|
||||
else:
|
||||
root_files.append(rel)
|
||||
|
||||
books: list[PackBook] = []
|
||||
if root_files:
|
||||
parsed = [
|
||||
parse_pack_book_name(f.stem, series_name=series_name, author_name=author_name)
|
||||
for f in root_files
|
||||
]
|
||||
positions = {p[1] for p in parsed if p[1] is not None}
|
||||
titles = {p[0].strip().lower() for p in parsed if p[0]}
|
||||
one_book_per_file = all(
|
||||
rel.suffix.lower().lstrip(".") not in _CHAPTERED_AUDIO_EXTENSIONS for rel in root_files
|
||||
)
|
||||
# Split a flat folder into a book per file only with real evidence of distinct
|
||||
# books: two or more series positions, more than one title, and no chaptered audio
|
||||
# (a bare list of `01 - Chapter.mp3` tracks is one book, not a pack).
|
||||
if len(positions) >= 2 and len(titles) >= 2 and one_book_per_file:
|
||||
books.extend(
|
||||
PackBook(title=title, series_position=position, year=year, files=[str(f)])
|
||||
for f, (title, position, year) in zip(root_files, parsed, strict=True)
|
||||
)
|
||||
elif len(root_files) == 1:
|
||||
books.append(
|
||||
_book_from_name(root_files[0].stem, [str(root_files[0])], series_name, author_name)
|
||||
)
|
||||
else:
|
||||
group_name = root_parts[-1] if root_parts else ""
|
||||
books.append(
|
||||
_book_from_name(group_name, [str(f) for f in root_files], series_name, author_name)
|
||||
)
|
||||
|
||||
books.extend(
|
||||
_book_from_name(folder, paths, series_name, author_name)
|
||||
for folder, paths in folders.items()
|
||||
)
|
||||
return PackPlan(books=books, ignored=ignored)
|
||||
|
||||
|
||||
def _relative_paths(
|
||||
book_files: list[Path], root: Path | None = None
|
||||
) -> tuple[Path, dict[Path, str]]:
|
||||
if root is None:
|
||||
root = Path(os.path.commonpath([str(f.parent) for f in book_files]))
|
||||
return root, {f: f.relative_to(root).as_posix() for f in book_files}
|
||||
|
||||
|
||||
def group_files_into_books(
|
||||
book_files: list[Path],
|
||||
*,
|
||||
series_name: str | None,
|
||||
author_name: str | None = None,
|
||||
root: Path | None = None,
|
||||
) -> list[BookGroup]:
|
||||
"""Heuristically split on-disk files into books (see `plan_pack`).
|
||||
|
||||
`root` pins the release root when grouping a subset of a larger file set.
|
||||
"""
|
||||
if not book_files:
|
||||
return []
|
||||
_root, rel_by_path = _relative_paths(book_files, root)
|
||||
path_by_rel = {rel: path for path, rel in rel_by_path.items()}
|
||||
extensions = {f.suffix.lower().lstrip(".") for f in book_files}
|
||||
plan = plan_pack(
|
||||
[PackFile(rel) for rel in rel_by_path.values()],
|
||||
supported_extensions=extensions,
|
||||
series_name=series_name,
|
||||
author_name=author_name,
|
||||
root_depth=None if root is None else 0,
|
||||
)
|
||||
return [
|
||||
BookGroup(
|
||||
title=book.title,
|
||||
series_position=book.series_position,
|
||||
year=book.year,
|
||||
files=[path_by_rel[rel] for rel in book.files],
|
||||
)
|
||||
for book in plan.books
|
||||
]
|
||||
|
||||
|
||||
def match_plan_to_files(
|
||||
plan: list[PackBook],
|
||||
book_files: list[Path],
|
||||
*,
|
||||
series_name: str | None = None,
|
||||
author_name: str | None = None,
|
||||
) -> list[BookGroup]:
|
||||
"""Apply an approved plan to on-disk files.
|
||||
|
||||
Files match by release-relative path first, then by basename (archive extraction
|
||||
and client save paths can shift the root), then by the on-disk basename being a
|
||||
suffix of the planned name (sources that glue folder and file names together).
|
||||
Book files the plan does not mention fall back to heuristic grouping so nothing
|
||||
is silently dropped.
|
||||
"""
|
||||
if not book_files:
|
||||
return []
|
||||
root, rel_by_path = _relative_paths(book_files)
|
||||
by_rel = {rel: path for path, rel in rel_by_path.items()}
|
||||
by_name: dict[str, list[Path]] = {}
|
||||
for path in book_files:
|
||||
by_name.setdefault(path.name, []).append(path)
|
||||
|
||||
claimed: set[Path] = set()
|
||||
groups: list[BookGroup] = []
|
||||
for book in plan:
|
||||
matched: list[Path] = []
|
||||
for wanted in book.files:
|
||||
wanted_rel = wanted.replace("\\", "/").lstrip("./")
|
||||
candidate = by_rel.get(wanted_rel)
|
||||
if candidate is None:
|
||||
candidates = [
|
||||
p for p in by_name.get(PurePosixPath(wanted_rel).name, []) if p not in claimed
|
||||
]
|
||||
candidate = candidates[0] if candidates else None
|
||||
if candidate is None:
|
||||
wanted_name = PurePosixPath(wanted_rel).name.lower()
|
||||
candidates = [
|
||||
p
|
||||
for p in book_files
|
||||
if p not in claimed and wanted_name.endswith(p.name.lower())
|
||||
]
|
||||
candidate = candidates[0] if len(candidates) == 1 else None
|
||||
if candidate is not None and candidate not in claimed:
|
||||
claimed.add(candidate)
|
||||
matched.append(candidate)
|
||||
if matched:
|
||||
groups.append(
|
||||
BookGroup(
|
||||
title=book.title,
|
||||
series_position=book.series_position,
|
||||
year=book.year,
|
||||
files=matched,
|
||||
)
|
||||
)
|
||||
|
||||
unmatched = [p for p in book_files if p not in claimed]
|
||||
if unmatched:
|
||||
groups.extend(
|
||||
group_files_into_books(
|
||||
unmatched, series_name=series_name, author_name=author_name, root=root
|
||||
)
|
||||
)
|
||||
return groups
|
||||
@@ -40,6 +40,7 @@ from .transfer import (
|
||||
build_metadata_dict,
|
||||
is_torrent_source,
|
||||
process_directory,
|
||||
resolve_book_groups,
|
||||
resolve_hardlink_source,
|
||||
should_hardlink,
|
||||
transfer_book_files,
|
||||
@@ -80,6 +81,7 @@ __all__ = [
|
||||
"process_directory",
|
||||
"record_step",
|
||||
"resolve_custom_script_target",
|
||||
"resolve_book_groups",
|
||||
"resolve_hardlink_source",
|
||||
"run_custom_script",
|
||||
"safe_cleanup_path",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -26,6 +27,7 @@ from shelfmark.download.fs import (
|
||||
)
|
||||
from shelfmark.download.postprocess.policy import get_file_organization, get_template
|
||||
|
||||
from .packs import BookGroup, PackBook, group_files_into_books, match_plan_to_files
|
||||
from .scan import collect_directory_files, scan_directory_tree
|
||||
from .types import TransferPlan
|
||||
from .workspace import safe_cleanup_path
|
||||
@@ -196,6 +198,19 @@ def transfer_book_files(
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
organization_mode = organization_mode or get_file_organization(is_audiobook=is_audiobook)
|
||||
|
||||
groups = resolve_book_groups(task, book_files, organization_mode=organization_mode)
|
||||
if groups is not None:
|
||||
return _transfer_book_groups(
|
||||
groups,
|
||||
destination,
|
||||
task,
|
||||
use_hardlink=use_hardlink,
|
||||
is_torrent=is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
organization_mode=organization_mode,
|
||||
)
|
||||
|
||||
max_attempts = _max_attempts_for_batch(len(book_files))
|
||||
|
||||
final_paths: list[Path] = []
|
||||
@@ -299,6 +314,101 @@ def transfer_book_files(
|
||||
return final_paths, None, op_counts
|
||||
|
||||
|
||||
def resolve_book_groups(
|
||||
task: DownloadTask,
|
||||
book_files: list[Path],
|
||||
*,
|
||||
organization_mode: str,
|
||||
) -> list[BookGroup] | None:
|
||||
"""Split a multi-book pack into per-book groups, or None to file as one book.
|
||||
|
||||
An approved `book_plan` wins; a bare `multi_book` flag falls back to heuristic
|
||||
grouping. Organization `none` keeps files as-is, and a split that yields a single
|
||||
group is not a pack at all.
|
||||
"""
|
||||
if organization_mode == "none" or not (task.book_plan or task.multi_book):
|
||||
return None
|
||||
if task.book_plan:
|
||||
plan = [
|
||||
PackBook(
|
||||
title=str(entry.get("title") or ""),
|
||||
series_position=entry.get("series_position"),
|
||||
year=entry.get("year"),
|
||||
files=list(entry.get("files") or []),
|
||||
)
|
||||
for entry in task.book_plan
|
||||
if isinstance(entry, dict)
|
||||
]
|
||||
groups = match_plan_to_files(
|
||||
plan, book_files, series_name=task.series_name, author_name=task.author
|
||||
)
|
||||
else:
|
||||
groups = group_files_into_books(
|
||||
book_files, series_name=task.series_name, author_name=task.author
|
||||
)
|
||||
return groups if len(groups) > 1 else None
|
||||
|
||||
|
||||
def _transfer_book_groups(
|
||||
groups: list[BookGroup],
|
||||
destination: Path,
|
||||
task: DownloadTask,
|
||||
*,
|
||||
use_hardlink: bool,
|
||||
is_torrent: bool,
|
||||
preserve_source: bool,
|
||||
organization_mode: str,
|
||||
) -> tuple[list[Path], str | None, dict[str, int]]:
|
||||
"""Transfer each book of a pack through the normal single-book path.
|
||||
|
||||
Each book gets an isolated task copy (the single-file path mutates `task.format`)
|
||||
carrying its own title, position and year; the searched book's position must not
|
||||
leak onto its siblings, while author and series name apply to all of them.
|
||||
"""
|
||||
all_paths: list[Path] = []
|
||||
totals: dict[str, int] = {"hardlink": 0, "copy": 0, "move": 0}
|
||||
errors: list[str] = []
|
||||
|
||||
for group in groups:
|
||||
book_task = dataclasses.replace(
|
||||
task,
|
||||
title=group.title or task.title,
|
||||
year=str(group.year) if group.year is not None else None,
|
||||
subtitle=None,
|
||||
series_position=group.series_position,
|
||||
multi_book=False,
|
||||
book_plan=None,
|
||||
)
|
||||
paths, error, op_counts = transfer_book_files(
|
||||
group.files,
|
||||
destination,
|
||||
book_task,
|
||||
use_hardlink=use_hardlink,
|
||||
is_torrent=is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
organization_mode=organization_mode,
|
||||
source_root=group.files[0].parent,
|
||||
)
|
||||
for op, count in op_counts.items():
|
||||
totals[op] = totals.get(op, 0) + count
|
||||
if error:
|
||||
errors.append(f"{group.title}: {error}")
|
||||
logger.warning("Task %s: pack book %r failed: %s", task.task_id, group.title, error)
|
||||
continue
|
||||
all_paths.extend(paths)
|
||||
|
||||
if not all_paths:
|
||||
return [], "; ".join(errors) or "No book files found", totals
|
||||
if errors:
|
||||
logger.warning(
|
||||
"Task %s: pack filed with %d failed book(s): %s",
|
||||
task.task_id,
|
||||
len(errors),
|
||||
"; ".join(errors),
|
||||
)
|
||||
return all_paths, None, totals
|
||||
|
||||
|
||||
def process_directory(
|
||||
directory: Path,
|
||||
ingest_dir: Path,
|
||||
|
||||
@@ -32,6 +32,19 @@ _DEFAULT_QUERY = "The Great Gatsby"
|
||||
_warmup_thread: threading.Thread | None = None
|
||||
_warmup_lock = threading.Lock()
|
||||
|
||||
# Set as soon as a real release search starts. The warm-up exists to pay the cold path
|
||||
# *before* the user does; once they have beaten it to the box there is nothing left to
|
||||
# pre-solve, and running anyway is actively harmful - the bypasser serializes on one
|
||||
# browser, so the warm-up's solve goes in front of the search the user is watching. In
|
||||
# the bundle on issue #1276 that cost a full minute of a 2m27s wait, on a container 16
|
||||
# seconds old, for a throwaway "The Great Gatsby" query nobody asked for.
|
||||
_user_search_seen = threading.Event()
|
||||
|
||||
|
||||
def note_user_search() -> None:
|
||||
"""Record that a real search has run, so a pending warm-up stands down."""
|
||||
_user_search_seen.set()
|
||||
|
||||
|
||||
def _as_bool(value: object, *, default: bool) -> bool:
|
||||
"""Coerce a config value that may arrive as a string, bool or None."""
|
||||
@@ -85,6 +98,12 @@ def run_warmup() -> bool:
|
||||
"""
|
||||
from shelfmark.core.mirrors import has_aa_mirror_configuration
|
||||
|
||||
# Checked here rather than only at schedule time: the delay is what this races with,
|
||||
# so the user's first search usually lands *during* the wait, not before it.
|
||||
if _user_search_seen.is_set():
|
||||
logger.info("Search warm-up skipped: a real search got there first")
|
||||
return False
|
||||
|
||||
if not has_aa_mirror_configuration():
|
||||
logger.debug("Search warm-up skipped: no Anna's Archive mirrors configured")
|
||||
return False
|
||||
|
||||
+46
-10
@@ -42,6 +42,7 @@ from shelfmark.config.settings import (
|
||||
_SUPPORTED_BOOK_LANGUAGE,
|
||||
migrate_audiobook_format_settings,
|
||||
)
|
||||
from shelfmark.core import search_deadline
|
||||
from shelfmark.core.activity_view_state_service import ActivityViewStateService
|
||||
from shelfmark.core.auth_modes import (
|
||||
get_auth_check_admin_status,
|
||||
@@ -62,6 +63,7 @@ from shelfmark.core.notifications import (
|
||||
notify_user,
|
||||
)
|
||||
from shelfmark.core.prefix_middleware import PrefixMiddleware
|
||||
from shelfmark.core.release_inspect_routes import register_release_inspect_routes
|
||||
from shelfmark.core.request_helpers import (
|
||||
coerce_bool,
|
||||
emit_ws_event,
|
||||
@@ -1024,6 +1026,9 @@ def _serialize_release(release: Release) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
register_release_inspect_routes(app, login_required)
|
||||
|
||||
|
||||
@app.route("/api/releases/download", methods=["POST"])
|
||||
@login_required
|
||||
def api_download_release() -> Response | tuple[Response, int]:
|
||||
@@ -1150,7 +1155,7 @@ def api_config() -> Response | tuple[Response, int]:
|
||||
"build_version": BUILD_VERSION,
|
||||
"release_version": RELEASE_VERSION,
|
||||
"book_languages": _SUPPORTED_BOOK_LANGUAGE,
|
||||
"default_language": app_config.BOOK_LANGUAGE,
|
||||
"default_language": app_config.get("BOOK_LANGUAGE", ["en"], user_id=db_user_id),
|
||||
"supported_formats": app_config.SUPPORTED_FORMATS,
|
||||
"supported_audiobook_formats": app_config.SUPPORTED_AUDIOBOOK_FORMATS,
|
||||
"search_mode": search_mode,
|
||||
@@ -1175,6 +1180,12 @@ def api_config() -> Response | tuple[Response, int]:
|
||||
[],
|
||||
user_id=db_user_id,
|
||||
),
|
||||
# The client must not give up before this budget does. `/api/releases`
|
||||
# answers a spent budget with a message naming the real cause (a protection
|
||||
# challenge nobody could solve); a browser that aborted first replaces it
|
||||
# with a generic network/proxy error and RELEASE_SEARCH_TIMEOUT becomes a
|
||||
# setting the user can raise with no visible effect. See issue #1285.
|
||||
"release_search_timeout": search_deadline.budget_seconds(),
|
||||
"settings_enabled": _is_config_dir_writable(),
|
||||
"onboarding_complete": _get_onboarding_complete(),
|
||||
# Default sort orders
|
||||
@@ -2840,6 +2851,7 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
manual_query=query_text if source_query_filters is not None else manual_query,
|
||||
indexers=indexers,
|
||||
source_filters=source_query_filters,
|
||||
user_id=db_user_id,
|
||||
)
|
||||
|
||||
if plan.source_filters is not None:
|
||||
@@ -2892,6 +2904,8 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
if languages_param
|
||||
else None
|
||||
)
|
||||
# Without an explicit filter the plan falls back to this user's default languages.
|
||||
db_user_id = get_session_db_user_id(session)
|
||||
# Content type for audiobook vs ebook search
|
||||
content_type = request.args.get("content_type", "ebook").strip()
|
||||
|
||||
@@ -2939,6 +2953,10 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
elif provider == "manual":
|
||||
resolved_title = title_param or manual_query or "Manual Search"
|
||||
resolved_author = author_param or ""
|
||||
# The release modal sends `authors.join(', ')` as `author`, so the commas here
|
||||
# are joins between contributors, not part of one name. This split is the only
|
||||
# place that knows that, so `search_author` comes from it rather than from the
|
||||
# joined text - see issue #1252.
|
||||
authors = [a.strip() for a in resolved_author.split(",") if a.strip()]
|
||||
|
||||
book = BookMetadata(
|
||||
@@ -2947,7 +2965,7 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
provider_display_name="Manual Search",
|
||||
title=resolved_title,
|
||||
search_title=resolved_title,
|
||||
search_author=resolved_author or None,
|
||||
search_author=authors[0] if authors else None,
|
||||
authors=authors,
|
||||
)
|
||||
else:
|
||||
@@ -2980,18 +2998,36 @@ def api_releases() -> Response | tuple[Response, int]:
|
||||
# Search only enabled sources
|
||||
sources_to_search = [src["name"] for src in list_available_sources() if src["enabled"]]
|
||||
|
||||
# Search each source for releases
|
||||
# Search each source for releases.
|
||||
#
|
||||
# Under a wall-clock budget: this endpoint is synchronous, and the bypass path it
|
||||
# can reach used to be allowed minutes per URL with nothing bounding the request
|
||||
# as a whole. A search that ran into an unsolvable protection challenge therefore
|
||||
# outlived every reverse proxy in front of it and surfaced to the user as
|
||||
# "Server unavailable (504)" - a gateway timeout that blames their proxy for a
|
||||
# challenge failure. The budget is shared across sources, so a stuck first source
|
||||
# cannot spend the whole request on its own. See issue #1276.
|
||||
all_releases = []
|
||||
errors = []
|
||||
source_instances = {} # Keep source instances for column config
|
||||
|
||||
for source_name in sources_to_search:
|
||||
source, releases, error = _search_source_releases(source_name, book)
|
||||
if source is not None:
|
||||
source_instances[source_name] = source
|
||||
all_releases.extend(releases)
|
||||
if error is not None:
|
||||
errors.append(error)
|
||||
# A real search is under way, so a warm-up still sitting on its start-up delay
|
||||
# should stand down rather than queue its throwaway solve in front of this one.
|
||||
warmup.note_user_search()
|
||||
|
||||
with search_deadline.search_deadline():
|
||||
for source_name in sources_to_search:
|
||||
if search_deadline.expired():
|
||||
logger.warning("Release search budget spent; %s not searched", source_name)
|
||||
errors.append(f"{source_name}: {search_deadline.deadline_message()}")
|
||||
continue
|
||||
|
||||
source, releases, error = _search_source_releases(source_name, book)
|
||||
if source is not None:
|
||||
source_instances[source_name] = source
|
||||
all_releases.extend(releases)
|
||||
if error is not None:
|
||||
errors.append(error)
|
||||
|
||||
# Convert Release objects to dicts
|
||||
releases_data = [_serialize_release(release) for release in all_releases]
|
||||
|
||||
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.core.search_plan import ReleaseSearchPlan
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
|
||||
@@ -400,6 +401,14 @@ class DownloadHandler(ABC):
|
||||
"""Return private queue-time fields needed for restart-safe retry."""
|
||||
return {}
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""Return the release's file list without downloading it.
|
||||
|
||||
Lets the UI review a multi-book pack before queueing. Return None when the
|
||||
source cannot know the files ahead of time (magnet links, usenet, ...).
|
||||
"""
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def cancel(self, task_id: str) -> bool:
|
||||
"""Cancel an in-progress download."""
|
||||
@@ -510,6 +519,10 @@ def browse_record_to_book_metadata(
|
||||
"""Convert a source-native browse record into generic book metadata."""
|
||||
resolved_title = title_override or str(record.title or "").strip() or "Unknown title"
|
||||
resolved_author = author_override or str(record.author or "").strip()
|
||||
# `author_override` is the frontend's display string, `authors.join(', ')` - every
|
||||
# contributor, translators included. The split below is the only place that knows the
|
||||
# commas were joins rather than part of a name, so `search_author` is taken from it
|
||||
# rather than from the joined text. See issue #1252.
|
||||
authors = [part.strip() for part in resolved_author.split(",") if part.strip()]
|
||||
publish_year = None
|
||||
|
||||
@@ -526,7 +539,7 @@ def browse_record_to_book_metadata(
|
||||
provider_display_name=get_source_display_name(record.source),
|
||||
title=resolved_title,
|
||||
search_title=resolved_title,
|
||||
search_author=resolved_author or None,
|
||||
search_author=authors[0] if authors else None,
|
||||
authors=authors,
|
||||
cover_url=record.preview,
|
||||
description=record.description,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""AudiobookBay download handler - resolves magnet links and uses shared client lifecycle."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.config import config
|
||||
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
DEFAULT_ABB_HOSTNAME = "audiobookbay.lu"
|
||||
@@ -68,6 +69,19 @@ class AudiobookBayHandler(ExternalClientHandler):
|
||||
return task_id
|
||||
return None
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""Read the torrent's file list off the detail page, without downloading."""
|
||||
raw_url = release_data.get("download_url") or release_data.get("source_url")
|
||||
detail_url = raw_url.strip() if isinstance(raw_url, str) else ""
|
||||
hostname = _resolve_allowed_detail_hostname()
|
||||
if not detail_url or not _detail_url_matches_host(detail_url, hostname):
|
||||
logger.debug("Cannot list files for AudiobookBay release without a valid detail URL")
|
||||
return None
|
||||
detail_html = scraper.fetch_detail_html(detail_url, hostname)
|
||||
if not detail_html:
|
||||
return None
|
||||
return scraper.extract_file_list(detail_html)
|
||||
|
||||
def _get_client(self, protocol: str) -> DownloadClient | None:
|
||||
"""Compatibility shim so module-level patching still works in tests."""
|
||||
return get_client(protocol)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import re
|
||||
import time
|
||||
from threading import Lock
|
||||
from urllib.parse import quote, quote_plus
|
||||
|
||||
import requests
|
||||
@@ -10,6 +11,7 @@ from bs4 import BeautifulSoup
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.download import http as downloader
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
@@ -32,6 +34,13 @@ FIRST_PAGE_SESSION_REFRESH_ATTEMPTS = 2
|
||||
# Legacy search parameter used by older ABB flows
|
||||
LEGACY_CATEGORY_QUERY = "undefined%2Cundefined"
|
||||
|
||||
# Detail pages are fetched once and shared by inspection (file list) and download
|
||||
# (magnet link) so a "review then download" round trip costs ABB a single request.
|
||||
DETAIL_PAGE_CACHE_TTL_SECONDS = 120.0
|
||||
DETAIL_PAGE_CACHE_MAX_ENTRIES = 8
|
||||
_detail_page_cache: dict[str, tuple[float, str]] = {}
|
||||
_detail_page_cache_lock = Lock()
|
||||
|
||||
# Precompiled patterns used while parsing result cards
|
||||
LANGUAGE_PATTERN = re.compile(r"Language:\s*([A-Za-z]+)")
|
||||
POSTED_PATTERN = re.compile(r"Posted:\s*(\d+\s+[A-Za-z]+\s+\d{4})")
|
||||
@@ -39,6 +48,11 @@ FORMAT_PATTERN = re.compile(r"Format:\s*([A-Za-z0-9]+)")
|
||||
BITRATE_PATTERN = re.compile(r"Bitrate:\s*([\d]+\s*[A-Za-z/]+)")
|
||||
SIZE_PATTERN = re.compile(r"File Size:\s*([\d.]+)\s*([A-Za-z]+)")
|
||||
INFO_HASH_LABEL_PATTERN = re.compile(r"Info Hash", re.IGNORECASE)
|
||||
FILE_ROW_SIZE_PATTERN = re.compile(
|
||||
r"^(?P<name>.+?)\s+(?P<size>\d+(?:\.\d+)?)\s*(?P<unit>Bytes?|KBs?|MBs?|GBs?|TBs?)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FILE_SIZE_MULTIPLIERS = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4}
|
||||
|
||||
|
||||
def _coerce_non_negative_float(value: object, default: float) -> float:
|
||||
@@ -348,6 +362,98 @@ def search_audiobookbay(
|
||||
return results
|
||||
|
||||
|
||||
def _get_cached_detail_page(details_url: str) -> str | None:
|
||||
with _detail_page_cache_lock:
|
||||
entry = _detail_page_cache.get(details_url)
|
||||
if entry is None:
|
||||
return None
|
||||
fetched_at, html = entry
|
||||
if time.monotonic() - fetched_at > DETAIL_PAGE_CACHE_TTL_SECONDS:
|
||||
del _detail_page_cache[details_url]
|
||||
return None
|
||||
return html
|
||||
|
||||
|
||||
def _store_cached_detail_page(details_url: str, html: str) -> None:
|
||||
with _detail_page_cache_lock:
|
||||
_detail_page_cache[details_url] = (time.monotonic(), html)
|
||||
while len(_detail_page_cache) > DETAIL_PAGE_CACHE_MAX_ENTRIES:
|
||||
oldest = min(_detail_page_cache, key=lambda key: _detail_page_cache[key][0])
|
||||
del _detail_page_cache[oldest]
|
||||
|
||||
|
||||
def clear_detail_page_cache() -> None:
|
||||
"""Drop cached detail pages (used by tests)."""
|
||||
with _detail_page_cache_lock:
|
||||
_detail_page_cache.clear()
|
||||
|
||||
|
||||
def _fetch_detail_page_once(details_url: str, hostname: str) -> str:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
return _coerce_markup_to_html(
|
||||
downloader.html_get_page(
|
||||
details_url,
|
||||
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fetch_detail_html(details_url: str, hostname: str = "audiobookbay.lu") -> str:
|
||||
"""Fetch a detail page (one retry with a fresh session), cached briefly per URL."""
|
||||
cached = _get_cached_detail_page(details_url)
|
||||
if cached is not None:
|
||||
logger.debug("Reusing recently fetched detail page: %s", details_url)
|
||||
return cached
|
||||
detail_html = _fetch_detail_page_once(details_url, hostname)
|
||||
if not detail_html:
|
||||
detail_html = _fetch_detail_page_once(details_url, hostname)
|
||||
if detail_html:
|
||||
_store_cached_detail_page(details_url, detail_html)
|
||||
return detail_html
|
||||
|
||||
|
||||
def _parse_file_row(text: str) -> PackFile | None:
|
||||
match = FILE_ROW_SIZE_PATTERN.match(text.strip())
|
||||
if not match:
|
||||
return None
|
||||
multiplier = _FILE_SIZE_MULTIPLIERS[match.group("unit")[0].lower()]
|
||||
return PackFile(match.group("name"), int(float(match.group("size")) * multiplier))
|
||||
|
||||
|
||||
def extract_file_list(detail_html: str) -> list[PackFile] | None:
|
||||
"""Read the torrent file rows off a detail page.
|
||||
|
||||
ABB renders the torrent's file table as single-cell rows between the
|
||||
"This is a Multifile Torrent" marker (absent for single-file torrents) and the
|
||||
"Combined File Size" row. Returns None when the page has no such table.
|
||||
"""
|
||||
soup = BeautifulSoup(detail_html, "html.parser")
|
||||
rows: list[PackFile] = []
|
||||
for row in soup.find_all("tr"):
|
||||
cells = row.find_all("td")
|
||||
if not cells:
|
||||
continue
|
||||
label = cells[0].get_text(" ", strip=True)
|
||||
if label.lower().startswith("combined file size"):
|
||||
return rows or None
|
||||
if len(cells) != 1:
|
||||
rows = [] # a two-column metadata row means we're not in the file table yet
|
||||
continue
|
||||
text = cells[0].get_text(" ", strip=True)
|
||||
if "multifile torrent" in text.lower():
|
||||
rows = []
|
||||
continue
|
||||
parsed = _parse_file_row(text)
|
||||
if parsed is not None:
|
||||
rows.append(parsed)
|
||||
return None
|
||||
|
||||
|
||||
def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") -> str | None:
|
||||
"""Extract info hash and trackers from book detail page, then construct magnet link.
|
||||
|
||||
@@ -360,35 +466,7 @@ def extract_magnet_link(details_url: str, hostname: str = "audiobookbay.lu") ->
|
||||
|
||||
"""
|
||||
try:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
|
||||
# Fetch detail page
|
||||
detail_html = _coerce_markup_to_html(
|
||||
downloader.html_get_page(
|
||||
details_url,
|
||||
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
)
|
||||
|
||||
if not detail_html:
|
||||
session = requests.Session()
|
||||
_bootstrap_abb_session(hostname, session, DETAIL_PAGE_RETRY_ATTEMPTS)
|
||||
detail_html = _coerce_markup_to_html(
|
||||
downloader.html_get_page(
|
||||
details_url,
|
||||
retry=DETAIL_PAGE_RETRY_ATTEMPTS,
|
||||
use_bypasser=False,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
session=session,
|
||||
)
|
||||
)
|
||||
|
||||
detail_html = fetch_detail_html(details_url, hostname)
|
||||
if not detail_html:
|
||||
logger.warning("Failed to fetch details page")
|
||||
return None
|
||||
|
||||
@@ -6,6 +6,8 @@ import re
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import replace
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
@@ -16,7 +18,9 @@ import requests
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from bs4.element import NavigableString
|
||||
|
||||
from shelfmark.bypass.challenge import MAX_CHALLENGE_HTML_CHARS, challenge_marker
|
||||
from shelfmark.config.env import DEBUG_SKIP_SOURCES, TMP_DIR
|
||||
from shelfmark.core import search_deadline
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.languages import language_alias_map
|
||||
from shelfmark.core.logger import setup_logger
|
||||
@@ -42,7 +46,7 @@ from shelfmark.release_sources import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
|
||||
@@ -111,6 +115,16 @@ def _html_response_text(response: str | tuple[str, str]) -> str:
|
||||
return response
|
||||
|
||||
|
||||
def _html_response_url(response: str | tuple[str, str]) -> str | None:
|
||||
"""The URL that actually answered, when the downloader was asked to report it.
|
||||
|
||||
None for the plain-string shape, so a caller can fall back to what it requested.
|
||||
"""
|
||||
if isinstance(response, tuple):
|
||||
return response[1] or None
|
||||
return None
|
||||
|
||||
|
||||
def _attr_to_str(value: object) -> str | None:
|
||||
"""Convert a BeautifulSoup attribute value to a plain string."""
|
||||
if isinstance(value, str):
|
||||
@@ -555,13 +569,6 @@ _AA_PAGE_MARKERS = (
|
||||
"/fast_download",
|
||||
"/slow_download",
|
||||
)
|
||||
_CHALLENGE_MARKERS = (
|
||||
"ddos-guard",
|
||||
"just a moment",
|
||||
"cloudflare",
|
||||
"checking your browser",
|
||||
"cf-browser-verification",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_aa_page(html: str) -> bool:
|
||||
@@ -571,12 +578,120 @@ def _looks_like_aa_page(html: str) -> bool:
|
||||
|
||||
|
||||
def _looks_like_challenge_page(html: str) -> bool:
|
||||
"""Whether ``html`` is a protection interstitial rather than the site behind it."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in _CHALLENGE_MARKERS)
|
||||
"""Whether ``html`` is a protection interstitial rather than the site behind it.
|
||||
|
||||
Delegates to the shared detector rather than substring-matching here. A bare
|
||||
"ddos-guard"/"cloudflare" scan flags the protected site's *own* pages: DDoS-Guard
|
||||
links its endpoints on everything it fronts, and AA ships a `DDOS-GUARD` comment in
|
||||
the inline JS on every page it serves. That misread every real AA response that was
|
||||
not a results table as an unsolved challenge, and sent users off to fix a bypasser
|
||||
that had just succeeded - see #1289/#1292. `challenge_marker` caps its scan at
|
||||
64 KB, which is what separates a few-KB interstitial from the page behind it.
|
||||
"""
|
||||
return challenge_marker(html) is not None
|
||||
|
||||
|
||||
# Pages already fetched during the search in flight, keyed by URL. Scoped to one
|
||||
# DirectDownload.search() so nothing is carried between requests.
|
||||
_search_page_cache: ContextVar[dict[str, tuple[str, Tag | None]] | None] = ContextVar(
|
||||
"aa_search_page_cache", default=None
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _search_page_reuse() -> Iterator[None]:
|
||||
"""Fetch each distinct AA search URL at most once per search.
|
||||
|
||||
One search asks AA for the same URL more than once. The language-filter retry in
|
||||
`search()` re-runs every title variant, and when DIRECT_DOWNLOAD_LANGUAGE_FROM_PATH
|
||||
is on the requested language is applied locally instead of as `&lang=`, so both
|
||||
passes build a byte-identical URL - the retry differs only in the filtering it does
|
||||
to the response it already had. A repeat is not a cheap round trip either: AA is
|
||||
behind DDoS-Guard, so each one is a fresh browser solve, tens of seconds that buy
|
||||
nothing. See issue #1285.
|
||||
"""
|
||||
token = _search_page_cache.set({})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_search_page_cache.reset(token)
|
||||
|
||||
|
||||
def _is_reusable_answer(result: tuple[str, Tag | None]) -> bool:
|
||||
"""Whether a fetched page is an answer, rather than a giving-up worth retrying.
|
||||
|
||||
`_fetch_search_table_uncached` exists to rotate past mirrors that are not actually AA,
|
||||
and when it runs out of them it *returns* instead of raising: a page with no results
|
||||
table and no marker. Storing that would hand the language-filter retry - the pass this
|
||||
cache exists for - a mirror set that may have recovered in between (DNS rotation, a
|
||||
mirror coming back), turning a transient outage into "this book has no releases". A
|
||||
real "No files found." is an answer and is worth keeping.
|
||||
"""
|
||||
html, tbody = result
|
||||
return tbody is not None or "No files found." in html or _looks_like_aa_page(html)
|
||||
|
||||
|
||||
# How much of an unreadable search page to quote in the debug log. Enough to carry the
|
||||
# <head> - title, injected challenge scripts - without pasting a 180 KB page into a log
|
||||
# file that ships inside the debug bundle.
|
||||
_PAGE_FINGERPRINT_CHARS = 700
|
||||
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def _log_untabled_search_page(url: str, html: str) -> None:
|
||||
"""Record why a search page with no results table is about to be classified.
|
||||
|
||||
#1289 cost a full investigation because the log said only "unsolved protection
|
||||
challenge" while FlareSolverr said "Challenge solved!", and the debug bundle carries
|
||||
no response bodies - there was no way to tell a real AA page from an interstitial
|
||||
after the fact. These are the facts that would have settled it in one line: the size
|
||||
(the 64 KB cap is what separates the two), which markers matched, and the head of
|
||||
the document.
|
||||
|
||||
Diagnostics must never be the reason a search fails, so this swallows its own errors.
|
||||
"""
|
||||
try:
|
||||
title_match = _TITLE_RE.search(html[: _PAGE_FINGERPRINT_CHARS * 4])
|
||||
title = " ".join(title_match.group(1).split())[:120] if title_match else "<none>"
|
||||
lowered = html.lower()
|
||||
aa_markers = [marker for marker in _AA_PAGE_MARKERS if marker in lowered]
|
||||
logger.info(
|
||||
"Search page has no results table: %s (bytes=%d, title=%r, aa_markers=%s, "
|
||||
"challenge_marker=%r, over_challenge_size_cap=%s)",
|
||||
url,
|
||||
len(html),
|
||||
title,
|
||||
aa_markers or "none",
|
||||
challenge_marker(html),
|
||||
len(html) > MAX_CHALLENGE_HTML_CHARS,
|
||||
)
|
||||
logger.debug(
|
||||
"Untabled search page head (%d of %d bytes): %s",
|
||||
min(len(html), _PAGE_FINGERPRINT_CHARS),
|
||||
len(html),
|
||||
html[:_PAGE_FINGERPRINT_CHARS],
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Could not fingerprint the untabled search page", exc_info=True)
|
||||
|
||||
|
||||
def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[str, Tag | None]:
|
||||
"""Fetch the AA search page, reusing one already fetched during this search."""
|
||||
cache = _search_page_cache.get()
|
||||
if cache is not None and url in cache:
|
||||
logger.debug("Reusing search page already fetched for this search: %s", url)
|
||||
return cache[url]
|
||||
|
||||
result = _fetch_search_table_uncached(url, selector)
|
||||
|
||||
if cache is not None and _is_reusable_answer(result):
|
||||
cache[url] = result
|
||||
return result
|
||||
|
||||
|
||||
def _fetch_search_table_uncached(
|
||||
url: str, selector: network.AAMirrorSelector
|
||||
) -> tuple[str, Tag | None]:
|
||||
"""Fetch the AA search page, retrying past mirrors that are not actually AA.
|
||||
|
||||
A parked or seized domain answers 200 with a page that has no results table and no
|
||||
@@ -586,10 +701,26 @@ def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[s
|
||||
"""
|
||||
attempt_url = url
|
||||
for _ in range(len(network.get_available_aa_urls()) or 1):
|
||||
# Every mirror shares the protection, so once the search budget is gone another
|
||||
# mirror is another full solve nobody is still waiting for.
|
||||
if search_deadline.expired():
|
||||
raise SearchUnavailableError(search_deadline.deadline_message())
|
||||
|
||||
# include_response_url is what makes the diagnostics below name the mirror that
|
||||
# actually answered. html_get_page rotates mirrors and follows redirects on its
|
||||
# own, so `attempt_url` is only where this iteration started: #1298's bundle
|
||||
# reported the untabled page against annas-archive.gl when the body had come
|
||||
# from .pk, which is precisely the triage cost #1289 added the line to remove.
|
||||
response = downloader.html_get_page(
|
||||
attempt_url, selector=selector, allow_bypasser_fallback=True
|
||||
attempt_url,
|
||||
selector=selector,
|
||||
allow_bypasser_fallback=True,
|
||||
include_response_url=True,
|
||||
)
|
||||
if not response:
|
||||
html = _html_response_text(response)
|
||||
# Checked on the body, not on `response`: with include_response_url the give-up
|
||||
# shape is the tuple ("", url), and a tuple is truthy.
|
||||
if not html:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients.
|
||||
# html_get_page records the concrete give-up reason on the selector; fall
|
||||
# back to the generic line only if nothing was recorded.
|
||||
@@ -598,7 +729,7 @@ def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[s
|
||||
)
|
||||
raise SearchUnavailableError(f"Unable to reach download source. {detail}")
|
||||
|
||||
html = _html_response_text(response)
|
||||
answered_url = _html_response_url(response) or attempt_url
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if isinstance(table, Tag):
|
||||
@@ -609,20 +740,38 @@ def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[s
|
||||
if "No files found." in html:
|
||||
# A real, genuinely empty answer from a healthy mirror.
|
||||
return html, None
|
||||
|
||||
# A search page with no table is the one shape we cannot read off the response
|
||||
# alone, and the response body is not in the debug bundle. Fingerprint it here
|
||||
# so the next report says which branch fired and why, rather than costing
|
||||
# another round of guesswork - see #1289.
|
||||
_log_untabled_search_page(answered_url, html)
|
||||
|
||||
if _looks_like_aa_page(html):
|
||||
# A real AA response in a shape the caller should report as drift. Checked
|
||||
# ahead of the challenge branch: AA's own pages carry the protection's
|
||||
# markers, so an interstitial is only the better explanation once the page
|
||||
# has nothing of AA's about it. A genuine interstitial has no AA markers.
|
||||
return html, None
|
||||
if _looks_like_challenge_page(html):
|
||||
# The bypass did not actually clear the protection - the interstitial is
|
||||
# what came back. Rotating is pointless (every mirror shares the same
|
||||
# protection) and reporting it as an empty result is worse: the user is
|
||||
# told their query found nothing when the search never ran.
|
||||
#
|
||||
# The wording no longer blames the bypasser outright. In #1292 it was
|
||||
# reachable and working, and the page it was handed was DDoS-Guard's manual
|
||||
# CAPTCHA - so "check that the bypasser is working" was the one piece of
|
||||
# advice guaranteed to waste the reporter's time. Name the marker instead
|
||||
# and let the two causes be told apart.
|
||||
msg = (
|
||||
"Anna's Archive answered with an unsolved protection challenge. "
|
||||
"Check that the bypasser is reachable and working."
|
||||
"Anna's Archive answered with a protection challenge that was not "
|
||||
f"cleared (marker={challenge_marker(html)!r}). If the bypasser reports "
|
||||
"solving it, the host is serving a manual CAPTCHA that no bypasser can "
|
||||
"answer - try again shortly. Otherwise check that the bypasser is "
|
||||
"reachable and working."
|
||||
)
|
||||
raise SearchUnavailableError(msg)
|
||||
if _looks_like_aa_page(html):
|
||||
# A real AA response in a shape the caller should report as drift.
|
||||
# Not the mirror's fault.
|
||||
return html, None
|
||||
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
fatal=True, reason="responded without an Anna's Archive page"
|
||||
@@ -1909,6 +2058,22 @@ class DirectDownloadSource(ReleaseSource):
|
||||
) -> list[Release]:
|
||||
"""Search for releases using the book's metadata.
|
||||
|
||||
The whole fan-out runs under one page cache, so a URL built twice by different
|
||||
passes is fetched once. See `_search_page_reuse`.
|
||||
"""
|
||||
with _search_page_reuse():
|
||||
return self._search(book, plan, expand_search=expand_search, content_type=content_type)
|
||||
|
||||
def _search(
|
||||
self,
|
||||
book: BookMetadata,
|
||||
plan: ReleaseSearchPlan,
|
||||
*,
|
||||
expand_search: bool = False,
|
||||
content_type: str = "ebook",
|
||||
) -> list[Release]:
|
||||
"""Search for releases using the book's metadata.
|
||||
|
||||
Priority: ISBN search first (most precise), then title+author fallback.
|
||||
For non-English languages, uses localized titles from book.titles_by_language.
|
||||
|
||||
@@ -1973,6 +2138,12 @@ class DirectDownloadSource(ReleaseSource):
|
||||
query = f"{title} {author}".strip()
|
||||
if not query:
|
||||
continue
|
||||
# `except Exception` below keeps this loop going past a failed variant, which
|
||||
# is right for a parse error and wrong for a spent budget: without this the
|
||||
# variants queue up behind each other and the request outlives the caller.
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent; skipping remaining title variants")
|
||||
break
|
||||
|
||||
logger.debug("Searching direct_download: title_author='%s', langs=%s", query, langs)
|
||||
filters = SearchFilters(lang=langs if langs is not None else [])
|
||||
@@ -1986,7 +2157,11 @@ class DirectDownloadSource(ReleaseSource):
|
||||
except Exception:
|
||||
logger.exception("Search error")
|
||||
|
||||
if not all_results and any(langs for _, langs in searches):
|
||||
if (
|
||||
not all_results
|
||||
and any(langs for _, langs in searches)
|
||||
and not search_deadline.expired()
|
||||
):
|
||||
logger.debug(
|
||||
"No title+author results with language filter, retrying without language filter"
|
||||
)
|
||||
@@ -1994,6 +2169,9 @@ class DirectDownloadSource(ReleaseSource):
|
||||
query = f"{title} {author}".strip()
|
||||
if not query:
|
||||
continue
|
||||
if search_deadline.expired():
|
||||
logger.info("Release search budget spent; skipping remaining retries")
|
||||
break
|
||||
|
||||
logger.debug("Searching direct_download: title_author='%s', langs=[]", query)
|
||||
try:
|
||||
|
||||
@@ -15,6 +15,7 @@ if TYPE_CHECKING:
|
||||
from shelfmark.api.websocket import ws_manager
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.search_plan import pick_search_author
|
||||
from shelfmark.core.utils import is_audiobook
|
||||
from shelfmark.release_sources import (
|
||||
ColumnColorHint,
|
||||
@@ -394,11 +395,13 @@ class IRCReleaseSource(ReleaseSource):
|
||||
if book.search_title or book.title:
|
||||
parts.append(book.search_title or book.title)
|
||||
|
||||
if book.search_author:
|
||||
parts.append(book.search_author)
|
||||
elif book.authors:
|
||||
# Use first author
|
||||
author = book.authors[0] if isinstance(book.authors, list) else book.authors
|
||||
# Only ever the first author: both metadata fields can arrive holding every
|
||||
# contributor joined with ", ", and an IRC query carrying an author plus two
|
||||
# translators matches nothing. The choice between them - and the narrowing - is
|
||||
# `pick_search_author`, shared with the search plan so this cannot drift from it
|
||||
# again. See issue #1252.
|
||||
author = pick_search_author(book)
|
||||
if author:
|
||||
parts.append(author)
|
||||
|
||||
return " ".join(parts)
|
||||
@@ -428,14 +431,15 @@ class IRCReleaseSource(ReleaseSource):
|
||||
"m4b": 0,
|
||||
"mp3": 1,
|
||||
"m4a": 2,
|
||||
"flac": 3,
|
||||
"opus": 4,
|
||||
"ogg": 5,
|
||||
"aac": 6,
|
||||
"wav": 7,
|
||||
"wma": 8,
|
||||
"rar": 9,
|
||||
"zip": 10,
|
||||
"mp4": 3,
|
||||
"flac": 4,
|
||||
"opus": 5,
|
||||
"ogg": 6,
|
||||
"aac": 7,
|
||||
"wav": 8,
|
||||
"wma": 9,
|
||||
"rar": 10,
|
||||
"zip": 11,
|
||||
}
|
||||
|
||||
def _convert_to_releases(
|
||||
|
||||
@@ -8,6 +8,7 @@ from shelfmark.core.settings_registry import (
|
||||
HeadingField,
|
||||
PasswordField,
|
||||
SettingsField,
|
||||
TableField,
|
||||
TagListField,
|
||||
TextField,
|
||||
register_settings,
|
||||
@@ -16,12 +17,36 @@ from shelfmark.core.utils import normalize_http_url
|
||||
|
||||
|
||||
def _test_newznab_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Test the Newznab connection using current form values."""
|
||||
"""Test all named Newznab connections, or the legacy connection as fallback."""
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.release_sources.newznab.api import NewznabClient
|
||||
from shelfmark.release_sources.newznab.source import _parse_indexer_rows
|
||||
|
||||
current_values = current_values or {}
|
||||
|
||||
raw_indexers = current_values.get("NEWZNAB_INDEXERS")
|
||||
if raw_indexers is None:
|
||||
raw_indexers = config.get("NEWZNAB_INDEXERS", [])
|
||||
indexers = _parse_indexer_rows(raw_indexers)
|
||||
|
||||
if indexers:
|
||||
details: list[str] = []
|
||||
all_successful = True
|
||||
for name, url, api_key in indexers:
|
||||
try:
|
||||
success, message = NewznabClient(url, api_key).test_connection()
|
||||
except Exception as e: # noqa: BLE001 — surface unexpected errors to the UI
|
||||
success, message = False, f"Connection failed: {e!s}"
|
||||
all_successful = all_successful and success
|
||||
details.append(f"{name}: {message}")
|
||||
|
||||
summary = (
|
||||
f"Connected to all {len(indexers)} indexers"
|
||||
if all_successful
|
||||
else "One or more Newznab indexers failed"
|
||||
)
|
||||
return {"success": all_successful, "message": summary, "details": details}
|
||||
|
||||
raw_url = str(current_values.get("NEWZNAB_URL") or config.get("NEWZNAB_URL", "") or "")
|
||||
api_key = str(current_values.get("NEWZNAB_API_KEY") or config.get("NEWZNAB_API_KEY", "") or "")
|
||||
|
||||
@@ -64,25 +89,60 @@ def newznab_config_settings() -> list[SettingsField]:
|
||||
default=False,
|
||||
description="Enable searching for books via a Newznab-compatible indexer",
|
||||
),
|
||||
TableField(
|
||||
key="NEWZNAB_INDEXERS",
|
||||
label="Named Indexers",
|
||||
description=(
|
||||
"Add each Newznab-compatible indexer separately. The configured name is shown "
|
||||
"beside every result from that indexer."
|
||||
),
|
||||
columns=[
|
||||
{
|
||||
"key": "name",
|
||||
"label": "Name",
|
||||
"type": "text",
|
||||
"placeholder": "NZBGeek",
|
||||
},
|
||||
{
|
||||
"key": "url",
|
||||
"label": "URL",
|
||||
"type": "text",
|
||||
"placeholder": "https://api.nzbgeek.info",
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"type": "password",
|
||||
"placeholder": "Optional",
|
||||
},
|
||||
],
|
||||
default=[],
|
||||
add_label="Add Indexer",
|
||||
empty_message=(
|
||||
"No named indexers configured. The legacy single-indexer fields below are used "
|
||||
"as a fallback."
|
||||
),
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
TextField(
|
||||
key="NEWZNAB_URL",
|
||||
label="Newznab URL",
|
||||
description="Base URL of your Newznab indexer or aggregator",
|
||||
label="Legacy Newznab URL",
|
||||
description="Used only when the named indexer list is empty",
|
||||
placeholder="http://nzbhydra:5076",
|
||||
required=True,
|
||||
required=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
PasswordField(
|
||||
key="NEWZNAB_API_KEY",
|
||||
label="API Key",
|
||||
description="Your Newznab API key (leave blank if not required)",
|
||||
label="Legacy API Key",
|
||||
description="Used only with the legacy Newznab URL",
|
||||
required=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
ActionButton(
|
||||
key="test_newznab",
|
||||
label="Test Connection",
|
||||
description="Verify your Newznab configuration",
|
||||
label="Test Connections",
|
||||
description="Verify every named indexer, or the legacy connection when the list is empty",
|
||||
style="primary",
|
||||
callback=_test_newznab_connection,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
|
||||
@@ -4,7 +4,10 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
from urllib.parse import urlparse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from shelfmark.core.search_plan import ReleaseSearchPlan
|
||||
@@ -48,6 +51,50 @@ _DEFAULT_BOOK_CATS = [7000]
|
||||
NEWZNAB_SEARCH_TIMEOUT_SECONDS = _SEARCH_TIMEOUT
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _NamedClient:
|
||||
"""A configured Newznab connection and its stable cache namespace."""
|
||||
|
||||
name: str
|
||||
connection_id: str
|
||||
client: NewznabClient
|
||||
|
||||
|
||||
def _parse_indexer_rows(raw: object) -> list[tuple[str, str, str]]:
|
||||
"""Normalize structured Newznab indexer settings.
|
||||
|
||||
Invalid/incomplete rows are ignored so one partially edited row cannot disable
|
||||
the other configured indexers.
|
||||
"""
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
|
||||
indexers: list[tuple[str, str, str]] = []
|
||||
seen_connections: set[tuple[str, str]] = set()
|
||||
for row in raw:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
raw_url = str(row.get("url") or "").strip()
|
||||
url = normalize_http_url(raw_url)
|
||||
if not url:
|
||||
if raw_url:
|
||||
logger.warning("Newznab: ignoring indexer row with invalid URL '%s'", raw_url)
|
||||
continue
|
||||
|
||||
api_key = str(row.get("api_key") or "").strip()
|
||||
connection_key = (url, api_key)
|
||||
if connection_key in seen_connections:
|
||||
continue
|
||||
seen_connections.add(connection_key)
|
||||
|
||||
configured_name = str(row.get("name") or "").strip()
|
||||
hostname = urlparse(url).hostname or ""
|
||||
name = configured_name or hostname or "Newznab"
|
||||
indexers.append((name, url, api_key))
|
||||
|
||||
return indexers
|
||||
|
||||
|
||||
def _parse_category_ids(raw: object) -> list[int]:
|
||||
"""Parse a configured category setting into Newznab category IDs.
|
||||
|
||||
@@ -146,8 +193,11 @@ def _newznab_result_to_release(
|
||||
else None
|
||||
)
|
||||
|
||||
# Build source_id from GUID
|
||||
source_id = result.get("guid") or f"newznab:{hash(raw_title)}"
|
||||
# Namespace IDs from named connections so identical GUIDs returned by two
|
||||
# indexers cannot overwrite one another in the private release cache.
|
||||
raw_source_id = result.get("guid") or f"newznab:{hash(raw_title)}"
|
||||
connection_id = str(result.get("_newznab_connection_id") or "").strip()
|
||||
source_id = f"newznab:{connection_id}:{raw_source_id}" if connection_id else raw_source_id
|
||||
|
||||
# Cache the raw result for the handler
|
||||
cache_release(source_id, result)
|
||||
@@ -272,6 +322,7 @@ class NewznabSource(ReleaseSource):
|
||||
)
|
||||
|
||||
def _get_client(self) -> NewznabClient | None:
|
||||
"""Build the legacy single-indexer client."""
|
||||
raw_url = str(config.get("NEWZNAB_URL", "") or "")
|
||||
api_key = str(config.get("NEWZNAB_API_KEY", "") or "")
|
||||
|
||||
@@ -284,6 +335,28 @@ class NewznabSource(ReleaseSource):
|
||||
|
||||
return NewznabClient(url, api_key or "")
|
||||
|
||||
def _get_clients(self) -> list[_NamedClient]:
|
||||
"""Build named clients, falling back to the legacy single connection."""
|
||||
configured = _parse_indexer_rows(config.get("NEWZNAB_INDEXERS", []))
|
||||
if configured:
|
||||
clients: list[_NamedClient] = []
|
||||
for name, url, api_key in configured:
|
||||
digest = sha256(f"{name}\0{url}\0{api_key}".encode()).hexdigest()[:16]
|
||||
clients.append(
|
||||
_NamedClient(
|
||||
name=name,
|
||||
connection_id=digest,
|
||||
client=NewznabClient(url, api_key),
|
||||
)
|
||||
)
|
||||
return clients
|
||||
|
||||
legacy_client = self._get_client()
|
||||
if legacy_client is None:
|
||||
return []
|
||||
legacy_name = str(config.get("NEWZNAB_NAME", "") or "").strip() or "Newznab"
|
||||
return [_NamedClient(name=legacy_name, connection_id="legacy", client=legacy_client)]
|
||||
|
||||
def search(
|
||||
self,
|
||||
book: BookMetadata,
|
||||
@@ -293,8 +366,8 @@ class NewznabSource(ReleaseSource):
|
||||
content_type: str = "ebook",
|
||||
) -> list[Release]:
|
||||
"""Search the Newznab indexer for releases matching the book."""
|
||||
client = self._get_client()
|
||||
if not client:
|
||||
clients = self._get_clients()
|
||||
if not clients:
|
||||
logger.warning("Newznab not configured - skipping search")
|
||||
return []
|
||||
|
||||
@@ -324,40 +397,60 @@ class NewznabSource(ReleaseSource):
|
||||
all_results: list[dict] = []
|
||||
|
||||
try:
|
||||
for idx, query in enumerate(queries, start=1):
|
||||
_check_timeout()
|
||||
if len(queries) > 1:
|
||||
logger.debug("Newznab query %d/%d: '%s'", idx, len(queries), query)
|
||||
for connection in clients:
|
||||
try:
|
||||
for idx, query in enumerate(queries, start=1):
|
||||
_check_timeout()
|
||||
if len(queries) > 1:
|
||||
logger.debug(
|
||||
"Newznab [%s] query %d/%d: '%s'",
|
||||
connection.name,
|
||||
idx,
|
||||
len(queries),
|
||||
query,
|
||||
)
|
||||
|
||||
raw = client.search(query=query, categories=categories)
|
||||
raw = connection.client.search(query=query, categories=categories)
|
||||
|
||||
# Auto-expand: retry without category filter if no results
|
||||
if not raw and categories and auto_expand:
|
||||
_check_timeout()
|
||||
logger.info(
|
||||
"Newznab: no results for '%s' with category filter, auto-expanding",
|
||||
query,
|
||||
)
|
||||
raw = client.search(query=query, categories=None)
|
||||
# Auto-expand: retry without category filter if no results
|
||||
if not raw and categories and auto_expand:
|
||||
_check_timeout()
|
||||
logger.info(
|
||||
"Newznab [%s]: no results for '%s' with category filter, "
|
||||
"auto-expanding",
|
||||
connection.name,
|
||||
query,
|
||||
)
|
||||
raw = connection.client.search(query=query, categories=None)
|
||||
|
||||
for r in raw:
|
||||
key = (
|
||||
r.get("guid")
|
||||
or r.get("downloadUrl")
|
||||
or f"{r.get('indexer')}:{r.get('title')}"
|
||||
)
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
for raw_result in raw:
|
||||
r = dict(raw_result)
|
||||
# Aggregators can identify the underlying indexer. Plain feeds
|
||||
# generally cannot, so use the user-configured connection name.
|
||||
r["indexer"] = r.get("indexer") or connection.name
|
||||
r["_newznab_connection_id"] = connection.connection_id
|
||||
key = (
|
||||
connection.connection_id,
|
||||
r.get("guid")
|
||||
or r.get("downloadUrl")
|
||||
or f"{r.get('indexer')}:{r.get('title')}",
|
||||
)
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
except TimeoutError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Newznab search failed for %s", connection.name)
|
||||
|
||||
except TimeoutError as e:
|
||||
logger.warning("Newznab search timed out: %s", e)
|
||||
except Exception:
|
||||
logger.exception("Newznab search failed")
|
||||
return []
|
||||
|
||||
results = [_newznab_result_to_release(r, content_type, categories) for r in all_results]
|
||||
if plan.indexers:
|
||||
selected_indexers = set(plan.indexers)
|
||||
results = [r for r in results if r.indexer in selected_indexers]
|
||||
|
||||
if results:
|
||||
nzb_count = sum(1 for r in results if r.protocol == ReleaseProtocol.NZB)
|
||||
@@ -379,5 +472,7 @@ class NewznabSource(ReleaseSource):
|
||||
def is_available(self) -> bool:
|
||||
if not config.get("NEWZNAB_ENABLED", False):
|
||||
return False
|
||||
if _parse_indexer_rows(config.get("NEWZNAB_INDEXERS", [])):
|
||||
return True
|
||||
url = normalize_http_url(str(config.get("NEWZNAB_URL", "") or ""))
|
||||
return bool(url)
|
||||
|
||||
@@ -28,6 +28,10 @@ from shelfmark.download.clients.base_handler import (
|
||||
DownloadRequest,
|
||||
ExternalClientHandler,
|
||||
)
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
extract_file_list_from_torrent,
|
||||
extract_torrent_info,
|
||||
)
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
from shelfmark.release_sources import register_handler
|
||||
from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, ProwlarrClient
|
||||
@@ -38,12 +42,14 @@ from shelfmark.release_sources.prowlarr.utils import (
|
||||
coerce_int_like,
|
||||
get_preferred_download_url,
|
||||
get_protocol,
|
||||
sanitize_download_url,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from shelfmark.core.models import DownloadTask
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -127,6 +133,24 @@ class ProwlarrHandler(ExternalClientHandler):
|
||||
|
||||
return settings.get(indexer_id)
|
||||
|
||||
def list_files(self, release_data: dict[str, Any]) -> list[PackFile] | None:
|
||||
"""List a cached torrent release's files from its .torrent, without downloading.
|
||||
|
||||
Magnet-only and usenet releases cannot be listed ahead of time.
|
||||
"""
|
||||
source_id = str(release_data.get("source_id") or "")
|
||||
prowlarr_result = get_release(source_id) if source_id else None
|
||||
if not prowlarr_result or get_protocol(prowlarr_result) != "torrent":
|
||||
return None
|
||||
download_url = sanitize_download_url(str(prowlarr_result.get("downloadUrl") or "").strip())
|
||||
if not download_url or download_url.startswith("magnet:"):
|
||||
return None
|
||||
expected_hash = str(prowlarr_result.get("infoHash") or "").strip() or None
|
||||
info = extract_torrent_info(download_url, expected_hash=expected_hash)
|
||||
if not info.torrent_data:
|
||||
return None
|
||||
return extract_file_list_from_torrent(info.torrent_data)
|
||||
|
||||
def _get_client(self, protocol: str) -> DownloadClient | None:
|
||||
"""Compatibility shim so module-level patching still works in tests."""
|
||||
return get_client(protocol)
|
||||
@@ -297,6 +321,8 @@ class ProwlarrHandler(ExternalClientHandler):
|
||||
search_title=title,
|
||||
search_author=task.author,
|
||||
)
|
||||
# No language default here on purpose: this re-finds one exact release by its
|
||||
# guid, and Prowlarr does not filter on plan.languages anyway.
|
||||
plan = build_release_search_plan(
|
||||
book,
|
||||
indexers=[indexer] if indexer is not None else None,
|
||||
|
||||
@@ -41,6 +41,8 @@ from shelfmark.release_sources.prowlarr.api import (
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.cache import cache_release
|
||||
from shelfmark.release_sources.prowlarr.utils import (
|
||||
AUTHOR_UNKNOWN,
|
||||
author_affinity,
|
||||
build_source_id,
|
||||
coerce_float_like,
|
||||
coerce_int_like,
|
||||
@@ -145,6 +147,36 @@ def _build_indexer_priority(indexers: list[dict]) -> dict[int, int]:
|
||||
return priority
|
||||
|
||||
|
||||
def _drop_unknown_indexer_ids(
|
||||
selected_ids: list[int] | None, indexers: list[dict]
|
||||
) -> list[int] | None:
|
||||
"""Keep only selected indexer ids Prowlarr still serves.
|
||||
|
||||
An indexer removed or disabled in Prowlarr stays in the saved selection,
|
||||
where settings can no longer show it - so it cannot be unselected, and every
|
||||
search keeps querying an indexer that is gone (#1283). Dropping it here
|
||||
keeps the saved selection intact for an indexer that comes back.
|
||||
"""
|
||||
if selected_ids is None:
|
||||
return None
|
||||
|
||||
live_ids = {
|
||||
indexer_id
|
||||
for indexer in indexers
|
||||
if (indexer_id := _coerce_indexer_id(indexer.get("id"))) is not None
|
||||
}
|
||||
kept = [indexer_id for indexer_id in selected_ids if indexer_id in live_ids]
|
||||
|
||||
stale = [indexer_id for indexer_id in selected_ids if indexer_id not in live_ids]
|
||||
if stale:
|
||||
logger.warning(
|
||||
"Skipping selected Prowlarr indexers that are no longer enabled in Prowlarr: %s",
|
||||
stale,
|
||||
)
|
||||
|
||||
return kept
|
||||
|
||||
|
||||
def _rank_for_indexer_id(indexer_id: object, priority: dict[int, int]) -> int:
|
||||
"""Preference rank for an indexer id. Lower wins, unknown ranks last."""
|
||||
coerced = _coerce_indexer_id(indexer_id)
|
||||
@@ -317,19 +349,25 @@ def _extract_mam_language(raw_title: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_mam_formats(raw_title: str) -> list[str]:
|
||||
"""Extract a list of formats from MyAnonamouse titles.
|
||||
def _split_mam_formats(raw_title: str) -> tuple[list[str], list[str]]:
|
||||
"""Split the format tokens of a MyAnonamouse title into (recognized, unrecognized).
|
||||
|
||||
Prowlarr's MAM parser appends a structured bracket segment like:
|
||||
[ENG / EPUB MOBI PDF]
|
||||
|
||||
We only trust this structured segment (and do not attempt generic title
|
||||
heuristics for other indexers).
|
||||
|
||||
Tokens after the "/" that Shelfmark does not know as a book or audiobook format
|
||||
(e.g. ``[ENG / AVI]``) are returned separately so the UI can warn that the release
|
||||
will download but cannot be processed, instead of showing a bare content-type icon
|
||||
that looks like an ordinary result.
|
||||
"""
|
||||
if not raw_title:
|
||||
return []
|
||||
return [], []
|
||||
|
||||
format_set = set(ALL_BOOK_FORMATS)
|
||||
first_unrecognized: list[str] | None = None
|
||||
for bracket in re.findall(r"\[([^\]]+)\]", raw_title):
|
||||
if "/" not in bracket:
|
||||
continue
|
||||
@@ -338,15 +376,26 @@ def _extract_mam_formats(raw_title: str) -> list[str]:
|
||||
tokens = re.findall(r"[A-Za-z0-9]+", after_slash)
|
||||
|
||||
formats: list[str] = []
|
||||
unrecognized: list[str] = []
|
||||
for token in tokens:
|
||||
fmt = token.lower()
|
||||
if fmt in format_set and fmt not in formats:
|
||||
formats.append(fmt)
|
||||
if fmt in format_set:
|
||||
if fmt not in formats:
|
||||
formats.append(fmt)
|
||||
elif fmt not in unrecognized:
|
||||
unrecognized.append(fmt)
|
||||
|
||||
if formats:
|
||||
return formats
|
||||
return formats, unrecognized
|
||||
if unrecognized and first_unrecognized is None:
|
||||
first_unrecognized = unrecognized
|
||||
|
||||
return []
|
||||
return [], first_unrecognized or []
|
||||
|
||||
|
||||
def _extract_mam_formats(raw_title: str) -> list[str]:
|
||||
"""Extract the recognized formats from a MyAnonamouse title (see _split_mam_formats)."""
|
||||
return _split_mam_formats(raw_title)[0]
|
||||
|
||||
|
||||
def _formats_display(formats: list[str]) -> str | None:
|
||||
@@ -485,6 +534,7 @@ def _prowlarr_result_to_release(
|
||||
|
||||
format_detected: str | None = None
|
||||
formats: list[str] = []
|
||||
unrecognized_formats: list[str] = []
|
||||
formats_display: str | None = None
|
||||
language_detected: str | None = None
|
||||
if enable_format_detection:
|
||||
@@ -492,7 +542,7 @@ def _prowlarr_result_to_release(
|
||||
if book_title:
|
||||
title = book_title
|
||||
|
||||
formats = _extract_mam_formats(str(raw_title or ""))
|
||||
formats, unrecognized_formats = _split_mam_formats(str(raw_title or ""))
|
||||
format_detected = formats[0] if formats else None
|
||||
formats_display = _formats_display(formats)
|
||||
language_detected = _extract_mam_language(str(raw_title or ""))
|
||||
@@ -554,6 +604,9 @@ def _prowlarr_result_to_release(
|
||||
"info_hash": result.get("infoHash"),
|
||||
"formats": formats or None,
|
||||
"formats_display": formats_display,
|
||||
# Format tokens the indexer declared but Shelfmark can't process (e.g. a MAM
|
||||
# "[ENG / AVI]"). Lets the UI warn instead of showing a bare content icon.
|
||||
"unrecognized_formats": unrecognized_formats or None,
|
||||
# Raw torznab attributes for rich tooltips (enriched indexers)
|
||||
"torznab_attrs": result.get("torznabAttrs"),
|
||||
},
|
||||
@@ -940,6 +993,7 @@ class ProwlarrSource(ReleaseSource):
|
||||
# found for this book" - the same lie as a swallowed timeout (#1249).
|
||||
msg = f"could not reach Prowlarr: {e}"
|
||||
raise SourceUnavailableError(msg) from e
|
||||
indexer_ids = _drop_unknown_indexer_ids(indexer_ids, enabled_indexers)
|
||||
indexer_priority = _build_indexer_priority(enabled_indexers)
|
||||
# Some indexers benefit from title+author queries and extra format detection.
|
||||
enriched_indexer_ids = client.get_enriched_indexer_ids(
|
||||
@@ -956,10 +1010,17 @@ class ProwlarrSource(ReleaseSource):
|
||||
if time.monotonic() > deadline:
|
||||
_raise_timeout_error(f"Prowlarr search timed out after {int(search_budget)}s")
|
||||
|
||||
def search_indexers(
|
||||
query: str, cats: list[int] | None, *, enriched_query: str | None = None
|
||||
) -> _IndexerSearchOutcome:
|
||||
"""Search indexers with given categories via Torznab/Newznab."""
|
||||
def search_indexers(query: str, cats: list[int] | None) -> _IndexerSearchOutcome:
|
||||
"""Search indexers with given categories via Torznab/Newznab.
|
||||
|
||||
Every indexer gets the same title-only query. Enriched indexers used
|
||||
to be sent "{title} {author}", but an indexer that ANDs its search
|
||||
terms (MyAnonamouse) returns nothing whenever the metadata provider
|
||||
spells the author differently to the tracker - "Timothy Ferriss" vs
|
||||
"Tim Ferriss" - and the UI reports the book as missing (#1293). The
|
||||
author still decides ordering below, where a spelling difference
|
||||
costs a release its position rather than its existence.
|
||||
"""
|
||||
outcome = _IndexerSearchOutcome(results=[])
|
||||
target_indexer_ids = self._get_search_indexer_ids(client, indexer_ids, cats)
|
||||
if not target_indexer_ids:
|
||||
@@ -967,16 +1028,11 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
for indexer_id in target_indexer_ids:
|
||||
_check_timeout()
|
||||
indexer_query = (
|
||||
enriched_query
|
||||
if indexer_id in enriched_indexer_ids_set and enriched_query
|
||||
else query
|
||||
)
|
||||
outcome.attempted += 1
|
||||
try:
|
||||
raw = client.torznab_search(
|
||||
indexer_id=indexer_id,
|
||||
query=indexer_query,
|
||||
query=query,
|
||||
categories=cats,
|
||||
search_type="book",
|
||||
)
|
||||
@@ -1001,14 +1057,11 @@ class ProwlarrSource(ReleaseSource):
|
||||
for idx, variant in enumerate(variants, start=1):
|
||||
_check_timeout()
|
||||
query = variant.title
|
||||
enriched_query = variant.query # title + author
|
||||
|
||||
if len(variants) > 1:
|
||||
logger.debug("Prowlarr query %s/%s: '%s'", idx, len(variants), query)
|
||||
|
||||
outcome = search_indexers(
|
||||
query=query, cats=categories, enriched_query=enriched_query
|
||||
)
|
||||
outcome = search_indexers(query=query, cats=categories)
|
||||
|
||||
# Auto-expand: if no results with categories and auto-expand enabled, retry without.
|
||||
# Only when every indexer actually answered: a failed search says nothing about
|
||||
@@ -1025,9 +1078,7 @@ class ProwlarrSource(ReleaseSource):
|
||||
"Prowlarr: no results for query '%s' with category filter, auto-expanding search",
|
||||
query,
|
||||
)
|
||||
expanded = search_indexers(
|
||||
query=query, cats=None, enriched_query=enriched_query
|
||||
)
|
||||
expanded = search_indexers(query=query, cats=None)
|
||||
outcome.results = expanded.results
|
||||
outcome.attempted += expanded.attempted
|
||||
outcome.failed += expanded.failed
|
||||
@@ -1065,6 +1116,10 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
results: list[Release] = []
|
||||
enriched_source_ids: set[str] = set()
|
||||
affinity_by_source_id: dict[str, int] = {}
|
||||
# A manual query is the user's own words; ranking it against the
|
||||
# metadata author would second-guess what they typed.
|
||||
wanted_author = "" if plan.manual_query else plan.author
|
||||
|
||||
for raw_result in all_results:
|
||||
result_with_seed_settings = _apply_indexer_seed_settings(
|
||||
@@ -1084,13 +1139,20 @@ class ProwlarrSource(ReleaseSource):
|
||||
if idx_id_int is not None and idx_id_int in indexer_priority:
|
||||
release.extra["indexer_priority"] = indexer_priority[idx_id_int]
|
||||
results.append(release)
|
||||
affinity_by_source_id[release.source_id] = author_affinity(
|
||||
wanted_author, release.extra.get("author")
|
||||
)
|
||||
|
||||
if is_enriched:
|
||||
enriched_source_ids.add(release.source_id)
|
||||
|
||||
# Indexer priority first: it is an explicit user preference. Author
|
||||
# agreement then orders what one indexer returned, so the editions that
|
||||
# match the requested author lead and the rest stay reachable below.
|
||||
results.sort(
|
||||
key=lambda r: (
|
||||
_release_indexer_rank(r, indexer_priority),
|
||||
affinity_by_source_id.get(r.source_id, AUTHOR_UNKNOWN),
|
||||
0 if r.source_id in enriched_source_ids else 1,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -14,6 +14,20 @@ if TYPE_CHECKING:
|
||||
|
||||
_INTEGER_LIKE_PATTERN = re.compile(r"^[+-]?\d+$")
|
||||
_FLOAT_LIKE_PATTERN = re.compile(r"^[+-]?(?:\d+(?:\.\d*)?|\.\d+)$")
|
||||
_AUTHOR_TOKEN_PATTERN = re.compile(r"\w+", re.UNICODE)
|
||||
_AUTHOR_NOISE_TOKENS = frozenset(
|
||||
{"jr", "sr", "ii", "iii", "iv", "phd", "md", "dr", "mr", "mrs", "ms", "et", "al", "and", "the"}
|
||||
)
|
||||
|
||||
# Ordering tiers for author agreement between the requested book and what an
|
||||
# indexer reported. Lower sorts first.
|
||||
AUTHOR_MATCH = 0
|
||||
AUTHOR_UNKNOWN = 1
|
||||
AUTHOR_MISMATCH = 2
|
||||
|
||||
# A mononym ("Homer") can only ever agree on one token; a longer name needs a
|
||||
# given name and a surname to agree before it counts as the same person.
|
||||
_AUTHOR_TOKENS_REQUIRED = 2
|
||||
|
||||
|
||||
def coerce_int_like(value: object) -> int | None:
|
||||
@@ -32,6 +46,49 @@ def coerce_int_like(value: object) -> int | None:
|
||||
return int(normalized)
|
||||
|
||||
|
||||
def _author_tokens(value: object) -> list[str]:
|
||||
"""Split an author string into comparable lowercase name tokens."""
|
||||
if not isinstance(value, str):
|
||||
return []
|
||||
tokens = [token.lower() for token in _AUTHOR_TOKEN_PATTERN.findall(value)]
|
||||
return [token for token in tokens if token not in _AUTHOR_NOISE_TOKENS]
|
||||
|
||||
|
||||
def _author_tokens_compatible(wanted: str, offered: str) -> bool:
|
||||
"""Treat an abbreviated given name as the name it abbreviates."""
|
||||
return wanted == offered or wanted.startswith(offered) or offered.startswith(wanted)
|
||||
|
||||
|
||||
def author_affinity(wanted: object, offered: object) -> int:
|
||||
"""Rank how far an indexer's author field is from the requested author.
|
||||
|
||||
Shelfmark ranks on this rather than filtering on it, so a wrong verdict only
|
||||
costs a release its position in the list, never its visibility. That is what
|
||||
makes the loose token comparison safe: "Tim"/"Timothy" and "T."/"Timothy"
|
||||
agree, while a transliteration ("Dostoevsky"/"Dostoyevsky") is merely sorted
|
||||
last instead of being hidden.
|
||||
|
||||
Three-way on purpose: an indexer that reports no author at all must not sort
|
||||
below one that reports a wrong author, so "no metadata" ranks between
|
||||
agreement and disagreement rather than counting as either.
|
||||
"""
|
||||
wanted_tokens = _author_tokens(wanted)
|
||||
offered_tokens = _author_tokens(offered)
|
||||
if not wanted_tokens or not offered_tokens:
|
||||
return AUTHOR_UNKNOWN
|
||||
|
||||
matched = sum(
|
||||
1
|
||||
for wanted_token in wanted_tokens
|
||||
if any(
|
||||
_author_tokens_compatible(wanted_token, offered_token)
|
||||
for offered_token in offered_tokens
|
||||
)
|
||||
)
|
||||
required = min(_AUTHOR_TOKENS_REQUIRED, len(wanted_tokens))
|
||||
return AUTHOR_MATCH if matched >= required else AUTHOR_MISMATCH
|
||||
|
||||
|
||||
def build_source_id(result: dict) -> str:
|
||||
"""Build the Release.source_id for a raw Prowlarr result.
|
||||
|
||||
|
||||
@@ -35,6 +35,15 @@
|
||||
"typescript/no-misused-promises": "error",
|
||||
"typescript/no-non-null-assertion": "error",
|
||||
"typescript/only-throw-error": "error",
|
||||
// React Compiler advisories, enforced everywhere with no per-file exemptions.
|
||||
// The violations inherited from the oxlint 1.70 -> 1.80 bump are all resolved:
|
||||
// three by widening a dependency to the object the compiler infers, and seven
|
||||
// by an `oxlint-disable-next-line` that says, at the callsite, why the flagged
|
||||
// dependency is load-bearing - five are re-run triggers that are never read,
|
||||
// and two are values the callback genuinely uses.
|
||||
"react/preserve-manual-memoization": "error",
|
||||
"react/exhaustive-effect-dependencies": "error",
|
||||
"react/memo-dependencies": "error",
|
||||
"react/no-danger": "error",
|
||||
"react/no-clone-element": "error",
|
||||
"react/no-react-children": "error",
|
||||
|
||||
Generated
+318
-287
File diff suppressed because it is too large
Load Diff
@@ -24,17 +24,17 @@
|
||||
"socket.io-client": "^4.7.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^26.2.0",
|
||||
"@types/node": "^26.3.0",
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.4",
|
||||
"@vitejs/plugin-react": "^6.0.5",
|
||||
"@types/react-dom": "^19.2.5",
|
||||
"@vitejs/plugin-react": "^6.1.0",
|
||||
"knip": "^6.32.2",
|
||||
"oxfmt": "^0.63.0",
|
||||
"oxlint": "^1.78.0",
|
||||
"oxfmt": "^0.65.0",
|
||||
"oxlint": "^1.80.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"typescript": "^7.0.2",
|
||||
"vite": "^8.2.1",
|
||||
"vitest": "^4.1.10"
|
||||
"vite": "^8.2.2",
|
||||
"vitest": "^4.1.11"
|
||||
}
|
||||
}
|
||||
|
||||
+117
-143
@@ -32,6 +32,7 @@ import {
|
||||
import { useActivity } from './hooks/useActivity';
|
||||
import { useAuth } from './hooks/useAuth';
|
||||
import { useDownloadTracking } from './hooks/useDownloadTracking';
|
||||
import { useLatestCallback } from './hooks/useLatestCallback';
|
||||
import { useMediaQuery } from './hooks/useMediaQuery';
|
||||
import { useMountEffect } from './hooks/useMountEffect';
|
||||
import { useRealtimeStatus } from './hooks/useRealtimeStatus';
|
||||
@@ -58,7 +59,6 @@ import {
|
||||
isApiResponseError,
|
||||
updateSelfUser,
|
||||
setBookTargetState,
|
||||
type DownloadReleasePayload,
|
||||
} from './services/api';
|
||||
import type {
|
||||
Book,
|
||||
@@ -74,6 +74,7 @@ import type {
|
||||
ActingAsUserSelection,
|
||||
MetadataProviderSummary,
|
||||
MetadataSearchConfig,
|
||||
MetadataSearchField,
|
||||
QueuedDownloadResult,
|
||||
QueryTargetOption,
|
||||
SearchMode,
|
||||
@@ -87,11 +88,13 @@ import { bookSupportsTargets } from './utils/bookTargetLoader';
|
||||
import { buildSearchQuery } from './utils/buildSearchQuery';
|
||||
import { wasDownloadQueuedAfterResponseError } from './utils/downloadRecovery';
|
||||
import { getDynamicOptionGroup } from './utils/dynamicFieldOptions';
|
||||
import { resolveDefaultLanguageCodes } from './utils/languageFilters';
|
||||
import { getConfiguredMetadataProviderForContentType } from './utils/metadataProviders';
|
||||
import { getEffectiveMetadataSort } from './utils/metadataSort';
|
||||
import { isRecord } from './utils/objectHelpers';
|
||||
import { policyTrace } from './utils/policyTrace';
|
||||
import { buildQueryTargets, getDefaultQueryTargetKey } from './utils/queryTargets';
|
||||
import { buildReleaseDownloadPayload, type ReleaseDownloadOptions } from './utils/releasePayload';
|
||||
import { applyRequestNoteToPayload } from './utils/requestConfirmation';
|
||||
import { bookFromRequestData } from './utils/requestFulfil';
|
||||
import {
|
||||
@@ -218,6 +221,7 @@ type PendingOnBehalfDownload =
|
||||
release: Release;
|
||||
releaseContentType: ContentType;
|
||||
actingAsUser: ActingAsUserSelection;
|
||||
options?: ReleaseDownloadOptions;
|
||||
}
|
||||
| {
|
||||
type: 'combined';
|
||||
@@ -490,8 +494,6 @@ function App() {
|
||||
});
|
||||
|
||||
// When a book is removed from the Hardcover list currently being browsed, remove it from results
|
||||
const searchFieldValuesRef = useRef(searchFieldValues);
|
||||
searchFieldValuesRef.current = searchFieldValues;
|
||||
useBookTargetDeselectSync({
|
||||
activeListValue: searchFieldValues.hardcover_list,
|
||||
setBooks,
|
||||
@@ -603,24 +605,6 @@ function App() {
|
||||
};
|
||||
}, [effectiveActingAsUser, pendingOnBehalfDownload]);
|
||||
|
||||
// Wire up logout callback to clear search state
|
||||
const handleLogoutWithCleanup = useCallback(async () => {
|
||||
await handleLogout();
|
||||
resetSearchResultsState();
|
||||
setActiveQueryTarget('general');
|
||||
setPendingRequestPayload(null);
|
||||
setPendingRequestExtraPayloads([]);
|
||||
setActingAsUser(null);
|
||||
setAdminUsers([]);
|
||||
setAdminUsersError(null);
|
||||
setHasLoadedAdminUsers(false);
|
||||
setPendingOnBehalfDownload(null);
|
||||
setFulfillingRequest(null);
|
||||
resetActivity();
|
||||
setSettingsOpen(false);
|
||||
setSelfSettingsOpen(false);
|
||||
}, [handleLogout, resetActivity, resetSearchResultsState]);
|
||||
|
||||
// Combined mode state (ebook + audiobook in one transaction)
|
||||
const [combinedState, setCombinedState] = useState<CombinedSelectionState | null>(null);
|
||||
|
||||
@@ -653,20 +637,6 @@ function App() {
|
||||
setDownloadsSidebarOpen(true);
|
||||
prefetchActivityHistory();
|
||||
}, [downloadsSidebarOpen, prefetchActivityHistory]);
|
||||
const handleSettingsClick = useCallback(() => {
|
||||
if (config?.settings_enabled) {
|
||||
if (authIsAdmin) {
|
||||
void primeUsersCache();
|
||||
void primeSettingsCache();
|
||||
setSettingsOpen(true);
|
||||
} else {
|
||||
setSelfSettingsOpen(true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
setConfigBannerOpen(true);
|
||||
}, [authIsAdmin, config?.settings_enabled]);
|
||||
|
||||
const headerRef = useCallback((el: HTMLDivElement | null) => {
|
||||
if (headerObserverRef.current) {
|
||||
headerObserverRef.current.disconnect();
|
||||
@@ -683,6 +653,39 @@ function App() {
|
||||
const [settingsOpen, setSettingsOpen] = useState(false);
|
||||
const [selfSettingsOpen, setSelfSettingsOpen] = useState(false);
|
||||
const [configBannerOpen, setConfigBannerOpen] = useState(false);
|
||||
|
||||
// Wire up logout callback to clear search state
|
||||
const handleLogoutWithCleanup = useCallback(async () => {
|
||||
await handleLogout();
|
||||
resetSearchResultsState();
|
||||
setActiveQueryTarget('general');
|
||||
setPendingRequestPayload(null);
|
||||
setPendingRequestExtraPayloads([]);
|
||||
setActingAsUser(null);
|
||||
setAdminUsers([]);
|
||||
setAdminUsersError(null);
|
||||
setHasLoadedAdminUsers(false);
|
||||
setPendingOnBehalfDownload(null);
|
||||
setFulfillingRequest(null);
|
||||
resetActivity();
|
||||
setSettingsOpen(false);
|
||||
setSelfSettingsOpen(false);
|
||||
}, [handleLogout, resetActivity, resetSearchResultsState]);
|
||||
|
||||
const handleSettingsClick = useCallback(() => {
|
||||
if (config?.settings_enabled) {
|
||||
if (authIsAdmin) {
|
||||
void primeUsersCache();
|
||||
void primeSettingsCache();
|
||||
setSettingsOpen(true);
|
||||
} else {
|
||||
setSelfSettingsOpen(true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
setConfigBannerOpen(true);
|
||||
}, [authIsAdmin, config?.settings_enabled]);
|
||||
|
||||
const [onboardingOpen, setOnboardingOpen] = useState(false);
|
||||
useShowOnboardingDebug({
|
||||
setOnboardingOpen,
|
||||
@@ -1071,85 +1074,45 @@ function App() {
|
||||
[],
|
||||
);
|
||||
|
||||
const buildReleaseDownloadPayload = useCallback(
|
||||
(book: Book, release: Release, releaseContentType: ContentType): DownloadReleasePayload => {
|
||||
const isManual = book.provider === 'manual';
|
||||
const releasePreview =
|
||||
typeof release.extra?.preview === 'string' ? release.extra.preview : undefined;
|
||||
const releaseAuthor =
|
||||
typeof release.extra?.author === 'string' ? release.extra.author : undefined;
|
||||
|
||||
return {
|
||||
source: release.source,
|
||||
source_id: release.source_id,
|
||||
title: isManual ? release.title : book.title,
|
||||
author: isManual ? releaseAuthor || '' : book.author,
|
||||
year: book.year,
|
||||
format: release.format,
|
||||
size: release.size,
|
||||
size_bytes: release.size_bytes,
|
||||
download_url: release.download_url,
|
||||
protocol: release.protocol,
|
||||
indexer: release.indexer,
|
||||
seeders: release.seeders,
|
||||
extra: release.extra,
|
||||
preview: isManual ? releasePreview || undefined : book.preview,
|
||||
content_type: releaseContentType,
|
||||
series_name: book.series_name,
|
||||
series_position: book.series_position,
|
||||
subtitle: book.subtitle,
|
||||
// From the release, never the book: book.language is the provider's
|
||||
// canonical edition, which would mislabel a translated release.
|
||||
language: release.language ?? undefined,
|
||||
};
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
// When downloading a book while browsing a Hardcover list the user owns,
|
||||
// automatically remove it from that list (fire-and-forget).
|
||||
const searchFieldLabelsRef = useRef(searchFieldLabels);
|
||||
searchFieldLabelsRef.current = searchFieldLabels;
|
||||
const metadataConfigRef = useRef(activeMetadataConfig);
|
||||
metadataConfigRef.current = activeMetadataConfig;
|
||||
// Stable identity for the download handlers below, while still reading the current
|
||||
// search field values, labels and metadata config. Not an Effect Event: the callers
|
||||
// are download handlers, not Effects. See useLatestCallback.
|
||||
const removeBookFromActiveList = useLatestCallback((book: Book) => {
|
||||
if (config?.hardcover_auto_remove_on_download === false) return;
|
||||
if (!bookSupportsTargets(book)) return;
|
||||
const activeList = searchFieldValues.hardcover_list;
|
||||
if (!activeList) return;
|
||||
const target = String(activeList);
|
||||
const provider = book.provider;
|
||||
const bookId = book.provider_id;
|
||||
if (!provider || !bookId) return;
|
||||
|
||||
const removeBookFromActiveList = useCallback(
|
||||
(book: Book) => {
|
||||
if (config?.hardcover_auto_remove_on_download === false) return;
|
||||
if (!bookSupportsTargets(book)) return;
|
||||
const activeList = searchFieldValuesRef.current.hardcover_list;
|
||||
if (!activeList) return;
|
||||
const target = String(activeList);
|
||||
const provider = book.provider;
|
||||
const bookId = book.provider_id;
|
||||
if (!provider || !bookId) return;
|
||||
// Only auto-remove from lists the user owns (Reading Status / My Lists)
|
||||
const listField = activeMetadataConfig?.search_fields.find(
|
||||
(f) => f.key === 'hardcover_list' && f.type === 'DynamicSelectSearchField',
|
||||
);
|
||||
if (listField && listField.type === 'DynamicSelectSearchField') {
|
||||
const group = getDynamicOptionGroup(listField.options_endpoint, target);
|
||||
if (group && group !== 'Reading Status' && group !== 'My Lists') return;
|
||||
}
|
||||
|
||||
// Only auto-remove from lists the user owns (Reading Status / My Lists)
|
||||
const listField = metadataConfigRef.current?.search_fields.find(
|
||||
(f) => f.key === 'hardcover_list' && f.type === 'DynamicSelectSearchField',
|
||||
);
|
||||
if (listField && listField.type === 'DynamicSelectSearchField') {
|
||||
const group = getDynamicOptionGroup(listField.options_endpoint, target);
|
||||
if (group && group !== 'Reading Status' && group !== 'My Lists') return;
|
||||
}
|
||||
|
||||
void setBookTargetState(provider, bookId, target, false)
|
||||
.then((result) => {
|
||||
if (result.changed) {
|
||||
emitBookTargetChange({
|
||||
provider,
|
||||
bookId,
|
||||
target,
|
||||
selected: false,
|
||||
});
|
||||
const listName = searchFieldLabelsRef.current['hardcover_list'];
|
||||
showToast(`Removed from ${listName || 'list'}`, 'info');
|
||||
}
|
||||
})
|
||||
.catch(() => undefined);
|
||||
},
|
||||
[config?.hardcover_auto_remove_on_download, showToast],
|
||||
);
|
||||
void setBookTargetState(provider, bookId, target, false)
|
||||
.then((result) => {
|
||||
if (result.changed) {
|
||||
emitBookTargetChange({
|
||||
provider,
|
||||
bookId,
|
||||
target,
|
||||
selected: false,
|
||||
});
|
||||
const listName = searchFieldLabels['hardcover_list'];
|
||||
showToast(`Removed from ${listName || 'list'}`, 'info');
|
||||
}
|
||||
})
|
||||
.catch(() => undefined);
|
||||
});
|
||||
|
||||
const executeBookDownload = useCallback(
|
||||
async (book: Book, onBehalfOfUserId?: number): Promise<void> => {
|
||||
@@ -1213,12 +1176,13 @@ function App() {
|
||||
release: Release,
|
||||
releaseContentType: ContentType,
|
||||
onBehalfOfUserId?: number,
|
||||
options?: ReleaseDownloadOptions,
|
||||
): Promise<void> => {
|
||||
const requestStartedAtSeconds = Date.now() / 1000;
|
||||
try {
|
||||
trackRelease(book.id, release.source_id);
|
||||
await downloadRelease(
|
||||
buildReleaseDownloadPayload(book, release, releaseContentType),
|
||||
buildReleaseDownloadPayload(book, release, releaseContentType, options),
|
||||
onBehalfOfUserId,
|
||||
);
|
||||
await fetchStatus();
|
||||
@@ -1300,7 +1264,6 @@ function App() {
|
||||
}
|
||||
},
|
||||
[
|
||||
buildReleaseDownloadPayload,
|
||||
fetchStatus,
|
||||
openRequestConfirmation,
|
||||
refreshRequestPolicy,
|
||||
@@ -1415,6 +1378,7 @@ function App() {
|
||||
effectivePendingOnBehalfDownload.release,
|
||||
effectivePendingOnBehalfDownload.releaseContentType,
|
||||
onBehalfOfUserId,
|
||||
effectivePendingOnBehalfDownload.options,
|
||||
);
|
||||
}
|
||||
setPendingOnBehalfDownload(null);
|
||||
@@ -1639,6 +1603,7 @@ function App() {
|
||||
book: Book,
|
||||
release: Release,
|
||||
releaseContentType: ContentType,
|
||||
options?: ReleaseDownloadOptions,
|
||||
) => {
|
||||
policyTrace('release.action:start', {
|
||||
bookId: book.id,
|
||||
@@ -1654,11 +1619,12 @@ function App() {
|
||||
release,
|
||||
releaseContentType,
|
||||
actingAsUser: effectiveActingAsUser,
|
||||
options,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await executeReleaseDownload(book, release, releaseContentType);
|
||||
await executeReleaseDownload(book, release, releaseContentType, undefined, options);
|
||||
};
|
||||
|
||||
const handleReleaseRequest = useCallback(
|
||||
@@ -1927,10 +1893,7 @@ function App() {
|
||||
);
|
||||
const supportedFormats = config?.supported_formats || DEFAULT_SUPPORTED_FORMATS;
|
||||
const defaultLanguageCodes = useMemo(
|
||||
() =>
|
||||
config?.default_language && config.default_language.length > 0
|
||||
? config.default_language
|
||||
: [bookLanguages[0]?.code || 'en'],
|
||||
() => resolveDefaultLanguageCodes(config?.default_language, bookLanguages),
|
||||
[config?.default_language, bookLanguages],
|
||||
);
|
||||
|
||||
@@ -1942,14 +1905,27 @@ function App() {
|
||||
effectiveSearchMode === 'universal' &&
|
||||
(universalDefaultMode === 'download' || universalDefaultMode === 'request_release');
|
||||
|
||||
// Keep the last known search fields so queryTargets doesn't collapse to
|
||||
// [general] while the metadata config briefly reloads on content type switch.
|
||||
// Held in state rather than a ref written during render: a ref read back in the same
|
||||
// pass is what `react/refs` forbids, and this is the adjust-state-during-render shape
|
||||
// React documents for exactly this - carry the previous value until a new one arrives.
|
||||
const [stableSearchFields, setStableSearchFields] = useState<MetadataSearchField[]>(
|
||||
() => activeMetadataConfig?.search_fields ?? [],
|
||||
);
|
||||
const incomingSearchFields = activeMetadataConfig?.search_fields;
|
||||
if (incomingSearchFields && incomingSearchFields !== stableSearchFields) {
|
||||
setStableSearchFields(incomingSearchFields);
|
||||
}
|
||||
|
||||
const queryTargets = useMemo<QueryTargetOption[]>(
|
||||
() =>
|
||||
buildQueryTargets({
|
||||
searchMode: effectiveSearchMode,
|
||||
metadataSearchFields: activeMetadataConfig?.search_fields ?? [],
|
||||
metadataSearchFields: stableSearchFields,
|
||||
manualSearchAllowed,
|
||||
}),
|
||||
[effectiveSearchMode, activeMetadataConfig?.search_fields, manualSearchAllowed],
|
||||
[effectiveSearchMode, stableSearchFields, manualSearchAllowed],
|
||||
);
|
||||
const effectiveActiveQueryTarget = useMemo(() => {
|
||||
if (queryTargets.some((target) => target.key === activeQueryTarget)) {
|
||||
@@ -1978,27 +1954,27 @@ function App() {
|
||||
? (queryTargets.find((target) => target.field?.key === seriesBrowseCapability.field_key) ??
|
||||
null)
|
||||
: null,
|
||||
[queryTargets, seriesBrowseCapability?.field_key],
|
||||
// `seriesBrowseCapability` whole: the body reads `.field_key` off it unguarded
|
||||
// inside the ternary, so that object is the dependency the compiler infers.
|
||||
[queryTargets, seriesBrowseCapability],
|
||||
);
|
||||
|
||||
const activeQueryValue = useMemo(() => {
|
||||
if (
|
||||
!activeQueryOption ||
|
||||
activeQueryOption.source === 'general' ||
|
||||
activeQueryOption.source === 'manual'
|
||||
activeQueryOption.source === 'manual' ||
|
||||
activeQueryOption.source === 'direct-field'
|
||||
) {
|
||||
return searchInput;
|
||||
}
|
||||
|
||||
if (activeQueryOption.source === 'direct-field') {
|
||||
if (activeQueryOption.key === 'isbn') return advancedFilters.isbn;
|
||||
if (activeQueryOption.key === 'author') return advancedFilters.author;
|
||||
if (activeQueryOption.key === 'title') return advancedFilters.title;
|
||||
if (!activeQueryOption.field) {
|
||||
return '';
|
||||
}
|
||||
|
||||
if (!activeQueryOption.field) {
|
||||
return '';
|
||||
if (activeQueryOption.field.type === 'TextSearchField') {
|
||||
return searchInput;
|
||||
}
|
||||
|
||||
if (activeQueryOption.field.type === 'CheckboxSearchField') {
|
||||
@@ -2008,7 +1984,7 @@ function App() {
|
||||
}
|
||||
|
||||
return searchFieldValues[activeQueryOption.field.key] ?? '';
|
||||
}, [activeQueryOption, searchInput, advancedFilters, searchFieldValues]);
|
||||
}, [activeQueryOption, searchInput, searchFieldValues]);
|
||||
|
||||
const activeQueryValueLabel = useMemo(() => {
|
||||
if (!activeQueryOption?.field) {
|
||||
@@ -2056,29 +2032,25 @@ function App() {
|
||||
if (
|
||||
!activeQueryOption ||
|
||||
activeQueryOption.source === 'general' ||
|
||||
activeQueryOption.source === 'manual'
|
||||
activeQueryOption.source === 'manual' ||
|
||||
activeQueryOption.source === 'direct-field'
|
||||
) {
|
||||
setSearchInput(typeof value === 'string' ? value : String(value ?? ''));
|
||||
return;
|
||||
}
|
||||
|
||||
if (activeQueryOption.source === 'direct-field') {
|
||||
const nextValue = typeof value === 'string' ? value : String(value ?? '');
|
||||
if (activeQueryOption.key === 'isbn') {
|
||||
updateAdvancedFilters({ isbn: nextValue });
|
||||
} else if (activeQueryOption.key === 'author') {
|
||||
updateAdvancedFilters({ author: nextValue });
|
||||
} else if (activeQueryOption.key === 'title') {
|
||||
updateAdvancedFilters({ title: nextValue });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (activeQueryOption.field) {
|
||||
if (activeQueryOption.field.type === 'TextSearchField') {
|
||||
setSearchInput(typeof value === 'string' ? value : String(value ?? ''));
|
||||
if (label !== undefined) {
|
||||
updateSearchFieldValue(activeQueryOption.field.key, value, label);
|
||||
}
|
||||
return;
|
||||
}
|
||||
updateSearchFieldValue(activeQueryOption.field.key, value, label);
|
||||
}
|
||||
},
|
||||
[activeQueryOption, setSearchInput, updateAdvancedFilters, updateSearchFieldValue],
|
||||
[activeQueryOption, setSearchInput, updateSearchFieldValue],
|
||||
);
|
||||
|
||||
const handleSearchModeChange = useCallback(
|
||||
@@ -2276,7 +2248,9 @@ function App() {
|
||||
|
||||
return book.provider === activeMetadataConfig.provider;
|
||||
},
|
||||
[activeMetadataConfig?.provider, seriesBrowseCapability?.sort, seriesBrowseTarget?.field],
|
||||
// `activeMetadataConfig` whole: the body reads `.provider` off it unguarded on
|
||||
// the last line, so that object is the dependency the compiler infers.
|
||||
[activeMetadataConfig, seriesBrowseCapability?.sort, seriesBrowseTarget?.field],
|
||||
);
|
||||
|
||||
const handleManualSearch = useCallback(() => {
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { useState } from 'react';
|
||||
|
||||
import type { PackBook, PackPlan, Release } from '../types';
|
||||
import {
|
||||
describePackPlan,
|
||||
parseSeriesPositionInput,
|
||||
toBookPlanPayload,
|
||||
updateReviewBook,
|
||||
} from '../utils/packReview';
|
||||
import { ToggleSwitch } from './shared/ToggleSwitch';
|
||||
|
||||
interface PackReviewPanelProps {
|
||||
release: Release;
|
||||
plan: PackPlan;
|
||||
books: PackBook[];
|
||||
onChange: (books: PackBook[]) => void;
|
||||
onBack: () => void;
|
||||
/** `null` means "treat the whole release as one book". */
|
||||
onConfirm: (books: PackBook[] | null) => Promise<void>;
|
||||
isSubmitting: boolean;
|
||||
}
|
||||
|
||||
const inputClassName =
|
||||
'w-full rounded-md border border-(--border-muted) bg-(--bg) px-2 py-1 text-sm text-(--text) focus:border-emerald-500 focus:outline-none';
|
||||
|
||||
export const PackReviewPanel = ({
|
||||
release,
|
||||
plan,
|
||||
books,
|
||||
onChange,
|
||||
onBack,
|
||||
onConfirm,
|
||||
isSubmitting,
|
||||
}: PackReviewPanelProps) => {
|
||||
const [singleBook, setSingleBook] = useState(false);
|
||||
const [expandedFiles, setExpandedFiles] = useState<number | null>(null);
|
||||
const [showIgnored, setShowIgnored] = useState(false);
|
||||
|
||||
const payloadBooks = toBookPlanPayload(books);
|
||||
const canConfirm = !isSubmitting && (singleBook || payloadBooks.length > 0);
|
||||
const confirmLabel = singleBook
|
||||
? 'Download as one book'
|
||||
: `Download ${payloadBooks.length} ${payloadBooks.length === 1 ? 'book' : 'books'}`;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4 px-5 py-4" data-testid="pack-review-panel">
|
||||
<div>
|
||||
<h3 className="text-base font-semibold text-(--text)">
|
||||
This release contains several books
|
||||
</h3>
|
||||
<p className="mt-1 text-sm text-zinc-500 dark:text-zinc-400">
|
||||
<span className="font-medium text-(--text)">{release.title}</span> ·{' '}
|
||||
{describePackPlan(books, plan.ignored)}
|
||||
</p>
|
||||
<p className="mt-1 text-xs text-zinc-500 dark:text-zinc-400">
|
||||
Each book below is filed separately with its own title. Fix any titles before downloading
|
||||
— the author and series come from the book you searched.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between rounded-lg border border-(--border-muted) px-3 py-2">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-(--text)">Treat as a single book</p>
|
||||
<p className="text-xs text-zinc-500 dark:text-zinc-400">
|
||||
Use this if the split is wrong and the files are really one audiobook.
|
||||
</p>
|
||||
</div>
|
||||
<ToggleSwitch
|
||||
checked={singleBook}
|
||||
onChange={setSingleBook}
|
||||
color="emerald"
|
||||
ariaLabel="Treat as a single book"
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div
|
||||
className={`flex flex-col divide-y divide-zinc-200/60 dark:divide-zinc-800/60 ${
|
||||
singleBook ? 'pointer-events-none opacity-40' : ''
|
||||
}`}
|
||||
>
|
||||
<div className="grid grid-cols-[minmax(0,1fr)_72px_72px_80px] gap-2 pb-1 text-xs font-medium tracking-wide text-zinc-500 uppercase dark:text-zinc-400">
|
||||
<span>Title</span>
|
||||
<span>Series #</span>
|
||||
<span>Year</span>
|
||||
<span className="text-right">Files</span>
|
||||
</div>
|
||||
{books.map((book, index) => (
|
||||
<div key={book.files[0] ?? index} className="py-2">
|
||||
<div className="grid grid-cols-[minmax(0,1fr)_72px_72px_80px] items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={book.title}
|
||||
onChange={(e) =>
|
||||
onChange(updateReviewBook(books, index, { title: e.target.value }))
|
||||
}
|
||||
aria-label={`Title for book ${index + 1}`}
|
||||
className={inputClassName}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
inputMode="decimal"
|
||||
value={book.series_position ?? ''}
|
||||
onChange={(e) =>
|
||||
onChange(
|
||||
updateReviewBook(books, index, {
|
||||
series_position: parseSeriesPositionInput(e.target.value),
|
||||
}),
|
||||
)
|
||||
}
|
||||
aria-label={`Series position for book ${index + 1}`}
|
||||
className={inputClassName}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={book.year ?? ''}
|
||||
onChange={(e) => {
|
||||
const parsed = parseSeriesPositionInput(e.target.value);
|
||||
onChange(
|
||||
updateReviewBook(books, index, {
|
||||
year: parsed === null ? null : Math.trunc(parsed),
|
||||
}),
|
||||
);
|
||||
}}
|
||||
aria-label={`Year for book ${index + 1}`}
|
||||
className={inputClassName}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setExpandedFiles(expandedFiles === index ? null : index)}
|
||||
className="hover-surface rounded-md px-2 py-1 text-right text-sm text-zinc-500 transition-colors dark:text-zinc-400"
|
||||
aria-expanded={expandedFiles === index}
|
||||
>
|
||||
{book.files.length} {book.files.length === 1 ? 'file' : 'files'}
|
||||
</button>
|
||||
</div>
|
||||
{expandedFiles === index && (
|
||||
<ul className="mt-2 max-h-40 overflow-y-auto rounded-md bg-(--bg-soft) px-3 py-2 font-mono text-xs break-all text-zinc-600 dark:text-zinc-300">
|
||||
{book.files.map((file) => (
|
||||
<li key={file}>{file}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{plan.ignored.length > 0 && (
|
||||
<div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowIgnored(!showIgnored)}
|
||||
className="text-xs text-zinc-500 underline-offset-2 hover:underline dark:text-zinc-400"
|
||||
aria-expanded={showIgnored}
|
||||
>
|
||||
{plan.ignored.length} {plan.ignored.length === 1 ? 'file' : 'files'} ignored (not a book
|
||||
format)
|
||||
</button>
|
||||
{showIgnored && (
|
||||
<ul className="mt-2 max-h-32 overflow-y-auto rounded-md bg-(--bg-soft) px-3 py-2 font-mono text-xs break-all text-zinc-600 dark:text-zinc-300">
|
||||
{plan.ignored.map((file) => (
|
||||
<li key={file}>{file}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-center justify-end gap-3 border-t border-(--border-muted) pt-4">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onBack}
|
||||
disabled={isSubmitting}
|
||||
className="hover-surface rounded-lg px-3 py-1.5 text-sm font-medium text-(--text) transition-colors disabled:opacity-50"
|
||||
>
|
||||
← Back
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void onConfirm(singleBook ? null : payloadBooks)}
|
||||
disabled={!canConfirm}
|
||||
className="rounded-lg bg-emerald-600 px-4 py-1.5 text-sm font-medium text-white transition-colors hover:bg-emerald-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{isSubmitting ? 'Queuing…' : confirmLabel}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
toStringArray,
|
||||
toStringValue,
|
||||
} from '../utils/objectHelpers';
|
||||
import { getUnrecognizedReleaseFormats } from '../utils/releaseFormats';
|
||||
import { Tooltip } from './shared/Tooltip';
|
||||
|
||||
interface ReleaseCellProps {
|
||||
@@ -424,6 +425,38 @@ export const ReleaseCell = ({
|
||||
const primaryFormat = formats?.[0] || null;
|
||||
const additionalFormats = formats?.slice(1) || [];
|
||||
|
||||
// The indexer named a format Shelfmark can't process (e.g. MAM "[ENG / AVI]").
|
||||
// Downloading it would only fail post-processing, so warn instead of showing the
|
||||
// bare content-type icon that makes it look like any other result.
|
||||
const unrecognizedFormats = primaryFormat ? [] : getUnrecognizedReleaseFormats(release);
|
||||
if (unrecognizedFormats.length > 0) {
|
||||
const unsupportedLabel = unrecognizedFormats.map((fmt) => fmt.toUpperCase()).join(', ');
|
||||
const unsupportedTitle = `Unsupported format (${unsupportedLabel}) - Shelfmark cannot process this release`;
|
||||
if (compact) {
|
||||
return (
|
||||
<span
|
||||
className="font-semibold text-amber-600 dark:text-amber-400"
|
||||
title={unsupportedTitle}
|
||||
>
|
||||
{unrecognizedFormats[0].toUpperCase()}
|
||||
{unrecognizedFormats.length > 1 && ` +${unrecognizedFormats.length - 1}`}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="flex items-center justify-start" title={unsupportedTitle}>
|
||||
<span className="inline-flex items-center gap-1">
|
||||
<span className="w-13 rounded-lg bg-amber-500/20 py-0.5 text-center text-[10px] font-semibold tracking-wide whitespace-nowrap text-amber-700 sm:text-[11px] dark:text-amber-400">
|
||||
{unrecognizedFormats[0].toUpperCase()}
|
||||
</span>
|
||||
<span className="text-[10px] font-medium whitespace-nowrap text-amber-700 sm:text-[11px] dark:text-amber-400">
|
||||
Unsupported
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Use blue for book, violet for audiobook when no format specified
|
||||
const noFormatStyle = isAudiobook
|
||||
? { bg: 'bg-violet-500/20', text: 'text-violet-600 dark:text-violet-400' }
|
||||
|
||||
@@ -7,6 +7,7 @@ import { useReleaseSearchSession } from '../hooks/releaseModal/useReleaseSearchS
|
||||
import { useTabIndicator } from '../hooks/ui/useTabIndicator';
|
||||
import { useBodyScrollLock } from '../hooks/useBodyScrollLock';
|
||||
import { useEscapeKey } from '../hooks/useEscapeKey';
|
||||
import { inspectRelease } from '../services/api';
|
||||
import type {
|
||||
Book,
|
||||
Release,
|
||||
@@ -18,6 +19,8 @@ import type {
|
||||
LeadingCellConfig,
|
||||
ContentType,
|
||||
RequestPolicyMode,
|
||||
PackBook,
|
||||
PackPlan,
|
||||
} from '../types';
|
||||
import { isMetadataBook } from '../types';
|
||||
import { bookSupportsTargets } from '../utils/bookTargetLoader';
|
||||
@@ -29,7 +32,10 @@ import {
|
||||
buildLanguageNormalizer,
|
||||
} from '../utils/languageFilters';
|
||||
import { getNestedValue, toComparableText, toStringValue } from '../utils/objectHelpers';
|
||||
import { toBookPlanPayload } from '../utils/packReview';
|
||||
import { getReleaseFormats } from '../utils/releaseFormats';
|
||||
import { INITIAL_ENTER_ANIMATION, nextEnterAnimation } from '../utils/releaseModalEnterAnimation';
|
||||
import { buildReleaseDownloadPayload, type ReleaseDownloadOptions } from '../utils/releasePayload';
|
||||
import {
|
||||
getBookTitleCandidates,
|
||||
getBookAuthorCandidates,
|
||||
@@ -50,6 +56,7 @@ import { BookTargetDropdown } from './BookTargetDropdown';
|
||||
import { Dropdown } from './Dropdown';
|
||||
import { DropdownList } from './DropdownList';
|
||||
import { LanguageMultiSelect } from './LanguageMultiSelect';
|
||||
import { PackReviewPanel } from './PackReviewPanel';
|
||||
import { ReleaseCell } from './ReleaseCell';
|
||||
|
||||
// Combined mode configuration for the ReleaseModal
|
||||
@@ -140,7 +147,12 @@ const DEFAULT_COLUMN_CONFIG: ReleaseColumnConfig = {
|
||||
interface ReleaseModalProps {
|
||||
book: Book | null;
|
||||
onClose: () => void;
|
||||
onDownload: (book: Book, release: Release, contentType: ContentType) => Promise<void>;
|
||||
onDownload: (
|
||||
book: Book,
|
||||
release: Release,
|
||||
contentType: ContentType,
|
||||
options?: ReleaseDownloadOptions,
|
||||
) => Promise<void>;
|
||||
onRequestRelease?: (book: Book, release: Release, contentType: ContentType) => Promise<void>;
|
||||
onRequestBook?: (book: Book, contentType: ContentType) => Promise<void>;
|
||||
getPolicyModeForSource?: (source: string, contentType: ContentType) => RequestPolicyMode;
|
||||
@@ -762,6 +774,15 @@ const ReleaseModalSession = ({
|
||||
: supportedFormats;
|
||||
const [isRequestingBook, setIsRequestingBook] = useState(false);
|
||||
const [selectedRelease, setSelectedRelease] = useState<Release | null>(null);
|
||||
// Multi-book packs: `multiBook` is the manual header toggle (heuristic split for
|
||||
// releases we can't inspect); `packReview` holds an inspected pack awaiting approval.
|
||||
const [multiBook, setMultiBook] = useState(false);
|
||||
const [packReview, setPackReview] = useState<{
|
||||
release: Release;
|
||||
plan: PackPlan;
|
||||
books: PackBook[];
|
||||
} | null>(null);
|
||||
const [packSubmitting, setPackSubmitting] = useState(false);
|
||||
const isCombinedMode = combinedMode != null;
|
||||
const combinedPhase = combinedMode?.phase ?? null;
|
||||
const combinedStepLabel = combinedMode?.stepLabel ?? '';
|
||||
@@ -869,6 +890,11 @@ const ReleaseModalSession = ({
|
||||
} finally {
|
||||
setIsRequestingBook(false);
|
||||
}
|
||||
// Kept against the advisory: the body really does read both. `handleClose` is aliased
|
||||
// from the `onClose` prop, which is why the compiler names the source instead, and
|
||||
// dropping `contentType` would let this close over a stale one and request the wrong
|
||||
// format. Correctness first; the cost is an extra callback identity.
|
||||
// oxlint-disable-next-line react/memo-dependencies
|
||||
}, [book, onRequestBook, isRequestingBook, contentType, handleClose]);
|
||||
|
||||
const tabRefs = useRef<Record<string, HTMLButtonElement | null>>({});
|
||||
@@ -1115,7 +1141,9 @@ const ReleaseModalSession = ({
|
||||
const narratorField = book.display_fields.find((f) => f.icon === 'microphone');
|
||||
|
||||
return { starField, ratingsField, usersField, pagesField, lengthField, narratorField };
|
||||
}, [book?.display_fields]);
|
||||
// `book`, not `book?.display_fields`: the body reads `book.display_fields`
|
||||
// unguarded after the early return, which is the dependency the compiler infers.
|
||||
}, [book]);
|
||||
|
||||
const getReleaseActionMode = useCallback(
|
||||
(release: Release): RequestPolicyMode => {
|
||||
@@ -1196,7 +1224,36 @@ const ReleaseModalSession = ({
|
||||
|
||||
const mode = getReleaseActionMode(release);
|
||||
if (mode === 'download') {
|
||||
await onDownload(book, release, contentType);
|
||||
// Look at the release's files before queueing so a whole-series pack can be
|
||||
// reviewed and filed as separate books instead of one mangled item.
|
||||
let inspected = false;
|
||||
let plan: PackPlan | null = null;
|
||||
let reason: string | null = null;
|
||||
try {
|
||||
const inspection = await inspectRelease(
|
||||
buildReleaseDownloadPayload(book, release, contentType),
|
||||
);
|
||||
inspected = inspection.inspected;
|
||||
plan = inspection.plan;
|
||||
reason = inspection.reason;
|
||||
} catch (error) {
|
||||
console.error('Release inspection failed:', error);
|
||||
}
|
||||
if (inspected && plan?.is_pack) {
|
||||
setPackReview({ release, plan, books: plan.books });
|
||||
return;
|
||||
}
|
||||
// Not a pack (or couldn't be inspected): queue exactly as before. A release we
|
||||
// couldn't inspect might still be an unnoticed pack, so leave a console breadcrumb
|
||||
// rather than interrupting the user; the multi-book toggle forces the split.
|
||||
if (!inspected && !multiBook) {
|
||||
console.warn(
|
||||
`Could not inspect release "${release.title}" before download${
|
||||
reason ? `: ${reason}` : ''
|
||||
}. If it contains several books, enable the multi-book pack toggle.`,
|
||||
);
|
||||
}
|
||||
await onDownload(book, release, contentType, multiBook ? { multiBook: true } : {});
|
||||
handleClose();
|
||||
return;
|
||||
}
|
||||
@@ -1215,9 +1272,34 @@ const ReleaseModalSession = ({
|
||||
onRequestRelease,
|
||||
contentType,
|
||||
handleClose,
|
||||
multiBook,
|
||||
],
|
||||
);
|
||||
|
||||
const handlePackConfirm = useCallback(
|
||||
async (books: PackBook[] | null): Promise<void> => {
|
||||
if (!book || !packReview) {
|
||||
return;
|
||||
}
|
||||
setPackSubmitting(true);
|
||||
try {
|
||||
await onDownload(
|
||||
book,
|
||||
packReview.release,
|
||||
contentType,
|
||||
books ? { multiBook: true, bookPlan: toBookPlanPayload(books) } : {},
|
||||
);
|
||||
handleClose();
|
||||
} finally {
|
||||
setPackSubmitting(false);
|
||||
}
|
||||
},
|
||||
// Same as handleRequestBook above: the body reads `onDownload`, `contentType` and
|
||||
// `handleClose`, so they stay in the list whatever the advisory infers.
|
||||
// oxlint-disable-next-line react/memo-dependencies
|
||||
[book, packReview, onDownload, contentType, handleClose],
|
||||
);
|
||||
|
||||
const titleId = `release-modal-title-${book.id}`;
|
||||
const providerDisplay =
|
||||
book.provider_display_name ||
|
||||
@@ -1720,6 +1802,37 @@ const ReleaseModalSession = ({
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3 pr-1 pl-2">
|
||||
{/* Multi-book pack toggle (fallback for releases that can't be inspected) */}
|
||||
{!isCombinedMode && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMultiBook((prev) => !prev)}
|
||||
className={`hover-surface relative rounded-full p-2.5 text-zinc-500 transition-colors dark:text-zinc-400 ${
|
||||
multiBook ? 'text-emerald-600 dark:text-emerald-400' : ''
|
||||
}`}
|
||||
aria-label="Multi-book pack"
|
||||
aria-pressed={multiBook}
|
||||
title="Multi-book pack: file each subfolder (or each file) as a separate book. Only needed when a release can't be inspected before download."
|
||||
>
|
||||
<svg
|
||||
className="h-4 w-4"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth={1.5}
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M6.429 9.75 2.25 12l4.179 2.25m0-4.5 5.571 3 5.571-3m-11.142 0L2.25 7.5 12 2.25l9.75 5.25-4.179 2.25m0 0L21.75 12l-4.179 2.25m0 0 4.179 2.25L12 21.75 2.25 16.5l4.179-2.25m11.142 0-5.571 3-5.571-3"
|
||||
/>
|
||||
</svg>
|
||||
{multiBook && (
|
||||
<span className="absolute top-1 right-1 h-2 w-2 rounded-full bg-emerald-500" />
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
|
||||
{/* Manual query button */}
|
||||
<button
|
||||
type="button"
|
||||
@@ -2120,6 +2233,19 @@ const ReleaseModalSession = ({
|
||||
{/* Release list content */}
|
||||
<div className="min-h-[200px]">
|
||||
{(() => {
|
||||
if (packReview) {
|
||||
return (
|
||||
<PackReviewPanel
|
||||
release={packReview.release}
|
||||
plan={packReview.plan}
|
||||
books={packReview.books}
|
||||
onChange={(books) => setPackReview({ ...packReview, books })}
|
||||
onBack={() => setPackReview(null)}
|
||||
onConfirm={handlePackConfirm}
|
||||
isSubmitting={packSubmitting}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (sourcesLoading) {
|
||||
return <ReleaseSkeleton />;
|
||||
}
|
||||
@@ -2335,7 +2461,7 @@ const ReleaseModalSession = ({
|
||||
|
||||
export const ReleaseModal = ({ book, onClose, ...rest }: ReleaseModalProps) => {
|
||||
const [isClosing, setIsClosing] = useState(false);
|
||||
const previousSessionKeyRef = useRef<string | null>(null);
|
||||
const [enterAnimation, setEnterAnimation] = useState(INITIAL_ENTER_ANIMATION);
|
||||
|
||||
const handleClose = useCallback(() => {
|
||||
setIsClosing(true);
|
||||
@@ -2359,12 +2485,13 @@ export const ReleaseModal = ({ book, onClose, ...rest }: ReleaseModalProps) => {
|
||||
].join('|')
|
||||
: null;
|
||||
|
||||
const animateEnter =
|
||||
!rest.combinedMode ||
|
||||
previousSessionKeyRef.current === null ||
|
||||
previousSessionKeyRef.current === sessionKey;
|
||||
|
||||
previousSessionKeyRef.current = sessionKey;
|
||||
// Decided once per session key and held for that session's lifetime, so a
|
||||
// re-render mid-session cannot restart the enter animation.
|
||||
const nextAnimation = nextEnterAnimation(enterAnimation, sessionKey, rest.combinedMode != null);
|
||||
if (nextAnimation !== enterAnimation) {
|
||||
setEnterAnimation(nextAnimation);
|
||||
}
|
||||
const animateEnter = nextAnimation.animate;
|
||||
|
||||
if (!book && !isClosing) return null;
|
||||
if (!book || !sessionKey) return null;
|
||||
|
||||
@@ -3,8 +3,8 @@ import { forwardRef, useImperativeHandle, useMemo, useRef, useState } from 'reac
|
||||
|
||||
import { useSearchMode } from '../contexts/SearchModeContext';
|
||||
import { useSearchBarAutocomplete } from '../hooks/searchBar/useSearchBarAutocomplete';
|
||||
import { useSearchBarHoverTimeout } from '../hooks/searchBar/useSearchBarHoverTimeout';
|
||||
import { useDismiss } from '../hooks/useDismiss';
|
||||
import { useLatestCallback } from '../hooks/useLatestCallback';
|
||||
import type { DynamicFieldOption } from '../services/api';
|
||||
import type { ContentType, MetadataSearchField, QueryTargetOption, SortOption } from '../types';
|
||||
import { SearchBarAutocompleteSession } from './SearchBarAutocompleteSession';
|
||||
@@ -51,6 +51,8 @@ const EMPTY_SORT_OPTIONS: SortOption[] = [];
|
||||
const EMPTY_AUTOCOMPLETE_OPTIONS: DynamicFieldOption[] = [];
|
||||
const EMPTY_QUERY_TARGETS: QueryTargetOption[] = [];
|
||||
|
||||
const SEARCH_CONTROLS_PANEL_ID = 'search-bar-controls-panel';
|
||||
|
||||
const BookIcon = () => (
|
||||
<svg
|
||||
className="h-5 w-5 shrink-0"
|
||||
@@ -195,8 +197,9 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
const { searchMode } = useSearchMode();
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const buttonRef = useRef<HTMLButtonElement>(null);
|
||||
const onSubmitRef = useRef(onSubmit);
|
||||
onSubmitRef.current = onSubmit;
|
||||
// Deferred submits below run from a timeout, not an Effect, so this is a latest-value
|
||||
// callback rather than an Effect Event. See useLatestCallback.
|
||||
const submitLatest = useLatestCallback(() => onSubmit());
|
||||
const selectorRef = useRef<HTMLDivElement>(null);
|
||||
const hasSearchQuery = hasActiveValue(value);
|
||||
const [isSelectorOpen, setIsSelectorOpen] = useState(false);
|
||||
@@ -205,7 +208,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
const selectTriggerRef = useRef<HTMLButtonElement>(null);
|
||||
const selectPanelRef = useRef<HTMLDivElement>(null);
|
||||
const autocompletePanelRef = useRef<HTMLDivElement>(null);
|
||||
const { hoverTimeoutRef: selectorHoverTimeout, clearHoverTimeout } = useSearchBarHoverTimeout();
|
||||
const controlsPanelRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const hasMultipleContentTypes = !allowedContentTypes || allowedContentTypes.length !== 1;
|
||||
const showContentTypeSelector =
|
||||
@@ -221,9 +224,20 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
() => queryTargets.find((target) => target.key === activeQueryTarget) ?? queryTargets[0],
|
||||
[queryTargets, activeQueryTarget],
|
||||
);
|
||||
const showActiveTargetLabel = queryTargets.length > 0 && activeTarget.source !== 'general';
|
||||
const showActiveTargetLabel = queryTargets.length > 0 && activeTarget?.source !== 'general';
|
||||
|
||||
useDismiss(isSelectorOpen, [selectorRef], () => setIsSelectorOpen(false));
|
||||
// Manual search browses release sources directly, one media type at a time — the
|
||||
// combined ("both") flow doesn't apply. Present a plain, switchable Books/Audiobooks
|
||||
// choice for it, even when combined search is forced on for metadata targets.
|
||||
const isManualTarget = activeTarget?.source === 'manual';
|
||||
const combinedSelectionActive = combinedMode && !isManualTarget;
|
||||
const combinedSelectorLocked = combinedModeLocked && !isManualTarget;
|
||||
const combinedToggleAvailable = !!onCombinedModeChange && !isManualTarget;
|
||||
const combinedLineColor = combinedSelectionActive
|
||||
? 'bg-emerald-500'
|
||||
: 'bg-(--border-muted) group-hover:bg-zinc-400 dark:group-hover:bg-zinc-500';
|
||||
|
||||
useDismiss(isSelectorOpen, [selectorRef, controlsPanelRef], () => setIsSelectorOpen(false));
|
||||
useDismiss(isSelectOpen, [selectPanelRef, selectTriggerRef], () => setIsSelectOpen(false));
|
||||
useDismiss(isAutocompleteOpen, [autocompletePanelRef, inputRef], () =>
|
||||
setIsAutocompleteOpen(false),
|
||||
@@ -325,18 +339,15 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
const handleContentTypeSelect = (type: ContentType) => {
|
||||
onContentTypeChange?.(type);
|
||||
onCombinedModeChange?.(false);
|
||||
setIsSelectorOpen(false);
|
||||
};
|
||||
|
||||
const handleCombinedModeSelect = () => {
|
||||
if (combinedMode) {
|
||||
// Toggle off — revert to ebook-only
|
||||
onCombinedModeChange?.(false);
|
||||
} else {
|
||||
onContentTypeChange?.('ebook');
|
||||
onCombinedModeChange?.(true);
|
||||
}
|
||||
setIsSelectorOpen(false);
|
||||
};
|
||||
|
||||
const handleQueryTargetSelect = (targetKey: string) => {
|
||||
@@ -349,14 +360,13 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
setIsSelectOpen(shouldOpenSelect);
|
||||
setIsAutocompleteOpen(false);
|
||||
resetAutocomplete();
|
||||
setIsSelectorOpen(false);
|
||||
};
|
||||
|
||||
const effectivePlaceholder = getDefaultPlaceholder(
|
||||
contentType,
|
||||
activeTarget,
|
||||
placeholder,
|
||||
combinedMode,
|
||||
combinedSelectionActive,
|
||||
);
|
||||
const effectiveInputAriaLabel = activeTarget
|
||||
? `${inputAriaLabel}: ${activeTarget.label}`
|
||||
@@ -404,7 +414,6 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
setAutocompleteDraftValue(nextValue);
|
||||
setIsAutocompleteOpen(nextValue.trim().length >= autocompleteMinQueryLength);
|
||||
setIsSelectOpen(false);
|
||||
setIsSelectorOpen(false);
|
||||
onChange(nextValue);
|
||||
return;
|
||||
}
|
||||
@@ -416,7 +425,6 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
textInputValue.trim().length >= autocompleteMinQueryLength
|
||||
) {
|
||||
setIsAutocompleteOpen(true);
|
||||
setIsSelectorOpen(false);
|
||||
}
|
||||
}}
|
||||
onKeyDown={handleKeyDown}
|
||||
@@ -482,7 +490,6 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
onClick={() => {
|
||||
if (!disabled && !isDynamicLoading) {
|
||||
setIsSelectOpen((prev) => !prev);
|
||||
setIsSelectorOpen(false);
|
||||
setIsAutocompleteOpen(false);
|
||||
}
|
||||
}}
|
||||
@@ -537,7 +544,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
|
||||
let selectorContentTypeLabel = 'audiobooks';
|
||||
let selectorIcon = <AudiobookIcon />;
|
||||
if (combinedMode) {
|
||||
if (combinedSelectionActive) {
|
||||
selectorContentTypeLabel = 'books and audiobooks';
|
||||
selectorIcon = <BothIcon />;
|
||||
} else if (contentType === 'ebook') {
|
||||
@@ -593,25 +600,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
}}
|
||||
>
|
||||
{showQueryTargetSelector && (
|
||||
<div
|
||||
className="relative flex shrink-0 self-stretch"
|
||||
ref={selectorRef}
|
||||
onPointerEnter={(e) => {
|
||||
if (e.pointerType !== 'mouse') return;
|
||||
clearHoverTimeout();
|
||||
setIsSelectorOpen(true);
|
||||
setIsSelectOpen(false);
|
||||
setIsAutocompleteOpen(false);
|
||||
}}
|
||||
onPointerLeave={(e) => {
|
||||
if (e.pointerType !== 'mouse') return;
|
||||
clearHoverTimeout();
|
||||
selectorHoverTimeout.current = setTimeout(() => {
|
||||
setIsSelectorOpen(false);
|
||||
selectorHoverTimeout.current = null;
|
||||
}, 150);
|
||||
}}
|
||||
>
|
||||
<div className="relative flex shrink-0 self-stretch" ref={selectorRef}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
@@ -619,11 +608,11 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
setIsSelectOpen(false);
|
||||
setIsAutocompleteOpen(false);
|
||||
}}
|
||||
className="hover-action flex items-center gap-1.5 rounded-l-full pr-2 pl-5 transition-colors"
|
||||
className="hover-action flex cursor-pointer items-center gap-1.5 rounded-l-full pr-2 pl-5 transition-colors"
|
||||
style={{ color: 'var(--text)' }}
|
||||
aria-label={`Searching ${selectorContentTypeLabel} by ${activeTarget?.label ?? 'general'}. Click to change.`}
|
||||
aria-expanded={isSelectorOpen}
|
||||
aria-haspopup="dialog"
|
||||
aria-controls={SEARCH_CONTROLS_PANEL_ID}
|
||||
>
|
||||
{selectorIcon}
|
||||
{showActiveTargetLabel && (
|
||||
@@ -646,266 +635,10 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
|
||||
<div
|
||||
className="absolute top-1/2 right-0 h-6 w-px -translate-y-1/2"
|
||||
style={{ background: 'var(--border-muted)' }}
|
||||
/>
|
||||
|
||||
{isSelectorOpen && (
|
||||
<div
|
||||
className="animate-fade-in-down absolute top-full left-0 z-50 mt-2 w-[min(20rem,calc(100vw-2rem))] overflow-hidden rounded-2xl border shadow-2xl"
|
||||
style={{
|
||||
background: 'var(--bg)',
|
||||
borderColor: 'var(--border-muted)',
|
||||
}}
|
||||
role="dialog"
|
||||
aria-label="Search context"
|
||||
>
|
||||
<div className="max-h-[min(24rem,calc(100vh-8rem))] overflow-y-auto p-3">
|
||||
{showContentTypeSelector && (
|
||||
<div
|
||||
className={`border-b ${onCombinedModeChange ? 'pb-0' : 'pb-3'}`}
|
||||
style={{ borderColor: 'var(--border-muted)' }}
|
||||
>
|
||||
<div className="flex items-center justify-between px-1 pb-2">
|
||||
<span className="text-xs font-medium tracking-wide uppercase opacity-60">
|
||||
Content
|
||||
</span>
|
||||
{onAdvancedToggle && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setIsSelectorOpen(false);
|
||||
onAdvancedToggle();
|
||||
}}
|
||||
className={`-mt-1.5 -mr-1 -mb-0.5 flex items-center gap-1.5 rounded-xl px-4 py-2.5 text-xs font-medium transition-colors ${
|
||||
isAdvancedActive ? 'bg-emerald-600 text-white' : 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
isAdvancedActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text-muted)' }
|
||||
}
|
||||
>
|
||||
<svg
|
||||
className="h-3.5 w-3.5"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth="1.5"
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M10.5 6h9.75M10.5 6a1.5 1.5 0 1 1-3 0m3 0a1.5 1.5 0 1 0-3 0M3.75 6H7.5m3 12h9.75m-9.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-3.75 0H7.5m9-6h3.75m-3.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-9.75 0h9.75"
|
||||
/>
|
||||
</svg>
|
||||
Options
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('ebook')}
|
||||
className={`flex items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'ebook' || combinedMode
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'ebook' || combinedMode
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{contentType === 'ebook' || combinedMode ? <CheckIcon /> : <BookIcon />}
|
||||
<span>Books</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('audiobook')}
|
||||
className={`flex items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'audiobook' || combinedMode
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'audiobook' || combinedMode
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{contentType === 'audiobook' || combinedMode ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<AudiobookIcon />
|
||||
)}
|
||||
<span>Audiobooks</span>
|
||||
</button>
|
||||
</div>
|
||||
{onCombinedModeChange &&
|
||||
(() => {
|
||||
const lineColor = combinedMode
|
||||
? 'bg-emerald-500'
|
||||
: 'bg-(--border-muted) group-hover:bg-zinc-400 dark:group-hover:bg-zinc-500';
|
||||
return (
|
||||
<Tooltip
|
||||
content="Combined search"
|
||||
position="bottom"
|
||||
triggerClassName="w-full"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleCombinedModeSelect}
|
||||
className="group w-full"
|
||||
aria-label="Combined search"
|
||||
>
|
||||
{/* Bracket connector: vertical drops + horizontal bar with icon */}
|
||||
<div className="relative flex h-7 items-end">
|
||||
{/* Left vertical */}
|
||||
<div
|
||||
className={`absolute top-1.5 bottom-[11px] left-[25%] w-px transition-colors ${lineColor}`}
|
||||
/>
|
||||
{/* Right vertical */}
|
||||
<div
|
||||
className={`absolute top-1.5 right-[25%] bottom-[11px] w-px transition-colors ${lineColor}`}
|
||||
/>
|
||||
{/* Horizontal bar – left segment */}
|
||||
<div
|
||||
className={`absolute bottom-[11px] left-[25%] h-px transition-colors ${lineColor}`}
|
||||
style={{ width: 'calc(25% - 16px)' }}
|
||||
/>
|
||||
{/* Horizontal bar – right segment */}
|
||||
<div
|
||||
className={`absolute right-[25%] bottom-[11px] h-px transition-colors ${lineColor}`}
|
||||
style={{ width: 'calc(25% - 16px)' }}
|
||||
/>
|
||||
{/* Chain icon centered at bottom */}
|
||||
<div
|
||||
className={`relative z-10 mx-auto rounded-full p-1 transition-colors ${
|
||||
combinedMode
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'bg-(--bg) text-zinc-400 group-hover:bg-zinc-200 group-hover:text-zinc-600 dark:text-zinc-500 dark:group-hover:bg-zinc-700 dark:group-hover:text-zinc-300'
|
||||
}`}
|
||||
>
|
||||
<svg
|
||||
className="h-3.5 w-3.5"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth="2"
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
{combinedModeLocked ? (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M16.5 10.5V6.75a4.5 4.5 0 1 0-9 0v3.75m-.75 11.25h10.5a2.25 2.25 0 0 0 2.25-2.25v-6.75a2.25 2.25 0 0 0-2.25-2.25H6.75a2.25 2.25 0 0 0-2.25 2.25v6.75a2.25 2.25 0 0 0 2.25 2.25Z"
|
||||
/>
|
||||
) : (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
|
||||
/>
|
||||
)}
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
</Tooltip>
|
||||
);
|
||||
})()}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className={showContentTypeSelector ? 'pt-2' : ''}>
|
||||
<div className="flex items-center justify-between px-1 pb-1.5">
|
||||
<span className="text-xs font-medium tracking-wide uppercase opacity-60">
|
||||
Search By
|
||||
</span>
|
||||
{!showContentTypeSelector && onAdvancedToggle && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setIsSelectorOpen(false);
|
||||
onAdvancedToggle();
|
||||
}}
|
||||
className={`-mt-1.5 -mr-1 -mb-0.5 flex items-center gap-1.5 rounded-xl px-4 py-2.5 text-xs font-medium transition-colors ${
|
||||
isAdvancedActive
|
||||
? `${searchMode === 'direct' ? 'bg-sky-700' : 'bg-emerald-600'} text-white`
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
isAdvancedActive
|
||||
? {
|
||||
borderColor:
|
||||
searchMode === 'direct'
|
||||
? 'rgb(3 105 161 / 0.7)'
|
||||
: 'rgb(16 185 129 / 0.7)',
|
||||
}
|
||||
: { color: 'var(--text-muted)' }
|
||||
}
|
||||
>
|
||||
<svg
|
||||
className="h-3.5 w-3.5"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth="1.5"
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M10.5 6h9.75M10.5 6a1.5 1.5 0 1 1-3 0m3 0a1.5 1.5 0 1 0-3 0M3.75 6H7.5m3 12h9.75m-9.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-3.75 0H7.5m9-6h3.75m-3.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-9.75 0h9.75"
|
||||
/>
|
||||
</svg>
|
||||
Options
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
{queryTargets.map((target) => {
|
||||
const isActive = target.key === activeTarget?.key;
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
key={target.key}
|
||||
onClick={() => handleQueryTargetSelect(target.key)}
|
||||
title={target.description || target.label}
|
||||
aria-label={target.label}
|
||||
className={`flex min-w-0 items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
isActive
|
||||
? `${searchMode === 'direct' ? 'bg-sky-700' : 'bg-emerald-600'} text-white`
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
isActive
|
||||
? {
|
||||
borderColor:
|
||||
searchMode === 'direct'
|
||||
? 'rgb(3 105 161 / 0.7)'
|
||||
: 'rgb(16 185 129 / 0.7)',
|
||||
}
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{isActive && <CheckIcon />}
|
||||
<span className="block truncate">{target.label}</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -999,7 +732,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
onClick={() => {
|
||||
onChange(option.value, option.label);
|
||||
setIsSelectOpen(false);
|
||||
setTimeout(() => onSubmitRef.current(), 0);
|
||||
setTimeout(() => submitLatest(), 0);
|
||||
}}
|
||||
className={`flex w-full items-center gap-3 px-5 py-2.5 text-left text-sm transition-colors ${
|
||||
isSelected ? '' : 'hover-surface'
|
||||
@@ -1069,7 +802,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
setAutocompleteSelection(option.value, option.label);
|
||||
onChange(option.value, option.label);
|
||||
setIsAutocompleteOpen(false);
|
||||
setTimeout(() => onSubmitRef.current(), 0);
|
||||
setTimeout(() => submitLatest(), 0);
|
||||
}}
|
||||
className="hover-surface w-full px-5 py-3 text-left text-sm transition-colors"
|
||||
style={{ color: 'var(--text)' }}
|
||||
@@ -1087,6 +820,246 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{showQueryTargetSelector && isSelectorOpen && (
|
||||
<div
|
||||
id={SEARCH_CONTROLS_PANEL_ID}
|
||||
className="animate-fade-in-down flex flex-wrap items-start gap-x-8 gap-y-2 px-1 pt-2"
|
||||
ref={controlsPanelRef}
|
||||
>
|
||||
{showContentTypeSelector && (
|
||||
<div className="shrink-0">
|
||||
<div className="flex items-center justify-between pb-1.5">
|
||||
<span className="text-xs font-medium tracking-wide uppercase opacity-60">
|
||||
Content
|
||||
</span>
|
||||
{onAdvancedToggle && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onAdvancedToggle}
|
||||
className={`-mt-1.5 -mr-1 -mb-0.5 flex cursor-pointer items-center gap-1.5 rounded-xl px-4 py-2.5 text-xs font-medium transition-colors ${
|
||||
isAdvancedActive ? 'bg-emerald-600 text-white' : 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
isAdvancedActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text-muted)' }
|
||||
}
|
||||
>
|
||||
<svg
|
||||
className="h-3.5 w-3.5"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth="1.5"
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M10.5 6h9.75M10.5 6a1.5 1.5 0 1 1-3 0m3 0a1.5 1.5 0 1 0-3 0M3.75 6H7.5m3 12h9.75m-9.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-3.75 0H7.5m9-6h3.75m-3.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-9.75 0h9.75"
|
||||
/>
|
||||
</svg>
|
||||
Options
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="grid w-fit grid-cols-2 gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('ebook')}
|
||||
className={`flex cursor-pointer items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'ebook' || combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'ebook' || combinedSelectionActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
<span className="flex w-4 justify-center">
|
||||
{contentType === 'ebook' || combinedSelectionActive ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<BookIcon />
|
||||
)}
|
||||
</span>
|
||||
<span>Books</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('audiobook')}
|
||||
className={`flex cursor-pointer items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'audiobook' || combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'audiobook' || combinedSelectionActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
<span className="flex w-4 justify-center">
|
||||
{contentType === 'audiobook' || combinedSelectionActive ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<AudiobookIcon />
|
||||
)}
|
||||
</span>
|
||||
<span>Audiobooks</span>
|
||||
</button>
|
||||
{combinedToggleAvailable && (
|
||||
<div className="col-span-2">
|
||||
<Tooltip
|
||||
content="Combined search"
|
||||
position="bottom"
|
||||
triggerClassName="w-full"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleCombinedModeSelect}
|
||||
className="group w-full cursor-pointer"
|
||||
aria-label="Combined search"
|
||||
>
|
||||
<div className="relative flex h-7 items-end">
|
||||
<div
|
||||
className={`absolute top-1.5 bottom-[11px] left-[25%] w-px transition-colors ${combinedLineColor}`}
|
||||
/>
|
||||
<div
|
||||
className={`absolute top-1.5 right-[25%] bottom-[11px] w-px transition-colors ${combinedLineColor}`}
|
||||
/>
|
||||
<div
|
||||
className={`absolute bottom-[11px] left-[25%] h-px transition-colors ${combinedLineColor}`}
|
||||
style={{ width: 'calc(25% - 16px)' }}
|
||||
/>
|
||||
<div
|
||||
className={`absolute right-[25%] bottom-[11px] h-px transition-colors ${combinedLineColor}`}
|
||||
style={{ width: 'calc(25% - 16px)' }}
|
||||
/>
|
||||
<div
|
||||
className={`relative z-10 mx-auto rounded-full p-1 transition-colors ${
|
||||
combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'bg-(--bg) text-zinc-400 group-hover:bg-zinc-200 group-hover:text-zinc-600 dark:text-zinc-500 dark:group-hover:bg-zinc-700 dark:group-hover:text-zinc-300'
|
||||
}`}
|
||||
>
|
||||
<svg
|
||||
className="h-3.5 w-3.5"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth="2"
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
{combinedSelectorLocked ? (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M16.5 10.5V6.75a4.5 4.5 0 1 0-9 0v3.75m-.75 11.25h10.5a2.25 2.25 0 0 0 2.25-2.25v-6.75a2.25 2.25 0 0 0-2.25-2.25H6.75a2.25 2.25 0 0 0-2.25 2.25v6.75a2.25 2.25 0 0 0 2.25 2.25Z"
|
||||
/>
|
||||
) : (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M13.19 8.688a4.5 4.5 0 0 1 1.242 7.244l-4.5 4.5a4.5 4.5 0 0 1-6.364-6.364l1.757-1.757m13.35-.622 1.757-1.757a4.5 4.5 0 0 0-6.364-6.364l-4.5 4.5a4.5 4.5 0 0 0 1.242 7.244"
|
||||
/>
|
||||
)}
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{queryTargets.length > 1 && (
|
||||
<div className="shrink-0">
|
||||
<div className="flex items-center justify-between pb-1.5">
|
||||
<span className="text-xs font-medium tracking-wide uppercase opacity-60">
|
||||
Search By
|
||||
</span>
|
||||
{!showContentTypeSelector && onAdvancedToggle && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onAdvancedToggle}
|
||||
className={`-mt-1.5 -mr-1 -mb-0.5 flex cursor-pointer items-center gap-1.5 rounded-xl px-4 py-2.5 text-xs font-medium transition-colors ${
|
||||
isAdvancedActive
|
||||
? `${searchMode === 'direct' ? 'bg-sky-700' : 'bg-emerald-600'} text-white`
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
isAdvancedActive
|
||||
? {
|
||||
borderColor:
|
||||
searchMode === 'direct'
|
||||
? 'rgb(3 105 161 / 0.7)'
|
||||
: 'rgb(16 185 129 / 0.7)',
|
||||
}
|
||||
: { color: 'var(--text-muted)' }
|
||||
}
|
||||
>
|
||||
<svg
|
||||
className="h-3.5 w-3.5"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth="1.5"
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M10.5 6h9.75M10.5 6a1.5 1.5 0 1 1-3 0m3 0a1.5 1.5 0 1 0-3 0M3.75 6H7.5m3 12h9.75m-9.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-3.75 0H7.5m9-6h3.75m-3.75 0a1.5 1.5 0 0 1-3 0m3 0a1.5 1.5 0 0 0-3 0m-9.75 0h9.75"
|
||||
/>
|
||||
</svg>
|
||||
Options
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{queryTargets.map((target) => {
|
||||
const isActive = target.key === activeTarget?.key;
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
key={target.key}
|
||||
onClick={() => handleQueryTargetSelect(target.key)}
|
||||
title={target.description || target.label}
|
||||
aria-label={target.label}
|
||||
className={`flex min-w-0 cursor-pointer items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
isActive
|
||||
? `${searchMode === 'direct' ? 'bg-sky-700' : 'bg-emerald-600'} text-white`
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
isActive
|
||||
? {
|
||||
borderColor:
|
||||
searchMode === 'direct'
|
||||
? 'rgb(3 105 161 / 0.7)'
|
||||
: 'rgb(16 185 129 / 0.7)',
|
||||
}
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{isActive && <CheckIcon />}
|
||||
<span className="block truncate">{target.label}</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
},
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { Dispatch, SetStateAction } from 'react';
|
||||
import { useMountEffect } from '@/hooks/useMountEffect';
|
||||
import type { AppConfig, AdvancedFilterState, ContentType, SearchMode, SortOption } from '@/types';
|
||||
import { buildSearchQuery } from '@/utils/buildSearchQuery';
|
||||
import { resolveDefaultLanguageCodes } from '@/utils/languageFilters';
|
||||
import { getEffectiveMetadataSort } from '@/utils/metadataSort';
|
||||
import type { ParsedUrlSearch } from '@/utils/parseUrlSearchParams';
|
||||
|
||||
@@ -73,10 +74,10 @@ export const UrlSearchBootstrapMount = ({
|
||||
}
|
||||
|
||||
const bookLanguages = config.book_languages || [];
|
||||
const defaultLanguageCodes =
|
||||
config.default_language && config.default_language.length > 0
|
||||
? config.default_language
|
||||
: [bookLanguages[0]?.code || 'en'];
|
||||
const defaultLanguageCodes = resolveDefaultLanguageCodes(
|
||||
config.default_language,
|
||||
bookLanguages,
|
||||
);
|
||||
|
||||
if (parsedParams.searchInput) {
|
||||
setSearchInput(parsedParams.searchInput);
|
||||
|
||||
@@ -562,6 +562,10 @@ export const ActivityCard = ({
|
||||
}
|
||||
|
||||
return () => observer.disconnect();
|
||||
// None of these are read here - they are all re-measure triggers. The title's overflow
|
||||
// depends on its text and on the width it is laid out in, and opening either panel
|
||||
// reflows the card. Drop them and the tooltip-on-truncation goes stale.
|
||||
// oxlint-disable-next-line react/exhaustive-effect-dependencies
|
||||
}, [item.title, item.author, isRequestDetailsOpen, isRequestRejectOpen]);
|
||||
|
||||
const reviewRecord = item.requestRecord;
|
||||
|
||||
@@ -347,6 +347,10 @@ function SettingsContentPanel({
|
||||
if (scrollRef.current) {
|
||||
scrollRef.current.scrollTop = 0;
|
||||
}
|
||||
// `tab.name` is never read here - it is the trigger, and the whole point: the scroll
|
||||
// position resets *because* the tab changed. Removing it strands the new tab at the
|
||||
// previous one's offset.
|
||||
// oxlint-disable-next-line react/exhaustive-effect-dependencies
|
||||
}, [embedded, tab.name]);
|
||||
|
||||
const updateCustomFieldUiState = useCallback((fieldKey: string, key: string, value: unknown) => {
|
||||
@@ -407,6 +411,9 @@ function SettingsContentPanel({
|
||||
if (scrollRef.current) {
|
||||
scrollRef.current.scrollTop = 0;
|
||||
}
|
||||
// `activeTakeOverFieldKey` is never read here - it is the trigger. Entering or leaving
|
||||
// a subpage takeover is exactly when the scroll must reset.
|
||||
// oxlint-disable-next-line react/exhaustive-effect-dependencies
|
||||
}, [embedded, activeTakeOverFieldKey]);
|
||||
|
||||
const visibleFields = useMemo(() => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useCallback, useLayoutEffect, useRef } from 'react';
|
||||
|
||||
import { useLatestCallback } from '../../../hooks/useLatestCallback';
|
||||
import { useMountEffect } from '../../../hooks/useMountEffect';
|
||||
import type { AdminUser } from '../../../services/api';
|
||||
import { testAdminUserNotificationPreferences } from '../../../services/api';
|
||||
@@ -169,12 +170,12 @@ export const UsersManagementField = ({
|
||||
}
|
||||
}, [backToList, onRefreshOverrideSummary, onSettingsSaved, onUiStateChange, saveEditedUser]);
|
||||
|
||||
const handleSaveUserOverridesRef = useRef(handleSaveUserOverrides);
|
||||
handleSaveUserOverridesRef.current = handleSaveUserOverrides;
|
||||
|
||||
const triggerSaveUserOverrides = useCallback(async () => {
|
||||
await handleSaveUserOverridesRef.current();
|
||||
}, []);
|
||||
// Stored in parent UI state, so it must keep a stable identity while still invoking the
|
||||
// latest handler. Not an Effect Event: those must not be handed to another component.
|
||||
// See useLatestCallback.
|
||||
const triggerSaveUserOverrides = useLatestCallback(async () => {
|
||||
await handleSaveUserOverrides();
|
||||
});
|
||||
|
||||
const handleOpenOverrides = () => {
|
||||
if (editingUser) {
|
||||
|
||||
@@ -44,162 +44,172 @@ const getOptionsIdentity = (options: MultiSelectFieldConfig['options']): string
|
||||
const getSelectionIdentity = (values: string[]): string =>
|
||||
values.toSorted((left, right) => left.localeCompare(right)).join('\u0001');
|
||||
|
||||
export const MultiSelectField = ({
|
||||
interface MultiSelectVariantProps {
|
||||
field: MultiSelectFieldConfig;
|
||||
selected: string[];
|
||||
onChange: (value: string[]) => void;
|
||||
isDisabled: boolean;
|
||||
}
|
||||
|
||||
// Dropdown variant - use DropdownList with checkboxes
|
||||
const MultiSelectDropdownField = ({
|
||||
field,
|
||||
value: fieldValue,
|
||||
selected,
|
||||
onChange,
|
||||
disabled,
|
||||
}: MultiSelectFieldProps) => {
|
||||
const selected = fieldValue ?? EMPTY_SELECTION;
|
||||
// disabled prop is already computed by SettingsContent.getDisabledState()
|
||||
const isDisabled = disabled ?? false;
|
||||
isDisabled,
|
||||
}: MultiSelectVariantProps) => {
|
||||
const optionValues = field.options.map((opt) => opt.value);
|
||||
const optionSet = new Set(optionValues);
|
||||
const hasAllOption = optionSet.has(ALL_OPTION_VALUE);
|
||||
const orderedOptions = hasAllOption
|
||||
? [
|
||||
...field.options.filter((opt) => opt.value === ALL_OPTION_VALUE),
|
||||
...field.options.filter((opt) => opt.value !== ALL_OPTION_VALUE),
|
||||
]
|
||||
: field.options;
|
||||
const nonAllValues = orderedOptions
|
||||
.map((opt) => opt.value)
|
||||
.filter((optValue) => optValue !== ALL_OPTION_VALUE);
|
||||
|
||||
// Dropdown variant - use DropdownList with checkboxes
|
||||
if (field.variant === 'dropdown') {
|
||||
const optionValues = field.options.map((opt) => opt.value);
|
||||
const optionSet = new Set(optionValues);
|
||||
const hasAllOption = optionSet.has(ALL_OPTION_VALUE);
|
||||
const orderedOptions = hasAllOption
|
||||
? [
|
||||
...field.options.filter((opt) => opt.value === ALL_OPTION_VALUE),
|
||||
...field.options.filter((opt) => opt.value !== ALL_OPTION_VALUE),
|
||||
]
|
||||
: field.options;
|
||||
const nonAllValues = orderedOptions
|
||||
.map((opt) => opt.value)
|
||||
.filter((optValue) => optValue !== ALL_OPTION_VALUE);
|
||||
const normalizeValues = (values: string[]): string[] => {
|
||||
const deduped = new Set(
|
||||
values
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry.length > 0 && optionSet.has(entry)),
|
||||
);
|
||||
return orderedOptions.map((opt) => opt.value).filter((optValue) => deduped.has(optValue));
|
||||
};
|
||||
|
||||
const normalizeValues = (values: string[]): string[] => {
|
||||
const deduped = new Set(
|
||||
values
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry.length > 0 && optionSet.has(entry)),
|
||||
);
|
||||
return orderedOptions.map((opt) => opt.value).filter((optValue) => deduped.has(optValue));
|
||||
};
|
||||
const selectedExplicit = normalizeValues(selected);
|
||||
const allSelected =
|
||||
hasAllOption &&
|
||||
(selectedExplicit.includes(ALL_OPTION_VALUE) ||
|
||||
(nonAllValues.length > 0 &&
|
||||
nonAllValues.every((optValue) => selectedExplicit.includes(optValue))));
|
||||
|
||||
const selectedExplicit = normalizeValues(selected);
|
||||
const allSelected =
|
||||
hasAllOption &&
|
||||
(selectedExplicit.includes(ALL_OPTION_VALUE) ||
|
||||
(nonAllValues.length > 0 &&
|
||||
nonAllValues.every((optValue) => selectedExplicit.includes(optValue))));
|
||||
// Build parent -> children map for cascading selection
|
||||
const parentChildMap = new Map<string, string[]>();
|
||||
orderedOptions.forEach((opt) => {
|
||||
if (opt.childOf) {
|
||||
const children = parentChildMap.get(opt.childOf) || [];
|
||||
children.push(opt.value);
|
||||
parentChildMap.set(opt.childOf, children);
|
||||
}
|
||||
});
|
||||
|
||||
// Build parent -> children map for cascading selection
|
||||
const parentChildMap = new Map<string, string[]>();
|
||||
orderedOptions.forEach((opt) => {
|
||||
if (opt.childOf) {
|
||||
const children = parentChildMap.get(opt.childOf) || [];
|
||||
children.push(opt.value);
|
||||
parentChildMap.set(opt.childOf, children);
|
||||
// Check which children are implicitly selected via parent
|
||||
const selectedForCascade = allSelected
|
||||
? selectedExplicit.filter((optValue) => optValue !== ALL_OPTION_VALUE)
|
||||
: selectedExplicit;
|
||||
const implicitlySelected = new Set<string>();
|
||||
selectedForCascade.forEach((val) => {
|
||||
const children = parentChildMap.get(val);
|
||||
if (children) {
|
||||
children.forEach((child) => implicitlySelected.add(child));
|
||||
}
|
||||
});
|
||||
|
||||
// Build options with disabled state for implicitly selected children
|
||||
const dropdownOptions = orderedOptions.map((opt) => ({
|
||||
value: opt.value,
|
||||
label: opt.label,
|
||||
disabled: !allSelected && implicitlySelected.has(opt.value),
|
||||
}));
|
||||
|
||||
// For display purposes:
|
||||
// - if "all" is active, check every option
|
||||
// - otherwise show explicit + implicit parent/child selections
|
||||
const displayValue = allSelected
|
||||
? [ALL_OPTION_VALUE, ...nonAllValues]
|
||||
: normalizeValues([...selectedExplicit, ...Array.from(implicitlySelected)]);
|
||||
|
||||
const handleDropdownChange = (newValue: string | string[]) => {
|
||||
const nextValues = normalizeValues(Array.isArray(newValue) ? newValue : [newValue]);
|
||||
|
||||
if (hasAllOption) {
|
||||
const includesAll = nextValues.includes(ALL_OPTION_VALUE);
|
||||
|
||||
// When currently "all" is active:
|
||||
// - unticking "all" clears everything
|
||||
// - unticking a specific option converts to explicit subset
|
||||
if (allSelected && !includesAll && nextValues.length === nonAllValues.length) {
|
||||
onChange([]);
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
// Check which children are implicitly selected via parent
|
||||
const selectedForCascade = allSelected
|
||||
? selectedExplicit.filter((optValue) => optValue !== ALL_OPTION_VALUE)
|
||||
: selectedExplicit;
|
||||
const implicitlySelected = new Set<string>();
|
||||
selectedForCascade.forEach((val) => {
|
||||
const children = parentChildMap.get(val);
|
||||
if (children) {
|
||||
children.forEach((child) => implicitlySelected.add(child));
|
||||
}
|
||||
});
|
||||
|
||||
// Build options with disabled state for implicitly selected children
|
||||
const dropdownOptions = orderedOptions.map((opt) => ({
|
||||
value: opt.value,
|
||||
label: opt.label,
|
||||
disabled: !allSelected && implicitlySelected.has(opt.value),
|
||||
}));
|
||||
|
||||
// For display purposes:
|
||||
// - if "all" is active, check every option
|
||||
// - otherwise show explicit + implicit parent/child selections
|
||||
const displayValue = allSelected
|
||||
? [ALL_OPTION_VALUE, ...nonAllValues]
|
||||
: normalizeValues([...selectedExplicit, ...Array.from(implicitlySelected)]);
|
||||
|
||||
const handleDropdownChange = (newValue: string | string[]) => {
|
||||
const nextValues = normalizeValues(Array.isArray(newValue) ? newValue : [newValue]);
|
||||
|
||||
if (hasAllOption) {
|
||||
const includesAll = nextValues.includes(ALL_OPTION_VALUE);
|
||||
|
||||
// When currently "all" is active:
|
||||
// - unticking "all" clears everything
|
||||
// - unticking a specific option converts to explicit subset
|
||||
if (allSelected && !includesAll && nextValues.length === nonAllValues.length) {
|
||||
onChange([]);
|
||||
return;
|
||||
}
|
||||
if (allSelected && includesAll && nextValues.length < optionValues.length) {
|
||||
onChange(nextValues.filter((entry) => entry !== ALL_OPTION_VALUE));
|
||||
return;
|
||||
}
|
||||
|
||||
if (includesAll) {
|
||||
onChange([ALL_OPTION_VALUE]);
|
||||
return;
|
||||
}
|
||||
|
||||
// If user selects every specific option individually, collapse to "all".
|
||||
if (
|
||||
nonAllValues.length > 0 &&
|
||||
nonAllValues.every((optValue) => nextValues.includes(optValue))
|
||||
) {
|
||||
onChange([ALL_OPTION_VALUE]);
|
||||
return;
|
||||
}
|
||||
if (allSelected && includesAll && nextValues.length < optionValues.length) {
|
||||
onChange(nextValues.filter((entry) => entry !== ALL_OPTION_VALUE));
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter out implicitly selected values - only store explicit selections.
|
||||
const explicitOnly = nextValues.filter((entry) => !implicitlySelected.has(entry));
|
||||
onChange(explicitOnly);
|
||||
};
|
||||
if (includesAll) {
|
||||
onChange([ALL_OPTION_VALUE]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Custom summary formatter - only count explicit selections
|
||||
const summaryFormatter = () => {
|
||||
if (allSelected) {
|
||||
return orderedOptions.find((opt) => opt.value === ALL_OPTION_VALUE)?.label || 'All';
|
||||
// If user selects every specific option individually, collapse to "all".
|
||||
if (
|
||||
nonAllValues.length > 0 &&
|
||||
nonAllValues.every((optValue) => nextValues.includes(optValue))
|
||||
) {
|
||||
onChange([ALL_OPTION_VALUE]);
|
||||
return;
|
||||
}
|
||||
if (selectedExplicit.length === 0) {
|
||||
return <span className="opacity-60">{field.placeholder || 'Select categories...'}</span>;
|
||||
}
|
||||
const selectedLabels = selectedExplicit
|
||||
.map((v) => orderedOptions.find((o) => o.value === v)?.label)
|
||||
.filter(Boolean);
|
||||
if (selectedLabels.length === 1) {
|
||||
return selectedLabels[0];
|
||||
}
|
||||
const [first, second, ...rest] = selectedLabels;
|
||||
const suffix = rest.length > 0 ? ` +${rest.length}` : '';
|
||||
return `${first}, ${second ?? ''}${suffix}`.trim();
|
||||
};
|
||||
|
||||
if (isDisabled) {
|
||||
return (
|
||||
<div className="w-full cursor-not-allowed rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm opacity-60">
|
||||
{summaryFormatter()}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Filter out implicitly selected values - only store explicit selections.
|
||||
const explicitOnly = nextValues.filter((entry) => !implicitlySelected.has(entry));
|
||||
onChange(explicitOnly);
|
||||
};
|
||||
|
||||
// Custom summary formatter - only count explicit selections
|
||||
const summaryFormatter = () => {
|
||||
if (allSelected) {
|
||||
return orderedOptions.find((opt) => opt.value === ALL_OPTION_VALUE)?.label || 'All';
|
||||
}
|
||||
if (selectedExplicit.length === 0) {
|
||||
return <span className="opacity-60">{field.placeholder || 'Select categories...'}</span>;
|
||||
}
|
||||
const selectedLabels = selectedExplicit
|
||||
.map((v) => orderedOptions.find((o) => o.value === v)?.label)
|
||||
.filter(Boolean);
|
||||
if (selectedLabels.length === 1) {
|
||||
return selectedLabels[0];
|
||||
}
|
||||
const [first, second, ...rest] = selectedLabels;
|
||||
const suffix = rest.length > 0 ? ` +${rest.length}` : '';
|
||||
return `${first}, ${second ?? ''}${suffix}`.trim();
|
||||
};
|
||||
|
||||
if (isDisabled) {
|
||||
return (
|
||||
<DropdownList
|
||||
options={dropdownOptions}
|
||||
value={displayValue}
|
||||
onChange={handleDropdownChange}
|
||||
multiple
|
||||
showCheckboxes
|
||||
keepOpenOnSelect
|
||||
placeholder={field.placeholder || 'Select categories...'}
|
||||
widthClassName="w-full"
|
||||
summaryFormatter={summaryFormatter}
|
||||
/>
|
||||
<div className="w-full cursor-not-allowed rounded-lg border border-(--border-muted) bg-(--bg-soft) px-3 py-2 text-sm opacity-60">
|
||||
{summaryFormatter()}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<DropdownList
|
||||
options={dropdownOptions}
|
||||
value={displayValue}
|
||||
onChange={handleDropdownChange}
|
||||
multiple
|
||||
showCheckboxes
|
||||
keepOpenOnSelect
|
||||
placeholder={field.placeholder || 'Select categories...'}
|
||||
widthClassName="w-full"
|
||||
summaryFormatter={summaryFormatter}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
// Pill variant - inline toggle buttons that collapse past a threshold
|
||||
const MultiSelectPillsField = ({
|
||||
field,
|
||||
selected,
|
||||
onChange,
|
||||
isDisabled,
|
||||
}: MultiSelectVariantProps) => {
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
// Initialize based on option count to avoid flash of expanded content
|
||||
const [needsCollapse, setNeedsCollapse] = useState(
|
||||
@@ -353,3 +363,35 @@ export const MultiSelectField = ({
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const MultiSelectField = ({
|
||||
field,
|
||||
value: fieldValue,
|
||||
onChange,
|
||||
disabled,
|
||||
}: MultiSelectFieldProps) => {
|
||||
const selected = fieldValue ?? EMPTY_SELECTION;
|
||||
// disabled prop is already computed by SettingsContent.getDisabledState()
|
||||
const isDisabled = disabled ?? false;
|
||||
|
||||
// Each variant is its own component so neither calls hooks conditionally.
|
||||
if (field.variant === 'dropdown') {
|
||||
return (
|
||||
<MultiSelectDropdownField
|
||||
field={field}
|
||||
selected={selected}
|
||||
onChange={onChange}
|
||||
isDisabled={isDisabled}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<MultiSelectPillsField
|
||||
field={field}
|
||||
selected={selected}
|
||||
onChange={onChange}
|
||||
isDisabled={isDisabled}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -353,12 +353,12 @@ export const TableField = ({ field, value, onChange, disabled }: TableFieldProps
|
||||
);
|
||||
}
|
||||
|
||||
// text/path
|
||||
// text/password/path
|
||||
return (
|
||||
<div key={col.key} className="flex min-w-0 flex-col gap-1">
|
||||
{mobileLabel}
|
||||
<input
|
||||
type="text"
|
||||
type={col.type === 'password' ? 'password' : 'text'}
|
||||
value={toPrimitiveString(cellValue)}
|
||||
onChange={(e) => updateCell(rowIndex, col.key, e.target.value)}
|
||||
placeholder={col.placeholder}
|
||||
|
||||
@@ -7,7 +7,12 @@ import type {
|
||||
} from '../../../types/settings';
|
||||
import { HeadingField, MultiSelectField, SelectField, TextField } from '../fields';
|
||||
import { FieldWrapper } from '../shared';
|
||||
import { getFieldByKey, toNormalizedLowercaseTextValue, toTextValue } from './fieldHelpers';
|
||||
import {
|
||||
getFieldByKey,
|
||||
resolveListOverride,
|
||||
toNormalizedLowercaseTextValue,
|
||||
toTextValue,
|
||||
} from './fieldHelpers';
|
||||
import type { PerUserSettings } from './types';
|
||||
|
||||
interface UserOverridesSectionProps {
|
||||
@@ -175,16 +180,6 @@ export const UserOverridesSection = ({
|
||||
label: 'Email Recipient',
|
||||
description: 'Email address used for this user in Email output mode.',
|
||||
};
|
||||
const browserDownloadGlobalValue = Array.isArray(globalValues.DOWNLOAD_TO_BROWSER_CONTENT_TYPES)
|
||||
? globalValues.DOWNLOAD_TO_BROWSER_CONTENT_TYPES.map((entry) => String(entry).trim()).filter(
|
||||
(entry) => entry.length > 0,
|
||||
)
|
||||
: [];
|
||||
const browserDownloadUserValue = Array.isArray(userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES)
|
||||
? userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES.map((entry) => entry.trim()).filter(
|
||||
(entry) => entry.length > 0,
|
||||
)
|
||||
: [];
|
||||
|
||||
const isOverridden = (key: DeliverySettingKey): boolean => {
|
||||
if (
|
||||
@@ -200,10 +195,12 @@ export const UserOverridesSection = ({
|
||||
return userValue !== globalValue;
|
||||
};
|
||||
|
||||
const isBrowserDownloadOverridden =
|
||||
Object.prototype.hasOwnProperty.call(userSettings, 'DOWNLOAD_TO_BROWSER_CONTENT_TYPES') &&
|
||||
userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES !== null &&
|
||||
JSON.stringify(browserDownloadUserValue) !== JSON.stringify(browserDownloadGlobalValue);
|
||||
const { value: browserDownloadContentTypes, isOverridden: isBrowserDownloadOverridden } =
|
||||
resolveListOverride(
|
||||
userSettings.DOWNLOAD_TO_BROWSER_CONTENT_TYPES,
|
||||
globalValues.DOWNLOAD_TO_BROWSER_CONTENT_TYPES,
|
||||
Object.prototype.hasOwnProperty.call(userSettings, 'DOWNLOAD_TO_BROWSER_CONTENT_TYPES'),
|
||||
);
|
||||
|
||||
const resetKeys = (keys: DeliverySettingKey[]) => {
|
||||
setUserSettings((prev) => {
|
||||
@@ -232,9 +229,6 @@ export const UserOverridesSection = ({
|
||||
const outputModeValue = readValue('BOOKS_OUTPUT_MODE', 'folder');
|
||||
const effectiveOutputMode = normalizeMode(outputModeValue);
|
||||
|
||||
const browserDownloadContentTypes = isBrowserDownloadOverridden
|
||||
? browserDownloadUserValue
|
||||
: browserDownloadGlobalValue;
|
||||
const destinationValue = readValue('DESTINATION');
|
||||
const destinationAudiobookValue = readValue('DESTINATION_AUDIOBOOK');
|
||||
const libraryValue = readValue('BOOKLORE_LIBRARY_ID');
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
import type { DeliveryPreferencesResponse } from '../../../services/api';
|
||||
import type { HeadingFieldConfig, SelectFieldConfig } from '../../../types/settings';
|
||||
import { HeadingField, SelectField } from '../fields';
|
||||
import type {
|
||||
HeadingFieldConfig,
|
||||
MultiSelectFieldConfig,
|
||||
SelectFieldConfig,
|
||||
} from '../../../types/settings';
|
||||
import { HeadingField, MultiSelectField, SelectField } from '../fields';
|
||||
import { FieldWrapper } from '../shared';
|
||||
import { getFieldByKey, toNormalizedLowercaseTextValue, toTextValue } from './fieldHelpers';
|
||||
import {
|
||||
getFieldByKey,
|
||||
resolveListOverride,
|
||||
toNormalizedLowercaseTextValue,
|
||||
toTextValue,
|
||||
} from './fieldHelpers';
|
||||
import type { PerUserSettings } from './types';
|
||||
|
||||
interface UserSearchPreferencesSectionProps {
|
||||
@@ -14,6 +23,7 @@ interface UserSearchPreferencesSectionProps {
|
||||
|
||||
type SearchSettingKey =
|
||||
| 'SEARCH_MODE'
|
||||
| 'BOOK_LANGUAGE'
|
||||
| 'METADATA_PROVIDER'
|
||||
| 'METADATA_PROVIDER_AUDIOBOOK'
|
||||
| 'DEFAULT_RELEASE_SOURCE'
|
||||
@@ -68,6 +78,15 @@ const fallbackDefaultAudiobookReleaseSourceField: SelectFieldConfig = {
|
||||
options: [{ value: '', label: 'Use book release source' }],
|
||||
};
|
||||
|
||||
const fallbackBookLanguageField: MultiSelectFieldConfig = {
|
||||
type: 'MultiSelectField',
|
||||
key: 'BOOK_LANGUAGE',
|
||||
label: 'Default Book Languages',
|
||||
description: 'Default language filter for searches.',
|
||||
value: [],
|
||||
options: [],
|
||||
};
|
||||
|
||||
const searchHeading: HeadingFieldConfig = {
|
||||
type: 'HeadingField',
|
||||
key: 'search_preferences_heading',
|
||||
@@ -120,6 +139,13 @@ export const UserSearchPreferencesSection = ({
|
||||
'DEFAULT_RELEASE_SOURCE_AUDIOBOOK',
|
||||
fallbackDefaultAudiobookReleaseSourceField,
|
||||
);
|
||||
const bookLanguageField = getFieldByKey(fields, 'BOOK_LANGUAGE', fallbackBookLanguageField);
|
||||
|
||||
const { value: bookLanguageValue, isOverridden: isBookLanguageOverridden } = resolveListOverride(
|
||||
userSettings.BOOK_LANGUAGE,
|
||||
globalValues.BOOK_LANGUAGE,
|
||||
Object.prototype.hasOwnProperty.call(userSettings, 'BOOK_LANGUAGE'),
|
||||
);
|
||||
|
||||
const isOverridden = (key: SearchSettingKey): boolean => {
|
||||
if (
|
||||
@@ -172,9 +198,12 @@ export const UserSearchPreferencesSection = ({
|
||||
const canOverrideDefaultAudiobookReleaseSource =
|
||||
isUserOverridable('DEFAULT_RELEASE_SOURCE_AUDIOBOOK') &&
|
||||
preferenceKeySet.has('DEFAULT_RELEASE_SOURCE_AUDIOBOOK');
|
||||
const canOverrideBookLanguage =
|
||||
isUserOverridable('BOOK_LANGUAGE') && preferenceKeySet.has('BOOK_LANGUAGE');
|
||||
|
||||
if (
|
||||
!canOverrideSearchMode &&
|
||||
!canOverrideBookLanguage &&
|
||||
!canOverrideMetadataProvider &&
|
||||
!canOverrideAudiobookMetadataProvider &&
|
||||
!canOverrideDefaultReleaseSource &&
|
||||
@@ -208,6 +237,27 @@ export const UserSearchPreferencesSection = ({
|
||||
</FieldWrapper>
|
||||
)}
|
||||
|
||||
{canOverrideBookLanguage && (
|
||||
<FieldWrapper
|
||||
field={bookLanguageField}
|
||||
resetAction={
|
||||
isBookLanguageOverridden
|
||||
? {
|
||||
disabled: Boolean(bookLanguageField.fromEnv),
|
||||
onClick: () => resetKeys(['BOOK_LANGUAGE']),
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
<MultiSelectField
|
||||
field={bookLanguageField}
|
||||
value={bookLanguageValue}
|
||||
onChange={(value) => setUserSettings((prev) => ({ ...prev, BOOK_LANGUAGE: value }))}
|
||||
disabled={Boolean(bookLanguageField.fromEnv)}
|
||||
/>
|
||||
</FieldWrapper>
|
||||
)}
|
||||
|
||||
{effectiveSearchMode === 'universal' && canOverrideMetadataProvider && (
|
||||
<FieldWrapper
|
||||
field={metadataProviderField}
|
||||
|
||||
@@ -31,6 +31,33 @@ export const toNormalizedLowercaseTextValue = (value: unknown): string => {
|
||||
return toTrimmedTextValue(value).toLowerCase();
|
||||
};
|
||||
|
||||
const toStringListValue = (value: unknown): string[] => {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.map((entry) => toTrimmedTextValue(entry)).filter((entry) => entry.length > 0);
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve a list-valued per-user override against its global value.
|
||||
*
|
||||
* A key absent from userSettings, or set to null, is not an override. A stored list
|
||||
* that matches the global one is treated as inherited, matching how
|
||||
* buildUserSettingsPayload clears it on save.
|
||||
*/
|
||||
export const resolveListOverride = (
|
||||
userValue: unknown,
|
||||
globalValue: unknown,
|
||||
hasUserKey: boolean,
|
||||
): { value: string[]; isOverridden: boolean } => {
|
||||
const globalList = toStringListValue(globalValue);
|
||||
const userList = toStringListValue(userValue);
|
||||
const isOverridden =
|
||||
hasUserKey && userValue !== null && JSON.stringify(userList) !== JSON.stringify(globalList);
|
||||
|
||||
return { value: isOverridden ? userList : globalList, isOverridden };
|
||||
};
|
||||
|
||||
export const toComparableValue = (value: unknown): string => {
|
||||
if (value === null || value === undefined) {
|
||||
return '';
|
||||
|
||||
@@ -10,6 +10,7 @@ export interface PerUserSettings {
|
||||
EMAIL_RECIPIENT?: string;
|
||||
DOWNLOAD_TO_BROWSER_CONTENT_TYPES?: string[];
|
||||
SEARCH_MODE?: string;
|
||||
BOOK_LANGUAGE?: string[];
|
||||
METADATA_PROVIDER?: string;
|
||||
METADATA_PROVIDER_AUDIOBOOK?: string;
|
||||
DEFAULT_RELEASE_SOURCE?: string;
|
||||
|
||||
@@ -139,6 +139,10 @@ export function Tooltip({
|
||||
}
|
||||
|
||||
if (deltaX !== 0 || deltaY !== 0) {
|
||||
// Genuine measure-and-adjust: the tooltip must be laid out before we know
|
||||
// whether it overflows the viewport. The loop converges in one pass because
|
||||
// the corrected position yields deltaX/deltaY of 0 on the next run.
|
||||
// oxlint-disable-next-line react/set-state-in-effect
|
||||
setCoords((current) => {
|
||||
if (!current) {
|
||||
return current;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { useRef } from 'react';
|
||||
import { useEffectEvent } from 'react';
|
||||
import type { Dispatch, SetStateAction } from 'react';
|
||||
|
||||
import type { Book } from '../../types';
|
||||
import { onBookTargetChange } from '../../utils/bookTargetEvents';
|
||||
import { onBookTargetChange, type BookTargetChangeEvent } from '../../utils/bookTargetEvents';
|
||||
import { useMountEffect } from '../useMountEffect';
|
||||
|
||||
interface UseBookTargetDeselectSyncOptions {
|
||||
@@ -14,15 +14,14 @@ export const useBookTargetDeselectSync = ({
|
||||
activeListValue,
|
||||
setBooks,
|
||||
}: UseBookTargetDeselectSyncOptions): void => {
|
||||
const activeListValueRef = useRef(activeListValue);
|
||||
activeListValueRef.current = activeListValue;
|
||||
|
||||
useMountEffect(() => {
|
||||
return onBookTargetChange((event) => {
|
||||
if (event.selected) return;
|
||||
const currentValue = activeListValueRef.current;
|
||||
if (!currentValue || String(currentValue) !== event.target) return;
|
||||
setBooks((prev) => prev.filter((book) => book.provider_id !== event.bookId));
|
||||
});
|
||||
const handleTargetChange = useEffectEvent((event: BookTargetChangeEvent) => {
|
||||
if (event.selected) return;
|
||||
if (!activeListValue || String(activeListValue) !== event.target) return;
|
||||
setBooks((prev) => prev.filter((book) => book.provider_id !== event.bookId));
|
||||
});
|
||||
|
||||
// Wrapped rather than handed over directly: an Effect Event must not be given to
|
||||
// something that stores it, and `onBookTargetChange` puts its argument in a
|
||||
// module-level listener set. Same shape as useDismiss.
|
||||
useMountEffect(() => onBookTargetChange((event) => handleTargetChange(event)));
|
||||
};
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
import { useCallback, useRef, useState } from 'react';
|
||||
import { useCallback, useState } from 'react';
|
||||
import type { Dispatch, SetStateAction } from 'react';
|
||||
|
||||
import type { ContentType } from '../../types';
|
||||
import { useDependencyEffect } from '../useMountEffect';
|
||||
|
||||
const CONTENT_TYPE_STORAGE_KEY = 'preferred-content-type';
|
||||
|
||||
const readInitialPreference = (): { contentType: ContentType; combinedMode: boolean } => {
|
||||
interface ContentTypePreference {
|
||||
contentType: ContentType;
|
||||
combinedMode: boolean;
|
||||
}
|
||||
|
||||
const readInitialPreference = (): ContentTypePreference => {
|
||||
try {
|
||||
const saved = localStorage.getItem(CONTENT_TYPE_STORAGE_KEY);
|
||||
if (saved === 'combined') {
|
||||
@@ -26,53 +32,32 @@ export const useContentTypePreferences = (): {
|
||||
combinedMode: boolean;
|
||||
setCombinedMode: Dispatch<SetStateAction<boolean>>;
|
||||
} => {
|
||||
const initialPreference = readInitialPreference();
|
||||
const [contentType, setContentTypeState] = useState<ContentType>(
|
||||
() => initialPreference.contentType,
|
||||
);
|
||||
const [combinedMode, setCombinedModeState] = useState<boolean>(
|
||||
() => initialPreference.combinedMode,
|
||||
);
|
||||
const contentTypeRef = useRef(contentType);
|
||||
const combinedModeRef = useRef(combinedMode);
|
||||
contentTypeRef.current = contentType;
|
||||
combinedModeRef.current = combinedMode;
|
||||
// Both values live in one state object so each setter can derive the other
|
||||
// from a pure updater instead of mirroring it into a ref during render.
|
||||
const [preference, setPreference] = useState<ContentTypePreference>(readInitialPreference);
|
||||
const { contentType, combinedMode } = preference;
|
||||
|
||||
const persistPreference = useCallback(
|
||||
(nextContentType: ContentType, nextCombinedMode: boolean) => {
|
||||
try {
|
||||
localStorage.setItem(
|
||||
CONTENT_TYPE_STORAGE_KEY,
|
||||
nextCombinedMode ? 'combined' : nextContentType,
|
||||
);
|
||||
} catch {
|
||||
// localStorage may be unavailable in private browsing
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
const setContentType: Dispatch<SetStateAction<ContentType>> = useCallback((value) => {
|
||||
setPreference((current) => ({
|
||||
...current,
|
||||
contentType: typeof value === 'function' ? value(current.contentType) : value,
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const setContentType: Dispatch<SetStateAction<ContentType>> = useCallback(
|
||||
(value) => {
|
||||
setContentTypeState((current) => {
|
||||
const nextContentType = typeof value === 'function' ? value(current) : value;
|
||||
persistPreference(nextContentType, combinedModeRef.current);
|
||||
return nextContentType;
|
||||
});
|
||||
},
|
||||
[persistPreference],
|
||||
);
|
||||
const setCombinedMode: Dispatch<SetStateAction<boolean>> = useCallback((value) => {
|
||||
setPreference((current) => ({
|
||||
...current,
|
||||
combinedMode: typeof value === 'function' ? value(current.combinedMode) : value,
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const setCombinedMode: Dispatch<SetStateAction<boolean>> = useCallback(
|
||||
(value) => {
|
||||
setCombinedModeState((current) => {
|
||||
const nextCombinedMode = typeof value === 'function' ? value(current) : value;
|
||||
persistPreference(contentTypeRef.current, nextCombinedMode);
|
||||
return nextCombinedMode;
|
||||
});
|
||||
},
|
||||
[persistPreference],
|
||||
);
|
||||
useDependencyEffect(() => {
|
||||
try {
|
||||
localStorage.setItem(CONTENT_TYPE_STORAGE_KEY, combinedMode ? 'combined' : contentType);
|
||||
} catch {
|
||||
// localStorage may be unavailable in private browsing
|
||||
}
|
||||
}, [contentType, combinedMode]);
|
||||
|
||||
return {
|
||||
contentType,
|
||||
|
||||
@@ -44,6 +44,10 @@ export function useDescriptionOverflow({
|
||||
return () => {
|
||||
observer.disconnect();
|
||||
};
|
||||
// `descriptionKey` is never read here - it is the trigger. When the modal swaps to a
|
||||
// different release the text changes under the same element, and the overflow has to
|
||||
// be measured again; drop it and the clamp keeps the previous release's answer.
|
||||
// oxlint-disable-next-line react/exhaustive-effect-dependencies
|
||||
}, [descriptionExpanded, descriptionKey, descriptionRef]);
|
||||
|
||||
return descriptionOverflows;
|
||||
|
||||
@@ -164,7 +164,6 @@ export function useReleaseSearchSession(
|
||||
const lastStatusTimeRef = useRef(0);
|
||||
const pendingStatusRef = useRef<SearchStatusData | null>(null);
|
||||
const statusTimeoutRef = useRef<NodeJS.Timeout | null>(null);
|
||||
activeTabRef.current = activeTab;
|
||||
|
||||
const allTabs = useMemo(() => {
|
||||
return buildReleaseTabs(
|
||||
@@ -363,7 +362,6 @@ export function useReleaseSearchSession(
|
||||
indexerFilterInitializedRef.current = new Set<string>();
|
||||
const nextInitialActiveTab = preferredDefaultReleaseSource || '';
|
||||
initialActiveTabRef.current = nextInitialActiveTab;
|
||||
activeTabRef.current = nextInitialActiveTab;
|
||||
pendingStatusRef.current = null;
|
||||
lastStatusTimeRef.current = 0;
|
||||
if (statusTimeoutRef.current) {
|
||||
@@ -381,6 +379,7 @@ export function useReleaseSearchSession(
|
||||
? nextInitialActiveTab
|
||||
: (tabs[0]?.name ?? '');
|
||||
|
||||
activeTabRef.current = nextActiveTab;
|
||||
setActiveTabState(nextActiveTab);
|
||||
setReleasesBySource({});
|
||||
setLoadingBySource({});
|
||||
@@ -458,6 +457,7 @@ export function useReleaseSearchSession(
|
||||
|
||||
const setActiveTab = useCallback(
|
||||
(tabName: string) => {
|
||||
activeTabRef.current = tabName;
|
||||
setActiveTabState(tabName);
|
||||
|
||||
if (!tabName) {
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { useCallback, useRef } from 'react';
|
||||
|
||||
import { useMountEffect } from '@/hooks/useMountEffect';
|
||||
|
||||
export const useSearchBarHoverTimeout = () => {
|
||||
const hoverTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
const clearHoverTimeout = useCallback(() => {
|
||||
if (hoverTimeoutRef.current) {
|
||||
clearTimeout(hoverTimeoutRef.current);
|
||||
hoverTimeoutRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
useMountEffect(() => {
|
||||
return () => {
|
||||
clearHoverTimeout();
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
hoverTimeoutRef,
|
||||
clearHoverTimeout,
|
||||
};
|
||||
};
|
||||
@@ -5,24 +5,30 @@ interface TabIndicatorStyle {
|
||||
width: number;
|
||||
}
|
||||
|
||||
// One shared instance, so the no-active-tab path below can set it repeatedly and React
|
||||
// bails out on reference equality instead of re-rendering on every resize event.
|
||||
const HIDDEN_INDICATOR: TabIndicatorStyle = { left: 0, width: 0 };
|
||||
|
||||
export function useTabIndicator(
|
||||
tabRefs: MutableRefObject<Record<string, HTMLButtonElement | null>>,
|
||||
activeTab: string,
|
||||
tabsDependency: unknown,
|
||||
): TabIndicatorStyle {
|
||||
const [tabIndicatorStyle, setTabIndicatorStyle] = useState({
|
||||
left: 0,
|
||||
width: 0,
|
||||
});
|
||||
const [tabIndicatorStyle, setTabIndicatorStyle] = useState<TabIndicatorStyle>(HIDDEN_INDICATOR);
|
||||
|
||||
useLayoutEffect(() => {
|
||||
const activeButton = tabRefs.current[activeTab];
|
||||
if (!activeButton) {
|
||||
setTabIndicatorStyle({ left: 0, width: 0 });
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Single measurement path, so a resize that removes the active tab also
|
||||
// resets the indicator instead of leaving it stranded.
|
||||
const updateIndicator = () => {
|
||||
const activeButton = tabRefs.current[activeTab];
|
||||
if (!activeButton) {
|
||||
// The shared constant, not a fresh literal: this path now runs on every resize
|
||||
// event, and a new object would never be Object.is-equal to the current state,
|
||||
// so React would re-render on every frame of a window drag for an unchanged value.
|
||||
setTabIndicatorStyle(HIDDEN_INDICATOR);
|
||||
return;
|
||||
}
|
||||
|
||||
const containerRect = activeButton.parentElement?.getBoundingClientRect();
|
||||
const buttonRect = activeButton.getBoundingClientRect();
|
||||
if (!containerRect) {
|
||||
@@ -41,6 +47,10 @@ export function useTabIndicator(
|
||||
return () => {
|
||||
window.removeEventListener('resize', updateIndicator);
|
||||
};
|
||||
// `tabsDependency` is never read here - it exists only to re-run the measurement when
|
||||
// the tab set changes (callers pass `allTabs` / `showRequestsTab`). The buttons move
|
||||
// when tabs are added or removed, so without it the indicator sits under the old one.
|
||||
// oxlint-disable-next-line react/exhaustive-effect-dependencies
|
||||
}, [activeTab, tabRefs, tabsDependency]);
|
||||
|
||||
return tabIndicatorStyle;
|
||||
|
||||
@@ -1,47 +1,43 @@
|
||||
import { useEffect, useEffectEvent, useRef, type RefObject } from 'react';
|
||||
import { useEffect, useEffectEvent, type RefObject } from 'react';
|
||||
|
||||
export const useDismiss = (
|
||||
isOpen: boolean,
|
||||
refs: RefObject<HTMLElement | null>[],
|
||||
onClose: () => void,
|
||||
) => {
|
||||
const handleClose = useEffectEvent(() => {
|
||||
const handlePointerDown = useEffectEvent((event: MouseEvent) => {
|
||||
const target = event.target;
|
||||
if (!(target instanceof Node)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (refs.some((ref) => ref.current?.contains(target))) {
|
||||
return;
|
||||
}
|
||||
|
||||
onClose();
|
||||
});
|
||||
|
||||
const refsRef = useRef(refs);
|
||||
refsRef.current = refs;
|
||||
const handleEscape = useEffectEvent((event: KeyboardEvent) => {
|
||||
if (event.key === 'Escape') {
|
||||
onClose();
|
||||
}
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOpen) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const handleClickOutside = (event: MouseEvent) => {
|
||||
const target = event.target;
|
||||
if (!(target instanceof Node)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (refsRef.current.some((ref) => ref.current?.contains(target))) {
|
||||
return;
|
||||
}
|
||||
|
||||
handleClose();
|
||||
};
|
||||
|
||||
const handleEscape = (event: KeyboardEvent) => {
|
||||
if (event.key === 'Escape') {
|
||||
handleClose();
|
||||
}
|
||||
};
|
||||
const handleClickOutside = (event: MouseEvent) => handlePointerDown(event);
|
||||
const handleKeyDown = (event: KeyboardEvent) => handleEscape(event);
|
||||
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
document.addEventListener('keydown', handleEscape);
|
||||
document.addEventListener('keydown', handleKeyDown);
|
||||
|
||||
return () => {
|
||||
document.removeEventListener('mousedown', handleClickOutside);
|
||||
document.removeEventListener('keydown', handleEscape);
|
||||
document.removeEventListener('keydown', handleKeyDown);
|
||||
};
|
||||
}, [isOpen]);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { useCallback, useLayoutEffect, useRef } from 'react';
|
||||
|
||||
/**
|
||||
* A callback with a stable identity that always runs the latest render's implementation.
|
||||
*
|
||||
* `useEffectEvent` is React's answer to this shape, but its contract is narrower than it
|
||||
* looks: an Effect Event may only be called from inside an Effect, and must not be handed
|
||||
* to another component, stored in state, or registered with something that outlives the
|
||||
* Effect. Handlers that run from a DOM event, from an async continuation, or from a parent
|
||||
* holding the function in its own UI state are all outside that contract - React documents
|
||||
* the behaviour there as undefined, and the React Compiler advisories oxlint reports
|
||||
* ("existing memoization could not be preserved") are the same fact from the other side.
|
||||
*
|
||||
* So this is the supported shape for those callers. The ref is published in a layout
|
||||
* effect - after commit, before paint - rather than assigned during render, so a render
|
||||
* React later throws away cannot leak its closure into a handler, and no event can
|
||||
* observe the gap.
|
||||
*
|
||||
* Use `useEffectEvent` when the caller really is an Effect; use this everywhere else.
|
||||
*/
|
||||
export function useLatestCallback<Args extends unknown[], Result>(
|
||||
callback: (...args: Args) => Result,
|
||||
): (...args: Args) => Result {
|
||||
const callbackRef = useRef(callback);
|
||||
|
||||
useLayoutEffect(() => {
|
||||
callbackRef.current = callback;
|
||||
});
|
||||
|
||||
return useCallback((...args: Args) => callbackRef.current(...args), []);
|
||||
}
|
||||
@@ -1,16 +1,14 @@
|
||||
import { useEffect, useRef, type DependencyList, type EffectCallback } from 'react';
|
||||
import { useEffect, useEffectEvent, type DependencyList, type EffectCallback } from 'react';
|
||||
|
||||
export function useMountEffect(effect: EffectCallback): void {
|
||||
const effectRef = useRef(effect);
|
||||
effectRef.current = effect;
|
||||
const runEffect = useEffectEvent(effect);
|
||||
|
||||
useEffect(() => effectRef.current(), []);
|
||||
useEffect(() => runEffect(), []);
|
||||
}
|
||||
|
||||
export function useDependencyEffect(effect: EffectCallback, deps: DependencyList): void {
|
||||
const effectRef = useRef(effect);
|
||||
effectRef.current = effect;
|
||||
const runEffect = useEffectEvent(effect);
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
useEffect(() => effectRef.current(), deps);
|
||||
useEffect(() => runEffect(), deps);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { DEFAULT_SUPPORTED_FORMATS } from '../data/languages';
|
||||
import { searchBooks, searchMetadata, AuthenticationError } from '../services/api';
|
||||
import type { Book, AppConfig, AdvancedFilterState, ContentType, SearchMode } from '../types';
|
||||
import { LANGUAGE_OPTION_DEFAULT } from '../utils/languageFilters';
|
||||
import { describeSearchFailure } from '../utils/searchFailureMessage';
|
||||
|
||||
const DEFAULT_FORMAT_SELECTION = DEFAULT_SUPPORTED_FORMATS;
|
||||
|
||||
@@ -263,12 +264,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn {
|
||||
handleSearchError(error, 'Search failed');
|
||||
} else {
|
||||
console.error('Search failed:', error);
|
||||
const message = error instanceof Error ? error.message : 'Search failed';
|
||||
const friendly =
|
||||
message.includes('Network restricted') || message.includes('Unable to reach')
|
||||
? message
|
||||
: 'Unable to reach download source. Network may be restricted or mirrors blocked.';
|
||||
showToast(friendly, 'error');
|
||||
showToast(describeSearchFailure(error), 'error');
|
||||
}
|
||||
} finally {
|
||||
setIsSearching(false);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState, useCallback, useRef } from 'react';
|
||||
import { useState, useCallback } from 'react';
|
||||
|
||||
import { getSettings, updateSettings, executeSettingsAction } from '../services/api';
|
||||
import type {
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
setThemePreference,
|
||||
THEME_FIELD,
|
||||
} from '../utils/themePreference';
|
||||
import { useLatestCallback } from './useLatestCallback';
|
||||
import { useMountEffect } from './useMountEffect';
|
||||
|
||||
interface FetchSettingsOptions {
|
||||
@@ -137,13 +138,9 @@ export function useSettings(): UseSettingsReturn {
|
||||
() => initialState?.originalValues ?? {},
|
||||
);
|
||||
const [isSaving, setIsSaving] = useState(false);
|
||||
const valuesRef = useRef<SettingsValues>({});
|
||||
const originalValuesRef = useRef<SettingsValues>({});
|
||||
|
||||
valuesRef.current = values;
|
||||
originalValuesRef.current = originalValues;
|
||||
|
||||
const applySettingsResponse = useCallback(
|
||||
// Stable identity, latest `values`/`originalValues`. Not an Effect Event: it is called
|
||||
// from async fetch and save continuations, not from an Effect. See useLatestCallback.
|
||||
const applySettingsResponse = useLatestCallback(
|
||||
(response: SettingsResponse, options: { preserveDirtyValues?: boolean } = {}) => {
|
||||
const { preserveDirtyValues = false } = options;
|
||||
cachedSettingsResponse = response;
|
||||
@@ -156,11 +153,7 @@ export function useSettings(): UseSettingsReturn {
|
||||
setError(null);
|
||||
|
||||
const nextValues = preserveDirtyValues
|
||||
? mergeFetchedSettingsWithDirtyValues(
|
||||
hydratedState.values,
|
||||
valuesRef.current,
|
||||
originalValuesRef.current,
|
||||
)
|
||||
? mergeFetchedSettingsWithDirtyValues(hydratedState.values, values, originalValues)
|
||||
: hydratedState.values;
|
||||
|
||||
setValues(nextValues);
|
||||
@@ -170,7 +163,6 @@ export function useSettings(): UseSettingsReturn {
|
||||
setSelectedTab((current) => current ?? hydratedState.selectedTab);
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const fetchSettings = useCallback(
|
||||
|
||||
@@ -12,6 +12,8 @@ import type {
|
||||
RequestSubmissionResult,
|
||||
MetadataProvidersResponse,
|
||||
MetadataSearchConfig,
|
||||
PackBook,
|
||||
InspectReleaseResponse,
|
||||
} from '../types';
|
||||
import type {
|
||||
ActionResult,
|
||||
@@ -82,6 +84,9 @@ type ApiResponseErrorShape = Error & {
|
||||
code?: string;
|
||||
requiredMode?: string;
|
||||
payload?: Record<string, unknown>;
|
||||
// Set only when the server explained itself, so callers can tell a real explanation
|
||||
// apart from the `503 SERVICE UNAVAILABLE` placeholder built from the status line.
|
||||
serverMessage?: string;
|
||||
};
|
||||
|
||||
class ApiResponseError extends Error {
|
||||
@@ -89,6 +94,7 @@ class ApiResponseError extends Error {
|
||||
code?: string;
|
||||
requiredMode?: string;
|
||||
payload?: Record<string, unknown>;
|
||||
serverMessage?: string;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
@@ -97,6 +103,7 @@ class ApiResponseError extends Error {
|
||||
code?: string;
|
||||
requiredMode?: string;
|
||||
payload?: Record<string, unknown>;
|
||||
serverMessage?: string;
|
||||
},
|
||||
) {
|
||||
super(message);
|
||||
@@ -105,6 +112,7 @@ class ApiResponseError extends Error {
|
||||
this.code = params.code;
|
||||
this.requiredMode = params.requiredMode;
|
||||
this.payload = params.payload;
|
||||
this.serverMessage = params.serverMessage;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,6 +120,13 @@ export const isApiResponseError = (error: unknown): error is ApiResponseErrorSha
|
||||
return error instanceof ApiResponseError;
|
||||
};
|
||||
|
||||
// The client gave up before the server answered. Distinguishable so callers can report
|
||||
// the wait rather than guessing at a cause: a search that hits this has told us nothing
|
||||
// about the network or the mirrors, and saying it did is what issue #1285 was about.
|
||||
export const isTimeoutError = (error: unknown): error is Error => {
|
||||
return error instanceof TimeoutError;
|
||||
};
|
||||
|
||||
const mapApiErrorToActionResult = (error: unknown): ActionResult | null => {
|
||||
if (!isApiResponseError(error) || !error.payload) {
|
||||
return null;
|
||||
@@ -147,7 +162,31 @@ const DEFAULT_TIMEOUT_MS = 30000;
|
||||
// Release searches can be long-running: a source behind Cloudflare/DDoS-Guard has
|
||||
// to spin up the bypasser and solve the challenge before any results come back,
|
||||
// which routinely takes well over the default timeout.
|
||||
const SEARCH_TIMEOUT_MS = 180000;
|
||||
//
|
||||
// The server bounds them itself (RELEASE_SEARCH_TIMEOUT, reported by /api/config) and
|
||||
// answers a spent budget with a message naming the real cause. This client abort is only
|
||||
// the backstop for a server that never answers at all, so it has to fire *after* the
|
||||
// server's own deadline - a fixed 180s here beat the 300s default, so the accurate
|
||||
// message was never reachable and raising the setting did nothing. See issue #1285.
|
||||
//
|
||||
// The margin has to cover what the server still has to do *after* its budget trips, not
|
||||
// just the budget itself. The deadline is cooperative: it is handed to the bypasser as a
|
||||
// cancel flag, and internal_bypasser._CDP_UNWIND_GRACE_SECONDS allows 15s on its own for a
|
||||
// cancelled solve to close its browser - before the handler has serialized releases, built
|
||||
// the column config and put bytes on the wire. A 15s margin is entirely spent by that
|
||||
// unwind, so give it room for the unwind plus the response.
|
||||
const SEARCH_TIMEOUT_MARGIN_MS = 45000;
|
||||
const FALLBACK_SEARCH_TIMEOUT_MS = 300000; // search_deadline.DEFAULT_SEARCH_BUDGET_SECONDS
|
||||
let searchTimeoutMs = FALLBACK_SEARCH_TIMEOUT_MS + SEARCH_TIMEOUT_MARGIN_MS;
|
||||
|
||||
// Exported for tests; callers get this applied automatically via getConfig().
|
||||
export const setSearchTimeoutFromConfig = (budgetSeconds: unknown): void => {
|
||||
if (typeof budgetSeconds === 'number' && Number.isFinite(budgetSeconds) && budgetSeconds > 0) {
|
||||
searchTimeoutMs = budgetSeconds * 1000 + SEARCH_TIMEOUT_MARGIN_MS;
|
||||
}
|
||||
};
|
||||
|
||||
export const getSearchTimeoutMs = (): number => searchTimeoutMs;
|
||||
|
||||
// Utility function for JSON fetch with credentials and timeout
|
||||
async function fetchJSON<T>(
|
||||
@@ -181,12 +220,15 @@ async function fetchJSON<T>(
|
||||
if (isRecord(parsed) && !Array.isArray(parsed)) {
|
||||
errorData = parsed;
|
||||
}
|
||||
// Prefer user-friendly 'message' field, fall back to 'error'
|
||||
if (typeof errorData?.message === 'string') {
|
||||
errorMessage = errorData.message;
|
||||
hasServerMessage = true;
|
||||
} else if (typeof errorData?.error === 'string') {
|
||||
errorMessage = errorData.error;
|
||||
// Prefer user-friendly 'message' field, fall back to 'error'. Both must carry
|
||||
// actual text: an empty string is not the server explaining itself, and treating
|
||||
// it as one suppresses the placeholder below and shows the user a blank toast.
|
||||
const explanation = [errorData?.message, errorData?.error].find(
|
||||
(candidate): candidate is string =>
|
||||
typeof candidate === 'string' && candidate.trim() !== '',
|
||||
);
|
||||
if (explanation !== undefined) {
|
||||
errorMessage = explanation;
|
||||
hasServerMessage = true;
|
||||
}
|
||||
} catch (e) {
|
||||
@@ -211,6 +253,7 @@ async function fetchJSON<T>(
|
||||
|
||||
throw new ApiResponseError(errorMessage, {
|
||||
status: res.status,
|
||||
serverMessage: hasServerMessage ? errorMessage : undefined,
|
||||
code: typeof errorData?.code === 'string' ? errorData.code : undefined,
|
||||
requiredMode:
|
||||
typeof errorData?.required_mode === 'string' ? errorData.required_mode : undefined,
|
||||
@@ -241,7 +284,7 @@ export const searchBooks = async (query: string): Promise<Book[]> => {
|
||||
const response = await fetchJSON<ReleasesResponse>(
|
||||
`${API_BASE}/releases?source=direct_download&${query}`,
|
||||
{},
|
||||
SEARCH_TIMEOUT_MS,
|
||||
searchTimeoutMs,
|
||||
);
|
||||
return response.releases.map(transformReleaseToDirectBook);
|
||||
};
|
||||
@@ -510,6 +553,18 @@ export type DownloadReleasePayload = {
|
||||
language?: string; // Release language code, for the {Language} naming variable
|
||||
search_author?: string;
|
||||
search_mode?: 'direct' | 'universal';
|
||||
multi_book?: boolean; // Split a multi-book pack into one book per subfolder/file
|
||||
book_plan?: PackBook[]; // The split the user approved before download
|
||||
};
|
||||
|
||||
/** Inspect a release's file list before download (same body as downloadRelease). */
|
||||
export const inspectRelease = async (
|
||||
release: DownloadReleasePayload,
|
||||
): Promise<InspectReleaseResponse> => {
|
||||
return fetchJSON<InspectReleaseResponse>(`${API_BASE}/releases/inspect`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(release),
|
||||
});
|
||||
};
|
||||
|
||||
export const downloadRelease = async (
|
||||
@@ -572,7 +627,9 @@ export const retryDownload = async (id: string): Promise<void> => {
|
||||
};
|
||||
|
||||
export const getConfig = async (): Promise<AppConfig> => {
|
||||
return fetchJSON<AppConfig>(API.config);
|
||||
const config = await fetchJSON<AppConfig>(API.config);
|
||||
setSearchTimeoutFromConfig(config.release_search_timeout);
|
||||
return config;
|
||||
};
|
||||
|
||||
interface ActivityDismissedItem {
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
buildLanguageNormalizer,
|
||||
getReleaseSearchLanguageParams,
|
||||
releaseLanguageMatchesFilter,
|
||||
resolveDefaultLanguageCodes,
|
||||
} from '../utils/languageFilters';
|
||||
|
||||
const supportedLanguages: Language[] = [
|
||||
@@ -67,3 +68,36 @@ describe('releaseLanguageMatchesFilter', () => {
|
||||
expect(visibleLanguages).toHaveLength(48);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveDefaultLanguageCodes', () => {
|
||||
it('keeps an explicitly empty default as "no default filter"', () => {
|
||||
// The backend stores [] to mean "do not filter"; substituting the first
|
||||
// supported language here would make the UI filter where the server does not.
|
||||
expect(resolveDefaultLanguageCodes([], supportedLanguages)).toEqual([]);
|
||||
});
|
||||
|
||||
it('leaves a configured default untouched', () => {
|
||||
expect(resolveDefaultLanguageCodes(['de', 'hu'], supportedLanguages)).toEqual(['de', 'hu']);
|
||||
});
|
||||
|
||||
it('falls back to the first supported language only when nothing is configured', () => {
|
||||
expect(resolveDefaultLanguageCodes(undefined, supportedLanguages)).toEqual(['en']);
|
||||
expect(resolveDefaultLanguageCodes(null, [])).toEqual(['en']);
|
||||
});
|
||||
|
||||
it('sends no language filter when an empty default is the whole selection', () => {
|
||||
const defaults = resolveDefaultLanguageCodes([], supportedLanguages);
|
||||
|
||||
expect(
|
||||
getReleaseSearchLanguageParams([LANGUAGE_OPTION_DEFAULT], supportedLanguages, defaults),
|
||||
).toBe(undefined);
|
||||
});
|
||||
|
||||
it('does not smuggle the first language into a Default+German selection', () => {
|
||||
const defaults = resolveDefaultLanguageCodes([], supportedLanguages);
|
||||
|
||||
expect(
|
||||
getReleaseSearchLanguageParams([LANGUAGE_OPTION_DEFAULT, 'de'], supportedLanguages, defaults),
|
||||
).toEqual(['de']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import type { PackBook } from '../types';
|
||||
import {
|
||||
describePackPlan,
|
||||
parseSeriesPositionInput,
|
||||
toBookPlanPayload,
|
||||
updateReviewBook,
|
||||
} from '../utils/packReview';
|
||||
|
||||
const books: PackBook[] = [
|
||||
{ title: 'Leviathan Wakes', series_position: 1, year: 2011, files: ['a.m4b'] },
|
||||
{ title: 'Caliban’s War', series_position: 2, year: 2012, files: ['b.m4b', 'b2.m4b'] },
|
||||
];
|
||||
|
||||
describe('packReview.updateReviewBook', () => {
|
||||
it('replaces one book without touching the others', () => {
|
||||
const next = updateReviewBook(books, 1, { title: 'Caliban’s War (Unabridged)' });
|
||||
expect(next[0]).toBe(books[0]);
|
||||
expect(next[1]).toEqual({ ...books[1], title: 'Caliban’s War (Unabridged)' });
|
||||
expect(books[1].title).toBe('Caliban’s War');
|
||||
});
|
||||
});
|
||||
|
||||
describe('packReview.parseSeriesPositionInput', () => {
|
||||
it('accepts whole and fractional positions', () => {
|
||||
expect(parseSeriesPositionInput('3')).toBe(3);
|
||||
expect(parseSeriesPositionInput('2.5')).toBe(2.5);
|
||||
});
|
||||
|
||||
it('treats blank or junk as no position', () => {
|
||||
expect(parseSeriesPositionInput('')).toBeNull();
|
||||
expect(parseSeriesPositionInput('abc')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('packReview.toBookPlanPayload', () => {
|
||||
it('trims titles, drops books without a title, and keeps file lists', () => {
|
||||
const edited = updateReviewBook(books, 0, { title: ' ' });
|
||||
expect(toBookPlanPayload(edited)).toEqual([
|
||||
{ title: 'Caliban’s War', series_position: 2, year: 2012, files: ['b.m4b', 'b2.m4b'] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('packReview.describePackPlan', () => {
|
||||
it('summarises books, files and ignored sidecars', () => {
|
||||
expect(describePackPlan(books, ['a.txt', 'cover.jpg'])).toBe(
|
||||
'2 books · 3 files · 2 files ignored',
|
||||
);
|
||||
expect(describePackPlan([books[0]], [])).toBe('1 book · 1 file');
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import type { Release } from '../types';
|
||||
import { getReleaseFormats } from '../utils/releaseFormats';
|
||||
import { getReleaseFormats, getUnrecognizedReleaseFormats } from '../utils/releaseFormats';
|
||||
|
||||
function buildRelease(overrides: Partial<Release>): Release {
|
||||
return {
|
||||
@@ -38,3 +38,26 @@ describe('releaseFormats.getReleaseFormats', () => {
|
||||
expect(getReleaseFormats(release)).toEqual(['pdf']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('releaseFormats.getUnrecognizedReleaseFormats', () => {
|
||||
it('returns normalized, deduplicated unrecognized formats from extra', () => {
|
||||
const release = buildRelease({
|
||||
extra: { unrecognized_formats: ['AVI', ' avi ', 'WEBM'] },
|
||||
});
|
||||
|
||||
expect(getUnrecognizedReleaseFormats(release)).toEqual(['avi', 'webm']);
|
||||
});
|
||||
|
||||
it('accepts a single string value', () => {
|
||||
const release = buildRelease({ extra: { unrecognized_formats: 'AVI' } });
|
||||
|
||||
expect(getUnrecognizedReleaseFormats(release)).toEqual(['avi']);
|
||||
});
|
||||
|
||||
it('returns an empty list when nothing was flagged', () => {
|
||||
expect(getUnrecognizedReleaseFormats(buildRelease({}))).toEqual([]);
|
||||
expect(
|
||||
getUnrecognizedReleaseFormats(buildRelease({ extra: { unrecognized_formats: null } })),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import {
|
||||
INITIAL_ENTER_ANIMATION,
|
||||
nextEnterAnimation,
|
||||
type EnterAnimationState,
|
||||
} from '../utils/releaseModalEnterAnimation';
|
||||
|
||||
describe('nextEnterAnimation', () => {
|
||||
it('animates the first session', () => {
|
||||
const next = nextEnterAnimation(INITIAL_ENTER_ANIMATION, 'book-1', false);
|
||||
expect(next).toEqual({ key: 'book-1', animate: true });
|
||||
});
|
||||
|
||||
it('animates the first session in combined mode too', () => {
|
||||
const next = nextEnterAnimation(INITIAL_ENTER_ANIMATION, 'book-1', true);
|
||||
expect(next).toEqual({ key: 'book-1', animate: true });
|
||||
});
|
||||
|
||||
it('does not animate a step transition between combined-mode sessions', () => {
|
||||
const current: EnterAnimationState = { key: 'book-1', animate: true };
|
||||
expect(nextEnterAnimation(current, 'book-2', true)).toEqual({
|
||||
key: 'book-2',
|
||||
animate: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('animates a session swap outside combined mode', () => {
|
||||
const current: EnterAnimationState = { key: 'book-1', animate: true };
|
||||
expect(nextEnterAnimation(current, 'book-2', false)).toEqual({
|
||||
key: 'book-2',
|
||||
animate: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('holds the decision across re-renders of the same session', () => {
|
||||
// Regression: the decision used to flip back to `true` on the next render,
|
||||
// replaying the enter animation mid-session.
|
||||
const stepped = nextEnterAnimation({ key: 'book-1', animate: true }, 'book-2', true);
|
||||
expect(stepped.animate).toBe(false);
|
||||
|
||||
let state = stepped;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
state = nextEnterAnimation(state, 'book-2', true);
|
||||
expect(state.animate).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('returns the same reference when the session is unchanged', () => {
|
||||
const current: EnterAnimationState = { key: 'book-1', animate: false };
|
||||
expect(nextEnterAnimation(current, 'book-1', true)).toBe(current);
|
||||
});
|
||||
|
||||
it('animates again after the modal closes and reopens', () => {
|
||||
const open = nextEnterAnimation(INITIAL_ENTER_ANIMATION, 'book-1', true);
|
||||
const closed = nextEnterAnimation(open, null, true);
|
||||
expect(closed.key).toBeNull();
|
||||
|
||||
const reopened = nextEnterAnimation(closed, 'book-1', true);
|
||||
expect(reopened.animate).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import type { Book, Release } from '../types';
|
||||
import { buildReleaseDownloadPayload } from '../utils/releasePayload';
|
||||
|
||||
const book: Book = {
|
||||
id: 'hc-1',
|
||||
title: 'Drive',
|
||||
author: 'James S. A. Corey',
|
||||
year: '2012',
|
||||
preview: 'https://img/drive.jpg',
|
||||
series_name: 'The Expanse',
|
||||
series_position: 2.6,
|
||||
subtitle: 'An Expanse Short Story',
|
||||
provider: 'hardcover',
|
||||
provider_id: 'hc-1',
|
||||
source: 'direct_download',
|
||||
};
|
||||
|
||||
const release: Release = {
|
||||
source: 'audiobookbay',
|
||||
source_id: 'abb-1',
|
||||
title: 'James S. A. Corey - The Expanse Complete 2.0',
|
||||
format: 'm4b',
|
||||
language: 'en',
|
||||
download_url: 'https://audiobookbay.lu/abss/expanse/',
|
||||
};
|
||||
|
||||
describe('buildReleaseDownloadPayload', () => {
|
||||
it('describes the searched book and the chosen release', () => {
|
||||
const payload = buildReleaseDownloadPayload(book, release, 'audiobook');
|
||||
expect(payload).toMatchObject({
|
||||
source: 'audiobookbay',
|
||||
source_id: 'abb-1',
|
||||
title: 'Drive',
|
||||
author: 'James S. A. Corey',
|
||||
series_name: 'The Expanse',
|
||||
series_position: 2.6,
|
||||
language: 'en',
|
||||
content_type: 'audiobook',
|
||||
});
|
||||
expect(payload.multi_book).toBeUndefined();
|
||||
expect(payload.book_plan).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses the release title and author for manual books', () => {
|
||||
const manual: Book = { ...book, provider: 'manual', title: 'ignored' };
|
||||
const withAuthor = { ...release, extra: { author: 'Release Author' } };
|
||||
const payload = buildReleaseDownloadPayload(manual, withAuthor, 'audiobook');
|
||||
expect(payload.title).toBe(release.title);
|
||||
expect(payload.author).toBe('Release Author');
|
||||
});
|
||||
|
||||
it('flags a manual multi-book pack', () => {
|
||||
const payload = buildReleaseDownloadPayload(book, release, 'audiobook', { multiBook: true });
|
||||
expect(payload.multi_book).toBe(true);
|
||||
expect(payload.book_plan).toBeUndefined();
|
||||
});
|
||||
|
||||
it('attaches the approved book plan', () => {
|
||||
const plan = [{ title: 'Leviathan Wakes', series_position: 1, year: 2011, files: ['a.m4b'] }];
|
||||
const payload = buildReleaseDownloadPayload(book, release, 'audiobook', {
|
||||
multiBook: true,
|
||||
bookPlan: plan,
|
||||
});
|
||||
expect(payload.multi_book).toBe(true);
|
||||
expect(payload.book_plan).toEqual(plan);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
|
||||
import { searchBooks } from '../services/api';
|
||||
import {
|
||||
describeSearchFailure,
|
||||
CLIENT_TIMEOUT_MESSAGE,
|
||||
UNREACHABLE_SOURCE_MESSAGE,
|
||||
} from '../utils/searchFailureMessage';
|
||||
|
||||
/**
|
||||
* What a failed direct-mode search tells the user.
|
||||
*
|
||||
* Every non-auth failure used to be relabelled "Unable to reach download source. Network
|
||||
* may be restricted or mirrors blocked.", which threw away the server's explanation and
|
||||
* blamed the user's network for a protection challenge. See issue #1285.
|
||||
*/
|
||||
|
||||
const jsonResponse = (body: unknown, status: number): Response =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status,
|
||||
statusText: 'SERVICE UNAVAILABLE',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
/** Drive a real searchBooks() failure so the error is the one the hook actually sees. */
|
||||
const failedSearch = async (respond: () => Promise<Response>): Promise<unknown> => {
|
||||
vi.stubGlobal('fetch', vi.fn(respond));
|
||||
return searchBooks('q=dune').catch((error: unknown) => error);
|
||||
};
|
||||
|
||||
describe('describeSearchFailure', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('shows the sentence the server sent', async () => {
|
||||
const sentence = 'The release search ran out of time (300s).';
|
||||
const error = await failedSearch(() => Promise.resolve(jsonResponse({ error: sentence }, 503)));
|
||||
|
||||
expect(describeSearchFailure(error)).toBe(sentence);
|
||||
});
|
||||
|
||||
it('names the wait when the client gave up first', async () => {
|
||||
// The client's abort is the backstop for a server that never answered. It tells us
|
||||
// nothing about mirrors or the network, and the old chain reported it as if it did.
|
||||
const abort = Object.assign(new Error('The operation was aborted.'), { name: 'AbortError' });
|
||||
const error = await failedSearch(() => Promise.reject(abort));
|
||||
|
||||
expect(describeSearchFailure(error)).toBe(CLIENT_TIMEOUT_MESSAGE);
|
||||
expect(describeSearchFailure(error)).not.toBe(UNREACHABLE_SOURCE_MESSAGE);
|
||||
expect(describeSearchFailure(error)).not.toContain('mirrors');
|
||||
});
|
||||
|
||||
it('falls back to the mirrors line only when nothing explained itself', async () => {
|
||||
const error = await failedSearch(() => Promise.resolve(jsonResponse({}, 503)));
|
||||
|
||||
expect(describeSearchFailure(error)).toBe(UNREACHABLE_SOURCE_MESSAGE);
|
||||
});
|
||||
|
||||
it('keeps a reachability message that already says the right thing', () => {
|
||||
const error = new Error('Unable to reach download source. Every mirror was quarantined.');
|
||||
|
||||
expect(describeSearchFailure(error)).toBe(error.message);
|
||||
});
|
||||
|
||||
it('never produces an empty sentence from a blank server message', async () => {
|
||||
// `{"message": ""}` is not the server explaining itself. Treating it as one used to
|
||||
// reach showToast('') and render an empty error toast.
|
||||
const error = await failedSearch(() => Promise.resolve(jsonResponse({ message: '' }, 503)));
|
||||
|
||||
expect(describeSearchFailure(error)).toBe(UNREACHABLE_SOURCE_MESSAGE);
|
||||
});
|
||||
|
||||
it('handles a non-Error rejection without inventing detail', () => {
|
||||
expect(describeSearchFailure('something odd')).toBe(UNREACHABLE_SOURCE_MESSAGE);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,197 @@
|
||||
import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest';
|
||||
|
||||
import {
|
||||
getConfig,
|
||||
getSearchTimeoutMs,
|
||||
setSearchTimeoutFromConfig,
|
||||
searchBooks,
|
||||
isApiResponseError,
|
||||
} from '../services/api';
|
||||
|
||||
/**
|
||||
* The client's abort must fire *after* the server's own search deadline.
|
||||
*
|
||||
* `/api/releases` bounds itself with RELEASE_SEARCH_TIMEOUT and answers a spent budget
|
||||
* with a message naming the real cause. The client aborted at a fixed 180s against a
|
||||
* 300s default, so it always won the race and replaced that message with "Request timed
|
||||
* out. Check your network connection or proxy configuration." Raising the setting had no
|
||||
* visible effect either, the 180s being baked into the hashed bundle. See issue #1285.
|
||||
*/
|
||||
|
||||
// Must stay ahead of what the server still has to do after its budget trips: the deadline
|
||||
// is cooperative, and internal_bypasser._CDP_UNWIND_GRACE_SECONDS alone allows 15s for a
|
||||
// cancelled solve to close its browser before the response is even built.
|
||||
const MARGIN_MS = 45_000;
|
||||
const SERVER_UNWIND_GRACE_MS = 15_000;
|
||||
|
||||
const jsonResponse = (body: unknown, status = 200): Response =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status,
|
||||
statusText: status === 200 ? 'OK' : 'SERVICE UNAVAILABLE',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
const configBody = (releaseSearchTimeout: number): Record<string, unknown> => ({
|
||||
release_search_timeout: releaseSearchTimeout,
|
||||
});
|
||||
|
||||
describe('release search timeout', () => {
|
||||
beforeEach(() => {
|
||||
setSearchTimeoutFromConfig(300);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
setSearchTimeoutFromConfig(300);
|
||||
});
|
||||
|
||||
it('defaults behind the server default rather than ahead of it', () => {
|
||||
expect(getSearchTimeoutMs()).toBe(300 * 1000 + MARGIN_MS);
|
||||
});
|
||||
|
||||
it('follows the budget the server reports', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse(configBody(900)))),
|
||||
);
|
||||
|
||||
await getConfig();
|
||||
|
||||
expect(getSearchTimeoutMs()).toBe(900 * 1000 + MARGIN_MS);
|
||||
});
|
||||
|
||||
it('still outlasts the server when the budget is lowered', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse(configBody(30)))),
|
||||
);
|
||||
|
||||
await getConfig();
|
||||
|
||||
// Exact, not a lower bound: `> 30_000` is also satisfied by the 345_000 left over
|
||||
// from the previous budget, so a setter that silently stopped applying the config
|
||||
// would pass it.
|
||||
expect(getSearchTimeoutMs()).toBe(30 * 1000 + MARGIN_MS);
|
||||
});
|
||||
|
||||
it('leaves the server room to unwind a cancelled solve and answer', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse(configBody(300)))),
|
||||
);
|
||||
|
||||
await getConfig();
|
||||
|
||||
// The failure this exists for is a budget spent mid-solve. The server then has to
|
||||
// close a browser before it can serialize anything, so a margin merely equal to that
|
||||
// unwind is entirely spent by it and the client aborts first all over again.
|
||||
expect(getSearchTimeoutMs() - 300 * 1000).toBeGreaterThan(SERVER_UNWIND_GRACE_MS);
|
||||
});
|
||||
|
||||
it('ignores a missing or nonsensical budget instead of disabling the backstop', () => {
|
||||
const before = getSearchTimeoutMs();
|
||||
|
||||
setSearchTimeoutFromConfig(undefined);
|
||||
setSearchTimeoutFromConfig(0);
|
||||
setSearchTimeoutFromConfig(-1);
|
||||
setSearchTimeoutFromConfig('600');
|
||||
setSearchTimeoutFromConfig(Number.NaN);
|
||||
|
||||
expect(getSearchTimeoutMs()).toBe(before);
|
||||
});
|
||||
|
||||
it('applies the derived timeout to the direct_download search', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse(configBody(600)))),
|
||||
);
|
||||
await getConfig();
|
||||
|
||||
const seen: Array<AbortSignal | undefined> = [];
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn((_url: string, init: RequestInit) => {
|
||||
seen.push(init.signal ?? undefined);
|
||||
return Promise.resolve(jsonResponse({ releases: [] }));
|
||||
}),
|
||||
);
|
||||
|
||||
await searchBooks('q=dune');
|
||||
|
||||
// The request carries an abort signal, and it is not yet aborted: the point is that
|
||||
// the clock it runs on is the server's, not a constant.
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(seen[0]?.aborted).toBe(false);
|
||||
expect(getSearchTimeoutMs()).toBe(600 * 1000 + MARGIN_MS);
|
||||
});
|
||||
});
|
||||
|
||||
describe('server-provided failure messages', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('carries the server sentence through instead of a status placeholder', async () => {
|
||||
const sentence =
|
||||
'The release search ran out of time (300s). Anna’s Archive is behind a ' +
|
||||
'protection challenge the bypasser could not solve in that window.';
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse({ error: sentence }, 503))),
|
||||
);
|
||||
|
||||
const error = await searchBooks('q=dune').catch((e: unknown) => e);
|
||||
|
||||
expect(isApiResponseError(error)).toBe(true);
|
||||
if (isApiResponseError(error)) {
|
||||
expect(error.serverMessage).toBe(sentence);
|
||||
}
|
||||
});
|
||||
|
||||
it('leaves serverMessage unset when the server explained nothing', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse({}, 503))),
|
||||
);
|
||||
|
||||
const error = await searchBooks('q=dune').catch((e: unknown) => e);
|
||||
|
||||
expect(isApiResponseError(error)).toBe(true);
|
||||
if (isApiResponseError(error)) {
|
||||
expect(error.serverMessage).toBeUndefined();
|
||||
// Without this the UI would show a bare "503 SERVICE UNAVAILABLE".
|
||||
expect(error.message).toContain('Server unavailable');
|
||||
}
|
||||
});
|
||||
|
||||
it('treats a blank message as no explanation at all', async () => {
|
||||
// An empty string is not the server explaining itself. Taking it as one suppresses
|
||||
// the placeholder below *and* survives a `??` fallback, leaving an empty toast.
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse({ message: ' ' }, 503))),
|
||||
);
|
||||
|
||||
const error = await searchBooks('q=dune').catch((e: unknown) => e);
|
||||
|
||||
expect(isApiResponseError(error)).toBe(true);
|
||||
if (isApiResponseError(error)) {
|
||||
expect(error.serverMessage).toBeUndefined();
|
||||
expect(error.message).toContain('Server unavailable');
|
||||
}
|
||||
});
|
||||
|
||||
it('falls back to `error` when `message` is blank', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(jsonResponse({ message: '', error: 'the real reason' }, 503))),
|
||||
);
|
||||
|
||||
const error = await searchBooks('q=dune').catch((e: unknown) => e);
|
||||
|
||||
expect(isApiResponseError(error)).toBe(true);
|
||||
if (isApiResponseError(error)) {
|
||||
expect(error.serverMessage).toBe('the real reason');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -285,6 +285,7 @@ export interface AppConfig {
|
||||
auto_open_downloads_sidebar: boolean; // Auto-open sidebar when download is queued
|
||||
hardcover_auto_remove_on_download: boolean; // Auto-remove from active Hardcover list on download
|
||||
download_to_browser_content_types: string[]; // Auto-download completed files to browser for selected content types
|
||||
release_search_timeout: number; // Server-side budget for one release search, in seconds
|
||||
settings_enabled: boolean; // Whether config directory is mounted and writable
|
||||
onboarding_complete: boolean; // Whether the user has completed initial setup
|
||||
default_sort: string; // Default sort for direct mode
|
||||
@@ -460,6 +461,27 @@ export interface SourceSearchInfo {
|
||||
}
|
||||
|
||||
// Response from /api/releases endpoint
|
||||
/** One book split out of a multi-book pack release, files as release-relative paths. */
|
||||
export interface PackBook {
|
||||
title: string;
|
||||
series_position: number | null;
|
||||
year: number | null;
|
||||
files: string[];
|
||||
}
|
||||
|
||||
export interface PackPlan {
|
||||
is_pack: boolean;
|
||||
books: PackBook[];
|
||||
ignored: string[];
|
||||
}
|
||||
|
||||
export interface InspectReleaseResponse {
|
||||
inspected: boolean;
|
||||
reason: string | null;
|
||||
files: { path: string; size: number | null }[];
|
||||
plan: PackPlan | null;
|
||||
}
|
||||
|
||||
export interface ReleasesResponse {
|
||||
releases: Release[];
|
||||
book: {
|
||||
|
||||
@@ -146,7 +146,13 @@ export interface TableFieldColumnOption {
|
||||
childOf?: string;
|
||||
}
|
||||
|
||||
export type TableFieldColumnType = 'text' | 'select' | 'multiselect' | 'checkbox' | 'path';
|
||||
export type TableFieldColumnType =
|
||||
| 'text'
|
||||
| 'password'
|
||||
| 'select'
|
||||
| 'multiselect'
|
||||
| 'checkbox'
|
||||
| 'path';
|
||||
|
||||
export interface TableFieldColumn {
|
||||
key: string;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
type BookTargetChangeEvent = {
|
||||
export type BookTargetChangeEvent = {
|
||||
provider: string;
|
||||
bookId: string;
|
||||
target: string;
|
||||
|
||||
@@ -27,6 +27,24 @@ export const normalizeLanguageSelection = (selected: string[]): string[] => {
|
||||
return unique.length ? unique : [LANGUAGE_OPTION_DEFAULT];
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve the language codes the "Default" filter option stands for.
|
||||
*
|
||||
* An explicitly empty list is a deliberate "no default filter" and is returned as-is;
|
||||
* only a missing value falls back to the first supported language. Substituting a
|
||||
* language for the empty list would make the UI filter by a language the backend
|
||||
* does not apply.
|
||||
*/
|
||||
export const resolveDefaultLanguageCodes = (
|
||||
configuredDefault: string[] | null | undefined,
|
||||
supportedLanguages: Language[],
|
||||
): string[] => {
|
||||
if (Array.isArray(configuredDefault)) {
|
||||
return configuredDefault;
|
||||
}
|
||||
return [supportedLanguages[0]?.code || 'en'];
|
||||
};
|
||||
|
||||
export const getLanguageFilterValues = (
|
||||
selection: string[],
|
||||
supportedLanguages: Language[],
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { PackBook } from '../types';
|
||||
|
||||
/** Return a copy of `books` with one entry patched; the input is not mutated. */
|
||||
export function updateReviewBook(
|
||||
books: PackBook[],
|
||||
index: number,
|
||||
patch: Partial<PackBook>,
|
||||
): PackBook[] {
|
||||
return books.map((book, i) => (i === index ? { ...book, ...patch } : book));
|
||||
}
|
||||
|
||||
/** Parse a series-position text field: "3" → 3, "2.5" → 2.5, blank/junk → null. */
|
||||
export function parseSeriesPositionInput(value: string): number | null {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return null;
|
||||
const parsed = Number(trimmed);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
/** The plan sent with the download: trimmed titles, untitled books dropped. */
|
||||
export function toBookPlanPayload(books: PackBook[]): PackBook[] {
|
||||
return books
|
||||
.map((book) => ({ ...book, title: book.title.trim() }))
|
||||
.filter((book) => book.title.length > 0 && book.files.length > 0);
|
||||
}
|
||||
|
||||
function plural(count: number, noun: string): string {
|
||||
return `${count} ${noun}${count === 1 ? '' : 's'}`;
|
||||
}
|
||||
|
||||
/** "2 books · 3 files · 2 files ignored" */
|
||||
export function describePackPlan(books: PackBook[], ignored: string[]): string {
|
||||
const fileCount = books.reduce((sum, book) => sum + book.files.length, 0);
|
||||
const parts = [plural(books.length, 'book'), plural(fileCount, 'file')];
|
||||
if (ignored.length > 0) {
|
||||
parts.push(`${plural(ignored.length, 'file')} ignored`);
|
||||
}
|
||||
return parts.join(' · ');
|
||||
}
|
||||
@@ -33,3 +33,27 @@ export function getReleaseFormats(release: Release): string[] {
|
||||
|
||||
return formats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format tokens the indexer declared but the backend could not map to a known
|
||||
* book/audiobook format (e.g. MyAnonamouse "[ENG / AVI]"). Such a release will
|
||||
* download but fail post-processing, so the UI warns instead of showing a bare
|
||||
* content-type icon.
|
||||
*/
|
||||
export function getUnrecognizedReleaseFormats(release: Release): string[] {
|
||||
const raw = release.extra?.unrecognized_formats;
|
||||
const values = Array.isArray(raw) ? raw : [raw];
|
||||
const formats: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
values.forEach((value) => {
|
||||
const normalized = normalizeFormatValue(value);
|
||||
if (!normalized || seen.has(normalized)) {
|
||||
return;
|
||||
}
|
||||
seen.add(normalized);
|
||||
formats.push(normalized);
|
||||
});
|
||||
|
||||
return formats;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
export interface EnterAnimationState {
|
||||
key: string | null;
|
||||
animate: boolean;
|
||||
}
|
||||
|
||||
export const INITIAL_ENTER_ANIMATION: EnterAnimationState = { key: null, animate: true };
|
||||
|
||||
/**
|
||||
* Decide whether a release modal session should play its enter animation.
|
||||
*
|
||||
* The decision is made once, when a session key first appears, and then held for
|
||||
* that session's lifetime — re-rendering mid-session must not restart the
|
||||
* animation. Swapping between sessions in combined mode is a step transition
|
||||
* rather than an entrance, so it does not animate.
|
||||
*/
|
||||
export const nextEnterAnimation = (
|
||||
current: EnterAnimationState,
|
||||
sessionKey: string | null,
|
||||
isCombinedMode: boolean,
|
||||
): EnterAnimationState => {
|
||||
if (current.key === sessionKey) {
|
||||
return current;
|
||||
}
|
||||
|
||||
return {
|
||||
key: sessionKey,
|
||||
animate: !isCombinedMode || current.key === null,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { DownloadReleasePayload } from '../services/api';
|
||||
import type { Book, ContentType, PackBook, Release } from '../types';
|
||||
|
||||
export interface ReleaseDownloadOptions {
|
||||
/** Ask post-processing to split the release into one book per subfolder/file. */
|
||||
multiBook?: boolean;
|
||||
/** The split the user approved in the pack review panel. */
|
||||
bookPlan?: PackBook[];
|
||||
}
|
||||
|
||||
/** Build the body for /api/releases/download (and /api/releases/inspect). */
|
||||
export function buildReleaseDownloadPayload(
|
||||
book: Book,
|
||||
release: Release,
|
||||
releaseContentType: ContentType,
|
||||
options: ReleaseDownloadOptions = {},
|
||||
): DownloadReleasePayload {
|
||||
const isManual = book.provider === 'manual';
|
||||
const releasePreview =
|
||||
typeof release.extra?.preview === 'string' ? release.extra.preview : undefined;
|
||||
const releaseAuthor =
|
||||
typeof release.extra?.author === 'string' ? release.extra.author : undefined;
|
||||
|
||||
const payload: DownloadReleasePayload = {
|
||||
source: release.source,
|
||||
source_id: release.source_id,
|
||||
title: isManual ? release.title : book.title,
|
||||
author: isManual ? releaseAuthor || '' : book.author,
|
||||
year: book.year,
|
||||
format: release.format,
|
||||
size: release.size,
|
||||
size_bytes: release.size_bytes,
|
||||
download_url: release.download_url,
|
||||
protocol: release.protocol,
|
||||
indexer: release.indexer,
|
||||
seeders: release.seeders,
|
||||
extra: release.extra,
|
||||
preview: isManual ? releasePreview || undefined : book.preview,
|
||||
content_type: releaseContentType,
|
||||
series_name: book.series_name,
|
||||
series_position: book.series_position,
|
||||
subtitle: book.subtitle,
|
||||
// From the release, never the book: book.language is the provider's
|
||||
// canonical edition, which would mislabel a translated release.
|
||||
language: release.language ?? undefined,
|
||||
};
|
||||
|
||||
if (options.multiBook || options.bookPlan) {
|
||||
payload.multi_book = true;
|
||||
}
|
||||
if (options.bookPlan) {
|
||||
payload.book_plan = options.bookPlan;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { isApiResponseError, isTimeoutError } from '../services/api';
|
||||
|
||||
// Shown when we genuinely have nothing better: the request failed without the server
|
||||
// saying why, which really can mean blocked mirrors or a restricted network.
|
||||
export const UNREACHABLE_SOURCE_MESSAGE =
|
||||
'Unable to reach download source. Network may be restricted or mirrors blocked.';
|
||||
|
||||
// Shown when the client's own abort fired. It is the backstop for a server that never
|
||||
// answered at all, and it says nothing about mirrors - the budget the client waited out
|
||||
// is the server's own, which the user can raise. See issue #1285.
|
||||
export const CLIENT_TIMEOUT_MESSAGE =
|
||||
'The search took longer than the server said it would. Raise the release search ' +
|
||||
'timeout if your setup is simply slow.';
|
||||
|
||||
/**
|
||||
* The sentence to show for a failed direct-mode search.
|
||||
*
|
||||
* The server knows why a search failed - a spent search budget, an unsolved protection
|
||||
* challenge - and blanket-replacing that with the mirrors line told users their network
|
||||
* was broken when it was not. So: the server's own words when it explained itself, the
|
||||
* timeout line when we gave up before it answered, and the mirrors line only when
|
||||
* neither applies. See issue #1285.
|
||||
*/
|
||||
export const describeSearchFailure = (error: unknown): string => {
|
||||
if (isTimeoutError(error)) {
|
||||
return CLIENT_TIMEOUT_MESSAGE;
|
||||
}
|
||||
|
||||
if (isApiResponseError(error) && error.serverMessage) {
|
||||
return error.serverMessage;
|
||||
}
|
||||
|
||||
const message = error instanceof Error ? error.message : '';
|
||||
if (message.includes('Network restricted') || message.includes('Unable to reach')) {
|
||||
return message;
|
||||
}
|
||||
|
||||
return UNREACHABLE_SOURCE_MESSAGE;
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
"""AudiobookBay test fixtures."""
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_detail_page_cache():
|
||||
"""Detail pages are cached briefly in production; tests must not share them."""
|
||||
scraper.clear_detail_page_cache()
|
||||
yield
|
||||
scraper.clear_detail_page_cache()
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Tests for reading the torrent file list off an AudiobookBay detail page."""
|
||||
|
||||
from shelfmark.download.postprocess.packs import PackFile
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
|
||||
# Trimmed from a real detail page (2026-08): the file rows sit between the
|
||||
# "Multifile Torrent" marker and the "Combined File Size" row.
|
||||
MULTIFILE_DETAIL_HTML = """
|
||||
<table>
|
||||
<tr><td>Tracker:</td><td>udp://tracker.torrent.eu.org:451/announce</td></tr>
|
||||
<tr><td>Creation Date:</td><td>Sun, 29 Mar 2026 21:09:39 +0200</td></tr>
|
||||
<tr><td colspan='2'>This is a Multifile Torrent</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 9.0 - Leviathan Falls (2021).m4b 1.05 GBs</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 0.1 - An Expanse Novella - Drive (2012).txt 340 Bytes</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 0.2 - An Expanse Novella - The Churn (2014).m4b 125.72 MBs</td></tr>
|
||||
<tr><td colspan='2'>The Expanse 2.0 - Caliban’s War (2012).m4b 578.97 MBs</td></tr>
|
||||
<tr><td>Combined File Size:</td><td><span style='color:#00f;'>7.87</span> GBs</td></tr>
|
||||
<tr><td>Info Hash:</td><td>e4a5538e26987ee58a43aa629ec2c4f2b2d46526</td></tr>
|
||||
</table>
|
||||
"""
|
||||
|
||||
SINGLE_FILE_DETAIL_HTML = """
|
||||
<table>
|
||||
<tr><td>Creation Date:</td><td>Sun, 29 Mar 2026 21:09:39 +0200</td></tr>
|
||||
<tr><td colspan='2'>Drive.m4b 41.55 MBs</td></tr>
|
||||
<tr><td>Combined File Size:</td><td><span style='color:#00f;'>41.55</span> MBs</td></tr>
|
||||
<tr><td>Info Hash:</td><td>e4a5538e26987ee58a43aa629ec2c4f2b2d46526</td></tr>
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
def test_extracts_multifile_rows_with_byte_sizes():
|
||||
files = scraper.extract_file_list(MULTIFILE_DETAIL_HTML)
|
||||
assert files == [
|
||||
PackFile("The Expanse 9.0 - Leviathan Falls (2021).m4b", int(1.05 * 1024**3)),
|
||||
PackFile("The Expanse 0.1 - An Expanse Novella - Drive (2012).txt", 340),
|
||||
PackFile(
|
||||
"The Expanse 0.2 - An Expanse Novella - The Churn (2014).m4b", int(125.72 * 1024**2)
|
||||
),
|
||||
PackFile("The Expanse 2.0 - Caliban’s War (2012).m4b", int(578.97 * 1024**2)),
|
||||
]
|
||||
|
||||
|
||||
def test_single_file_torrent_lists_the_row_before_combined_size():
|
||||
assert scraper.extract_file_list(SINGLE_FILE_DETAIL_HTML) == [
|
||||
PackFile("Drive.m4b", int(41.55 * 1024**2))
|
||||
]
|
||||
|
||||
|
||||
def test_page_without_file_table_returns_none():
|
||||
assert scraper.extract_file_list("<html><body><p>nothing here</p></body></html>") is None
|
||||
|
||||
|
||||
class TestHandlerListFiles:
|
||||
def test_lists_files_from_detail_page(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
from shelfmark.release_sources.audiobookbay.handler import AudiobookBayHandler
|
||||
|
||||
with patch(
|
||||
"shelfmark.release_sources.audiobookbay.handler.scraper.fetch_detail_html",
|
||||
return_value=SINGLE_FILE_DETAIL_HTML,
|
||||
) as fetch:
|
||||
files = AudiobookBayHandler().list_files(
|
||||
{"source_id": "abc", "download_url": "https://audiobookbay.lu/abss/drive/"}
|
||||
)
|
||||
assert files == [PackFile("Drive.m4b", int(41.55 * 1024**2))]
|
||||
fetch.assert_called_once_with("https://audiobookbay.lu/abss/drive/", "audiobookbay.lu")
|
||||
|
||||
def test_rejects_detail_url_on_other_host(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
from shelfmark.release_sources.audiobookbay.handler import AudiobookBayHandler
|
||||
|
||||
with patch(
|
||||
"shelfmark.release_sources.audiobookbay.handler.scraper.fetch_detail_html"
|
||||
) as fetch:
|
||||
files = AudiobookBayHandler().list_files(
|
||||
{"source_id": "abc", "download_url": "https://evil.example/abss/drive/"}
|
||||
)
|
||||
assert files is None
|
||||
fetch.assert_not_called()
|
||||
|
||||
|
||||
def test_extract_magnet_link_and_file_list_share_one_page_fetch():
|
||||
from unittest.mock import patch
|
||||
|
||||
page = MULTIFILE_DETAIL_HTML
|
||||
with (
|
||||
patch(
|
||||
"shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page",
|
||||
return_value=page,
|
||||
) as get_page,
|
||||
patch("shelfmark.release_sources.audiobookbay.scraper._bootstrap_abb_session"),
|
||||
):
|
||||
scraper.clear_detail_page_cache()
|
||||
url = "https://audiobookbay.lu/abss/expanse/"
|
||||
assert scraper.fetch_detail_html(url, "audiobookbay.lu") == page
|
||||
magnet = scraper.extract_magnet_link(url, "audiobookbay.lu")
|
||||
assert magnet is not None
|
||||
assert "e4a5538e26987ee58a43aa629ec2c4f2b2d46526".upper() in magnet
|
||||
assert get_page.call_count == 1
|
||||
@@ -0,0 +1,315 @@
|
||||
"""How long the internal bypasser is allowed to spend, and on what.
|
||||
|
||||
Issue #1276: MAX_RETRY drove *both* the outer page-load loop and the per-page method
|
||||
loop, so the default of 10 meant ~40 solve attempts on one browser. That overran the
|
||||
worker deadline, and the failure reached the user as `RuntimeError: TimeoutError` - a
|
||||
message that says nothing about a protection challenge and sent people looking at their
|
||||
reverse proxy instead.
|
||||
|
||||
Also covered: the undisturbed window a passive challenge gets before anything touches the
|
||||
page. Anna's Archive's DDoS-Guard check has no click target and clears itself; going
|
||||
straight to the click/reload methods meant the one thing that solves it was never tried.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bypass(monkeypatch):
|
||||
"""internal_bypasser with sleeps and jitter removed."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
async def _no_sleep(_seconds) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.asyncio, "sleep", _no_sleep)
|
||||
monkeypatch.setattr(internal_bypasser._RNG, "uniform", lambda _a, _b: 0)
|
||||
return internal_bypasser
|
||||
|
||||
|
||||
def _recording_methods(calls: list[str], count: int = 4):
|
||||
def _make(name: str):
|
||||
async def _method(_page) -> bool:
|
||||
calls.append(name)
|
||||
return False
|
||||
|
||||
_method.__name__ = name
|
||||
return _method
|
||||
|
||||
return [_make(f"m{i}") for i in range(count)]
|
||||
|
||||
|
||||
def _stub_page_state(monkeypatch, bypass, *, bypassed=False, challenge="ddos_guard"):
|
||||
async def _is_bypassed(*_args, **_kwargs) -> bool:
|
||||
return bypassed
|
||||
|
||||
async def _detect(*_args, **_kwargs) -> str:
|
||||
return challenge
|
||||
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _is_bypassed)
|
||||
monkeypatch.setattr(bypass, "_detect_challenge_type", _detect)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The method loop must not read MAX_RETRY
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_method_loop_budget_is_independent_of_max_retry(monkeypatch, bypass):
|
||||
"""MAX_RETRY is the outer page-load retry; reading it here squared the budget.
|
||||
|
||||
Exercised against a challenge whose *type* keeps changing, because that is the case
|
||||
where max_retries is what bounds the loop: the stuck-challenge guard only fires on a
|
||||
run of the same type, so with a stable challenge it hid the real budget entirely.
|
||||
"""
|
||||
monkeypatch.setattr(type(bypass.app_config), "MAX_RETRY", 50, raising=False)
|
||||
|
||||
types = iter(["ddos_guard", "cloudflare"] * 100)
|
||||
|
||||
async def _alternating(*_args, **_kwargs) -> str:
|
||||
return next(types)
|
||||
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _never_passes)
|
||||
_stub_page_state(monkeypatch, bypass)
|
||||
monkeypatch.setattr(bypass, "_detect_challenge_type", _alternating)
|
||||
|
||||
assert asyncio.run(bypass._bypass(object())) is False
|
||||
assert len(calls) == bypass._BYPASS_METHOD_ATTEMPTS
|
||||
assert len(calls) < 50, "MAX_RETRY must not reach the method loop"
|
||||
|
||||
|
||||
def test_method_attempt_budget_is_reachable(bypass):
|
||||
"""The number reported as `attempt N/X` must be a number the loop can reach.
|
||||
|
||||
It used to be MAX_RETRY (10) while the stuck-challenge guard capped the loop at 5,
|
||||
so logs showed `4/10` and stopped, which reads like six lost attempts.
|
||||
"""
|
||||
assert bypass._BYPASS_METHOD_ATTEMPTS == len(bypass.BYPASS_METHODS) + 1
|
||||
assert bypass._BYPASS_METHOD_ATTEMPTS >= (
|
||||
max(bypass.MAX_CONSECUTIVE_SAME_CHALLENGE, len(bypass.BYPASS_METHODS) + 1)
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# A passive challenge gets an undisturbed window first
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def _never_passes(*_args, **_kwargs) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def test_passive_challenge_is_given_time_before_any_method_runs(monkeypatch, bypass):
|
||||
"""DDoS-Guard's JS check clears itself; nothing should click or reload first."""
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
||||
_stub_page_state(monkeypatch, bypass)
|
||||
|
||||
async def _passes(*_args, **_kwargs) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _passes)
|
||||
|
||||
assert asyncio.run(bypass._bypass(object())) is True
|
||||
assert calls == [], "the page must not be touched while the check can still pass"
|
||||
|
||||
|
||||
def test_passive_wait_happens_once_not_before_every_method(monkeypatch, bypass):
|
||||
"""It is a settling window, not a delay bolted onto each attempt."""
|
||||
waits: list[int] = []
|
||||
calls: list[str] = []
|
||||
|
||||
async def _count_wait(*_args, **_kwargs) -> bool:
|
||||
waits.append(1)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(bypass, "BYPASS_METHODS", _recording_methods(calls))
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _count_wait)
|
||||
_stub_page_state(monkeypatch, bypass)
|
||||
|
||||
asyncio.run(bypass._bypass(object()))
|
||||
|
||||
assert len(waits) == 1
|
||||
assert calls == ["m0", "m1", "m2", "m3"]
|
||||
|
||||
|
||||
def test_no_passive_wait_when_no_challenge_is_detected(monkeypatch, bypass):
|
||||
"""The 'none' branch has its own settle-and-refresh handling."""
|
||||
waits: list[int] = []
|
||||
|
||||
async def _count_wait(*_args, **_kwargs) -> bool:
|
||||
waits.append(1)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(bypass, "_wait_for_passive_solve", _count_wait)
|
||||
_stub_page_state(monkeypatch, bypass, challenge="none")
|
||||
|
||||
class _Page:
|
||||
async def reload(self, **_kwargs) -> None:
|
||||
return None
|
||||
|
||||
asyncio.run(bypass._bypass(_Page(), max_retries=1))
|
||||
|
||||
assert waits == []
|
||||
|
||||
|
||||
def test_wait_for_passive_solve_returns_as_soon_as_the_page_clears(monkeypatch, bypass):
|
||||
polls = {"n": 0}
|
||||
|
||||
async def _is_bypassed(*_args, **_kwargs) -> bool:
|
||||
polls["n"] += 1
|
||||
return polls["n"] >= 3
|
||||
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _is_bypassed)
|
||||
|
||||
assert asyncio.run(bypass._wait_for_passive_solve(object())) is True
|
||||
assert polls["n"] == 3
|
||||
|
||||
|
||||
def test_wait_for_passive_solve_gives_up_at_the_window(monkeypatch, bypass):
|
||||
"""It must not poll forever - the methods still need their share of the budget."""
|
||||
clock = {"now": 0.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
|
||||
async def _tick(*_args, **_kwargs) -> bool:
|
||||
clock["now"] += 1.0
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _tick)
|
||||
|
||||
assert asyncio.run(bypass._wait_for_passive_solve(object())) is False
|
||||
assert clock["now"] >= bypass._PASSIVE_SOLVE_SECONDS
|
||||
|
||||
|
||||
def test_wait_for_passive_solve_honours_cancellation(monkeypatch, bypass):
|
||||
import threading
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
|
||||
cancel = threading.Event()
|
||||
cancel.set()
|
||||
monkeypatch.setattr(bypass, "_is_bypassed", _never_passes)
|
||||
|
||||
with pytest.raises(BypassCancelledError):
|
||||
asyncio.run(bypass._wait_for_passive_solve(object(), cancel))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The page-load loop stops while there is still time to report a real failure
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_page_load_loop_stops_before_the_worker_deadline(monkeypatch, bypass):
|
||||
"""A stubborn challenge must produce "bypass failed", not a cancelled coroutine."""
|
||||
clock = {"now": 0.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
monkeypatch.delenv(bypass._BYPASS_CHILD_ENV, raising=False)
|
||||
|
||||
attempts = {"n": 0}
|
||||
|
||||
async def _create(_url):
|
||||
return object()
|
||||
|
||||
async def _get(_url, _driver, _cancel=None) -> str:
|
||||
attempts["n"] += 1
|
||||
# Each pass eats a realistic slice of the budget.
|
||||
clock["now"] += 120.0
|
||||
return ""
|
||||
|
||||
async def _close(_driver) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(bypass, "_create_cdp_browser", _create)
|
||||
monkeypatch.setattr(bypass, "_get", _get)
|
||||
monkeypatch.setattr(bypass, "_close_cdp_driver", _close)
|
||||
|
||||
class _RealWorker:
|
||||
def run(self, coro, timeout=None):
|
||||
return asyncio.run(coro)
|
||||
|
||||
monkeypatch.setattr(bypass, "_CDP_WORKER", _RealWorker())
|
||||
|
||||
result = bypass._run_bypass_in_current_process("https://example.com", 10)
|
||||
|
||||
assert result == ""
|
||||
# Well short of the 10 it was asked for, and short of the deadline it had.
|
||||
assert attempts["n"] < 10
|
||||
budget = bypass._IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
assert clock["now"] < budget, "the loop must leave room to report the failure"
|
||||
|
||||
|
||||
def test_page_load_loop_still_makes_one_attempt_on_a_spent_budget(monkeypatch, bypass):
|
||||
"""The deadline check must never skip the request entirely."""
|
||||
clock = {"now": 10_000.0}
|
||||
monkeypatch.setattr(bypass.time, "monotonic", lambda: clock["now"])
|
||||
monkeypatch.delenv(bypass._BYPASS_CHILD_ENV, raising=False)
|
||||
|
||||
attempts = {"n": 0}
|
||||
|
||||
async def _create(_url):
|
||||
return object()
|
||||
|
||||
async def _get(_url, _driver, _cancel=None) -> str:
|
||||
attempts["n"] += 1
|
||||
return "<html>solved</html>"
|
||||
|
||||
async def _close(_driver) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(bypass, "_create_cdp_browser", _create)
|
||||
monkeypatch.setattr(bypass, "_get", _get)
|
||||
monkeypatch.setattr(bypass, "_close_cdp_driver", _close)
|
||||
|
||||
class _RealWorker:
|
||||
def run(self, coro, timeout=None):
|
||||
return asyncio.run(coro)
|
||||
|
||||
monkeypatch.setattr(bypass, "_CDP_WORKER", _RealWorker())
|
||||
|
||||
assert bypass._run_bypass_in_current_process("https://example.com", 10) == "<html>solved</html>"
|
||||
assert attempts["n"] == 1
|
||||
|
||||
|
||||
class _FakeElement:
|
||||
async def get_html_async(self) -> str:
|
||||
return "<html>solved</html>"
|
||||
|
||||
|
||||
class _FakePage:
|
||||
"""A page that only produces its document after `ready_after` seconds of waiting."""
|
||||
|
||||
def __init__(self, ready_after: float = 0.0) -> None:
|
||||
self.ready_after = ready_after
|
||||
self.waited_with: list[float] = []
|
||||
|
||||
async def find(self, selector: str, timeout: float = 1):
|
||||
self.waited_with.append(timeout)
|
||||
if timeout < self.ready_after:
|
||||
msg = f"Time ran out while waiting for: {{{selector}}}"
|
||||
raise TimeoutError(msg)
|
||||
return _FakeElement()
|
||||
|
||||
|
||||
def test_page_source_waits_longer_than_seleniumbases_one_second(bypass):
|
||||
"""A page still navigating after a solve must not lose the solve.
|
||||
|
||||
SeleniumBase's get_page_source() allows one second for the document. Anna's Archive
|
||||
hands back a redirect to the real content instead, so the read raised TimeoutError
|
||||
while the challenge had in fact been cleared.
|
||||
"""
|
||||
page = _FakePage(ready_after=5.0)
|
||||
|
||||
assert asyncio.run(bypass._read_page_source(page)) == "<html>solved</html>"
|
||||
assert page.waited_with == [bypass._PAGE_SOURCE_TIMEOUT_DEFAULT]
|
||||
|
||||
|
||||
def test_page_source_timeout_is_configurable(bypass, monkeypatch):
|
||||
"""BYPASS_PAGE_SOURCE_TIMEOUT overrides the default for slow or fast setups."""
|
||||
monkeypatch.setattr(
|
||||
bypass.app_config,
|
||||
"get",
|
||||
lambda key, default=None: 45 if key == "BYPASS_PAGE_SOURCE_TIMEOUT" else default,
|
||||
)
|
||||
page = _FakePage()
|
||||
|
||||
asyncio.run(bypass._read_page_source(page))
|
||||
|
||||
assert page.waited_with == [45.0]
|
||||
@@ -0,0 +1,88 @@
|
||||
"""No method in the list may be a step another method already takes first.
|
||||
|
||||
`BYPASS_METHODS` used to open with a solve-only entry that called `page.solve_captcha()`
|
||||
and checked the result. `_bypass_method_cdp_gui_click`, the entry behind it, opens by
|
||||
doing exactly that and returns the moment it works - so against a challenge that
|
||||
`solve_captcha()` cannot clear, the first method could only repeat the half that had
|
||||
already failed, then charge the loop's backoff before the method that does work started.
|
||||
Measured on Anna's Archive at 0/19 successes and ~5.5s of the ~26s each solve cost
|
||||
(issue #1285).
|
||||
|
||||
The passive-solve window added in v1.3.13 keeps most solves away from this loop entirely,
|
||||
so this is about what the loop costs when it does run.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_sleep(monkeypatch):
|
||||
async def _no_sleep(_seconds) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(ib.asyncio, "sleep", _no_sleep)
|
||||
monkeypatch.setattr(ib._RNG, "uniform", lambda _a, _b: 0)
|
||||
|
||||
|
||||
class _Page:
|
||||
"""Records what a method asked the page to do."""
|
||||
|
||||
def __init__(self, *, solve_clears: bool) -> None:
|
||||
self.solve_clears = solve_clears
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def solve_captcha(self) -> None:
|
||||
self.calls.append("solve_captcha")
|
||||
|
||||
async def is_element_visible(self, selector: str) -> bool:
|
||||
self.calls.append(f"visible:{selector}")
|
||||
return False
|
||||
|
||||
async def click_with_offset(self, selector: str, _x, _y, center=True) -> None:
|
||||
self.calls.append(f"click:{selector}")
|
||||
|
||||
|
||||
def _stub_is_bypassed(monkeypatch, page: _Page) -> None:
|
||||
async def _is_bypassed(*_a, **_kw) -> bool:
|
||||
return page.solve_clears and "solve_captcha" in page.calls
|
||||
|
||||
monkeypatch.setattr(ib, "_is_bypassed", _is_bypassed)
|
||||
|
||||
|
||||
def test_no_solve_only_method_remains_in_the_list():
|
||||
names = [method.__name__ for method in ib.BYPASS_METHODS]
|
||||
assert "_bypass_method_cdp_solve" not in names
|
||||
assert names[0] == "_bypass_method_cdp_gui_click"
|
||||
|
||||
|
||||
def test_the_first_method_still_tries_solve_captcha_first(monkeypatch, no_sleep):
|
||||
"""Coverage is only preserved because gui_click opens with the same call."""
|
||||
page = _Page(solve_clears=True)
|
||||
_stub_is_bypassed(monkeypatch, page)
|
||||
|
||||
assert asyncio.run(ib.BYPASS_METHODS[0](page)) is True
|
||||
assert page.calls == ["solve_captcha"], "it must return before touching any selector"
|
||||
|
||||
|
||||
def test_it_falls_through_to_clicking_when_solve_does_not_clear(monkeypatch, no_sleep):
|
||||
"""The half that actually works on DDoS-Guard still runs in the same attempt."""
|
||||
page = _Page(solve_clears=False)
|
||||
_stub_is_bypassed(monkeypatch, page)
|
||||
|
||||
assert asyncio.run(ib.BYPASS_METHODS[0](page)) is False
|
||||
assert page.calls[0] == "solve_captcha"
|
||||
assert any(call.startswith("visible:") for call in page.calls), (
|
||||
"the selector pass should have been reached in the same attempt"
|
||||
)
|
||||
|
||||
|
||||
def test_the_derived_budgets_follow_the_shortened_list():
|
||||
"""Both budgets are computed from the list, so removing an entry must not strand them."""
|
||||
assert ib._BYPASS_METHOD_ATTEMPTS == len(ib.BYPASS_METHODS) + 1
|
||||
assert ib._BYPASS_METHOD_ATTEMPTS >= len(ib.BYPASS_METHODS), (
|
||||
"every method must still get a turn"
|
||||
)
|
||||
@@ -43,6 +43,37 @@ def _store(cookies, url="https://annas-archive.gl/search"):
|
||||
cs.store_extracted_cookies(url=url, cookies=cookies, user_agent="UA/1.0")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cookie_store_logs():
|
||||
"""Collect cookie-store log messages.
|
||||
|
||||
The store's logger is built outside the standard hierarchy, so its records never
|
||||
reach the root handler caplog installs.
|
||||
"""
|
||||
import logging
|
||||
|
||||
messages: list[str] = []
|
||||
|
||||
class _Capture(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
messages.append(record.getMessage())
|
||||
|
||||
handler = _Capture()
|
||||
cs.logger.addHandler(handler)
|
||||
previous = cs.logger.level
|
||||
cs.logger.setLevel(logging.DEBUG)
|
||||
# setup_logger builds its loggers with CustomLogger(name) rather than getLogger, so
|
||||
# they are not in the manager's hierarchy - and Logger.setLevel only invalidates the
|
||||
# is-enabled cache *through* the manager. Without this the logger keeps answering
|
||||
# "DEBUG is off" from a cache entry made while it was at INFO.
|
||||
cs.logger._cache.clear()
|
||||
try:
|
||||
yield messages
|
||||
finally:
|
||||
cs.logger.removeHandler(handler)
|
||||
cs.logger.setLevel(previous)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Per-check cookies must not be persisted for replay
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -220,3 +251,56 @@ def test_failure_only_clears_the_failing_host(monkeypatch):
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
assert ib.get_cf_cookies_for_domain("other-site.test") == {"__ddg1_": "other"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Settling what DDoS-Guard actually treats as clearance (issue #1276)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_per_check_cookies_can_be_kept_for_a_field_test(monkeypatch):
|
||||
"""Which __ddg* cookies are clearance is not settled, so it has to be testable.
|
||||
|
||||
The store's premise - that __ddg8_/__ddg9_/__ddg10_ describe one check and must not
|
||||
be replayed - is contradicted by the field reports on #1276, where every request
|
||||
after a successful solve was challenged again. This env-only switch is how that gets
|
||||
answered against a live host without building a branch.
|
||||
"""
|
||||
from shelfmark.config import env
|
||||
|
||||
monkeypatch.setattr(env, "DDG_REPLAY_PER_CHECK_COOKIES", True)
|
||||
_store(
|
||||
[
|
||||
_Cookie("__ddg1_", "clearance"),
|
||||
_Cookie("__ddg8_", "opaque"),
|
||||
_Cookie("__ddg9_", "203.0.113.7"),
|
||||
_Cookie("__ddg10_", "1786826304"),
|
||||
]
|
||||
)
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(stored) == {"__ddg1_", "__ddg8_", "__ddg9_", "__ddg10_"}
|
||||
|
||||
|
||||
def test_dropping_per_check_cookies_is_the_default(monkeypatch):
|
||||
"""The switch is for reproducing the question, not a behaviour change."""
|
||||
from shelfmark.config import env
|
||||
|
||||
assert env.DDG_REPLAY_PER_CHECK_COOKIES is False
|
||||
_store([_Cookie("__ddg1_", "clearance"), _Cookie("__ddg9_", "203.0.113.7")])
|
||||
|
||||
assert set(ib.get_cf_cookies_for_domain("annas-archive.gl")) == {"__ddg1_"}
|
||||
|
||||
|
||||
def test_a_solve_logs_which_cookies_it_won_and_which_were_held_back(cookie_store_logs):
|
||||
"""Without this, a debug log shows a solve succeed and the next request challenged,
|
||||
with nothing in between to explain why."""
|
||||
_store([_Cookie("__ddg1_", "clearance"), _Cookie("__ddg9_", "203.0.113.7")])
|
||||
|
||||
messages = cookie_store_logs
|
||||
line = next((m for m in messages if "won" in m and "dropping" in m), None)
|
||||
assert line is not None, messages
|
||||
assert "__ddg1_" in line
|
||||
assert "__ddg9_" in line
|
||||
# Names only - a clearance cookie's value is a credential.
|
||||
assert "clearance" not in line
|
||||
assert "203.0.113.7" not in line
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""A recording that never happened must say why.
|
||||
|
||||
Issue #1276: the debug bundle's recording/ directory was empty, and the only trace was
|
||||
three "FFmpeg already stopped" debug lines - one per bypass, each logged 20-56s after
|
||||
the recorder was started, meaning ffmpeg had exited almost immediately every time. It ran
|
||||
with `-loglevel 0` and no stderr capture, so nothing anywhere recorded the reason. The
|
||||
screen recording is the single most useful artifact for diagnosing a bypass failure.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_display():
|
||||
before = dict(ib.DISPLAY)
|
||||
ib.DISPLAY["ffmpeg"] = None
|
||||
ib.DISPLAY["ffmpeg_output"] = None
|
||||
ib.DISPLAY["ffmpeg_error_log"] = None
|
||||
yield
|
||||
ib.DISPLAY.update(before)
|
||||
|
||||
|
||||
class _Proc:
|
||||
def __init__(self, returncode):
|
||||
self.returncode = returncode
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
|
||||
def test_ffmpeg_errors_are_captured_to_a_file_beside_the_recording(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(ib, "RECORDING_DIR", tmp_path)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_popen(cmd, **kwargs):
|
||||
captured["cmd"] = cmd
|
||||
captured["stderr"] = kwargs.get("stderr")
|
||||
return _Proc(None)
|
||||
|
||||
monkeypatch.setattr(ib.subprocess, "Popen", fake_popen)
|
||||
|
||||
ib._start_ffmpeg_recording(display=":99")
|
||||
|
||||
cmd = captured["cmd"]
|
||||
# Errors must not be thrown away any more.
|
||||
assert "-loglevel" in cmd
|
||||
assert cmd[cmd.index("-loglevel") + 1] == "error"
|
||||
# stderr goes to a real file, not a pipe nothing would drain.
|
||||
assert captured["stderr"] is not None
|
||||
assert captured["stderr"] is not subprocess.PIPE
|
||||
|
||||
error_log = ib.DISPLAY["ffmpeg_error_log"]
|
||||
assert error_log is not None
|
||||
assert error_log.parent == tmp_path
|
||||
# It sits beside the mp4, so it travels in the debug bundle.
|
||||
assert error_log.name.startswith("screen_recording_")
|
||||
|
||||
|
||||
def test_an_early_exit_is_reported_with_ffmpegs_own_reason(monkeypatch, tmp_path, caplog):
|
||||
reason = "[x11grab @ 0x1] Cannot open display :99, error 1."
|
||||
error_log = tmp_path / "screen_recording_x.ffmpeg.log"
|
||||
error_log.write_text(reason, encoding="utf-8")
|
||||
|
||||
ib.DISPLAY["ffmpeg"] = _Proc(1)
|
||||
ib.DISPLAY["ffmpeg_output"] = tmp_path / "screen_recording_x.mp4"
|
||||
ib.DISPLAY["ffmpeg_error_log"] = error_log
|
||||
|
||||
messages: list[str] = []
|
||||
|
||||
class _Capture:
|
||||
def emit(self, record):
|
||||
messages.append(record.getMessage())
|
||||
|
||||
import logging
|
||||
|
||||
handler = logging.Handler()
|
||||
handler.emit = _Capture().emit # type: ignore[method-assign]
|
||||
ib.logger.addHandler(handler)
|
||||
previous = ib.logger.level
|
||||
ib.logger.setLevel(logging.DEBUG)
|
||||
ib.logger._cache.clear()
|
||||
try:
|
||||
ib._stop_ffmpeg_recording()
|
||||
finally:
|
||||
ib.logger.removeHandler(handler)
|
||||
ib.logger.setLevel(previous)
|
||||
|
||||
line = next((m for m in messages if "exited early" in m), None)
|
||||
assert line is not None, messages
|
||||
assert "code 1" in line
|
||||
assert "Cannot open display" in line
|
||||
assert ib.DISPLAY["ffmpeg"] is None
|
||||
|
||||
|
||||
def test_summary_is_explicit_when_ffmpeg_logged_nothing(tmp_path):
|
||||
empty = tmp_path / "screen_recording_y.ffmpeg.log"
|
||||
empty.write_text("", encoding="utf-8")
|
||||
ib.DISPLAY["ffmpeg_error_log"] = empty
|
||||
|
||||
assert "logged nothing" in ib._ffmpeg_error_summary()
|
||||
|
||||
|
||||
def test_summary_survives_a_missing_log():
|
||||
ib.DISPLAY["ffmpeg_error_log"] = None
|
||||
|
||||
assert "No FFmpeg error log" in ib._ffmpeg_error_summary()
|
||||
@@ -0,0 +1,113 @@
|
||||
"""A 429 is throttling, not a dead clearance cookie.
|
||||
|
||||
Issue #1276: every rejection of the cached cookies took the same exit, which cleared the
|
||||
host's clearance. That is right for a 403 and for the ?check=1 redirect loop - being
|
||||
challenged while presenting a cookie proves the cookie is dead - and wrong for a 429,
|
||||
where the origin is rate-limiting the IP and would answer a real browser holding the very
|
||||
same cookies identically.
|
||||
|
||||
The cost in the reported bundle: a solve completed at 13:41:23 and stored five cookies;
|
||||
six seconds later a 429 threw them away, and the next query bought its own 56-second
|
||||
browser solve. Reuse rate across the whole log was 0 of 2.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.cookie_store as cs
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
URL = "https://annas-archive.gl/search?q=dune"
|
||||
HOST = "annas-archive.gl"
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code, text="page"):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean(monkeypatch):
|
||||
monkeypatch.setattr(cs, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cs, "_cf_user_agents", {})
|
||||
monkeypatch.setattr(cs, "_get_full_cookie_domains", set)
|
||||
monkeypatch.setattr(ib, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(ib, "get_ssl_verify", lambda _url: True)
|
||||
cs._cf_cookies[HOST] = {
|
||||
"__ddg1_": {"value": "clearance", "expiry": None},
|
||||
"__ddg2_": {"value": "c2", "expiry": None},
|
||||
}
|
||||
|
||||
|
||||
def _cooldowns(monkeypatch):
|
||||
"""Record note_rate_limited calls without arming the real per-host ladder."""
|
||||
armed: list[str] = []
|
||||
monkeypatch.setattr(ib.network, "note_rate_limited", lambda url: armed.append(url) or 120.0)
|
||||
return armed
|
||||
|
||||
|
||||
def test_429_keeps_the_clearance(monkeypatch):
|
||||
armed = _cooldowns(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(429))
|
||||
|
||||
assert ib._try_with_cached_cookies(URL, HOST) is None
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {"__ddg1_": "clearance", "__ddg2_": "c2"}
|
||||
assert armed == [URL], "the backoff must still be armed"
|
||||
|
||||
|
||||
def test_403_still_discards_the_clearance(monkeypatch):
|
||||
"""The pre-existing behaviour for a genuine rejection must not regress."""
|
||||
_cooldowns(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
assert ib._try_with_cached_cookies(URL, HOST) is None
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {}
|
||||
|
||||
|
||||
def test_redirect_loop_still_discards_the_clearance(monkeypatch):
|
||||
_cooldowns(monkeypatch)
|
||||
|
||||
def boom(*_a, **_k):
|
||||
raise ib.requests.exceptions.TooManyRedirects("Exceeded 30 redirects")
|
||||
|
||||
monkeypatch.setattr(ib.requests, "get", boom)
|
||||
|
||||
assert ib._try_with_cached_cookies(URL, HOST) is None
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {}
|
||||
|
||||
|
||||
def test_a_throttled_host_is_not_handed_a_browser_solve(monkeypatch):
|
||||
"""A solve cannot clear a throttle, and is itself more traffic at a host asking for
|
||||
less. get_bypassed_page checks the cooldown before the queue; get() has to re-check
|
||||
after it, because a request can hold for LOCKED while another collects the 429."""
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(429))
|
||||
monkeypatch.setattr(ib.network, "note_rate_limited", lambda _url: 120.0)
|
||||
monkeypatch.setattr(ib.network, "host_cooldown_remaining", lambda _url: 118.0)
|
||||
|
||||
solved: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
ib, "_run_bypass_in_current_process", lambda url, *a, **k: solved.append(url) or "html"
|
||||
)
|
||||
monkeypatch.setattr(ib.env, "DOCKERMODE", False)
|
||||
|
||||
with pytest.raises(ib.network.RateLimitedError) as excinfo:
|
||||
ib.get(URL, retry=1)
|
||||
|
||||
assert solved == [], "no browser should have been started"
|
||||
assert "rate-limited" in str(excinfo.value)
|
||||
# And the clearance survives, ready for when the cooldown clears.
|
||||
assert ib.get_cf_cookies_for_domain(HOST) == {"__ddg1_": "clearance", "__ddg2_": "c2"}
|
||||
|
||||
|
||||
def test_a_host_that_is_not_throttled_still_solves(monkeypatch):
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
monkeypatch.setattr(ib.network, "host_cooldown_remaining", lambda _url: 0.0)
|
||||
|
||||
solved: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
ib, "_run_bypass_in_current_process", lambda url, *a, **k: solved.append(url) or "html"
|
||||
)
|
||||
monkeypatch.setattr(ib.env, "DOCKERMODE", False)
|
||||
|
||||
assert ib.get(URL, retry=1) == "html"
|
||||
assert solved == [URL]
|
||||
@@ -0,0 +1,214 @@
|
||||
"""A challenge page is a failed solve, whatever verdict the solver reports on itself.
|
||||
|
||||
Regression for #1292. FlareSolverr answers "Challenge solved!" for anything it does not
|
||||
recognise as a Cloudflare challenge, and DDoS-Guard's manual CAPTCHA page is one such
|
||||
thing. The external bypasser logged a warning that the solve had not cleared the
|
||||
protection and then returned the page as a success anyway, which had three consequences:
|
||||
the retry-and-rotate loop that could still have reached a working mirror was never
|
||||
entered, the CAPTCHA page's own __ddg cookies were filed as that host's clearance, and
|
||||
the user was told to go and check a bypasser that was working perfectly.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.bypass import ChallengeNotSolvedError
|
||||
|
||||
# Verbatim from the annas-archive.pk page in #1292, trimmed to the markers. This is the
|
||||
# *manual* CAPTCHA - "could not verify your browser automatically" - not the ~900 byte
|
||||
# JS interstitial that a browser clears on its own.
|
||||
DDOS_GUARD_CAPTCHA = (
|
||||
'<html><head><title>DDOS-GUARD</title><meta charset="utf-8">'
|
||||
'<link rel="stylesheet" href="/.well-known/ddos-guard/ddg-captcha-page/index.css">'
|
||||
'<script defer="defer" src="/.well-known/ddos-guard/ddg-captcha-page/index.js"></script>'
|
||||
'</head><body><div class="container"><h1 id="title">Checking your browser before '
|
||||
'accessing annas-archive.pk</h1><p id="description">Sorry, we could not verify your '
|
||||
"browser automatically. Complete the manual check to continue</p>"
|
||||
'<div id="ddg-captcha"></div></div></body></html>'
|
||||
)
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return self._payload
|
||||
|
||||
|
||||
def _stub_solution(monkeypatch, external_bypasser, solution: dict) -> None:
|
||||
"""Answer every bypass with `solution`, with config and SSL stubbed out."""
|
||||
|
||||
def fake_get(key, default=""):
|
||||
values = {
|
||||
"EXT_BYPASSER_URL": "https://bypass.example",
|
||||
"EXT_BYPASSER_PATH": "/v1",
|
||||
"EXT_BYPASSER_TIMEOUT": 60000,
|
||||
}
|
||||
return values.get(key, default)
|
||||
|
||||
monkeypatch.setattr(external_bypasser.config, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
external_bypasser.requests,
|
||||
"post",
|
||||
# "Challenge solved!" is the solver's verdict; the page is the evidence.
|
||||
lambda *_a, **_k: _FakeResponse(
|
||||
{"status": "ok", "message": "Challenge solved!", "solution": solution}
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(external_bypasser, "get_ssl_verify", lambda _url: False)
|
||||
|
||||
|
||||
def test_a_captcha_page_is_reported_as_unsolved_not_returned(monkeypatch):
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
_stub_solution(monkeypatch, external_bypasser, {"response": DDOS_GUARD_CAPTCHA})
|
||||
|
||||
with pytest.raises(ChallengeNotSolvedError) as excinfo:
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.pk/search?q=dune")
|
||||
|
||||
# The marker travels with the failure so the user-facing message can name it.
|
||||
assert str(excinfo.value) == "/.well-known/ddos-guard/"
|
||||
|
||||
|
||||
def test_cookies_from_a_captcha_page_are_never_filed_as_clearance(monkeypatch):
|
||||
"""They belong to an unsolved check, so replaying them only re-arms the gate."""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": DDOS_GUARD_CAPTCHA,
|
||||
"userAgent": "Mozilla/5.0 (solver)",
|
||||
"cookies": [{"name": "__ddg1_", "value": "from-a-captcha"}],
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(ChallengeNotSolvedError):
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.pk/search?q=dune")
|
||||
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.pk") == {}
|
||||
assert cookie_store.get_cf_user_agent_for_domain("annas-archive.pk") is None
|
||||
|
||||
|
||||
class _FakeSelector:
|
||||
"""Two mirrors, rotated on demand - each is its own DDoS-Guard host."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.current_base = "https://mirror-one.example"
|
||||
self.rotate_calls = 0
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
return url.replace("https://orig.example", self.current_base, 1)
|
||||
|
||||
def next_mirror_or_rotate_dns(self) -> tuple[str | None, str]:
|
||||
self.rotate_calls += 1
|
||||
self.current_base = "https://mirror-two.example"
|
||||
return self.current_base, "mirror"
|
||||
|
||||
|
||||
def _no_sleeping(monkeypatch, external_bypasser) -> None:
|
||||
monkeypatch.setattr(external_bypasser, "_sleep_with_cancellation", lambda _seconds, _flag: None)
|
||||
|
||||
|
||||
def test_an_unsolved_challenge_rotates_to_the_next_mirror(monkeypatch):
|
||||
"""The recovery the old code skipped by calling the CAPTCHA page a success."""
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
_no_sleeping(monkeypatch, external_bypasser)
|
||||
fetched: list[str] = []
|
||||
|
||||
def fake_fetch(url: str) -> str | None:
|
||||
fetched.append(url)
|
||||
if "mirror-one" in url:
|
||||
raise ChallengeNotSolvedError("/.well-known/ddos-guard/")
|
||||
return "<html>real page</html>"
|
||||
|
||||
monkeypatch.setattr(external_bypasser, "_fetch_via_bypasser", fake_fetch)
|
||||
|
||||
selector = _FakeSelector()
|
||||
result = external_bypasser.get_bypassed_page("https://orig.example/search", selector=selector)
|
||||
|
||||
assert result == "<html>real page</html>"
|
||||
assert fetched == [
|
||||
"https://mirror-one.example/search",
|
||||
"https://mirror-two.example/search",
|
||||
]
|
||||
assert selector.rotate_calls == 1
|
||||
|
||||
|
||||
def test_every_attempt_challenged_blames_the_host_not_the_bypasser(monkeypatch):
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
_no_sleeping(monkeypatch, external_bypasser)
|
||||
|
||||
def always_challenged(_url: str) -> str | None:
|
||||
raise ChallengeNotSolvedError("/.well-known/ddos-guard/")
|
||||
|
||||
monkeypatch.setattr(external_bypasser, "_fetch_via_bypasser", always_challenged)
|
||||
|
||||
with pytest.raises(ChallengeNotSolvedError) as excinfo:
|
||||
external_bypasser.get_bypassed_page("https://orig.example/search", selector=_FakeSelector())
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "manual CAPTCHA" in message
|
||||
assert "the bypasser itself is working" in message
|
||||
|
||||
|
||||
def test_an_unreachable_bypasser_still_reports_as_such(monkeypatch):
|
||||
"""The other cause must stay distinguishable: None, not an unsolved challenge."""
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
_no_sleeping(monkeypatch, external_bypasser)
|
||||
monkeypatch.setattr(external_bypasser, "_fetch_via_bypasser", lambda _url: None)
|
||||
|
||||
assert (
|
||||
external_bypasser.get_bypassed_page("https://orig.example/search", selector=_FakeSelector())
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_html_get_page_surfaces_the_host_as_the_cause(monkeypatch):
|
||||
"""The message the user actually reads must not send them to fix FlareSolverr.
|
||||
|
||||
`_run_bypasser`'s generic handler says "the protection bypasser failed", and the
|
||||
search layer's give-up used to add "check that the bypasser is reachable and
|
||||
working" - which is what #1292 spent its investigation doing.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
||||
|
||||
def challenged(*_args, **_kwargs):
|
||||
msg = "the site kept answering with a protection challenge - manual CAPTCHA"
|
||||
raise ChallengeNotSolvedError(msg)
|
||||
|
||||
monkeypatch.setattr(http, "get_bypassed_page", challenged)
|
||||
|
||||
statuses: list[tuple[str, str | None]] = []
|
||||
selector = network.AAMirrorSelector()
|
||||
|
||||
html = http.html_get_page(
|
||||
"https://annas-archive.pk/search?q=dune",
|
||||
retry=1,
|
||||
selector=selector,
|
||||
status_callback=lambda stage, detail: statuses.append((stage, detail)),
|
||||
use_bypasser=True,
|
||||
success_delay=0,
|
||||
)
|
||||
|
||||
assert html == ""
|
||||
assert selector.last_failure is not None
|
||||
assert "manual CAPTCHA" in selector.last_failure
|
||||
assert "reachable" not in selector.last_failure
|
||||
assert ("error", "the site kept answering with a protection challenge - manual CAPTCHA") in (
|
||||
statuses
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user