mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 22:05:20 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5b3df2a463 | ||
|
|
5247ec6124 | ||
|
|
bd21ec1257 | ||
|
|
7b9c416df8 | ||
|
|
646b531669 | ||
|
|
eafb965662 | ||
|
|
7193036626 | ||
|
|
12d554a92f | ||
|
|
fae6140c6a | ||
|
|
63133097e4 | ||
|
|
82aeee387e | ||
|
|
4cd1091d16 | ||
|
|
651096ed7b | ||
|
|
ebb833a82c | ||
|
|
b7093f4594 | ||
|
|
b656f019be | ||
|
|
6e96ead519 | ||
|
|
7345f6be1a | ||
|
|
2b8b35bb52 | ||
|
|
58a5b5ed27 | ||
|
|
52c1702419 | ||
|
|
3e2a7a48d5 | ||
|
|
a178541561 | ||
|
|
78e1f4daba | ||
|
|
eeea92280c | ||
|
|
0a5256ecbb | ||
|
|
6d2af0ac28 | ||
|
|
056ddd372a | ||
|
|
d0e008adde | ||
|
|
03e219eb43 |
+10
-9
@@ -24,10 +24,14 @@ COPY src/frontend/ ./
|
||||
# Build the frontend
|
||||
RUN npm run build
|
||||
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14.7-slim@sha256:83c1cebb322d099ac9e3a3a532ba74b0146d702838b25e4c75c02fa81ffeb910 AS base
|
||||
# uv is a build-time tool only, so it is mounted into the RUNs that need it rather
|
||||
# than copied into the image. A COPY here would land ~24 MB in a `base` layer that
|
||||
# every published image inherits, and a later `rm` cannot take it back out again --
|
||||
# a RUN adds a layer, it does not rewrite the one underneath.
|
||||
FROM ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 AS uv
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 /uv /uvx /bin/
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14.7-slim@sha256:ce40764625a4ff50df3548277632e7f96c4e77fe75fa848aae9885476e7df5a4 AS base
|
||||
|
||||
# Add build argument for version
|
||||
ARG BUILD_VERSION
|
||||
@@ -111,6 +115,7 @@ WORKDIR /app
|
||||
# Install core Python dependencies first for better layer caching
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=from=uv,source=/uv,target=/usr/local/bin/uv \
|
||||
uv sync --locked --no-default-groups
|
||||
|
||||
# Runtime dependencies are installed into /app/.venv during the build. Remove the
|
||||
@@ -199,6 +204,7 @@ RUN echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-
|
||||
|
||||
# Install the browser automation stack used by the full image
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=from=uv,source=/uv,target=/usr/local/bin/uv \
|
||||
uv sync --locked --no-default-groups --extra browser
|
||||
|
||||
# Deterministically resolve the Xlib namespace collision.
|
||||
@@ -212,13 +218,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
# and force python-xlib 0.33 to own the namespace. pyautogui runs fine against
|
||||
# 0.33 (superset API).
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=from=uv,source=/uv,target=/usr/local/bin/uv \
|
||||
uv pip uninstall --python /app/.venv/bin/python python3-xlib && \
|
||||
uv pip install --python /app/.venv/bin/python --reinstall python-xlib==0.33 && \
|
||||
/app/.venv/bin/python -c "import Xlib.X; assert hasattr(Xlib.X, 'FamilyServerInterpreted'), 'Xlib.X.FamilyServerInterpreted missing after fix'; print('Xlib namespace OK:', Xlib.__version__)"
|
||||
|
||||
# uv is only needed while building the image.
|
||||
RUN rm -f /usr/bin/uv /usr/bin/uvx
|
||||
|
||||
# Keep SeleniumBase's bundled driver cache writable for the fixed non-root user.
|
||||
RUN SELENIUMBASE_DRIVERS_DIR=$(/app/.venv/bin/python -c "import pathlib, seleniumbase; print(pathlib.Path(seleniumbase.__file__).resolve().parent / 'drivers')") && \
|
||||
chown -R 1000:1000 "${SELENIUMBASE_DRIVERS_DIR}" && \
|
||||
@@ -235,7 +239,4 @@ FROM base AS shelfmark-lite
|
||||
|
||||
ENV USING_EXTERNAL_BYPASSER=true
|
||||
|
||||
# uv is only needed while building the image.
|
||||
RUN rm -f /usr/bin/uv /usr/bin/uvx
|
||||
|
||||
CMD ["/app/entrypoint.sh"]
|
||||
|
||||
@@ -247,7 +247,7 @@ Seconds since the last WireGuard handshake before the healthcheck bounces the tu
|
||||
| `CALIBRE_WEB_URL` | Adds a navigation button to your book library (Calibre-Web Automated, Grimmory, etc). | string | _none_ |
|
||||
| `AUDIOBOOK_LIBRARY_URL` | Adds a separate navigation button for your audiobook library (Audiobookshelf, Plex, etc). When both URLs are set, icons are shown instead of text. | string | _none_ |
|
||||
| `SUPPORTED_FORMATS` | Book formats to include in search results. ZIP/RAR archives are extracted automatically and book files are used if found. | string (comma-separated) | `epub,mobi,azw3,fb2,djvu,cbz,cbr` |
|
||||
| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3` |
|
||||
| `SUPPORTED_AUDIOBOOK_FORMATS` | Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found. | string (comma-separated) | `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar` |
|
||||
| `BOOK_LANGUAGE` | Default language filter for searches. | string (comma-separated) | `en` |
|
||||
|
||||
<details>
|
||||
@@ -296,7 +296,7 @@ Book formats to include in search results. ZIP/RAR archives are extracted automa
|
||||
Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `m4b,mp3`
|
||||
- **Default:** `m4b,mp3,m4a,flac,ogg,wma,aac,wav,opus,zip,rar`
|
||||
|
||||
#### `BOOK_LANGUAGE`
|
||||
|
||||
@@ -1304,6 +1304,8 @@ Apply per-indexer seed time and ratio preferences from Prowlarr when sending tor
|
||||
| `NEWZNAB_ENABLED` | Enable searching for books via a Newznab-compatible indexer | boolean | `false` |
|
||||
| `NEWZNAB_URL` | Base URL of your Newznab indexer or aggregator | string | _none_ |
|
||||
| `NEWZNAB_API_KEY` | Your Newznab API key (leave blank if not required) | string (secret) | _none_ |
|
||||
| `NEWZNAB_EBOOK_CATEGORIES` | Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000. | string (comma-separated) | `7000` |
|
||||
| `NEWZNAB_AUDIOBOOK_CATEGORIES` | Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030. | string (comma-separated) | `3030` |
|
||||
| `NEWZNAB_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
|
||||
|
||||
<details>
|
||||
@@ -1337,6 +1339,24 @@ Your Newznab API key (leave blank if not required)
|
||||
- **Type:** string (secret)
|
||||
- **Default:** _none_
|
||||
|
||||
#### `NEWZNAB_EBOOK_CATEGORIES`
|
||||
|
||||
**Ebook Categories**
|
||||
|
||||
Newznab category IDs searched for ebooks. Most indexers use the standard 7000, but some use custom IDs. Leave empty to use 7000.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `7000`
|
||||
|
||||
#### `NEWZNAB_AUDIOBOOK_CATEGORIES`
|
||||
|
||||
**Audiobook Categories**
|
||||
|
||||
Newznab category IDs searched for audiobooks. Most indexers use the standard 3030, but some use custom IDs. Leave empty to use 3030.
|
||||
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** `3030`
|
||||
|
||||
#### `NEWZNAB_AUTO_EXPAND`
|
||||
|
||||
**Auto-expand search on no results**
|
||||
@@ -1421,7 +1441,7 @@ Delay between requests in seconds to avoid rate limiting (0-10).
|
||||
| `IRC_CHANNEL` | Channel name without the # prefix. Used for all searches unless a separate audiobook channel is configured below. | string | _none_ |
|
||||
| `IRC_NICK` | Your IRC nickname (required). Must be unique on the IRC network. | string | _none_ |
|
||||
| `IRC_SEARCH_BOT` | The search bot to address queries to (required). Searches are sent as "@<bot> <query>". | string | _none_ |
|
||||
| `IRC_AUDIOBOOK_CHANNEL` | Optional. Channel name (without the # prefix) to use for audiobook searches. Leave blank to use the main channel above for audiobooks too. | string | _none_ |
|
||||
| `IRC_AUDIOBOOK_CHANNEL` | Optional. Channel name (without the # prefix) for networks that index audiobooks separately, such as Undernet's bookz. Leave blank (the usual setting) to search the main channel above for audiobooks too. | string | _none_ |
|
||||
| `IRC_AUDIOBOOK_SEARCH_BOT` | Optional. Search bot for the audiobook channel. Leave blank to reuse the main search bot above. Only used when an audiobook channel is set. | string | _none_ |
|
||||
| `IRC_CACHE_TTL` | How long to keep cached search results before they expire. | string (choice) | `2592000` |
|
||||
|
||||
@@ -1490,7 +1510,7 @@ The search bot to address queries to (required). Searches are sent as "@<bot> <q
|
||||
|
||||
**Audiobook channel**
|
||||
|
||||
Optional. Channel name (without the # prefix) to use for audiobook searches. Leave blank to use the main channel above for audiobooks too.
|
||||
Optional. Channel name (without the # prefix) for networks that index audiobooks separately, such as Undernet's bookz. Leave blank (the usual setting) to search the main channel above for audiobooks too.
|
||||
|
||||
- **Type:** string
|
||||
- **Default:** _none_
|
||||
@@ -1957,7 +1977,7 @@ Move deletes the job from your usenet client after import; Copy keeps it in the
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `HARDCOVER_ENABLED` | Enable Hardcover as a metadata provider for book searches | boolean | `false` |
|
||||
| `HARDCOVER_API_KEY` | Get your API key from hardcover.app/account/api | string (secret) | _none_ |
|
||||
| `HARDCOVER_API_KEY` | Get your API key from hardcover.app/account/api (starts with hc_pat_) | string (secret) | _none_ |
|
||||
| `HARDCOVER_DEFAULT_SORT` | Default sort order for Hardcover search results. | string (choice) | `relevance` |
|
||||
| `HARDCOVER_EXCLUDE_COMPILATIONS` | Filter out compilations, anthologies, and omnibus editions from search results | boolean | `false` |
|
||||
| `HARDCOVER_EXCLUDE_UNRELEASED` | Filter out books with a release year in the future | boolean | `false` |
|
||||
@@ -1979,7 +1999,7 @@ Enable Hardcover as a metadata provider for book searches
|
||||
|
||||
**API Key**
|
||||
|
||||
Get your API key from hardcover.app/account/api
|
||||
Get your API key from hardcover.app/account/api (starts with hc_pat_)
|
||||
|
||||
- **Type:** string (secret)
|
||||
- **Default:** _none_
|
||||
@@ -2284,6 +2304,7 @@ Override destination based on content type metadata.
|
||||
| `EXT_BYPASSER_URL` | URL of the external bypasser service (e.g., FlareSolverr). | string | `http://flaresolverr:8191` |
|
||||
| `EXT_BYPASSER_PATH` | API path for the external bypasser. | string | `/v1` |
|
||||
| `EXT_BYPASSER_TIMEOUT` | Timeout for external bypasser requests in milliseconds. | number | `60000` |
|
||||
| `BYPASS_BROWSER_IDLE_TIMEOUT` | How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner. | number | `180` |
|
||||
|
||||
<details>
|
||||
<summary>Detailed descriptions</summary>
|
||||
@@ -2339,6 +2360,17 @@ Timeout for external bypasser requests in milliseconds.
|
||||
- **Requires restart:** Yes
|
||||
- **Constraints:** min: 10000, max: 300000
|
||||
|
||||
#### `BYPASS_BROWSER_IDLE_TIMEOUT`
|
||||
|
||||
**Bypasser Idle Timeout (seconds)**
|
||||
|
||||
How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `180`
|
||||
- **Requires restart:** Yes
|
||||
- **Constraints:** min: 30, max: 3600
|
||||
|
||||
</details>
|
||||
|
||||
### Direct Download: Mirrors
|
||||
|
||||
+7
-4
@@ -19,28 +19,31 @@ dependencies = [
|
||||
"psutil",
|
||||
"emoji",
|
||||
"rarfile",
|
||||
"qbittorrent-api>=2026.8.0",
|
||||
"qbittorrent-api>=2026.8.1",
|
||||
"transmission-rpc",
|
||||
"authlib>=1.7.2,<1.8",
|
||||
"apprise>=1.12.0",
|
||||
# HTTP/2 client for RFC 8484 DoH: quad9 rejects HTTP/1.1 outright (505), which
|
||||
# requests cannot speak. See shelfmark/download/doh_wireformat.py.
|
||||
"httpx[http2]>=0.28.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
browser = [
|
||||
"pyvirtualdisplay",
|
||||
"pyautogui",
|
||||
"seleniumbase==4.51.11",
|
||||
"seleniumbase==4.51.12",
|
||||
"python-xlib",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"basedpyright>=1.39.9",
|
||||
"basedpyright>=1.39.10",
|
||||
"prek",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pytest-xdist>=3.8.0",
|
||||
"ruff==0.16.2",
|
||||
"ruff==0.16.3",
|
||||
"vulture>=2.14",
|
||||
]
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
|
||||
|
||||
> [!NOTE]
|
||||
> This project is in a stable state as of May 2026 but is not under active maintenance.
|
||||
> Shelfmark is feature stable and maintained on a best-effort basis. Bug fixes, security updates, and small quality-of-life improvements are still shipped, and pull requests are reviewed — including new features. There is no roadmap for new features for now.
|
||||
|
||||
Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
|
||||
|
||||
@@ -95,6 +95,30 @@ volumes:
|
||||
- Aggregates releases from multiple configured sources
|
||||
- Full audiobook support
|
||||
|
||||
### Hardcover API Key
|
||||
|
||||
Hardcover powers metadata search in Universal mode. Create a token at
|
||||
[hardcover.app/account/api](https://hardcover.app/account/api) — current keys start with `hc_pat_`
|
||||
and are far shorter than the JWTs Hardcover issued before August 2026.
|
||||
|
||||
Tick these seven scopes on the token screen:
|
||||
|
||||
| Scope | Used for |
|
||||
|-------|----------|
|
||||
| `read:catalog` | Metadata search, plus book, edition, author and series lookups |
|
||||
| `read:library` | Your reading status and shelf counts |
|
||||
| `read:lists` | Your lists and the books on them |
|
||||
| `read:me:content` | Test Connection and the "Connected as" label |
|
||||
| `read:users` | Usernames shown alongside lists |
|
||||
| `write:library` | Setting a book's reading status from Shelfmark |
|
||||
| `write:lists` | Adding and removing books from lists, including auto-remove on download |
|
||||
|
||||
The two `write:` scopes matter only if you set reading status from Shelfmark or leave
|
||||
**Auto-Remove from List on Download** enabled (it is on by default) — without them those actions
|
||||
fail silently. Everything else Hardcover offers (journal, goals, reviews, prompts, notifications,
|
||||
account) can stay unticked. The `all` scope works too, but it grants full account access including
|
||||
deletion, so prefer the list above.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
|
||||
@@ -123,7 +147,7 @@ See the full [Environment Variables Reference](docs/environment-variables.md) fo
|
||||
Some of the additional options available in Settings:
|
||||
- **Prowlarr** - Configure indexers and download clients to download books and audiobooks
|
||||
- **Additional audiobook sources** - Configure additional sources for audiobook discovery
|
||||
- **IRC** - Add details for IRC book sources and download directly from the UI
|
||||
- **IRC** - Add details for IRC book sources and download directly from the UI. Most networks serve audiobooks from the same channel as ebooks (on `irc.irchighway.net` that's `#ebooks`, while `#bookz` is effectively inactive), so leave the separate audiobook channel blank unless your network actually indexes one. IRC audiobooks usually arrive as ZIP/RAR archives — keep those enabled under Supported Audiobook Formats or the releases are filtered out of results
|
||||
- **Library Link** - Add a link to your Calibre-Web or Grimmory instance in the UI header
|
||||
- **File processing** - Customiseable download paths, file renaming and directory creation with template-based renaming
|
||||
- **Network Settings** - Custom proxy support (SOCKS5 + HTTP/S) and configurable DNS
|
||||
@@ -238,9 +262,11 @@ These are non-goals, not missing features.
|
||||
|
||||
## Contributing
|
||||
|
||||
Shelfmark's core feature set is complete. Development focuses on stability, bug fixes, quality-of-life improvements, and refining the search experience. Contributions in these areas are welcome, please file issues or submit pull requests on GitHub.
|
||||
Shelfmark's core feature set is complete.
|
||||
|
||||
Feature requests that fall outside the project scope (library integration, automation, collection management) will be closed. If you're unsure whether something fits, open a discussion first.
|
||||
Pull requests are welcome and all of them get reviewed, new features included. If you want a feature, the fastest path is to send a PR for it rather than to file a request.
|
||||
|
||||
Feature requests that fall outside the project scope (library integration, automation, collection management) will be closed, and PRs implementing them won't be merged. If you're unsure whether something fits, open a discussion first.
|
||||
|
||||
## Health Monitoring
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Challenge-page detection shared by the bypassers and the HTTP retry path.
|
||||
|
||||
Kept out of `internal_bypasser` so the HTTP layer can recognise an interstitial
|
||||
without importing SeleniumBase: that module is imported lazily precisely because its
|
||||
browser dependencies are optional, and external-bypasser setups run without them.
|
||||
"""
|
||||
|
||||
# Matched against lowercased text, so every entry must be lowercase.
|
||||
CLOUDFLARE_INDICATORS = [
|
||||
"just a moment",
|
||||
"verify you are human",
|
||||
"verifying you are human",
|
||||
"cloudflare.com/products/turnstile",
|
||||
]
|
||||
|
||||
DDOS_GUARD_INDICATORS = [
|
||||
"ddos-guard",
|
||||
"ddos guard",
|
||||
"checking your browser before accessing",
|
||||
"complete the manual check to continue",
|
||||
"could not verify your browser automatically",
|
||||
]
|
||||
|
||||
# Markers that exist only in raw markup: the bypassers scan rendered innerText, where
|
||||
# a script src or a <title> never appears. The title match is scoped to the tag on
|
||||
# purpose - hosts word the rest of that sentence differently, and matching "checking
|
||||
# your browser" as free text would trip on any page that merely discusses a challenge.
|
||||
_RAW_HTML_MARKERS = (
|
||||
"<title>checking your browser",
|
||||
"/cdn-cgi/challenge-platform",
|
||||
"/.well-known/ddos-guard/",
|
||||
)
|
||||
|
||||
# An interstitial is a few KB of markup. Past that it is a real page that happens to
|
||||
# mention a marker - a protected site links its own DDoS-Guard endpoints on every page.
|
||||
MAX_CHALLENGE_HTML_CHARS = 64 * 1024
|
||||
|
||||
|
||||
def challenge_marker(html: str) -> str | None:
|
||||
"""Return the marker proving `html` is an unsolved challenge page, or None.
|
||||
|
||||
Only meaningful for a response that already carries a challenge status: the
|
||||
markers appear on protected sites' real pages too, so the status is what
|
||||
separates "blocked" from "served".
|
||||
"""
|
||||
if not html or len(html) > MAX_CHALLENGE_HTML_CHARS:
|
||||
return None
|
||||
lowered = html.lower()
|
||||
for marker in (*_RAW_HTML_MARKERS, *DDOS_GUARD_INDICATORS, *CLOUDFLARE_INDICATORS):
|
||||
if marker in lowered:
|
||||
return marker
|
||||
return None
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Clearance cookies won by a bypass, shared by every bypasser implementation.
|
||||
|
||||
Kept in its own module rather than inside a bypasser because both of them feed it and
|
||||
both read from it. The internal bypasser cannot host it: it imports seleniumbase at
|
||||
module scope, which is exactly the dependency an external-bypasser deployment is
|
||||
entitled not to have installed.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Cookie storage - shared with requests library for Cloudflare bypass
|
||||
# Nested mapping of domain to cookie name to cookie metadata.
|
||||
_cf_cookies: dict[str, dict] = {}
|
||||
_cf_cookies_lock = threading.Lock()
|
||||
|
||||
# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
|
||||
_cf_user_agents: dict[str, str] = {}
|
||||
|
||||
# Protection cookie names we care about (Cloudflare and DDoS-Guard)
|
||||
CF_COOKIE_NAMES = {"cf_clearance", "__cf_bm", "cf_chl_2", "cf_chl_prog"}
|
||||
DDG_COOKIE_NAMES = {
|
||||
"__ddg1_",
|
||||
"__ddg2_",
|
||||
"__ddg5_",
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"__ddgid_",
|
||||
"__ddgmark_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
# DDoS-Guard cookies that describe *one* check rather than granting clearance, and so
|
||||
# must never be replayed on a later request. Observed live on Anna's Archive:
|
||||
#
|
||||
# __ddg9_ the client IP address
|
||||
# __ddg10_ the unix timestamp the check was issued
|
||||
# __ddg8_ an opaque token issued with them, same ~40 minute expiry
|
||||
#
|
||||
# Clearance itself lives in __ddg1_/__ddg2_/__ddgid_ (roughly a year) and __ddg5_.
|
||||
# Replaying the trio is actively harmful: once the timestamp ages out - or the egress
|
||||
# IP changes, which happens routinely behind a VPN - the values no longer describe the
|
||||
# caller, DDoS-Guard re-arms its check and answers every request with a ?check=1
|
||||
# redirect. That is the redirect loop, and it is self-inflicted. Dropping them simply
|
||||
# lets DDoS-Guard issue a fresh set, exactly as it does for a browser.
|
||||
DDG_EPHEMERAL_COOKIE_NAMES = {
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
|
||||
def _get_base_domain(domain: str) -> str:
|
||||
"""Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
|
||||
return ".".join(domain.split(".")[-2:]) if "." in domain else domain
|
||||
|
||||
|
||||
def _get_full_cookie_domains() -> set[str]:
|
||||
"""Return mirror domains that need full-session cookie extraction."""
|
||||
from shelfmark.core.mirrors import get_zlib_cookie_domains
|
||||
|
||||
return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()}
|
||||
|
||||
|
||||
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
|
||||
"""Determine if a cookie should be extracted based on its name."""
|
||||
# Checked before extract_all: a per-check token is wrong to replay for every
|
||||
# domain, including the full-session ones.
|
||||
if name in DDG_EPHEMERAL_COOKIE_NAMES:
|
||||
return False
|
||||
if extract_all:
|
||||
return True
|
||||
is_cf = name in CF_COOKIE_NAMES or name.startswith("cf_")
|
||||
is_ddg = name in DDG_COOKIE_NAMES or name.startswith("__ddg")
|
||||
return is_cf or is_ddg
|
||||
|
||||
|
||||
def _cookie_field(cookie: Any, name: str) -> Any:
|
||||
"""Read one field from a cookie in either shape we are handed.
|
||||
|
||||
The internal bypasser extracts CDP cookie objects; an external bypasser returns
|
||||
the same fields as JSON objects, so the difference is attribute versus key access.
|
||||
"""
|
||||
if isinstance(cookie, Mapping):
|
||||
return cookie.get(name)
|
||||
return getattr(cookie, name, None)
|
||||
|
||||
|
||||
def _cookie_expiry(cookie: Any) -> float | None:
|
||||
"""A cookie's absolute expiry, or None when it is a session cookie.
|
||||
|
||||
The two spellings are not interchangeable and both reach this store. CDP and
|
||||
Playwright cookies carry `expires`; the WebDriver cookie object - what a
|
||||
Selenium-based solver such as FlareSolverr returns - carries `expiry`. Reading
|
||||
only one silently turns every cookie from the other into a never-expiring one,
|
||||
which is exactly how dead clearance ends up replayed forever (see
|
||||
get_cf_cookies_for_domain).
|
||||
|
||||
The value is coerced rather than trusted: it arrives as JSON from a service we
|
||||
do not control, and a string here used to raise straight out of the store.
|
||||
"""
|
||||
for field in ("expires", "expiry"):
|
||||
raw = _cookie_field(cookie, field)
|
||||
if raw is None:
|
||||
continue
|
||||
try:
|
||||
expiry = float(raw)
|
||||
except TypeError, ValueError:
|
||||
logger.debug("Unreadable cookie expiry %r; treating as a session cookie", raw)
|
||||
return None
|
||||
# <= 0 is how both shapes spell "session cookie", not "expired in 1970".
|
||||
return expiry if expiry > 0 else None
|
||||
return None
|
||||
|
||||
|
||||
def store_extracted_cookies(
|
||||
*,
|
||||
url: str,
|
||||
cookies: list[Any],
|
||||
user_agent: str | None = None,
|
||||
) -> None:
|
||||
"""Store filtered bypass cookies (and optional UA) for a URL domain."""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.hostname or ""
|
||||
if not domain:
|
||||
return
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
extract_all = base_domain in _get_full_cookie_domains()
|
||||
|
||||
cookies_found: dict[str, dict[str, Any]] = {}
|
||||
for cookie in cookies:
|
||||
name = _cookie_field(cookie, "name") or ""
|
||||
if not _should_extract_cookie(name, extract_all=extract_all):
|
||||
continue
|
||||
secure = _cookie_field(cookie, "secure")
|
||||
cookies_found[name] = {
|
||||
"value": _cookie_field(cookie, "value") or "",
|
||||
"domain": _cookie_field(cookie, "domain") or domain,
|
||||
"path": _cookie_field(cookie, "path") or "/",
|
||||
"expiry": _cookie_expiry(cookie),
|
||||
"secure": True if secure is None else bool(secure),
|
||||
"httpOnly": True,
|
||||
}
|
||||
|
||||
if not cookies_found:
|
||||
return
|
||||
|
||||
with _cf_cookies_lock:
|
||||
_cf_cookies[base_domain] = cookies_found
|
||||
if user_agent:
|
||||
_cf_user_agents[base_domain] = user_agent
|
||||
logger.debug("Stored UA for %s: %s...", base_domain, str(user_agent)[:60])
|
||||
else:
|
||||
logger.debug("No UA captured for %s", base_domain)
|
||||
|
||||
cookie_type = "all" if extract_all else "protection"
|
||||
logger.debug("Extracted %s %s cookies for %s", len(cookies_found), cookie_type, base_domain)
|
||||
|
||||
|
||||
def _is_cookie_expired(cookie: dict[str, Any]) -> bool:
|
||||
"""Whether a stored cookie's expiry has passed. Session cookies never expire here."""
|
||||
expiry = cookie.get("expiry")
|
||||
if expiry is None:
|
||||
expiry = cookie.get("expires")
|
||||
if not expiry or expiry <= 0:
|
||||
return False
|
||||
return time.time() > expiry
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get stored cookies for a domain. Returns empty dict if none available."""
|
||||
if not domain:
|
||||
return {}
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
|
||||
with _cf_cookies_lock:
|
||||
cookies = _cf_cookies.get(base_domain, {})
|
||||
if not cookies:
|
||||
return {}
|
||||
|
||||
cf_clearance = cookies.get("cf_clearance", {})
|
||||
if cf_clearance and _is_cookie_expired(cf_clearance):
|
||||
logger.debug("CF cookies expired for %s", base_domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
return {}
|
||||
|
||||
# Expiry applies to every cookie, not just Cloudflare's. DDoS-Guard domains
|
||||
# have no cf_clearance, so the check above never fired for them and dead
|
||||
# cookies were replayed indefinitely - the server answers those with a
|
||||
# challenge, which is indistinguishable from having sent nothing at all.
|
||||
live = {name: c for name, c in cookies.items() if not _is_cookie_expired(c)}
|
||||
if len(live) != len(cookies):
|
||||
expired = sorted(set(cookies) - set(live))
|
||||
logger.debug("Dropping expired cookies for %s: %s", base_domain, expired)
|
||||
if live:
|
||||
_cf_cookies[base_domain] = live
|
||||
else:
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
|
||||
return {name: c["value"] for name, c in live.items()}
|
||||
|
||||
|
||||
def has_valid_cf_cookies(domain: str) -> bool:
|
||||
"""Check if we have valid Cloudflare cookies for a domain."""
|
||||
return bool(get_cf_cookies_for_domain(domain))
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get the User-Agent that was used during bypass for a domain."""
|
||||
if not domain:
|
||||
return None
|
||||
with _cf_cookies_lock:
|
||||
return _cf_user_agents.get(_get_base_domain(domain))
|
||||
|
||||
|
||||
def export_store() -> tuple[dict[str, dict], dict[str, str]]:
|
||||
"""Snapshot the whole store, for handing to another process.
|
||||
|
||||
The internal bypasser's Docker helper solves in a subprocess, so the clearance it
|
||||
wins has to be serialized back to the parent or the solve is lost with the child.
|
||||
"""
|
||||
with _cf_cookies_lock:
|
||||
return (
|
||||
{domain: dict(cookies) for domain, cookies in _cf_cookies.items()},
|
||||
dict(_cf_user_agents),
|
||||
)
|
||||
|
||||
|
||||
def import_store(cookies: object, user_agents: object) -> None:
|
||||
"""Merge a snapshot produced by :func:`export_store` into this process's store."""
|
||||
with _cf_cookies_lock:
|
||||
if isinstance(cookies, dict):
|
||||
_cf_cookies.update(cookies)
|
||||
if isinstance(user_agents, dict):
|
||||
_cf_user_agents.update(
|
||||
{str(domain): str(agent) for domain, agent in user_agents.items()}
|
||||
)
|
||||
|
||||
|
||||
def clear_cf_cookies(domain: str | None = None) -> None:
|
||||
"""Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
|
||||
with _cf_cookies_lock:
|
||||
if domain:
|
||||
base_domain = _get_base_domain(domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
_cf_user_agents.pop(base_domain, None)
|
||||
else:
|
||||
_cf_cookies.clear()
|
||||
_cf_user_agents.clear()
|
||||
@@ -2,17 +2,19 @@
|
||||
|
||||
import random
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass.cookie_store import store_extracted_cookies
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
from threading import Event
|
||||
|
||||
from shelfmark.download import network
|
||||
@@ -63,6 +65,31 @@ def max_duration_seconds() -> float:
|
||||
return MAX_RETRY * read_timeout + backoff_total
|
||||
|
||||
|
||||
def _store_solution_clearance(target_url: str, solution: Mapping[str, Any]) -> None:
|
||||
"""Keep the clearance the solver won, so later requests do not re-solve.
|
||||
|
||||
A solve is the expensive part of an external bypass - tens of seconds of real
|
||||
browser - and FlareSolverr-compatible services hand back the cookies and the
|
||||
User-Agent that earned it. Dropping them meant every single request paid a 403
|
||||
plus a full solve, and a file download (which the solver cannot proxy, being
|
||||
binary) never presented clearance at all.
|
||||
|
||||
The UA matters as much as the cookies: Cloudflare ties cf_clearance to the UA
|
||||
that solved the challenge, so replaying the cookie under our own UA is rejected.
|
||||
"""
|
||||
cookies = solution.get("cookies") or []
|
||||
if not isinstance(cookies, list):
|
||||
logger.debug("External bypasser returned no usable cookie list for '%s'", target_url)
|
||||
return
|
||||
|
||||
user_agent = solution.get("userAgent")
|
||||
store_extracted_cookies(
|
||||
url=target_url,
|
||||
cookies=cookies,
|
||||
user_agent=user_agent if isinstance(user_agent, str) else None,
|
||||
)
|
||||
|
||||
|
||||
def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
"""Make a single request to the external bypasser service. Returns HTML or None."""
|
||||
raw_bypasser_url = _coerce_config_str(
|
||||
@@ -116,6 +143,15 @@ def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
logger.warning("External bypasser returned empty response for '%s'", target_url)
|
||||
return None
|
||||
|
||||
try:
|
||||
_store_solution_clearance(target_url, solution)
|
||||
except AttributeError, KeyError, TypeError, ValueError:
|
||||
# Storing clearance is an optimisation; the page is the product. The
|
||||
# solution JSON comes from a service we do not control, so a surprise in
|
||||
# its cookie shape must not discard HTML that already cost a ~30s solve
|
||||
# and send the caller round for up to MAX_RETRY more of them.
|
||||
logger.debug("Could not store bypass clearance for '%s'", target_url, exc_info=True)
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning(
|
||||
"External bypasser timed out for '%s' (connect: %ss, read: %.0fs)",
|
||||
|
||||
@@ -5,7 +5,6 @@ import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import stat
|
||||
@@ -28,6 +27,15 @@ from seleniumbase import cdp_driver
|
||||
from seleniumbase.undetected.cdp_driver.connection import ProtocolException
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass.challenge import CLOUDFLARE_INDICATORS, DDOS_GUARD_INDICATORS
|
||||
from shelfmark.bypass.cookie_store import (
|
||||
clear_cf_cookies,
|
||||
export_store,
|
||||
get_cf_cookies_for_domain,
|
||||
get_cf_user_agent_for_domain,
|
||||
import_store,
|
||||
store_extracted_cookies,
|
||||
)
|
||||
from shelfmark.bypass.fingerprint import get_screen_size
|
||||
from shelfmark.config import env
|
||||
from shelfmark.config.env import LOG_DIR
|
||||
@@ -50,26 +58,31 @@ _LOADING_BODY_LENGTH_MAX = 50
|
||||
_PAGE_BODY_PREVIEW_CHARS = 500
|
||||
_BROWSER_START_TIMEOUT_SECONDS = 45.0
|
||||
_BYPASS_SUBPROCESS_TIMEOUT_SECONDS = 420.0
|
||||
# Same budget as the Docker helper process, applied to the in-process CDP path so both
|
||||
# branches of get() are bounded the same way.
|
||||
_IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS
|
||||
# How long a cancelled bypass may take to close its browser before the calling thread
|
||||
# stops waiting for it. Counted on top of the bypass deadline, so every budget below is
|
||||
# set to leave room for it.
|
||||
_CDP_UNWIND_GRACE_SECONDS = 15.0
|
||||
# Same wall-clock budget as the Docker helper process, applied to the in-process CDP path
|
||||
# so both branches of get() are bounded the same way: the deadline plus the unwind grace
|
||||
# comes to _BYPASS_SUBPROCESS_TIMEOUT_SECONDS either way.
|
||||
_IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS - _CDP_UNWIND_GRACE_SECONDS
|
||||
_BYPASS_CHILD_ENV = "SHELFMARK_INTERNAL_BYPASSER_CHILD"
|
||||
|
||||
# Challenge detection indicators
|
||||
CLOUDFLARE_INDICATORS = [
|
||||
"just a moment",
|
||||
"verify you are human",
|
||||
"verifying you are human",
|
||||
"cloudflare.com/products/turnstile",
|
||||
]
|
||||
|
||||
DDOS_GUARD_INDICATORS = [
|
||||
"ddos-guard",
|
||||
"ddos guard",
|
||||
"checking your browser before accessing",
|
||||
"complete the manual check to continue",
|
||||
"could not verify your browser automatically",
|
||||
]
|
||||
# The helper bounds each bypass below the parent's deadline, so it is the side that gives
|
||||
# up first: it still gets to report the timeout and close its browser, and stays available
|
||||
# for the next request. A parent that hit its deadline first could only kill the helper,
|
||||
# throwing away a process the next request would have to start again. The 30s covers the
|
||||
# unwind grace as well, so a helper that times out and closes its browser as slowly as it
|
||||
# is allowed to still answers with 15s to spare.
|
||||
_CHILD_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS - 30.0
|
||||
# The helper publishes its answer by writing the result file the request named, so the
|
||||
# parent waits by watching for that file rather than by reading a stream it would have to
|
||||
# demultiplex from the helper's log output.
|
||||
_HELPER_RESULT_POLL_SECONDS = 0.05
|
||||
# Closing the helper's stdin asks it to shut down; this is how long it may take to finish
|
||||
# what it is doing and exit before its session is killed instead.
|
||||
_HELPER_SHUTDOWN_GRACE_SECONDS = 15.0
|
||||
_HELPER_IDLE_TIMEOUT_DEFAULT = 180.0
|
||||
_PARENT_WATCHDOG_INTERVAL_SECONDS = 5.0
|
||||
|
||||
|
||||
class _DisplayState(TypedDict):
|
||||
@@ -90,8 +103,8 @@ DISPLAY: _DisplayState = {
|
||||
"ffmpeg_output": None,
|
||||
}
|
||||
LOCKED = threading.Lock()
|
||||
_PGREP_PATH = shutil.which("pgrep")
|
||||
_PKILL_PATH = shutil.which("pkill")
|
||||
_PROC_ROOT = Path("/proc")
|
||||
_BROWSER_PROCESS_PATTERNS = ("chrome", "chromium", "Xvfb", "ffmpeg")
|
||||
_RNG = random.SystemRandom()
|
||||
|
||||
_CDP_OPERATION_ERRORS = (
|
||||
@@ -214,14 +227,33 @@ class _CdpWorker:
|
||||
msg = "CDP worker loop failed to start"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
@staticmethod
|
||||
async def _bounded(coro: Any, timeout: float | None) -> Any:
|
||||
"""Run the coroutine under its deadline, on the loop that owns it.
|
||||
|
||||
The deadline has to be enforced from inside the loop rather than by the calling
|
||||
thread: asyncio.wait_for() cancels the bypass and then *waits for it to unwind*,
|
||||
so `finally: await _close_cdp_driver(driver)` has finished by the time this
|
||||
raises. Cancelling from outside returns the moment the cancellation is scheduled,
|
||||
which in a helper serving many requests let the abandoned bypass close its browser
|
||||
while the next one was already opening its own - on the same loop, sharing the
|
||||
DISPLAY globals and one process group.
|
||||
"""
|
||||
if timeout is None:
|
||||
return await coro
|
||||
return await asyncio.wait_for(coro, timeout)
|
||||
|
||||
def run(self, coro: Any, timeout: float | None = None) -> Any:
|
||||
self.start()
|
||||
if not self._loop or self._loop.is_closed():
|
||||
msg = "CDP worker loop not available"
|
||||
raise RuntimeError(msg)
|
||||
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
|
||||
future = asyncio.run_coroutine_threadsafe(self._bounded(coro, timeout), self._loop)
|
||||
# Backstop for an unwind that wedges too: _close_cdp_driver awaits websockets that
|
||||
# a dead browser may never answer, and _bounded cannot outlive its own cleanup.
|
||||
wait_for = None if timeout is None else timeout + _CDP_UNWIND_GRACE_SECONDS
|
||||
try:
|
||||
return future.result(timeout=timeout)
|
||||
return future.result(timeout=wait_for)
|
||||
except TimeoutError:
|
||||
# Otherwise the coroutine keeps running in the worker loop after we stop
|
||||
# waiting, holding the browser and racing the next bypass.
|
||||
@@ -231,96 +263,6 @@ class _CdpWorker:
|
||||
|
||||
_CDP_WORKER = _CdpWorker()
|
||||
|
||||
# Cookie storage - shared with requests library for Cloudflare bypass
|
||||
# Nested mapping of domain to cookie name to cookie metadata.
|
||||
_cf_cookies: dict[str, dict] = {}
|
||||
_cf_cookies_lock = threading.Lock()
|
||||
|
||||
# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
|
||||
_cf_user_agents: dict[str, str] = {}
|
||||
|
||||
# Protection cookie names we care about (Cloudflare and DDoS-Guard)
|
||||
CF_COOKIE_NAMES = {"cf_clearance", "__cf_bm", "cf_chl_2", "cf_chl_prog"}
|
||||
DDG_COOKIE_NAMES = {
|
||||
"__ddg1_",
|
||||
"__ddg2_",
|
||||
"__ddg5_",
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"__ddgid_",
|
||||
"__ddgmark_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
|
||||
def _get_base_domain(domain: str) -> str:
|
||||
"""Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
|
||||
return ".".join(domain.split(".")[-2:]) if "." in domain else domain
|
||||
|
||||
|
||||
def _get_full_cookie_domains() -> set[str]:
|
||||
"""Return mirror domains that need full-session cookie extraction."""
|
||||
from shelfmark.core.mirrors import get_zlib_cookie_domains
|
||||
|
||||
return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()}
|
||||
|
||||
|
||||
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
|
||||
"""Determine if a cookie should be extracted based on its name."""
|
||||
if extract_all:
|
||||
return True
|
||||
is_cf = name in CF_COOKIE_NAMES or name.startswith("cf_")
|
||||
is_ddg = name in DDG_COOKIE_NAMES or name.startswith("__ddg")
|
||||
return is_cf or is_ddg
|
||||
|
||||
|
||||
def _store_extracted_cookies(
|
||||
*,
|
||||
url: str,
|
||||
cookies: list[Any],
|
||||
user_agent: str | None = None,
|
||||
) -> None:
|
||||
"""Store filtered bypass cookies (and optional UA) for a URL domain."""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.hostname or ""
|
||||
if not domain:
|
||||
return
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
extract_all = base_domain in _get_full_cookie_domains()
|
||||
|
||||
cookies_found: dict[str, dict[str, Any]] = {}
|
||||
for cookie in cookies:
|
||||
name = getattr(cookie, "name", "") or ""
|
||||
if not _should_extract_cookie(name, extract_all=extract_all):
|
||||
continue
|
||||
expires = getattr(cookie, "expires", None)
|
||||
if expires is not None and expires <= 0:
|
||||
expires = None
|
||||
cookies_found[name] = {
|
||||
"value": getattr(cookie, "value", ""),
|
||||
"domain": getattr(cookie, "domain", None) or domain,
|
||||
"path": getattr(cookie, "path", None) or "/",
|
||||
"expiry": expires,
|
||||
"secure": bool(getattr(cookie, "secure", True)),
|
||||
"httpOnly": True,
|
||||
}
|
||||
|
||||
if not cookies_found:
|
||||
return
|
||||
|
||||
with _cf_cookies_lock:
|
||||
_cf_cookies[base_domain] = cookies_found
|
||||
if user_agent:
|
||||
_cf_user_agents[base_domain] = user_agent
|
||||
logger.debug("Stored UA for %s: %s...", base_domain, str(user_agent)[:60])
|
||||
else:
|
||||
logger.debug("No UA captured for %s", base_domain)
|
||||
|
||||
cookie_type = "all" if extract_all else "protection"
|
||||
logger.debug("Extracted %s %s cookies for %s", len(cookies_found), cookie_type, base_domain)
|
||||
|
||||
|
||||
async def _extract_cookies_from_cdp(driver: Any, page: Any, url: str) -> None:
|
||||
"""Extract cookies from a CDP browser after successful bypass."""
|
||||
@@ -336,117 +278,116 @@ async def _extract_cookies_from_cdp(driver: Any, page: Any, url: str) -> None:
|
||||
except _CDP_OPERATION_ERRORS:
|
||||
user_agent = None
|
||||
|
||||
_store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
|
||||
store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
|
||||
|
||||
except _CDP_OPERATION_ERRORS as e:
|
||||
logger.debug("Failed to extract cookies: %s", e)
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get stored cookies for a domain. Returns empty dict if none available."""
|
||||
if not domain:
|
||||
return {}
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
|
||||
with _cf_cookies_lock:
|
||||
cookies = _cf_cookies.get(base_domain, {})
|
||||
if not cookies:
|
||||
return {}
|
||||
|
||||
cf_clearance = cookies.get("cf_clearance", {})
|
||||
if cf_clearance:
|
||||
expiry = cf_clearance.get("expiry")
|
||||
if expiry is None:
|
||||
expiry = cf_clearance.get("expires")
|
||||
if expiry and expiry > 0 and time.time() > expiry:
|
||||
logger.debug("CF cookies expired for %s", base_domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
return {}
|
||||
|
||||
return {name: c["value"] for name, c in cookies.items()}
|
||||
def _read_process_cmdline(proc_dir: Path) -> str:
|
||||
"""Return a process's full command line, or "" when it cannot be read."""
|
||||
try:
|
||||
raw = (proc_dir / "cmdline").read_bytes()
|
||||
except OSError:
|
||||
return ""
|
||||
return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def has_valid_cf_cookies(domain: str) -> bool:
|
||||
"""Check if we have valid Cloudflare cookies for a domain."""
|
||||
return bool(get_cf_cookies_for_domain(domain))
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get the User-Agent that was used during bypass for a domain."""
|
||||
if not domain:
|
||||
def _read_process_pgid(proc_dir: Path) -> int | None:
|
||||
"""Return a process's group id from /proc/<pid>/stat, or None when unreadable."""
|
||||
try:
|
||||
stat_line = (proc_dir / "stat").read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return None
|
||||
# Field 2 (comm) is parenthesised and may itself contain spaces and parens, so the
|
||||
# fields are only unambiguous after the last ')': state, ppid, pgrp, ...
|
||||
fields = stat_line.rpartition(")")[2].split()
|
||||
pgrp_index = 2
|
||||
if len(fields) <= pgrp_index:
|
||||
return None
|
||||
try:
|
||||
return int(fields[pgrp_index])
|
||||
except ValueError:
|
||||
return None
|
||||
with _cf_cookies_lock:
|
||||
return _cf_user_agents.get(_get_base_domain(domain))
|
||||
|
||||
|
||||
def clear_cf_cookies(domain: str | None = None) -> None:
|
||||
"""Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
|
||||
with _cf_cookies_lock:
|
||||
if domain:
|
||||
base_domain = _get_base_domain(domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
_cf_user_agents.pop(base_domain, None)
|
||||
else:
|
||||
_cf_cookies.clear()
|
||||
_cf_user_agents.clear()
|
||||
def _find_browser_processes() -> list[tuple[int, int, str]]:
|
||||
"""Return (pid, pgid, cmdline) for every browser-ish process visible in /proc."""
|
||||
found: list[tuple[int, int, str]] = []
|
||||
try:
|
||||
entries = list(_PROC_ROOT.iterdir())
|
||||
except OSError as e:
|
||||
logger.debug("Could not list %s: %s", _PROC_ROOT, e)
|
||||
return found
|
||||
|
||||
for entry in entries:
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
cmdline = _read_process_cmdline(entry)
|
||||
if not cmdline or not any(name in cmdline for name in _BROWSER_PROCESS_PATTERNS):
|
||||
continue
|
||||
pgid = _read_process_pgid(entry)
|
||||
if pgid is None:
|
||||
continue
|
||||
found.append((int(entry.name), pgid, cmdline))
|
||||
return found
|
||||
|
||||
|
||||
def _kill_process(pid: int, cmdline: str) -> bool:
|
||||
"""SIGKILL one process, reporting whether it was actually signalled."""
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except OSError as e:
|
||||
logger.warning("Failed to kill pid %s: %s", pid, e)
|
||||
return False
|
||||
logger.debug("Killed leftover process %s: %s", pid, cmdline[:120])
|
||||
return True
|
||||
|
||||
|
||||
def _cleanup_orphan_processes() -> int:
|
||||
"""Kill orphan Chrome/Xvfb/ffmpeg processes. Only runs in Docker mode."""
|
||||
"""Kill leftover Chrome/Xvfb/ffmpeg processes. Only runs in Docker mode.
|
||||
|
||||
Scoped to this bypass session's process group plus groups whose leader has died.
|
||||
A container-wide sweep (the old `pkill -9 -f chrome`) also matched the browsers a
|
||||
concurrently running bypass was still driving, so with MAX_CONCURRENT_DOWNLOADS > 1
|
||||
every worker that started a solve killed the others' browsers (#1231).
|
||||
"""
|
||||
if not env.DOCKERMODE:
|
||||
return 0
|
||||
|
||||
_stop_ffmpeg_recording()
|
||||
|
||||
processes_to_kill = ["chrome", "chromium", "Xvfb", "ffmpeg"]
|
||||
total_killed = 0
|
||||
|
||||
logger.debug("Checking for orphan processes...")
|
||||
logger.debug("Checking for leftover browser processes...")
|
||||
logger.log_resource_usage()
|
||||
|
||||
if _PGREP_PATH is None or _PKILL_PATH is None:
|
||||
logger.warning("Skipping orphan-process cleanup because pgrep/pkill are unavailable")
|
||||
if not _PROC_ROOT.is_dir():
|
||||
logger.warning("Skipping browser-process cleanup because %s is unavailable", _PROC_ROOT)
|
||||
return 0
|
||||
|
||||
for proc_name in processes_to_kill:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[_PGREP_PATH, "-f", proc_name],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
continue
|
||||
own_pid = os.getpid()
|
||||
own_pgid = os.getpgrp()
|
||||
total_killed = 0
|
||||
|
||||
pids = result.stdout.strip().split("\n")
|
||||
count = len(pids)
|
||||
logger.info("Found %s orphan %s process(es), killing...", count, proc_name)
|
||||
|
||||
kill_result = subprocess.run(
|
||||
[_PKILL_PATH, "-9", "-f", proc_name],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=5,
|
||||
)
|
||||
if kill_result.returncode == 0:
|
||||
total_killed += count
|
||||
else:
|
||||
logger.warning("pkill for %s returned %s", proc_name, kill_result.returncode)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Timeout while checking for %s processes", proc_name)
|
||||
except _SUBPROCESS_OPERATION_ERRORS as e:
|
||||
logger.debug("Error checking for %s processes: %s", proc_name, e)
|
||||
for pid, pgid, cmdline in _find_browser_processes():
|
||||
if pid == own_pid:
|
||||
continue
|
||||
# Another live process group means another bypass session: its browsers are in
|
||||
# use, not orphans. Only our own group and groups whose leader is gone (a helper
|
||||
# that died or was killed, leaving its browser behind) are ours to clean up.
|
||||
if pgid != own_pgid and (_PROC_ROOT / str(pgid)).exists():
|
||||
logger.debug("Leaving pid %s to its live bypass session (pgid %s)", pid, pgid)
|
||||
continue
|
||||
if _kill_process(pid, cmdline):
|
||||
total_killed += 1
|
||||
|
||||
if total_killed > 0:
|
||||
time.sleep(1)
|
||||
logger.info("Cleaned up %s orphan process(es)", total_killed)
|
||||
logger.info("Cleaned up %s leftover browser process(es)", total_killed)
|
||||
logger.log_resource_usage()
|
||||
else:
|
||||
logger.debug("No orphan processes found")
|
||||
logger.debug("No leftover browser processes found")
|
||||
|
||||
return total_killed
|
||||
|
||||
@@ -903,27 +844,25 @@ def _run_bypass_in_current_process(url: str, retry: int, cancel_flag: Event | No
|
||||
if driver:
|
||||
await _close_cdp_driver(driver)
|
||||
|
||||
if os.environ.get(_BYPASS_CHILD_ENV) == "1":
|
||||
return asyncio.run(_run_bypass())
|
||||
# Bound the wait: this path runs in-process (non-Docker installs), holds the module-wide
|
||||
# LOCKED for its whole duration, and neither page.get() nor page.wait() has a timeout of
|
||||
# its own. Without a deadline here a single wedged CDP session blocks every subsequent
|
||||
# bypass in the process forever.
|
||||
return _CDP_WORKER.run(_run_bypass(), timeout=_IN_PROCESS_BYPASS_TIMEOUT_SECONDS)
|
||||
# Bound the wait: this holds the module-wide LOCKED for its whole duration, and neither
|
||||
# page.get() nor page.wait() has a timeout of its own. Without a deadline here a single
|
||||
# wedged CDP session blocks every subsequent bypass in the process forever.
|
||||
#
|
||||
# The helper goes through the worker too, rather than asyncio.run: that owns a loop for
|
||||
# one call and closes it on the way out, so a helper serving many requests would build
|
||||
# and tear down a loop per bypass and would carry no deadline of its own. The worker's
|
||||
# loop lives in a thread, outlives any single bypass, and cancels the coroutine when the
|
||||
# deadline passes.
|
||||
timeout = (
|
||||
_CHILD_BYPASS_TIMEOUT_SECONDS
|
||||
if os.environ.get(_BYPASS_CHILD_ENV) == "1"
|
||||
else _IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
)
|
||||
return _CDP_WORKER.run(_run_bypass(), timeout=timeout)
|
||||
|
||||
|
||||
def _store_child_bypass_state(payload: dict[str, Any]) -> None:
|
||||
cookies = payload.get("cookies")
|
||||
if isinstance(cookies, dict):
|
||||
with _cf_cookies_lock:
|
||||
_cf_cookies.update(cookies)
|
||||
|
||||
user_agents = payload.get("user_agents")
|
||||
if isinstance(user_agents, dict):
|
||||
with _cf_cookies_lock:
|
||||
_cf_user_agents.update(
|
||||
{str(domain): str(agent) for domain, agent in user_agents.items()}
|
||||
)
|
||||
import_store(payload.get("cookies"), payload.get("user_agents"))
|
||||
|
||||
|
||||
def _prepare_child_browser_env(env_vars: dict[str, str]) -> dict[str, str]:
|
||||
@@ -946,6 +885,228 @@ def _prepare_child_browser_env(env_vars: dict[str, str]) -> dict[str, str]:
|
||||
return env_vars
|
||||
|
||||
|
||||
def _terminate_helper_session(proc: subprocess.Popen[str]) -> None:
|
||||
"""Kill the bypass helper and every process it spawned.
|
||||
|
||||
start_new_session makes the helper a session leader, so its pid doubles as the
|
||||
process-group id of the browser tree underneath it and one killpg reaches all of it.
|
||||
"""
|
||||
if hasattr(os, "killpg"):
|
||||
with suppress(OSError):
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
with suppress(OSError):
|
||||
proc.kill()
|
||||
with suppress(OSError, subprocess.SubprocessError):
|
||||
proc.wait(timeout=5)
|
||||
|
||||
|
||||
def _part_path(result_path: Path) -> Path:
|
||||
"""Where the helper stages a result before renaming it into place."""
|
||||
return result_path.with_name(result_path.name + ".part")
|
||||
|
||||
|
||||
class _BypassHelper:
|
||||
"""The helper subprocess that runs the bypasses, kept alive across them.
|
||||
|
||||
Spawning it costs about 4.5 seconds of interpreter start and imports before any work
|
||||
begins, paid on every protected request - and a single search issues several. What it
|
||||
keeps is the process, not the browser: each bypass still starts and closes its own
|
||||
Chrome, so nothing accumulates between requests.
|
||||
|
||||
Protocol: one JSON request per line on stdin, answered by writing the result file that
|
||||
request named. stdout and stderr stay attached to the parent's, so helper logs keep
|
||||
showing up in `docker logs` as before.
|
||||
|
||||
Only one request is ever in flight - get() serializes every bypass behind LOCKED. The
|
||||
lock here is for the idle reaper, which runs on a timer thread.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._proc: subprocess.Popen[str] | None = None
|
||||
self._last_used = 0.0
|
||||
self._idle_timer: threading.Timer | None = None
|
||||
|
||||
def _idle_timeout(self) -> float:
|
||||
return _coerce_non_negative_float(
|
||||
app_config.get("BYPASS_BROWSER_IDLE_TIMEOUT", _HELPER_IDLE_TIMEOUT_DEFAULT),
|
||||
_HELPER_IDLE_TIMEOUT_DEFAULT,
|
||||
)
|
||||
|
||||
def _spawn(self) -> subprocess.Popen[str]:
|
||||
env_vars = os.environ.copy()
|
||||
env_vars[_BYPASS_CHILD_ENV] = "1"
|
||||
env_vars = _prepare_child_browser_env(env_vars)
|
||||
return subprocess.Popen(
|
||||
[sys.executable, "-m", "shelfmark.bypass.internal_bypasser"],
|
||||
stdin=subprocess.PIPE,
|
||||
text=True,
|
||||
env=env_vars,
|
||||
# Give the helper its own session: Chrome, Xvfb and ffmpeg inherit its process
|
||||
# group, which is what lets the cleanup sweep tell this helper's browsers apart
|
||||
# from a concurrent worker's (#1231) and lets us kill the whole tree below.
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
def _running(self) -> subprocess.Popen[str] | None:
|
||||
proc = self._proc
|
||||
if proc is None:
|
||||
return None
|
||||
if proc.poll() is not None or proc.stdin is None or proc.stdin.closed:
|
||||
return None
|
||||
return proc
|
||||
|
||||
def _ensure_running(self) -> subprocess.Popen[str]:
|
||||
proc = self._running()
|
||||
if proc is not None:
|
||||
return proc
|
||||
if self._proc is not None:
|
||||
logger.info("Bypass helper exited (code %s), starting a new one", self._proc.returncode)
|
||||
self._discard()
|
||||
self._proc = self._spawn()
|
||||
return self._proc
|
||||
|
||||
def _discard(self, *, wait_for_exit: bool = True) -> None:
|
||||
"""Stop the helper and forget it.
|
||||
|
||||
`wait_for_exit` belongs to a helper that could still act on the closed pipe: an
|
||||
idle one is sitting in its stdin read, notices EOF and exits on its own. A helper
|
||||
dropped mid-bypass is blocked inside the solve and will not return to that read,
|
||||
so the grace cannot end in anything but the kill below - and the caller waiting it
|
||||
out is a user cancelling a download, holding LOCKED while every other bypass in
|
||||
the worker queues behind them.
|
||||
"""
|
||||
proc = self._proc
|
||||
self._proc = None
|
||||
if proc is None:
|
||||
return
|
||||
|
||||
# Closing stdin ends the helper's request loop, so an idle helper gets to exit on
|
||||
# its own. One mid-bypass cannot answer, and is killed below.
|
||||
with suppress(OSError):
|
||||
if proc.stdin is not None and not proc.stdin.closed:
|
||||
proc.stdin.close()
|
||||
if wait_for_exit:
|
||||
try:
|
||||
proc.wait(timeout=_HELPER_SHUTDOWN_GRACE_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Bypass helper did not exit on request, killing its session")
|
||||
|
||||
# Tear the session down either way: a helper killed mid-bypass leaves its Chrome
|
||||
# and Xvfb running, and those leftovers are what made the next worker's browser
|
||||
# fail to start. Harmless once it has already exited.
|
||||
_terminate_helper_session(proc)
|
||||
|
||||
def _cancel_idle_timer(self) -> None:
|
||||
if self._idle_timer is not None:
|
||||
self._idle_timer.cancel()
|
||||
self._idle_timer = None
|
||||
|
||||
def _arm_idle_timer(self) -> None:
|
||||
self._cancel_idle_timer()
|
||||
timeout = self._idle_timeout()
|
||||
if self._proc is None or timeout <= 0:
|
||||
return
|
||||
timer = threading.Timer(timeout, self._reap_if_idle)
|
||||
timer.daemon = True
|
||||
self._idle_timer = timer
|
||||
timer.start()
|
||||
|
||||
def _reap_if_idle(self) -> None:
|
||||
with self._lock:
|
||||
if self._proc is None:
|
||||
return
|
||||
idle_for = time.monotonic() - self._last_used
|
||||
timeout = self._idle_timeout()
|
||||
if idle_for < timeout:
|
||||
# A bypass started while this timer was waiting for the lock.
|
||||
self._arm_idle_timer()
|
||||
return
|
||||
logger.info("Closing idle bypass helper after %.0fs without work", idle_for)
|
||||
self._discard()
|
||||
|
||||
def run(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
timeout: float,
|
||||
cancel_flag: Event | None,
|
||||
) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
self._cancel_idle_timer()
|
||||
try:
|
||||
return self._exchange(payload, timeout, cancel_flag)
|
||||
finally:
|
||||
self._last_used = time.monotonic()
|
||||
self._arm_idle_timer()
|
||||
|
||||
def _exchange(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
timeout: float,
|
||||
cancel_flag: Event | None,
|
||||
) -> dict[str, Any]:
|
||||
request_line = json.dumps(payload) + "\n"
|
||||
proc = self._ensure_running()
|
||||
try:
|
||||
self._write(proc, request_line)
|
||||
except OSError as exc:
|
||||
# A live helper can die between the liveness check and the write, so one retry
|
||||
# on a fresh process. A fresh one failing here is a real failure.
|
||||
logger.info("Bypass helper closed its pipe (%s), retrying on a new one", exc)
|
||||
# Nothing to ask of a helper we cannot write to: its read end is already gone.
|
||||
self._discard(wait_for_exit=False)
|
||||
proc = self._ensure_running()
|
||||
self._write(proc, request_line)
|
||||
|
||||
return self._await_result(proc, Path(str(payload["result_path"])), timeout, cancel_flag)
|
||||
|
||||
def _write(self, proc: subprocess.Popen[str], request_line: str) -> None:
|
||||
if proc.stdin is None:
|
||||
msg = "Bypass helper has no stdin pipe"
|
||||
raise OSError(msg)
|
||||
proc.stdin.write(request_line)
|
||||
proc.stdin.flush()
|
||||
|
||||
def _await_result(
|
||||
self,
|
||||
proc: subprocess.Popen[str],
|
||||
result_path: Path,
|
||||
timeout: float,
|
||||
cancel_flag: Event | None,
|
||||
) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout
|
||||
try:
|
||||
while not result_path.exists():
|
||||
if proc.poll() is not None:
|
||||
returncode = proc.returncode
|
||||
self._discard(wait_for_exit=False)
|
||||
msg = f"Internal bypasser helper exited without a result (code {returncode})"
|
||||
raise RuntimeError(msg)
|
||||
if cancel_flag is not None and cancel_flag.is_set():
|
||||
# The helper is mid-bypass and cannot be told to stop, so it goes.
|
||||
self._discard(wait_for_exit=False)
|
||||
_check_cancellation(cancel_flag, "Bypass cancelled while waiting for helper")
|
||||
if time.monotonic() >= deadline:
|
||||
self._discard(wait_for_exit=False)
|
||||
msg = "Internal bypasser helper process timed out"
|
||||
raise TimeoutError(msg)
|
||||
time.sleep(_HELPER_RESULT_POLL_SECONDS)
|
||||
|
||||
return json.loads(result_path.read_text(encoding="utf-8"))
|
||||
finally:
|
||||
# Every way out of here is final for this request: either the answer has been
|
||||
# read, or the helper that would have written it has just been killed. Nothing
|
||||
# will write these paths afterwards and nothing will come looking for them, so
|
||||
# they are cleaned on the failure paths too - otherwise every cancelled
|
||||
# download and every wedged solve leaves one behind for the container's life.
|
||||
for path in (result_path, _part_path(result_path)):
|
||||
with suppress(OSError):
|
||||
path.unlink()
|
||||
|
||||
|
||||
_BYPASS_HELPER = _BypassHelper()
|
||||
|
||||
|
||||
def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None) -> str:
|
||||
"""Run the browser bypass in a helper process isolated from gunicorn/gevent."""
|
||||
_check_cancellation(cancel_flag, "Bypass cancelled before helper process")
|
||||
@@ -956,39 +1117,15 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None)
|
||||
# freshly spawned helper would otherwise pre-resolve AA hostnames against the system
|
||||
# resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's
|
||||
# active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host
|
||||
# resolver rules.
|
||||
# resolver rules. Sent with every request, not just at spawn, because a helper outlives
|
||||
# changes the parent makes to its DNS provider.
|
||||
payload = {
|
||||
"url": url,
|
||||
"retry": retry,
|
||||
"result_path": str(result_path),
|
||||
"dns_config": network.get_dns_config(),
|
||||
}
|
||||
env_vars = os.environ.copy()
|
||||
env_vars[_BYPASS_CHILD_ENV] = "1"
|
||||
env_vars = _prepare_child_browser_env(env_vars)
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-m", "shelfmark.bypass.internal_bypasser"],
|
||||
stdin=subprocess.PIPE,
|
||||
text=True,
|
||||
env=env_vars,
|
||||
)
|
||||
try:
|
||||
proc.communicate(json.dumps(payload), timeout=_BYPASS_SUBPROCESS_TIMEOUT_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
msg = "Internal bypasser helper process timed out"
|
||||
raise TimeoutError(msg) from None
|
||||
|
||||
try:
|
||||
result = json.loads(result_path.read_text())
|
||||
except FileNotFoundError as exc:
|
||||
msg = f"Internal bypasser helper exited without a result (code {proc.returncode})"
|
||||
raise RuntimeError(msg) from exc
|
||||
finally:
|
||||
with suppress(OSError):
|
||||
result_path.unlink()
|
||||
result = _BYPASS_HELPER.run(payload, _BYPASS_SUBPROCESS_TIMEOUT_SECONDS, cancel_flag)
|
||||
|
||||
if not isinstance(result, dict):
|
||||
msg = "Internal bypasser helper returned an invalid result"
|
||||
@@ -1263,9 +1400,23 @@ def _try_with_cached_cookies(url: str, hostname: str) -> str | None:
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
logger.debug("Cached cookies worked, skipped Chrome bypass")
|
||||
return response.text
|
||||
logger.debug(
|
||||
"Cached cookies rejected (%s) for %s; discarding them",
|
||||
response.status_code,
|
||||
url,
|
||||
)
|
||||
except _REQUEST_OPERATION_ERRORS as exc:
|
||||
# A redirect loop lands here too: DDoS-Guard answers a dead clearance cookie
|
||||
# with an endless ?check=1 bounce rather than a status we can read.
|
||||
logger.debug("Cached cookie retry failed for %s: %s", url, exc)
|
||||
|
||||
# Reached only when the cached cookies did not produce a page, so they are no
|
||||
# longer clearance. Dropping them now means the imminent Chrome solve starts from
|
||||
# a clean slate and later requests cannot re-present the same rejected cookie.
|
||||
# Guarded because clear_cf_cookies("") means "every host", which would wipe
|
||||
# clearance for sites that are working fine.
|
||||
if hostname:
|
||||
clear_cf_cookies(hostname)
|
||||
return None
|
||||
|
||||
|
||||
@@ -1311,33 +1462,88 @@ def get_bypassed_page(
|
||||
return response_html
|
||||
|
||||
|
||||
def _dns_fingerprint(dns_config: dict[str, Any]) -> tuple[str, tuple[str, ...], bool]:
|
||||
"""Reduce a DNS config to what has to match for two of them to be the same one."""
|
||||
provider = str(dns_config.get("provider") or "").strip().lower()
|
||||
servers = dns_config.get("servers") if provider == "manual" else None
|
||||
server_list = tuple(str(server) for server in servers) if isinstance(servers, list) else ()
|
||||
return (provider, server_list, bool(dns_config.get("doh_enabled")))
|
||||
|
||||
|
||||
def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None:
|
||||
"""Mirror the parent process's active DNS provider in this helper subprocess.
|
||||
|
||||
DNS state is in-memory only, so a fresh helper defaults to system DNS and would
|
||||
pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver
|
||||
that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on
|
||||
the same DoH/custom resolver the parent already validated.
|
||||
DNS state is in-memory only, so a helper left to itself would pre-resolve AA hostnames
|
||||
(for Chrome's --host-resolver-rules) against a resolver that may be blocked or
|
||||
hijacked. Re-applying the parent's provider keeps the helper on the same DoH/custom
|
||||
resolver the parent already validated.
|
||||
|
||||
Compared against what this process is *actually* resolving through, rather than
|
||||
against the last config it happened to be handed. The helper now outlives the request,
|
||||
so it has to be able to travel back to auto as well as away from it - which a user
|
||||
flipping CUSTOM_DNS in settings does live, without a restart - and asking the network
|
||||
module what it is doing beats keeping a second, drifting copy of that answer here.
|
||||
"""
|
||||
provider = str(dns_config.get("provider") or "").strip().lower()
|
||||
# "auto" means the parent has not rotated off system DNS yet, so the helper's own
|
||||
# default initialization already matches it - nothing to override.
|
||||
if not provider or provider == "auto":
|
||||
wanted = _dns_fingerprint(dns_config)
|
||||
provider, servers, use_doh = wanted
|
||||
if not provider:
|
||||
return
|
||||
manual_servers = dns_config.get("servers") if provider == "manual" else None
|
||||
# set_dns_provider() rebuilds the resolvers, so it is worth doing only on a real change.
|
||||
if wanted == _dns_fingerprint(network.get_dns_config()):
|
||||
return
|
||||
|
||||
try:
|
||||
network.set_dns_provider(
|
||||
provider,
|
||||
manual_servers,
|
||||
use_doh=bool(dns_config.get("doh_enabled")),
|
||||
)
|
||||
network.set_dns_provider(provider, list(servers) or None, use_doh=use_doh)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
logger.warning("Could not apply parent DNS config (%s): %s", provider, exc)
|
||||
|
||||
|
||||
def _run_child_process() -> int:
|
||||
"""CLI entrypoint used by the Docker helper subprocess."""
|
||||
request = json.loads(sys.stdin.read() or "{}")
|
||||
def _terminate_own_session() -> None:
|
||||
"""SIGKILL this process and every process it spawned, browser included."""
|
||||
if hasattr(os, "killpg") and os.getpgrp() == os.getpid():
|
||||
with suppress(OSError):
|
||||
os.killpg(os.getpgrp(), signal.SIGKILL)
|
||||
# A thread cannot end the process any other way; sys.exit would only end itself.
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def _watch_parent_process(original_ppid: int, interval: float) -> None:
|
||||
"""Take the browser down with us once the app process that spawned us is gone.
|
||||
|
||||
Cleanup only reclaims process groups whose leader has died, so a helper that outlives
|
||||
its parent (worker restart, OOM kill) would sit there holding a browser that no later
|
||||
bypass is allowed to touch.
|
||||
"""
|
||||
while os.getppid() == original_ppid:
|
||||
time.sleep(interval)
|
||||
logger.warning("Bypass helper lost its parent process; taking the browser down")
|
||||
_terminate_own_session()
|
||||
|
||||
|
||||
def _start_parent_watchdog() -> None:
|
||||
"""Watch the spawning process in the background for the life of this helper."""
|
||||
threading.Thread(
|
||||
target=_watch_parent_process,
|
||||
args=(os.getppid(), _PARENT_WATCHDOG_INTERVAL_SECONDS),
|
||||
daemon=True,
|
||||
name="BypassParentWatchdog",
|
||||
).start()
|
||||
|
||||
|
||||
def _publish_result(result_path: Path, payload: dict[str, Any]) -> None:
|
||||
"""Write the result file atomically.
|
||||
|
||||
The parent decides the request is answered the moment this path exists, so it must
|
||||
never observe a half-written file. Rename within the same directory is atomic.
|
||||
"""
|
||||
tmp_path = _part_path(result_path)
|
||||
tmp_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
tmp_path.replace(result_path)
|
||||
|
||||
|
||||
def _handle_child_request(request_line: str) -> int:
|
||||
"""Answer one request from the parent."""
|
||||
request = json.loads(request_line or "{}")
|
||||
result_path = Path(str(request["result_path"]))
|
||||
url = str(request["url"])
|
||||
retry = _coerce_positive_int(
|
||||
@@ -1348,15 +1554,25 @@ def _run_child_process() -> int:
|
||||
if isinstance(dns_config, dict):
|
||||
_apply_parent_dns_config(dns_config)
|
||||
|
||||
# The parent owns the cookie store; this process only solves. Starting each request
|
||||
# from an empty store is what a helper spawned per request gave for free, and losing
|
||||
# it is what let clearance the parent had deliberately purged for some *other* host
|
||||
# survive here and get merged back over the parent's copy by the export below - the
|
||||
# dead-cookie resurrection that http.py's _redirect_loop_handoff purges to avoid.
|
||||
# Nothing is lost by dropping it: get() below re-checks cached cookies, and the
|
||||
# parent already ran that same check against a store that is a superset of this one.
|
||||
clear_cf_cookies()
|
||||
|
||||
try:
|
||||
html = get(url, retry=retry)
|
||||
cookies, user_agents = export_store()
|
||||
payload = {
|
||||
"ok": True,
|
||||
"html": html,
|
||||
"cookies": _cf_cookies,
|
||||
"user_agents": _cf_user_agents,
|
||||
"cookies": cookies,
|
||||
"user_agents": user_agents,
|
||||
}
|
||||
result_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
_publish_result(result_path, payload)
|
||||
except Exception as exc: # noqa: BLE001 - helper boundary must serialize failures.
|
||||
payload = {
|
||||
"ok": False,
|
||||
@@ -1364,10 +1580,30 @@ def _run_child_process() -> int:
|
||||
"error": str(exc),
|
||||
"traceback": traceback.format_exc(),
|
||||
}
|
||||
result_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
_publish_result(result_path, payload)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _run_child_process() -> int:
|
||||
"""CLI entrypoint used by the Docker helper subprocess.
|
||||
|
||||
Serves one request per line of stdin until the parent closes the pipe, so a burst of
|
||||
protected requests - a single search is several - pays the interpreter start and imports
|
||||
once instead of per request. Each bypass still gets its own browser, closed before the
|
||||
answer is published.
|
||||
"""
|
||||
exit_code = 0
|
||||
for line in sys.stdin:
|
||||
request_line = line.strip()
|
||||
if not request_line:
|
||||
continue
|
||||
exit_code = _handle_child_request(request_line)
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Started here rather than in _run_child_process() so it only ever watches a real
|
||||
# spawned helper, never a test or an embedded call.
|
||||
_start_parent_watchdog()
|
||||
raise SystemExit(_run_child_process())
|
||||
|
||||
@@ -7,7 +7,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Sequence
|
||||
from os import PathLike
|
||||
|
||||
_DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
|
||||
@@ -16,6 +16,13 @@ _DEPRECATED_SETTINGS_RESTRICTION_KEYS = (
|
||||
"RESTRICT_SETTINGS_TO_ADMIN",
|
||||
)
|
||||
|
||||
# The audiobook format list shipped as the default until the format sets were unified.
|
||||
# It only covered m4b/mp3, so FLAC/OPUS/OGG/M4A releases were dropped from search results
|
||||
# and rejected after download - and the wider default alone would never reach existing
|
||||
# installs, because initialize_default_configs() only writes defaults when the config
|
||||
# file does not exist yet.
|
||||
_LEGACY_AUDIOBOOK_FORMATS_DEFAULT = ("m4b", "mp3")
|
||||
|
||||
|
||||
class MigrationLogger(Protocol):
|
||||
"""Logger surface used by config migration helpers."""
|
||||
@@ -57,6 +64,54 @@ def _pick_legacy_settings_restriction(config: dict[str, Any]) -> bool | None:
|
||||
return None
|
||||
|
||||
|
||||
def migrate_audiobook_formats(
|
||||
*,
|
||||
load_general_config: Callable[[], dict[str, Any]],
|
||||
# `object` rather than `None`: the result is discarded, and savers that report
|
||||
# success (settings_registry.save_config_file returns bool) are not assignable to a
|
||||
# `-> None` callable.
|
||||
save_general_config: Callable[[dict[str, Any]], object],
|
||||
widened_formats: Sequence[str],
|
||||
logger: MigrationLogger,
|
||||
) -> None:
|
||||
"""Widen an untouched audiobook format list to the current, fuller default.
|
||||
|
||||
Only a list that still matches the old default exactly is rewritten. Any other value
|
||||
means someone chose it deliberately, and a migration that "helpfully" re-enabled
|
||||
formats a user had turned off would be worse than leaving them on the narrow list.
|
||||
"""
|
||||
try:
|
||||
config = load_general_config()
|
||||
|
||||
if "SUPPORTED_AUDIOBOOK_FORMATS" not in config:
|
||||
# Nothing persisted, so the field default already applies.
|
||||
logger.debug("No persisted audiobook formats - the current default applies")
|
||||
return
|
||||
|
||||
current = config.get("SUPPORTED_AUDIOBOOK_FORMATS")
|
||||
if not isinstance(current, list):
|
||||
return
|
||||
|
||||
normalized = {str(fmt).strip().lower() for fmt in current if str(fmt).strip()}
|
||||
if normalized != set(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT):
|
||||
logger.debug(
|
||||
"Audiobook formats were customized (%s) - left unchanged", sorted(normalized)
|
||||
)
|
||||
return
|
||||
|
||||
save_general_config({"SUPPORTED_AUDIOBOOK_FORMATS": list(widened_formats)})
|
||||
logger.info(
|
||||
"Widened audiobook formats from the legacy default %s to %s",
|
||||
list(_LEGACY_AUDIOBOOK_FORMATS_DEFAULT),
|
||||
list(widened_formats),
|
||||
)
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.debug("No existing general config file found - nothing to migrate")
|
||||
except Exception:
|
||||
logger.exception("Failed to migrate audiobook formats")
|
||||
|
||||
|
||||
def migrate_security_settings(
|
||||
*,
|
||||
load_security_config: Callable[[], dict[str, Any]],
|
||||
|
||||
@@ -14,6 +14,7 @@ from shelfmark.config.download_settings_handlers import (
|
||||
check_books_destination,
|
||||
)
|
||||
from shelfmark.config.email_settings import check_email_connection
|
||||
from shelfmark.config.migrations import migrate_audiobook_formats
|
||||
from shelfmark.core.languages import supported_book_languages
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.settings_registry import (
|
||||
@@ -35,6 +36,7 @@ from shelfmark.core.settings_registry import (
|
||||
register_on_save,
|
||||
register_settings,
|
||||
)
|
||||
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
|
||||
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_DEFAULT = 60
|
||||
_DOWNLOAD_CLIENT_COMPLETED_PATH_TIMEOUT_MAX = 3600
|
||||
@@ -134,6 +136,20 @@ def _on_save_advanced(values: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def migrate_audiobook_format_settings() -> None:
|
||||
"""Bring installs created before the audiobook format sets were unified up to date."""
|
||||
from shelfmark.core.settings_registry import load_config_file, save_config_file
|
||||
|
||||
migrate_audiobook_formats(
|
||||
load_general_config=lambda: load_config_file("general"),
|
||||
save_general_config=lambda values: save_config_file("general", values),
|
||||
widened_formats=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS],
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
|
||||
_SMTP_PORT_MAX = 65535
|
||||
_EMAIL_ATTACHMENT_LIMIT_MB_MAX = 600
|
||||
|
||||
@@ -215,11 +231,7 @@ _FORMAT_OPTIONS = [
|
||||
]
|
||||
|
||||
_AUDIOBOOK_FORMAT_OPTIONS = [
|
||||
{"value": "m4b", "label": "M4B"},
|
||||
{"value": "mp3", "label": "MP3"},
|
||||
{"value": "m4a", "label": "M4A"},
|
||||
{"value": "zip", "label": "ZIP"},
|
||||
{"value": "rar", "label": "RAR"},
|
||||
{"value": fmt, "label": fmt.upper()} for fmt in (*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS)
|
||||
]
|
||||
|
||||
_DOWNLOAD_TO_BROWSER_CONTENT_TYPE_OPTIONS = [
|
||||
@@ -416,7 +428,7 @@ def general_settings() -> list[SettingsField]:
|
||||
label="Supported Audiobook Formats",
|
||||
description="Audiobook formats to include in search results. ZIP/RAR archives are extracted automatically and audiobook files are used if found.",
|
||||
options=_AUDIOBOOK_FORMAT_OPTIONS,
|
||||
default=["m4b", "mp3"],
|
||||
default=[*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS],
|
||||
),
|
||||
MultiSelectField(
|
||||
key="BOOK_LANGUAGE",
|
||||
@@ -1643,6 +1655,19 @@ def cloudflare_bypass_settings() -> list[SettingsField]:
|
||||
requires_restart=True,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
|
||||
),
|
||||
NumberField(
|
||||
key="BYPASS_BROWSER_IDLE_TIMEOUT",
|
||||
label="Bypasser Idle Timeout (seconds)",
|
||||
description=(
|
||||
"How long the bypass helper process may sit unused before it is shut down. "
|
||||
"Higher keeps more searches fast, lower frees memory sooner."
|
||||
),
|
||||
default=180,
|
||||
min_value=30,
|
||||
max_value=3600,
|
||||
requires_restart=True,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": False},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ def upsert_cwa_user(
|
||||
email=normalized_email,
|
||||
role=role,
|
||||
allow_email_link=True,
|
||||
sync_username=True,
|
||||
collision_strategy=collision_strategy,
|
||||
alias_suffix=_CWA_ALIAS_SUFFIX,
|
||||
context=context,
|
||||
|
||||
@@ -108,6 +108,7 @@ def _build_updates(
|
||||
auth_source: str,
|
||||
role: str,
|
||||
sync_role: bool,
|
||||
username: str | object,
|
||||
email: str | None | object,
|
||||
display_name: str | None | object,
|
||||
subject_field: str | None,
|
||||
@@ -116,6 +117,8 @@ def _build_updates(
|
||||
updates: dict[str, Any] = {"auth_source": auth_source}
|
||||
if sync_role:
|
||||
updates["role"] = _normalize_role(role)
|
||||
if username is not UNSET:
|
||||
updates["username"] = _normalize_username(username)
|
||||
if email is not UNSET:
|
||||
updates["email"] = _normalize_email(email)
|
||||
if display_name is not UNSET:
|
||||
@@ -125,10 +128,17 @@ def _build_updates(
|
||||
return updates
|
||||
|
||||
|
||||
def _next_suffix_username(user_db: UserDB, base_username: str) -> str:
|
||||
def _next_suffix_username(
|
||||
user_db: UserDB,
|
||||
base_username: str,
|
||||
*,
|
||||
exclude_user_id: int | None = None,
|
||||
) -> str:
|
||||
candidate = base_username
|
||||
suffix = 1
|
||||
while user_db.get_user(username=candidate):
|
||||
while existing := user_db.get_user(username=candidate):
|
||||
if exclude_user_id is not None and int(existing.get("id") or 0) == exclude_user_id:
|
||||
return candidate
|
||||
candidate = f"{base_username}_{suffix}"
|
||||
suffix += 1
|
||||
return candidate
|
||||
@@ -185,6 +195,38 @@ def _resolve_create_username(
|
||||
return _next_suffix_username(user_db, alias_base), None, "username_collision_alias"
|
||||
|
||||
|
||||
def _resolve_update_username(
|
||||
user_db: UserDB,
|
||||
*,
|
||||
current_user: dict[str, Any],
|
||||
requested_username: str,
|
||||
strategy: CollisionStrategy,
|
||||
alias_suffix: str,
|
||||
) -> str:
|
||||
current_user_id = int(current_user["id"])
|
||||
existing = user_db.get_user(username=requested_username)
|
||||
if existing is None or int(existing.get("id") or 0) == current_user_id:
|
||||
return requested_username
|
||||
|
||||
if strategy == "suffix":
|
||||
return _next_suffix_username(
|
||||
user_db,
|
||||
requested_username,
|
||||
exclude_user_id=current_user_id,
|
||||
)
|
||||
if strategy == "alias":
|
||||
return _next_suffix_username(
|
||||
user_db,
|
||||
f"{requested_username}{alias_suffix}",
|
||||
exclude_user_id=current_user_id,
|
||||
)
|
||||
|
||||
# `takeover` can select an existing row during creation, but once an
|
||||
# identity is already matched it must never replace a different username
|
||||
# owner. Preserve the matched row's current collision-free name instead.
|
||||
return str(current_user["username"])
|
||||
|
||||
|
||||
def upsert_external_user(
|
||||
user_db: UserDB,
|
||||
*,
|
||||
@@ -197,6 +239,7 @@ def upsert_external_user(
|
||||
subject: str | None = None,
|
||||
allow_email_link: bool = False,
|
||||
sync_role: bool = True,
|
||||
sync_username: bool = False,
|
||||
allow_create: bool = True,
|
||||
collision_strategy: CollisionStrategy = "takeover",
|
||||
alias_suffix: str | None = None,
|
||||
@@ -229,10 +272,26 @@ def upsert_external_user(
|
||||
subject=subject,
|
||||
allow_email_link=allow_email_link,
|
||||
)
|
||||
resolved_alias_suffix = alias_suffix or f"__{auth_source}"
|
||||
update_username: str | object = UNSET
|
||||
if (
|
||||
matched is not None
|
||||
and sync_username
|
||||
and normalize_auth_source(matched.get("auth_source"), matched.get("oidc_subject"))
|
||||
== auth_source
|
||||
):
|
||||
update_username = _resolve_update_username(
|
||||
user_db,
|
||||
current_user=matched,
|
||||
requested_username=normalized_username,
|
||||
strategy=collision_strategy,
|
||||
alias_suffix=resolved_alias_suffix,
|
||||
)
|
||||
updates = _build_updates(
|
||||
auth_source=auth_source,
|
||||
role=normalized_role,
|
||||
sync_role=sync_role,
|
||||
username=update_username,
|
||||
email=normalized_email if email is not UNSET else UNSET,
|
||||
display_name=normalized_display_name if display_name is not UNSET else UNSET,
|
||||
subject_field=subject_field,
|
||||
@@ -261,7 +320,6 @@ def upsert_external_user(
|
||||
)
|
||||
return None, "not_found"
|
||||
|
||||
resolved_alias_suffix = alias_suffix or f"__{auth_source}"
|
||||
create_username, takeover_target, create_reason = _resolve_create_username(
|
||||
user_db,
|
||||
auth_source=auth_source,
|
||||
|
||||
@@ -344,6 +344,7 @@ class UserDB:
|
||||
|
||||
_ALLOWED_UPDATE_COLUMNS: ClassVar[frozenset[str]] = frozenset(
|
||||
{
|
||||
"username",
|
||||
"email",
|
||||
"display_name",
|
||||
"password_hash",
|
||||
@@ -353,6 +354,7 @@ class UserDB:
|
||||
}
|
||||
)
|
||||
_USER_UPDATE_STATEMENTS: ClassVar[dict[str, str]] = {
|
||||
"username": "UPDATE users SET username = ? WHERE id = ?",
|
||||
"email": "UPDATE users SET email = ? WHERE id = ?",
|
||||
"display_name": "UPDATE users SET display_name = ? WHERE id = ?",
|
||||
"password_hash": "UPDATE users SET password_hash = ? WHERE id = ?",
|
||||
|
||||
@@ -115,6 +115,21 @@ def is_audiobook(content_type: str | None) -> bool:
|
||||
return bool(content_type and "audiobook" in content_type.lower())
|
||||
|
||||
|
||||
# Every audio format an audiobook can legitimately arrive in, and the single source of
|
||||
# truth for that list. The settings UI, release-source parsing, archive extraction and
|
||||
# post-download scanning all derive from it, so a format added here becomes selectable,
|
||||
# searchable AND downloadable at once. These used to be four hand-maintained copies that
|
||||
# had drifted apart: the settings UI only offered m4b/mp3/m4a, which meant a FLAC
|
||||
# audiobook could never be enabled, was silently dropped from every search result, and
|
||||
# was rejected after download as "format not supported".
|
||||
AUDIOBOOK_FORMATS = ("m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus")
|
||||
|
||||
# Multi-file audiobooks are almost always distributed as an archive. These are containers
|
||||
# rather than formats: they are what a *release* looks like, and the formats above are
|
||||
# what comes out of one after extraction.
|
||||
ARCHIVE_FORMATS = ("zip", "rar")
|
||||
|
||||
|
||||
CONTENT_TYPES = [
|
||||
"book (fiction)",
|
||||
"book (non-fiction)",
|
||||
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.fs import atomic_move
|
||||
from shelfmark.download.postprocess.policy import (
|
||||
@@ -98,7 +99,7 @@ ALL_EBOOK_EXTENSIONS = {
|
||||
}
|
||||
|
||||
# All known audio extensions (superset of what user might enable for audiobooks)
|
||||
ALL_AUDIO_EXTENSIONS = {".m4b", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".wma", ".wav", ".opus"}
|
||||
ALL_AUDIO_EXTENSIONS = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS}
|
||||
|
||||
|
||||
def _filter_files(
|
||||
|
||||
@@ -843,6 +843,9 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
expected_hash=request.expected_hash,
|
||||
seeding_time_limit=request.seeding_time_limit,
|
||||
ratio_limit=request.ratio_limit,
|
||||
# rTorrent has no category concept, so its audiobook label
|
||||
# can only be chosen from the content type (#1235).
|
||||
content_type=task.content_type,
|
||||
)
|
||||
except Exception as e:
|
||||
if not refresh_attempted:
|
||||
|
||||
@@ -11,7 +11,12 @@ from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import get_hardened_xmlrpc_client
|
||||
from shelfmark.core.utils import (
|
||||
get_hardened_xmlrpc_client,
|
||||
)
|
||||
from shelfmark.core.utils import (
|
||||
is_audiobook as check_audiobook,
|
||||
)
|
||||
from shelfmark.download.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
@@ -173,7 +178,8 @@ class RTorrentClient(DownloadClient):
|
||||
|
||||
commands = []
|
||||
|
||||
is_audiobook = kwargs.get("content_type") == "audiobook"
|
||||
content_type = kwargs.get("content_type")
|
||||
is_audiobook = check_audiobook(content_type if isinstance(content_type, str) else None)
|
||||
default_label = (
|
||||
self._audiobook_label if is_audiobook and self._audiobook_label else self._label
|
||||
)
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""RFC 8484 DNS wireformat encoding/decoding for DoH providers.
|
||||
|
||||
Providers split into two incompatible camps and the difference is not cosmetic:
|
||||
|
||||
* **JSON** (Cloudflare, Google) - ``?name=<host>&type=A`` returning a JSON body. A
|
||||
convention, not a standard, and the only one Shelfmark used to speak.
|
||||
* **Wireformat** (Quad9, OpenDNS) - RFC 8484 proper: a base64url-encoded DNS message
|
||||
in ``?dns=``, answered with ``application/dns-message``. Quad9 additionally
|
||||
*requires HTTP/2* per RFC 8484 section 5.2 and answers HTTP/1.1 with 505.
|
||||
|
||||
This module carries the codec only; the transport choice lives in the resolver.
|
||||
Encoding a query is a handful of bytes, and parsing an answer needs message
|
||||
compression support (RFC 1035 section 4.1.4) because answer names are almost always
|
||||
pointers back into the question.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import secrets
|
||||
import struct
|
||||
|
||||
# Record types we resolve.
|
||||
TYPE_A = 1
|
||||
TYPE_AAAA = 28
|
||||
|
||||
_CLASS_IN = 1
|
||||
_HEADER = struct.Struct(">HHHHHH")
|
||||
_RR_FIXED = struct.Struct(">HHIH") # type, class, ttl, rdlength
|
||||
_FLAG_RECURSION_DESIRED = 0x0100
|
||||
_MAX_LABEL_JUMPS = 64 # cap pointer-following so a malicious answer cannot loop
|
||||
_MAX_NAME_LENGTH = 255
|
||||
|
||||
|
||||
class WireformatError(ValueError):
|
||||
"""Raised when a DNS wireformat message cannot be parsed."""
|
||||
|
||||
|
||||
def encode_query(hostname: str, record_type: int) -> bytes:
|
||||
"""Build a DNS query message for ``hostname``.
|
||||
|
||||
The ID is zero because RFC 8484 section 4.1 requires it for cacheability, but the
|
||||
caller may randomise it when not using a cache.
|
||||
"""
|
||||
if not hostname:
|
||||
msg = "hostname must not be empty"
|
||||
raise WireformatError(msg)
|
||||
|
||||
question = bytearray()
|
||||
for label in hostname.rstrip(".").split("."):
|
||||
encoded = label.encode("idna") if not label.isascii() else label.encode("ascii")
|
||||
if not encoded or len(encoded) > 63:
|
||||
msg = f"invalid DNS label in {hostname!r}"
|
||||
raise WireformatError(msg)
|
||||
question.append(len(encoded))
|
||||
question.extend(encoded)
|
||||
question.append(0)
|
||||
question.extend(struct.pack(">HH", record_type, _CLASS_IN))
|
||||
|
||||
header = _HEADER.pack(0, _FLAG_RECURSION_DESIRED, 1, 0, 0, 0)
|
||||
return header + bytes(question)
|
||||
|
||||
|
||||
def encode_query_param(hostname: str, record_type: int) -> str:
|
||||
"""Return the base64url ``dns=`` parameter value for a query (padding stripped)."""
|
||||
return base64.urlsafe_b64encode(encode_query(hostname, record_type)).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def _read_name(message: bytes, offset: int) -> int:
|
||||
"""Skip over a (possibly compressed) name, returning the offset after it."""
|
||||
jumps = 0
|
||||
length = 0
|
||||
while True:
|
||||
if offset >= len(message):
|
||||
msg = "truncated DNS name"
|
||||
raise WireformatError(msg)
|
||||
label_len = message[offset]
|
||||
if label_len == 0:
|
||||
return offset + 1
|
||||
if label_len & 0xC0 == 0xC0:
|
||||
# A pointer ends this name; the rest of the record follows the 2 bytes.
|
||||
if offset + 1 >= len(message):
|
||||
msg = "truncated DNS name pointer"
|
||||
raise WireformatError(msg)
|
||||
return offset + 2
|
||||
offset += 1 + label_len
|
||||
length += 1 + label_len
|
||||
jumps += 1
|
||||
if jumps > _MAX_LABEL_JUMPS or length > _MAX_NAME_LENGTH:
|
||||
msg = "malformed DNS name"
|
||||
raise WireformatError(msg)
|
||||
|
||||
|
||||
def decode_answer(message: bytes, record_type: int) -> list[str]:
|
||||
"""Extract the IP addresses of ``record_type`` from a DNS response message.
|
||||
|
||||
Returns an empty list for a well-formed response that carries no matching record
|
||||
(NXDOMAIN, or only CNAMEs), and raises WireformatError for a malformed one - the
|
||||
caller treats those differently.
|
||||
"""
|
||||
if len(message) < _HEADER.size:
|
||||
msg = "DNS response shorter than its header"
|
||||
raise WireformatError(msg)
|
||||
|
||||
_id, _flags, qdcount, ancount, _ns, _ar = _HEADER.unpack_from(message, 0)
|
||||
offset = _HEADER.size
|
||||
|
||||
for _ in range(qdcount):
|
||||
offset = _read_name(message, offset)
|
||||
offset += 4 # QTYPE + QCLASS
|
||||
|
||||
results: list[str] = []
|
||||
for _ in range(ancount):
|
||||
offset = _read_name(message, offset)
|
||||
if offset + _RR_FIXED.size > len(message):
|
||||
msg = "truncated resource record"
|
||||
raise WireformatError(msg)
|
||||
rtype, rclass, _ttl, rdlength = _RR_FIXED.unpack_from(message, offset)
|
||||
offset += _RR_FIXED.size
|
||||
rdata = message[offset : offset + rdlength]
|
||||
if len(rdata) != rdlength:
|
||||
msg = "truncated record data"
|
||||
raise WireformatError(msg)
|
||||
offset += rdlength
|
||||
|
||||
if rclass != _CLASS_IN or rtype != record_type:
|
||||
continue
|
||||
if rtype == TYPE_A and rdlength == 4:
|
||||
results.append(".".join(str(b) for b in rdata))
|
||||
elif rtype == TYPE_AAAA and rdlength == 16:
|
||||
groups = struct.unpack(">8H", rdata)
|
||||
results.append(_compress_ipv6(groups))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _compress_ipv6(groups: tuple[int, ...]) -> str:
|
||||
"""Render an IPv6 address with the longest zero run collapsed to '::'."""
|
||||
best_start = best_len = -1
|
||||
run_start = -1
|
||||
for i, group in enumerate([*list(groups), 1]): # sentinel closes a trailing run
|
||||
if group == 0 and i < len(groups):
|
||||
if run_start < 0:
|
||||
run_start = i
|
||||
elif run_start >= 0:
|
||||
if i - run_start > best_len:
|
||||
best_start, best_len = run_start, i - run_start
|
||||
run_start = -1
|
||||
|
||||
parts = [format(g, "x") for g in groups]
|
||||
if best_len > 1:
|
||||
return ":".join(parts[:best_start]) + "::" + ":".join(parts[best_start + best_len :])
|
||||
return ":".join(parts)
|
||||
|
||||
|
||||
def random_query_id() -> int:
|
||||
"""A random DNS message ID, for callers that do not want the RFC 8484 zero."""
|
||||
return secrets.randbelow(0x10000)
|
||||
+232
-48
@@ -10,7 +10,8 @@ from urllib.parse import urljoin, urlparse
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass import BypassCancelledError, cookie_store
|
||||
from shelfmark.bypass.challenge import challenge_marker
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int
|
||||
@@ -145,19 +146,13 @@ def get_bypassed_page(
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get CF cookies - only available with internal bypasser."""
|
||||
if _is_using_external_bypasser():
|
||||
logger.debug("External bypasser in use, CF cookies not available for %s", domain)
|
||||
return {}
|
||||
return _get_internal_bypasser().get_cf_cookies_for_domain(domain)
|
||||
"""Get the clearance cookies won by whichever bypasser solved this domain."""
|
||||
return cookie_store.get_cf_cookies_for_domain(domain)
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get CF user agent - only available with internal bypasser."""
|
||||
if _is_using_external_bypasser():
|
||||
logger.debug("External bypasser in use, CF user agent not available for %s", domain)
|
||||
return None
|
||||
return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
|
||||
"""Get the User-Agent that solved this domain's challenge, if one is stored."""
|
||||
return cookie_store.get_cf_user_agent_for_domain(domain)
|
||||
|
||||
|
||||
def _apply_cf_bypass(url: str, headers: dict) -> dict:
|
||||
@@ -234,13 +229,66 @@ def _is_retryable_error(e: Exception) -> bool:
|
||||
return status is not None and status in RETRYABLE_CODES
|
||||
|
||||
|
||||
# Statuses that mean the host is gone rather than busy: 410 Gone and 451 Unavailable
|
||||
# For Legal Reasons are what a seized domain answers with.
|
||||
_DEAD_MIRROR_CODES = (410, 451)
|
||||
|
||||
|
||||
def _response_challenge_marker(response: requests.Response) -> str | None:
|
||||
"""The challenge marker in a response body, or None if it carries no challenge.
|
||||
|
||||
Content type is checked first so a JSON or octet-stream error body is never
|
||||
decoded just to be scanned; a missing header is scanned anyway, since an
|
||||
interstitial served without one is still an interstitial.
|
||||
"""
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if content_type and "html" not in content_type.lower():
|
||||
return None
|
||||
try:
|
||||
return challenge_marker(response.text)
|
||||
except UnicodeDecodeError, ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _fatal_mirror_reason(e: Exception) -> str | None:
|
||||
"""Return why ``e`` proves the mirror is unusable, or None if it may recover.
|
||||
|
||||
Hard evidence only - the name does not resolve, nothing is listening, or the host
|
||||
says it is gone for good. A timeout, a 5xx or a challenge all mean the mirror is
|
||||
alive, and rotating off it discards the bypass clearance held for that domain.
|
||||
"""
|
||||
status = _get_status_code(e)
|
||||
if status is not None and status in _DEAD_MIRROR_CODES:
|
||||
return f"HTTP {status}"
|
||||
|
||||
# requests wraps the real cause; a read timeout subclasses ConnectionError for
|
||||
# some adapters, so exclude timeouts explicitly before inspecting the message.
|
||||
if isinstance(e, requests.exceptions.Timeout):
|
||||
return None
|
||||
if not isinstance(e, requests.exceptions.ConnectionError):
|
||||
return None
|
||||
|
||||
text = str(e).lower()
|
||||
if "nameresolutionerror" in text or "failed to resolve" in text or "name or service" in text:
|
||||
return "DNS does not resolve"
|
||||
if "connection refused" in text or "no route to host" in text:
|
||||
return "connection refused"
|
||||
return None
|
||||
|
||||
|
||||
def _try_rotation(
|
||||
original_url: str, current_url: str, selector: network.AAMirrorSelector
|
||||
original_url: str,
|
||||
current_url: str,
|
||||
selector: network.AAMirrorSelector,
|
||||
*,
|
||||
fatal_reason: str | None = None,
|
||||
) -> str | None:
|
||||
"""Try mirror/DNS rotation. Returns new URL or None."""
|
||||
aa_base_url = network.get_aa_base_url()
|
||||
if aa_base_url and current_url.startswith(aa_base_url):
|
||||
new_base, action = selector.next_mirror_or_rotate_dns()
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
fatal=fatal_reason is not None, reason=fatal_reason or ""
|
||||
)
|
||||
if action in ("mirror", "dns") and new_base:
|
||||
new_url = selector.rewrite(original_url)
|
||||
logger.info("[%s] switching to: %s", action, new_url)
|
||||
@@ -272,8 +320,11 @@ def html_get_page(
|
||||
selector: Mirror selector used for AA mirror and DNS rotation.
|
||||
cancel_flag: Optional event used to abort retries early.
|
||||
status_callback: Optional callback for UI status updates.
|
||||
allow_bypasser_fallback: If False, 403 errors will trigger mirror rotation
|
||||
instead of switching to the bypasser. Use for search operations.
|
||||
allow_bypasser_fallback: Whether a challenge may be handed to the bypasser.
|
||||
If False, a 403 triggers mirror rotation instead, and an AA redirect loop
|
||||
gives up immediately rather than waiting on a browser solve. Use False for
|
||||
best-effort fetches whose result is optional (e.g. the download count on
|
||||
the details modal); search and detail pages pass True.
|
||||
use_bypasser: Whether to start with the bypasser instead of direct HTTP.
|
||||
include_response_url: If True, return `(html, final_url)` to expose the
|
||||
resolved response URL after redirects.
|
||||
@@ -287,6 +338,72 @@ def html_get_page(
|
||||
return html, response_url
|
||||
return html
|
||||
|
||||
def _run_bypasser(bypass_url: str) -> str | tuple[str, str]:
|
||||
"""Run the active bypasser for one URL and return its result.
|
||||
|
||||
Factored out so the redirect-loop handoff below can invoke it directly. That
|
||||
call site sits inside the inner redirect `while`, so it cannot reach the
|
||||
retry-loop branch above with `continue`, and with MAX_RETRY=1 there is no
|
||||
later attempt for that branch to run on either.
|
||||
"""
|
||||
if status_callback:
|
||||
status_callback("resolving", "Bypassing protection...")
|
||||
try:
|
||||
# A bypass is one long blocking call with no incremental progress, so
|
||||
# tell the orchestrator up front how long it may legitimately take
|
||||
# instead of trying to fake activity while it runs. Inside the try so a
|
||||
# bypasser that fails to load is still reported as a bypasser error.
|
||||
request_activity_grace(status_callback, _bypass_grace_seconds())
|
||||
result = get_bypassed_page(bypass_url, selector, cancel_flag)
|
||||
return _result(result or "", bypass_url)
|
||||
except _BYPASSER_ERRORS as e:
|
||||
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
|
||||
# Surface the real reason. Without this the caller only sees an empty
|
||||
# page and the download dies with a generic failure, hiding e.g. a
|
||||
# FlareSolverr 500 behind a silent wait.
|
||||
if status_callback and not isinstance(e, BypassCancelledError):
|
||||
try:
|
||||
status_callback("error", f"Bypass failed: {type(e).__name__}: {e}")
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Bypass error status callback failed", exc_info=True)
|
||||
return _result("", bypass_url)
|
||||
finally:
|
||||
release_activity_grace(status_callback)
|
||||
|
||||
def _bypass_handoff_allowed() -> bool:
|
||||
"""Whether a challenge on the current URL may be handed to the bypasser.
|
||||
|
||||
allow_bypasser_fallback is honoured for the same reason the 403 path honours it:
|
||||
callers such as the /dyn/md5/summary fetch behind the details modal pass False
|
||||
precisely so a best-effort request fails fast instead of holding the UI open for
|
||||
a minutes-long browser solve.
|
||||
"""
|
||||
return allow_bypasser_fallback and _is_cf_bypass_enabled() and not use_bypasser_now
|
||||
|
||||
def _purge_clearance(target_url: str) -> None:
|
||||
"""Drop the host's stored clearance cookies.
|
||||
|
||||
Called whenever the protection answered a request that *carried* cookies:
|
||||
being challenged while presenting them proves they no longer work, so keeping
|
||||
them only guarantees the same rejection on every later request. Applies to
|
||||
either bypasser, since both fill the same store.
|
||||
"""
|
||||
hostname = urlparse(target_url).hostname or ""
|
||||
# An empty domain means "clear every host" to the store, so skip the purge
|
||||
# rather than wipe clearance for sites that are working fine.
|
||||
if hostname:
|
||||
cookie_store.clear_cf_cookies(hostname)
|
||||
|
||||
def _redirect_loop_handoff(bypass_url: str) -> str | tuple[str, str]:
|
||||
"""Drop the host's stale clearance cookies, then bypass `bypass_url`.
|
||||
|
||||
A `?check=1` loop is how DDoS-Guard answers a clearance cookie that has gone
|
||||
stale, so the dead cookie has to go before the solve — otherwise it is merged
|
||||
back over the fresh one on the next request and the loop simply resumes.
|
||||
"""
|
||||
_purge_clearance(bypass_url)
|
||||
return _run_bypasser(bypass_url)
|
||||
|
||||
configured_retry = normalize_positive_int(app_config.MAX_RETRY)
|
||||
retry_limit = (
|
||||
retry if retry is not None else (configured_retry if configured_retry is not None else 1)
|
||||
@@ -308,29 +425,7 @@ def html_get_page(
|
||||
cookies: dict[str, str] = {}
|
||||
try:
|
||||
if use_bypasser_now and _is_cf_bypass_enabled():
|
||||
if status_callback:
|
||||
status_callback("resolving", "Bypassing protection...")
|
||||
try:
|
||||
# A bypass is one long blocking call with no incremental progress, so
|
||||
# tell the orchestrator up front how long it may legitimately take
|
||||
# instead of trying to fake activity while it runs. Inside the try so a
|
||||
# bypasser that fails to load is still reported as a bypasser error.
|
||||
request_activity_grace(status_callback, _bypass_grace_seconds())
|
||||
result = get_bypassed_page(current_url, selector, cancel_flag)
|
||||
return _result(result or "", current_url)
|
||||
except _BYPASSER_ERRORS as e:
|
||||
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
|
||||
# Surface the real reason. Without this the caller only sees an empty
|
||||
# page and the download dies with a generic failure, hiding e.g. a
|
||||
# FlareSolverr 500 behind a silent wait.
|
||||
if status_callback and not isinstance(e, BypassCancelledError):
|
||||
try:
|
||||
status_callback("error", f"Bypass failed: {type(e).__name__}: {e}")
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Bypass error status callback failed", exc_info=True)
|
||||
return _result("", current_url)
|
||||
finally:
|
||||
release_activity_grace(status_callback)
|
||||
return _run_bypasser(current_url)
|
||||
|
||||
logger.debug("GET: %s", current_url)
|
||||
|
||||
@@ -377,6 +472,35 @@ def html_get_page(
|
||||
)
|
||||
continue
|
||||
|
||||
# A 503 still serving a challenge is protection, not a busy origin. The
|
||||
# handshake above has nothing left to echo back, and 503 is in
|
||||
# RETRYABLE_CODES, so without this the request spends every attempt on
|
||||
# the same wall: the bypasser is only ever reached from the 403 branch
|
||||
# and the AA redirect rescues. Gate on the body, not the status, so a
|
||||
# genuine overloaded-origin 503 keeps its retry path.
|
||||
if response.status_code == _HTTP_STATUS_SERVICE_UNAVAILABLE:
|
||||
marker = _response_challenge_marker(response)
|
||||
if marker and _bypass_handoff_allowed():
|
||||
if cookies:
|
||||
# Challenged while presenting clearance means those cookies
|
||||
# are dead; same reasoning as the 403 branch below.
|
||||
logger.debug(
|
||||
"503 challenge with cookies presented; purging: %s", current_url
|
||||
)
|
||||
_purge_clearance(current_url)
|
||||
logger.info(
|
||||
"503 challenge detected (%s); switching to bypasser: %s",
|
||||
marker,
|
||||
current_url,
|
||||
)
|
||||
return _run_bypasser(current_url)
|
||||
if marker:
|
||||
logger.debug(
|
||||
"503 challenge (%s) but no bypasser handoff available: %s",
|
||||
marker,
|
||||
current_url,
|
||||
)
|
||||
|
||||
if is_aa_url and response.is_redirect:
|
||||
location = response.headers.get("Location", "")
|
||||
if not location:
|
||||
@@ -420,9 +544,35 @@ def html_get_page(
|
||||
return _result("", current_url)
|
||||
|
||||
# Same-host redirect (relative or absolute) - follow manually.
|
||||
# DDoS-Guard gates AA /search behind a cookie probe: the 302 to
|
||||
# ?check=1 carries Set-Cookie (__ddg*) which must be echoed back on
|
||||
# the next hop, or the server just re-issues the redirect forever.
|
||||
issued = _new_cookies(response, handshake_cookies)
|
||||
if issued:
|
||||
handshake_cookies.update(issued)
|
||||
redirects_followed += 1
|
||||
if redirects_followed > _MAX_REDIRECTS:
|
||||
_raise_too_many_redirects(f"Too many redirects for {current_url}")
|
||||
# A same-host redirect loop on AA is not a network fault — it is
|
||||
# how DDoS-Guard presents a handshake that is unsolved, or whose
|
||||
# clearance cookie has gone stale: /search redirects to
|
||||
# /search&check=1, which redirects back, indefinitely. Hand it
|
||||
# straight to the bypasser rather than raising, which would send it
|
||||
# down the retry path to re-run the whole loop on every attempt
|
||||
# (10 x 6 = ~60 requests to AA) without ever offering the URL to the
|
||||
# bypasser. `continue` is no use here either — it would target this
|
||||
# inner redirect loop rather than the retry branch below.
|
||||
if _bypass_handoff_allowed():
|
||||
logger.info(
|
||||
"Redirect loop detected; switching to bypasser: %s", current_url
|
||||
)
|
||||
return _redirect_loop_handoff(current_url)
|
||||
# No bypasser to hand it to. Every AA mirror shares the challenge,
|
||||
# so rotating only collects another loop — give up now instead of
|
||||
# raising and burning the same ~60 requests over the retry budget.
|
||||
logger.warning(
|
||||
"Redirect loop and no bypasser available, giving up: %s", current_url
|
||||
)
|
||||
return _result("", current_url)
|
||||
current_url = redirect_url
|
||||
continue
|
||||
|
||||
@@ -434,6 +584,21 @@ def html_get_page(
|
||||
except Exception as e:
|
||||
status = _get_status_code(e)
|
||||
|
||||
# The same DDoS-Guard rescue, for the loops the manual AA follower above hands
|
||||
# back rather than resolving inline — an AA redirect missing its Location
|
||||
# header. TooManyRedirects carries no status, so the 403 rescue below never
|
||||
# fires and every retry would re-send the dead cookies. Scoped to the hosts
|
||||
# whose redirects we follow manually: elsewhere `requests` follows them itself,
|
||||
# and a loop there is an ordinary misconfiguration that a cookie purge and a
|
||||
# minutes-long browser solve would be the wrong answer to.
|
||||
if (
|
||||
isinstance(e, requests.exceptions.TooManyRedirects)
|
||||
and network.should_rotate_dns_for_url(current_url)
|
||||
and _bypass_handoff_allowed()
|
||||
):
|
||||
logger.info("Redirect loop detected; switching to bypasser: %s", current_url)
|
||||
return _redirect_loop_handoff(current_url)
|
||||
|
||||
# 403 = Cloudflare/DDoS-Guard protection
|
||||
if status == _HTTP_STATUS_FORBIDDEN:
|
||||
# If bypasser fallback is disabled, try mirrors instead
|
||||
@@ -450,18 +615,32 @@ def html_get_page(
|
||||
# (another concurrent download may have completed bypass and extracted cookies)
|
||||
parsed = urlparse(current_url)
|
||||
fresh_cookies = get_cf_cookies_for_domain(parsed.hostname or "")
|
||||
if fresh_cookies and not cookies:
|
||||
# Cookies are now available - retry with cookies before using bypasser
|
||||
if fresh_cookies and not cookies and attempt < retry_limit:
|
||||
# Cookies are now available - retry with cookies before using bypasser.
|
||||
# Guarded on there being a next attempt: `continue` on the last one
|
||||
# ends the retry loop and abandons the request without ever offering
|
||||
# the URL to the bypasser, and MAX_RETRY=1 is the supported setting.
|
||||
# Same reasoning as the bypasser invocation below.
|
||||
logger.debug(
|
||||
"403 but cookies now available - retrying with cookies: %s",
|
||||
current_url,
|
||||
)
|
||||
continue
|
||||
if cookies:
|
||||
# Challenged *while presenting* clearance: those cookies are
|
||||
# dead. Without this they survive the solve and get merged back
|
||||
# over the fresh ones, so every later request re-presents a
|
||||
# known-rejected cookie and is challenged again - the stale
|
||||
# retry that never ends.
|
||||
logger.debug("403 with cookies presented; purging: %s", current_url)
|
||||
_purge_clearance(current_url)
|
||||
logger.info("403 detected; switching to bypasser: %s", current_url)
|
||||
if status_callback:
|
||||
status_callback("resolving", "Bypassing protection...")
|
||||
use_bypasser_now = True
|
||||
continue
|
||||
# Invoke it here rather than setting use_bypasser_now and continuing.
|
||||
# The branch that acts on that flag runs at the top of the *next* retry
|
||||
# attempt, so under the supported MAX_RETRY=1 there is no next attempt
|
||||
# and the bypasser was never reached — a 403 simply ended the search.
|
||||
# Same reasoning as the redirect-loop handoffs.
|
||||
return _run_bypasser(current_url)
|
||||
logger.warning("403 error, giving up: %s", current_url)
|
||||
return _result("", current_url)
|
||||
|
||||
@@ -470,9 +649,14 @@ def html_get_page(
|
||||
logger.warning("404 error: %s", current_url)
|
||||
return _result("", current_url)
|
||||
|
||||
# Try mirror/DNS rotation on retryable errors
|
||||
if _is_retryable_error(e):
|
||||
new_url = _try_rotation(original_url, current_url, selector)
|
||||
# Try mirror/DNS rotation on retryable errors. A failure that proves the
|
||||
# mirror is unusable also drops it from this process's rotation, so the
|
||||
# next search does not pay for it again.
|
||||
fatal_reason = _fatal_mirror_reason(e)
|
||||
if fatal_reason or _is_retryable_error(e):
|
||||
new_url = _try_rotation(
|
||||
original_url, current_url, selector, fatal_reason=fatal_reason
|
||||
)
|
||||
if new_url:
|
||||
current_url = new_url
|
||||
handshake_cookies.clear()
|
||||
|
||||
+220
-54
@@ -11,6 +11,7 @@ from socket import AddressFamily, SocketKind
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import dns.resolver
|
||||
import httpx
|
||||
import requests
|
||||
from dns.exception import DNSException
|
||||
|
||||
@@ -277,6 +278,14 @@ _current_aa_url_index = 0
|
||||
_aa_urls: list[str] = [] # Initialized lazily in _initialize_aa_state()
|
||||
_aa_base_url: str = "" # Current active AA URL
|
||||
|
||||
# Mirrors quarantined for this process: domains that are not a working AA mirror at
|
||||
# all (NXDOMAIN, refused, or a 200 that isn't AA - seized/parked/for-sale domains all
|
||||
# land here). Kept separate from ordinary failures: a 403 challenge or a 5xx means the
|
||||
# mirror is alive and rotating away from it only discards the DDoS-Guard clearance we
|
||||
# hold for it. Deliberately in-memory only, so a restart re-probes everything.
|
||||
_dead_aa_urls: set[str] = set()
|
||||
_dead_aa_urls_lock = _RLock()
|
||||
|
||||
|
||||
def _ensure_initialized() -> None:
|
||||
"""Lazy guard so runtime setup happens once and late calls still work."""
|
||||
@@ -298,6 +307,24 @@ DNS_PROVIDERS = [
|
||||
("opendns", ["208.67.222.222", "208.67.220.220"], "https://doh.opendns.com/dns-query"),
|
||||
]
|
||||
|
||||
# httpx raises its own hierarchy, which shares no base class with requests', so a
|
||||
# wireformat failure would escape a requests-only except clause.
|
||||
_DOH_REQUEST_ERRORS = (OSError, ValueError, requests.RequestException, httpx.HTTPError)
|
||||
|
||||
|
||||
def _first_proxy(proxies: dict[str, str] | None) -> str | None:
|
||||
"""Pick a single proxy URL from a requests-style mapping, for httpx."""
|
||||
if not proxies:
|
||||
return None
|
||||
return proxies.get("https") or proxies.get("http") or None
|
||||
|
||||
|
||||
# DoH providers that speak RFC 8484 wireformat rather than the (non-standard) JSON API
|
||||
# Cloudflare and Google popularised. Verified against the live services: both reject a
|
||||
# ?name=&type= query outright - Quad9 with 505 (it also mandates HTTP/2 per RFC 8484
|
||||
# section 5.2, which requests cannot speak), OpenDNS with 400 "No valid query received".
|
||||
_DOH_WIREFORMAT_HOSTS = frozenset({"dns.quad9.net", "doh.opendns.com"})
|
||||
|
||||
# Domain patterns that should trigger DNS rotation on failure
|
||||
DNS_ROTATION_DOMAINS = [
|
||||
"annas-archive",
|
||||
@@ -462,8 +489,16 @@ class DoHResolver:
|
||||
# DNS cache: {(hostname, record_type): (ip_list, timestamp)}
|
||||
self._cache: dict[tuple[str, str], tuple[list[str], datetime]] = {}
|
||||
|
||||
# Different headers based on provider
|
||||
if "google" in self.base_url:
|
||||
# RFC 8484 providers get a separate transport: they need wireformat, and Quad9
|
||||
# additionally refuses HTTP/1.1, which requests has no way to upgrade from.
|
||||
self.use_wireformat = urllib.parse.urlparse(self.base_url).hostname in (
|
||||
_DOH_WIREFORMAT_HOSTS
|
||||
)
|
||||
self._http2_client: Any | None = None
|
||||
|
||||
if self.use_wireformat:
|
||||
self.session.headers.update({"Accept": "application/dns-message"})
|
||||
elif "google" in self.base_url:
|
||||
self.session.headers.update(
|
||||
{
|
||||
"Accept": "application/json",
|
||||
@@ -476,6 +511,35 @@ class DoHResolver:
|
||||
}
|
||||
)
|
||||
|
||||
def _get_http2_client(self) -> Any:
|
||||
"""Lazily build the HTTP/2 client used for RFC 8484 providers.
|
||||
|
||||
Built on first use so a resolver pointed at a JSON provider never opens an
|
||||
HTTP/2 connection pool it will not use.
|
||||
"""
|
||||
if self._http2_client is None:
|
||||
self._http2_client = httpx.Client(
|
||||
http2=True,
|
||||
timeout=10,
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
proxy=_first_proxy(get_proxies(self.base_url)),
|
||||
)
|
||||
return self._http2_client
|
||||
|
||||
def _resolve_wireformat(self, hostname: str, record_type: str) -> list[str]:
|
||||
"""Resolve via RFC 8484: base64url query in, DNS message out."""
|
||||
from shelfmark.download import doh_wireformat
|
||||
|
||||
qtype = doh_wireformat.TYPE_AAAA if record_type == "AAAA" else doh_wireformat.TYPE_A
|
||||
param = doh_wireformat.encode_query_param(hostname, qtype)
|
||||
response = self._get_http2_client().get(
|
||||
self.base_url,
|
||||
params={"dns": param},
|
||||
headers={"Accept": "application/dns-message"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return doh_wireformat.decode_answer(response.content, qtype)
|
||||
|
||||
def _get_cached(self, hostname: str, record_type: str) -> list[str] | None:
|
||||
"""Get cached DNS result if still valid."""
|
||||
key = (hostname, record_type)
|
||||
@@ -525,34 +589,37 @@ class DoHResolver:
|
||||
return cached
|
||||
|
||||
try:
|
||||
params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"}
|
||||
if self.use_wireformat:
|
||||
answers = self._resolve_wireformat(hostname, record_type)
|
||||
else:
|
||||
params = {"name": hostname, "type": "AAAA" if record_type == "AAAA" else "A"}
|
||||
|
||||
response = self.session.get(
|
||||
self.base_url,
|
||||
params=params,
|
||||
proxies=get_proxies(self.base_url),
|
||||
timeout=10, # Increased from 5s to handle slow network conditions
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
)
|
||||
response.raise_for_status()
|
||||
response = self.session.get(
|
||||
self.base_url,
|
||||
params=params,
|
||||
proxies=get_proxies(self.base_url),
|
||||
timeout=10, # Increased from 5s to handle slow network conditions
|
||||
verify=get_ssl_verify(self.base_url),
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
if "Answer" not in data:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, data)
|
||||
return []
|
||||
data = response.json()
|
||||
if "Answer" not in data:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, data)
|
||||
return []
|
||||
|
||||
# Extract IP addresses from the response
|
||||
answers = [
|
||||
answer["data"]
|
||||
for answer in data["Answer"]
|
||||
if answer.get("type") == (28 if record_type == "AAAA" else 1)
|
||||
]
|
||||
# Extract IP addresses from the response
|
||||
answers = [
|
||||
answer["data"]
|
||||
for answer in data["Answer"]
|
||||
if answer.get("type") == (28 if record_type == "AAAA" else 1)
|
||||
]
|
||||
|
||||
# Cache the result
|
||||
self._set_cached(hostname, record_type, answers)
|
||||
|
||||
# Don't log here - the caller (custom_getaddrinfo) will log the final result
|
||||
except (OSError, ValueError, requests.RequestException) as e:
|
||||
except _DOH_REQUEST_ERRORS as e:
|
||||
logger.warning("DoH resolution failed for %s: %s", hostname, e)
|
||||
return []
|
||||
else:
|
||||
@@ -616,8 +683,6 @@ def create_custom_getaddrinfo(
|
||||
source: str,
|
||||
provider_label: str,
|
||||
res: Sequence[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]],
|
||||
*,
|
||||
is_bypass: bool = False,
|
||||
) -> None:
|
||||
"""Emit a unified resolver log with the IPs returned.
|
||||
|
||||
@@ -625,7 +690,6 @@ def create_custom_getaddrinfo(
|
||||
source: Description of resolver source
|
||||
provider_label: Label for the DNS provider
|
||||
res: Resolution results
|
||||
is_bypass: If True, log at DEBUG level (for local/IP addresses)
|
||||
|
||||
"""
|
||||
# Skip logging entirely for localhost to reduce noise
|
||||
@@ -641,11 +705,7 @@ def create_custom_getaddrinfo(
|
||||
ip = sockaddr[0]
|
||||
if isinstance(ip, str):
|
||||
ips.append(ip)
|
||||
msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}"
|
||||
if is_bypass:
|
||||
logger.debug(msg)
|
||||
else:
|
||||
logger.info(msg)
|
||||
logger.debug("Resolved %s via %s [%s]: %s", host_str, source, provider_label, ips)
|
||||
|
||||
# Skip custom resolution for IP addresses, local addresses, or if skip check passes
|
||||
if (
|
||||
@@ -655,7 +715,7 @@ def create_custom_getaddrinfo(
|
||||
):
|
||||
# Quietly bypass custom resolution for IP/local targets
|
||||
res = original_getaddrinfo(host, port, family, socket_type, proto, flags)
|
||||
_log_results("system resolver (bypass)", "system", res, is_bypass=True)
|
||||
_log_results("system resolver (bypass)", "system", res)
|
||||
return res
|
||||
|
||||
results: list[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]] = []
|
||||
@@ -1015,14 +1075,18 @@ def rotate_dns_and_reset_aa() -> bool:
|
||||
configured_url = _get_configured_aa_url()
|
||||
|
||||
if configured_url == "auto":
|
||||
# Auto mode always resets to the first mirror to restart the cascade
|
||||
_current_aa_url_index = 0
|
||||
if _aa_urls:
|
||||
_aa_base_url = _aa_urls[0]
|
||||
# Auto mode always resets to the first mirror to restart the cascade. Skip any
|
||||
# quarantined ones: a new DNS provider cannot revive a parked or seized domain.
|
||||
with _dead_aa_urls_lock:
|
||||
restart_urls = [url for url in _aa_urls if url not in _dead_aa_urls] or _aa_urls
|
||||
if restart_urls:
|
||||
_aa_base_url = restart_urls[0]
|
||||
_current_aa_url_index = _aa_urls.index(_aa_base_url)
|
||||
logger.info("After DNS switch, resetting AA URL to: %s", _aa_base_url)
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
else:
|
||||
_aa_base_url = ""
|
||||
_current_aa_url_index = 0
|
||||
logger.info("After DNS switch, AA URL remains unconfigured")
|
||||
else:
|
||||
# Keep the user's configured primary mirror (if it exists in the list),
|
||||
@@ -1192,8 +1256,17 @@ def _initialize_aa_state() -> None:
|
||||
global _aa_base_url, _current_aa_url_index, _aa_urls
|
||||
|
||||
# Build URL list from config
|
||||
previous_urls = _aa_urls
|
||||
_aa_urls = _build_aa_urls()
|
||||
|
||||
# Drop quarantine decisions only when the mirror list itself changed - they were
|
||||
# made about a list that no longer applies. This runs on every re-init (settings
|
||||
# sync, DNS rotation, helper subprocess startup), and clearing unconditionally
|
||||
# would resurrect a parked mirror mid-session.
|
||||
if previous_urls != _aa_urls:
|
||||
with _dead_aa_urls_lock:
|
||||
_dead_aa_urls.clear()
|
||||
|
||||
# Get configured base URL from config
|
||||
configured_url = _get_configured_aa_url()
|
||||
|
||||
@@ -1209,26 +1282,34 @@ def _initialize_aa_state() -> None:
|
||||
return
|
||||
|
||||
if configured_url == "auto":
|
||||
if state.get("aa_base_url") and state["aa_base_url"] in _aa_urls:
|
||||
_current_aa_url_index = _aa_urls.index(state["aa_base_url"])
|
||||
_aa_base_url = state["aa_base_url"]
|
||||
# Never restore or probe a mirror quarantined this session: re-init happens
|
||||
# often, and re-electing a parked domain costs a wasted request every time
|
||||
# (its parking page answers 200, so the probe would happily pick it).
|
||||
with _dead_aa_urls_lock:
|
||||
candidates = [url for url in _aa_urls if url not in _dead_aa_urls]
|
||||
restored = state.get("aa_base_url")
|
||||
if restored and restored in candidates:
|
||||
_current_aa_url_index = _aa_urls.index(restored)
|
||||
_aa_base_url = restored
|
||||
else:
|
||||
logger.debug("AA_BASE_URL: auto, checking available urls %s", _aa_urls)
|
||||
for i, url in enumerate(_aa_urls):
|
||||
logger.debug("AA_BASE_URL: auto, checking available urls %s", candidates)
|
||||
for url in candidates:
|
||||
try:
|
||||
response = requests.get(
|
||||
url, proxies=get_proxies(url), timeout=3, verify=get_ssl_verify(url)
|
||||
)
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
_current_aa_url_index = i
|
||||
_current_aa_url_index = _aa_urls.index(url)
|
||||
_aa_base_url = url
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
break
|
||||
except (OSError, requests.RequestException) as exc:
|
||||
logger.debug("Could not reach AA mirror candidate %s: %s", url, exc)
|
||||
if not _aa_base_url or _aa_base_url == "auto":
|
||||
_aa_base_url = _aa_urls[0]
|
||||
_current_aa_url_index = 0
|
||||
# Also covers the case where every probe failed and the previous base is
|
||||
# itself quarantined - keeping it would aim the next search at a dead host.
|
||||
if not _aa_base_url or _aa_base_url == "auto" or _aa_base_url not in candidates:
|
||||
_aa_base_url = (candidates or _aa_urls)[0]
|
||||
_current_aa_url_index = _aa_urls.index(_aa_base_url)
|
||||
elif configured_url not in _aa_urls:
|
||||
logger.info("AA_BASE_URL set to custom value %s; skipping auto-switch", configured_url)
|
||||
_aa_base_url = configured_url
|
||||
@@ -1326,24 +1407,77 @@ def is_aa_auto_mode() -> bool:
|
||||
|
||||
|
||||
def get_available_aa_urls() -> list[str]:
|
||||
"""Get list of configured AA URLs (copy)."""
|
||||
"""Get configured AA URLs (copy), minus any quarantined this process.
|
||||
|
||||
Falls back to the full list when every mirror has been quarantined: a wrong
|
||||
classification must not leave the app with nowhere to search.
|
||||
"""
|
||||
_ensure_initialized()
|
||||
return _aa_urls.copy()
|
||||
with _dead_aa_urls_lock:
|
||||
alive = [url for url in _aa_urls if url not in _dead_aa_urls]
|
||||
if not alive and _aa_urls:
|
||||
logger.warning("All AA mirrors quarantined; retrying the full list")
|
||||
_dead_aa_urls.clear()
|
||||
return _aa_urls.copy()
|
||||
return alive
|
||||
|
||||
|
||||
def set_aa_url_index(new_index: int) -> bool:
|
||||
"""Set AA base URL by index in available list; returns True if applied."""
|
||||
def _aa_base_for_url(url: str) -> str:
|
||||
"""Return the configured mirror base that ``url`` belongs to, if any."""
|
||||
for base in _aa_urls:
|
||||
if base and url.startswith(base):
|
||||
return base
|
||||
return ""
|
||||
|
||||
|
||||
def mark_aa_url_dead(url: str, reason: str) -> bool:
|
||||
"""Quarantine an AA mirror for the rest of this process.
|
||||
|
||||
Only for hard evidence that the host is not a working AA mirror. Transient
|
||||
failures (403 challenge, 429, 5xx, timeouts) must never come through here -
|
||||
quarantining a live mirror throws away its bypass clearance.
|
||||
"""
|
||||
_ensure_initialized()
|
||||
base = _aa_base_for_url(url) or url
|
||||
with _dead_aa_urls_lock:
|
||||
if base not in _aa_urls or base in _dead_aa_urls:
|
||||
return False
|
||||
# Keep at least one mirror in play, even if it is the failing one.
|
||||
if len([u for u in _aa_urls if u not in _dead_aa_urls]) <= 1:
|
||||
logger.warning("Not quarantining last remaining AA mirror %s (%s)", base, reason)
|
||||
return False
|
||||
_dead_aa_urls.add(base)
|
||||
logger.warning("Quarantined AA mirror %s for this session: %s", base, reason)
|
||||
return True
|
||||
|
||||
|
||||
def get_dead_aa_urls() -> set[str]:
|
||||
"""Return the mirrors quarantined this process (copy)."""
|
||||
with _dead_aa_urls_lock:
|
||||
return set(_dead_aa_urls)
|
||||
|
||||
|
||||
def set_aa_url(url: str) -> bool:
|
||||
"""Set the active AA base URL; returns True if applied."""
|
||||
_ensure_initialized()
|
||||
global _aa_base_url, _current_aa_url_index
|
||||
if new_index < 0 or new_index >= len(_aa_urls):
|
||||
if url not in _aa_urls:
|
||||
return False
|
||||
_current_aa_url_index = new_index
|
||||
_aa_base_url = _aa_urls[_current_aa_url_index]
|
||||
_current_aa_url_index = _aa_urls.index(url)
|
||||
_aa_base_url = url
|
||||
logger.info("Set AA URL to: %s", _aa_base_url)
|
||||
_save_state(aa_url=_aa_base_url)
|
||||
return True
|
||||
|
||||
|
||||
def set_aa_url_index(new_index: int) -> bool:
|
||||
"""Set AA base URL by index in the full configured list; True if applied."""
|
||||
_ensure_initialized()
|
||||
if new_index < 0 or new_index >= len(_aa_urls):
|
||||
return False
|
||||
return set_aa_url(_aa_urls[new_index])
|
||||
|
||||
|
||||
class AAMirrorSelector:
|
||||
"""Keep AA mirror switching consistent across call sites.
|
||||
|
||||
@@ -1357,6 +1491,10 @@ class AAMirrorSelector:
|
||||
def _ensure_fresh_state(self, *, reset_attempts: bool = False) -> None:
|
||||
_ensure_initialized()
|
||||
self.aa_urls = get_available_aa_urls()
|
||||
# Rotation walks the live mirrors, but rewriting has to recognise every
|
||||
# configured base: a URL built before a mirror was quarantined still points at
|
||||
# it, and failing to rewrite would send the retry back to the dead host.
|
||||
self.all_aa_urls = _aa_urls.copy()
|
||||
self._index = self._safe_index(get_aa_base_url())
|
||||
self.current_base = self.aa_urls[self._index] if self.aa_urls else ""
|
||||
if reset_attempts:
|
||||
@@ -1369,16 +1507,41 @@ class AAMirrorSelector:
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
"""Replace any known AA base in url with current_base."""
|
||||
for base in self.aa_urls:
|
||||
for base in self.all_aa_urls:
|
||||
if url.startswith(base):
|
||||
return url.replace(base, self.current_base, 1)
|
||||
return url
|
||||
|
||||
def next_mirror_or_rotate_dns(self, *, allow_dns: bool = True) -> tuple[str | None, str]:
|
||||
def quarantine_current(self, reason: str) -> bool:
|
||||
"""Quarantine the mirror this selector is on (hard failures only)."""
|
||||
if not self.current_base:
|
||||
return False
|
||||
dropped = mark_aa_url_dead(self.current_base, reason)
|
||||
if dropped:
|
||||
# Rebuild from the surviving mirrors so the dead one is out of the cycle.
|
||||
self._ensure_fresh_state(reset_attempts=False)
|
||||
return dropped
|
||||
|
||||
def next_mirror_or_rotate_dns(
|
||||
self, *, allow_dns: bool = True, fatal: bool = False, reason: str = ""
|
||||
) -> tuple[str | None, str]:
|
||||
"""Advance to the next mirror or rotate DNS if needed.
|
||||
|
||||
``fatal`` marks the current mirror as not-an-AA-mirror (NXDOMAIN, refused, a
|
||||
200 that isn't AA) and drops it from this process's rotation. Leave it False
|
||||
for anything the mirror can recover from - a challenge or a 5xx means the host
|
||||
is alive, and quarantining it would discard its bypass clearance.
|
||||
|
||||
Returns (new_base, action) where action is 'mirror', 'dns', or 'exhausted'.
|
||||
"""
|
||||
if fatal and self.quarantine_current(reason or "unusable mirror"):
|
||||
# Quarantining rebuilt the state onto a surviving mirror, so that mirror is
|
||||
# the next one to try - advancing again here would skip straight past it.
|
||||
self.attempts_this_dns += 1
|
||||
if self.current_base and is_aa_auto_mode():
|
||||
set_aa_url(self.current_base)
|
||||
return self.current_base, "mirror"
|
||||
|
||||
self.attempts_this_dns += 1
|
||||
max_attempts = len(self.aa_urls) if is_aa_auto_mode() else 1
|
||||
if self.attempts_this_dns >= max_attempts:
|
||||
@@ -1391,8 +1554,11 @@ class AAMirrorSelector:
|
||||
# Mirror is explicitly configured; do not fail over to other mirrors.
|
||||
return None, "exhausted"
|
||||
|
||||
if not self.aa_urls:
|
||||
return None, "exhausted"
|
||||
|
||||
next_index = (self._index + 1) % len(self.aa_urls)
|
||||
set_aa_url_index(next_index)
|
||||
set_aa_url(self.aa_urls[next_index])
|
||||
self._ensure_fresh_state(reset_attempts=False)
|
||||
return self.current_base, "mirror"
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.archive import ArchiveExtractionError, extract_archive, is_archive
|
||||
from shelfmark.download.fs import run_blocking_io
|
||||
@@ -142,7 +143,7 @@ def scan_directory_tree(
|
||||
|
||||
is_audiobook = check_audiobook(content_type)
|
||||
if is_audiobook:
|
||||
trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"}
|
||||
trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS}
|
||||
else:
|
||||
trackable_exts = {
|
||||
".pdf",
|
||||
@@ -367,7 +368,7 @@ def collect_staged_files(
|
||||
|
||||
is_audiobook = check_audiobook(task.content_type)
|
||||
if is_audiobook:
|
||||
trackable_exts = {".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".wma", ".aac", ".wav"}
|
||||
trackable_exts = {f".{fmt}" for fmt in AUDIOBOOK_FORMATS}
|
||||
else:
|
||||
trackable_exts = {
|
||||
".pdf",
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Boot-time warm-up of the direct-download source.
|
||||
|
||||
The first AA search after a cold start pays for the whole cold path at once: DNS
|
||||
resolution, electing a live mirror, spinning up headless Chrome and solving the
|
||||
DDoS-Guard challenge. That is tens of seconds with the user sat at the search box.
|
||||
|
||||
Running one throwaway search shortly after boot moves that cost off the user's first
|
||||
search. It primes the DNS cache, elects (and quarantines) mirrors, and leaves the
|
||||
clearance cookie in the bypasser's per-domain cache, so the first real search reuses
|
||||
it instead of solving from scratch.
|
||||
|
||||
Runs on a daemon thread and swallows every failure: this is an optimisation, and a
|
||||
source that is down at boot must not affect startup or health.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Delay before the warm-up fires. Long enough that it does not compete with the rest
|
||||
# of startup (and with a container's own health probe) for the first request.
|
||||
_DEFAULT_DELAY_SECONDS = 15.0
|
||||
|
||||
_DEFAULT_QUERY = "The Great Gatsby"
|
||||
|
||||
_warmup_thread: threading.Thread | None = None
|
||||
_warmup_lock = threading.Lock()
|
||||
|
||||
|
||||
def _as_bool(value: object, *, default: bool) -> bool:
|
||||
"""Coerce a config value that may arrive as a string, bool or None."""
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, str):
|
||||
from shelfmark.config.env import string_to_bool
|
||||
|
||||
return string_to_bool(value)
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _setting(key: str, default: object) -> object:
|
||||
"""Read a warm-up setting, preferring the deployment environment.
|
||||
|
||||
These keys are not in the settings registry, and ``config.get`` only consults the
|
||||
environment for keys it knows about - so reading config alone silently ignored
|
||||
SEARCH_WARMUP_ENABLED and always returned the default. Check os.environ first so
|
||||
the documented switches actually work.
|
||||
"""
|
||||
raw = os.environ.get(key)
|
||||
if raw is not None and raw.strip():
|
||||
return raw
|
||||
return config.get(key, default)
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""Whether the boot-time warm-up search should run."""
|
||||
if not _as_bool(_setting("SEARCH_WARMUP_ENABLED", True), default=True):
|
||||
return False
|
||||
# Nothing to warm if the source is off, and no challenge to pre-solve without
|
||||
# the bypasser - a plain search is fast enough not to need this.
|
||||
if not _as_bool(_setting("DIRECT_DOWNLOAD_ENABLED", True), default=True):
|
||||
logger.debug("Search warm-up skipped: direct download disabled")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def warmup_query() -> str:
|
||||
"""The query used to warm the source."""
|
||||
raw = _setting("SEARCH_WARMUP_QUERY", _DEFAULT_QUERY)
|
||||
query = str(raw).strip() if raw else ""
|
||||
return query or _DEFAULT_QUERY
|
||||
|
||||
|
||||
def run_warmup() -> bool:
|
||||
"""Run one warm-up search. Returns True if it produced results.
|
||||
|
||||
Never raises: every failure mode here is one the next real search would hit
|
||||
anyway, and reporting it is the search path's job, not the warm-up's.
|
||||
"""
|
||||
from shelfmark.core.mirrors import has_aa_mirror_configuration
|
||||
|
||||
if not has_aa_mirror_configuration():
|
||||
logger.debug("Search warm-up skipped: no Anna's Archive mirrors configured")
|
||||
return False
|
||||
|
||||
query = warmup_query()
|
||||
logger.info("Warming up direct download search (%r)", query)
|
||||
try:
|
||||
from shelfmark.core.models import SearchFilters
|
||||
from shelfmark.release_sources.direct_download import search_books
|
||||
|
||||
results = search_books(query, SearchFilters())
|
||||
except Exception:
|
||||
# Broad by design: a warm-up must never take the app down, and the source
|
||||
# raises everything from network errors to parse failures.
|
||||
logger.warning("Search warm-up did not complete; first user search may be slow")
|
||||
logger.debug("Search warm-up failure detail", exc_info=True)
|
||||
return False
|
||||
|
||||
if results:
|
||||
logger.info("Search warm-up complete: %s results, source is ready", len(results))
|
||||
return True
|
||||
logger.info("Search warm-up returned no results; source reachable but empty")
|
||||
return False
|
||||
|
||||
|
||||
def start(delay_seconds: float = _DEFAULT_DELAY_SECONDS) -> bool:
|
||||
"""Schedule the warm-up on a daemon thread. Safe to call multiple times."""
|
||||
global _warmup_thread
|
||||
|
||||
if not is_enabled():
|
||||
return False
|
||||
|
||||
with _warmup_lock:
|
||||
if _warmup_thread is not None and _warmup_thread.is_alive():
|
||||
logger.debug("Search warm-up already scheduled")
|
||||
return False
|
||||
|
||||
def _run() -> None:
|
||||
run_warmup()
|
||||
|
||||
_warmup_thread = threading.Timer(delay_seconds, _run)
|
||||
_warmup_thread.daemon = True
|
||||
_warmup_thread.name = "SearchWarmup"
|
||||
_warmup_thread.start()
|
||||
|
||||
logger.debug("Search warm-up scheduled in %ss", delay_seconds)
|
||||
return True
|
||||
+14
-15
@@ -38,7 +38,10 @@ from shelfmark.config.env import (
|
||||
string_to_bool,
|
||||
)
|
||||
from shelfmark.config.security import _migrate_security_settings
|
||||
from shelfmark.config.settings import _SUPPORTED_BOOK_LANGUAGE
|
||||
from shelfmark.config.settings import (
|
||||
_SUPPORTED_BOOK_LANGUAGE,
|
||||
migrate_audiobook_format_settings,
|
||||
)
|
||||
from shelfmark.core.activity_view_state_service import ActivityViewStateService
|
||||
from shelfmark.core.auth_modes import (
|
||||
get_auth_check_admin_status,
|
||||
@@ -80,8 +83,9 @@ from shelfmark.core.requests_service import (
|
||||
sync_delivery_states_from_queue_status,
|
||||
)
|
||||
from shelfmark.core.user_db import UserDB
|
||||
from shelfmark.core.utils import normalize_base_path
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS, normalize_base_path
|
||||
from shelfmark.download import orchestrator as backend
|
||||
from shelfmark.download import warmup
|
||||
from shelfmark.release_sources import (
|
||||
BrowseRecord,
|
||||
Release,
|
||||
@@ -168,6 +172,9 @@ except ImportError as e:
|
||||
# Migrate legacy security settings if needed
|
||||
_migrate_security_settings()
|
||||
|
||||
# Widen audiobook formats for installs that still carry the old m4b/mp3-only default
|
||||
migrate_audiobook_format_settings()
|
||||
|
||||
# Initialize user database and register multi-user routes
|
||||
# If CONFIG_DIR doesn't exist or is read-only, multi-user features will be disabled
|
||||
_user_db_path = str(Path(os.environ.get("CONFIG_DIR", "/config")) / "users.db")
|
||||
@@ -200,6 +207,10 @@ except (sqlite3.OperationalError, OSError) as e:
|
||||
# Start download coordinator
|
||||
backend.start()
|
||||
|
||||
# Pre-solve the direct-download source's protection challenge in the background so the
|
||||
# first user search does not pay for a cold Chrome bypass. Never blocks startup.
|
||||
warmup.start()
|
||||
|
||||
# Rate limiting for login attempts
|
||||
# Map usernames to their failed-attempt counters and lockout timestamps.
|
||||
failed_login_attempts: dict[str, dict[str, Any]] = {}
|
||||
@@ -319,19 +330,7 @@ def get_auth_mode() -> str:
|
||||
|
||||
|
||||
_AUDIOBOOK_CATEGORY_RANGE = (3030, 3049)
|
||||
_AUDIOBOOK_FORMAT_HINTS = frozenset(
|
||||
{
|
||||
"m4b",
|
||||
"mp3",
|
||||
"m4a",
|
||||
"flac",
|
||||
"ogg",
|
||||
"wma",
|
||||
"aac",
|
||||
"wav",
|
||||
"opus",
|
||||
}
|
||||
)
|
||||
_AUDIOBOOK_FORMAT_HINTS = frozenset(AUDIOBOOK_FORMATS)
|
||||
|
||||
|
||||
def _contains_audiobook_format_hint(value: Any) -> bool:
|
||||
|
||||
@@ -48,6 +48,10 @@ HARDCOVER_PAGE_SIZE = 25 # Hardcover API returns max 25 results per page
|
||||
HARDCOVER_MIN_AUTHOR_PARTS = 2
|
||||
HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH = 2
|
||||
HARDCOVER_MAX_SERIES_OPTIONS = 7
|
||||
# Hardcover hands out short opaque tokens now ("hc_pat_...") instead of the ~500 char
|
||||
# JWTs it used to, so the length floor only applies to keys without that prefix.
|
||||
HARDCOVER_API_KEY_PREFIX = "hc_pat_"
|
||||
HARDCOVER_BEARER_PREFIX_PATTERN = re.compile(r"^bearer\s+", re.IGNORECASE)
|
||||
HARDCOVER_API_KEY_MIN_LENGTH = 100
|
||||
HARDCOVER_LIST_URL_PATTERN = re.compile(
|
||||
r"^/(?:@([\w.-]+)/)?lists?/([\w-]+)/?$",
|
||||
@@ -318,6 +322,7 @@ query SearchFieldOptions(
|
||||
fields: $fields,
|
||||
weights: $weights
|
||||
) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -536,13 +541,19 @@ SORT_MAPPING: dict[SortOrder, str] = {
|
||||
SortOrder.OLDEST: "release_year:asc",
|
||||
}
|
||||
|
||||
# Mapping from abstract search type to Hardcover fields parameter
|
||||
SEARCH_TYPE_FIELDS: dict[SearchType, str] = {
|
||||
SearchType.GENERAL: "title,isbns,series_names,author_names,alternative_titles",
|
||||
SearchType.TITLE: "title,alternative_titles",
|
||||
SearchType.AUTHOR: "author_names",
|
||||
# ISBN is handled separately via search_by_isbn()
|
||||
}
|
||||
# `fields` becomes Typesense's `query_by`, but Hardcover keeps `num_typos` and
|
||||
# `query_by_weights` as fixed-length presets per query_type. Passing a different
|
||||
# number of fields than the preset expects makes Typesense reject the whole search,
|
||||
# complaining that the number of num_typos values does not match the number of
|
||||
# query_by fields. So a Book search may only ever narrow to *these five* names --
|
||||
# a shorter list is rejected outright rather than searched, and any weights sent
|
||||
# alongside must match one-for-one.
|
||||
# Weights only bias ranking: a field weighted 0 still matches, so `fields` can no
|
||||
# longer restrict which fields a Book query looks at.
|
||||
BOOK_SEARCH_FIELDS = "title,alternative_titles,author_names,series_names,isbns"
|
||||
BOOK_SEARCH_FIELD_COUNT = 5
|
||||
BOOK_TITLE_WEIGHTS = "5,1,0,0,0"
|
||||
BOOK_TITLE_AUTHOR_WEIGHTS = "5,1,3,0,0"
|
||||
|
||||
SERIES_SEARCH_FIELDS = "name,books,author_name"
|
||||
SERIES_SEARCH_WEIGHTS = "2,1,1"
|
||||
@@ -550,22 +561,28 @@ SERIES_SEARCH_SORT = "_text_match:desc,readers_count:desc"
|
||||
AUTHOR_SUGGESTION_FIELDS = "name,name_personal,alternate_names"
|
||||
AUTHOR_SUGGESTION_WEIGHTS = "4,3,2"
|
||||
AUTHOR_SUGGESTION_SORT = "_text_match:desc,books_count:desc"
|
||||
TITLE_SUGGESTION_FIELDS = "title,alternative_titles"
|
||||
TITLE_SUGGESTION_WEIGHTS = "5,2"
|
||||
TITLE_SUGGESTION_FIELDS = BOOK_SEARCH_FIELDS
|
||||
TITLE_SUGGESTION_WEIGHTS = "5,2,0,0,0"
|
||||
TITLE_SUGGESTION_SORT = "_text_match:desc,users_count:desc"
|
||||
|
||||
# Hardcover forwards `sort` to Typesense's `sort_by` and rejects the whole search
|
||||
# if it does not like the value -- an unknown field, a bare field name with no
|
||||
# direction, more than three keys. A rejected search comes back as HTTP 200 with
|
||||
# no GraphQL errors and a null `results` body, which is otherwise indistinguishable
|
||||
# from "nothing matched". An empty sort is always accepted, so fall back to it and
|
||||
# keep the fallback sticky for a while rather than paying for a doomed request on
|
||||
# every search.
|
||||
SORT_FALLBACK = ""
|
||||
# from "nothing matched"; the reason only shows up in the sibling `error` field,
|
||||
# so every search asks for it. Dropping `sort` from the request is the one shape
|
||||
# Hardcover always accepts -- an empty string is a value like any other and has
|
||||
# been rejected too -- so retry that way and keep the fallback sticky for a while
|
||||
# rather than paying for a doomed request on every search.
|
||||
SORT_FALLBACK_TTL = 900.0
|
||||
_sort_fallback_until = 0.0
|
||||
|
||||
|
||||
def _without_sort(variables: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop `sort` entirely so Hardcover applies its own default ordering."""
|
||||
return {key: value for key, value in variables.items() if key != "sort"}
|
||||
|
||||
|
||||
def _search_payload_rejected(result: dict[str, Any] | None) -> bool:
|
||||
"""Report whether Hardcover answered a search with a null results body.
|
||||
|
||||
@@ -580,6 +597,17 @@ def _search_payload_rejected(result: dict[str, Any] | None) -> bool:
|
||||
return root["results"] is None
|
||||
|
||||
|
||||
def _search_rejection_reason(result: dict[str, Any] | None) -> str:
|
||||
"""Return Hardcover's explanation for a rejected search, if it sent one."""
|
||||
if not isinstance(result, dict):
|
||||
return ""
|
||||
root = result.get("search", result)
|
||||
if not isinstance(root, dict):
|
||||
return ""
|
||||
error = root.get("error")
|
||||
return error.strip() if isinstance(error, str) else ""
|
||||
|
||||
|
||||
def _combine_headline_description(headline: str | None, description: str | None) -> str | None:
|
||||
"""Combine headline (tagline) and description into a single description."""
|
||||
if headline and description:
|
||||
@@ -646,7 +674,7 @@ def _normalize_series_position(value: Any) -> float | None:
|
||||
def _normalize_hardcover_api_key(value: object) -> str:
|
||||
"""Normalize Hardcover API keys, stripping copied auth-header prefixes."""
|
||||
normalized_value = normalize_optional_text(value) or ""
|
||||
return normalized_value.removeprefix("Bearer ").strip()
|
||||
return HARDCOVER_BEARER_PREFIX_PATTERN.sub("", normalized_value.strip()).strip()
|
||||
|
||||
|
||||
def _normalize_search_text(value: str) -> str:
|
||||
@@ -1012,13 +1040,15 @@ class HardcoverProvider(MetadataProvider):
|
||||
"""Build search query, fields, and weights based on provided values.
|
||||
|
||||
Returns (query, fields, weights) tuple. Fields/weights are None for general search.
|
||||
A narrowed search still sends all of BOOK_SEARCH_FIELDS -- Hardcover rejects a
|
||||
shorter list outright -- and leans on the weights to rank the wanted field first.
|
||||
"""
|
||||
if author and not title and not series:
|
||||
return author, None, None
|
||||
if title and not author and not series:
|
||||
return title, "title,alternative_titles", "5,1"
|
||||
return title, BOOK_SEARCH_FIELDS, BOOK_TITLE_WEIGHTS
|
||||
if author and title and not series:
|
||||
return f"{title} {author}", "title,alternative_titles,author_names", "5,1,3"
|
||||
return f"{title} {author}", BOOK_SEARCH_FIELDS, BOOK_TITLE_AUTHOR_WEIGHTS
|
||||
return default_query, None, None
|
||||
|
||||
def _detect_list_url(self, query: str) -> tuple[str | None, str] | None:
|
||||
@@ -2384,6 +2414,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
graphql_query = """
|
||||
query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String, $fields: String, $weights: String) {
|
||||
search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort, fields: $fields, weights: $weights) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -2392,6 +2423,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
graphql_query = """
|
||||
query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String) {
|
||||
search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -2690,33 +2722,42 @@ class HardcoverProvider(MetadataProvider):
|
||||
|
||||
sort = variables.get("sort")
|
||||
if sort and time.monotonic() < _sort_fallback_until:
|
||||
variables = {**variables, "sort": SORT_FALLBACK}
|
||||
variables = _without_sort(variables)
|
||||
sort = None
|
||||
|
||||
result = self._execute_query(query, variables)
|
||||
if not _search_payload_rejected(result):
|
||||
return result
|
||||
|
||||
reason = _search_rejection_reason(result)
|
||||
if not sort:
|
||||
logger.error(
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s) and returned "
|
||||
"no result body",
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s): %s",
|
||||
variables.get("queryType", "Book"),
|
||||
variables.get("fields"),
|
||||
reason or "no error message",
|
||||
)
|
||||
return None
|
||||
|
||||
retry = self._execute_query(query, _without_sort(variables))
|
||||
if _search_payload_rejected(retry):
|
||||
# The sort was not the culprit, so leave sorting alone for other searches.
|
||||
logger.error(
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s) with and without "
|
||||
"a sort order: %s",
|
||||
variables.get("queryType", "Book"),
|
||||
variables.get("fields"),
|
||||
_search_rejection_reason(retry) or reason or "no error message",
|
||||
)
|
||||
return None
|
||||
|
||||
logger.warning(
|
||||
"Hardcover rejected sort '%s'; retrying searches without a sort order for %ss",
|
||||
"Hardcover rejected sort '%s' (%s); dropping the sort order from searches for %ss",
|
||||
sort,
|
||||
reason or "no error message",
|
||||
int(SORT_FALLBACK_TTL),
|
||||
)
|
||||
_sort_fallback_until = time.monotonic() + SORT_FALLBACK_TTL
|
||||
|
||||
retry = self._execute_query(query, {**variables, "sort": SORT_FALLBACK})
|
||||
if _search_payload_rejected(retry):
|
||||
logger.error("Hardcover rejected this search even without a sort order")
|
||||
return None
|
||||
return retry
|
||||
|
||||
def _parse_search_result(self, item: dict) -> BookMetadata | None:
|
||||
@@ -2982,12 +3023,13 @@ def _test_hardcover_connection(current_values: dict[str, Any] | None = None) ->
|
||||
_save_connected_user(None, None)
|
||||
return {"success": False, "message": "API key is required"}
|
||||
|
||||
if key_len < HARDCOVER_API_KEY_MIN_LENGTH:
|
||||
is_prefixed_key = api_key.startswith(HARDCOVER_API_KEY_PREFIX)
|
||||
if not is_prefixed_key and key_len < HARDCOVER_API_KEY_MIN_LENGTH:
|
||||
return {
|
||||
"success": False,
|
||||
"message": (
|
||||
f"API key seems too short ({key_len} chars). "
|
||||
f"Expected {HARDCOVER_API_KEY_MIN_LENGTH}+ chars."
|
||||
f"API key seems too short ({key_len} chars). Expected a key starting "
|
||||
f"with {HARDCOVER_API_KEY_PREFIX} or {HARDCOVER_API_KEY_MIN_LENGTH}+ chars."
|
||||
),
|
||||
}
|
||||
|
||||
@@ -3094,7 +3136,7 @@ def hardcover_settings() -> list[SettingsField]:
|
||||
PasswordField(
|
||||
key="HARDCOVER_API_KEY",
|
||||
label="API Key",
|
||||
description="Get your API key from hardcover.app/account/api",
|
||||
description="Get your API key from hardcover.app/account/api (starts with hc_pat_)",
|
||||
required=True,
|
||||
),
|
||||
ActionButton(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
from urllib.parse import quote, quote_plus
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
@@ -10,6 +10,7 @@ from bs4 import BeautifulSoup
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.download import http as downloader
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -98,8 +99,10 @@ def _encode_search_query(query: str, *, exact_phrase: bool) -> str:
|
||||
and not (search_query.startswith('"') and search_query.endswith('"'))
|
||||
):
|
||||
search_query = f'"{search_query}"'
|
||||
# Keep ABB-friendly encoding style (spaces as '+') while percent-encoding quotes.
|
||||
return search_query.replace('"', "%22").replace(" ", "+")
|
||||
# Keep ABB's space-as-'+' style, but percent-encode everything else: a bare
|
||||
# '&' would otherwise start a new query parameter, '%' would open an invalid
|
||||
# escape, and a literal '+' would arrive as a space.
|
||||
return quote_plus(search_query)
|
||||
|
||||
|
||||
def _normalize_result_url(url: str, hostname: str) -> str:
|
||||
@@ -153,6 +156,9 @@ def search_audiobookbay(
|
||||
|
||||
"""
|
||||
results = []
|
||||
# ABB matches the stored, untexturized title, so a curly apostrophe reaching
|
||||
# the search returns nothing at all rather than merely ranking worse.
|
||||
query = normalize_search_punctuation(query)
|
||||
rate_limit_delay = _coerce_non_negative_float(config.get("ABB_RATE_LIMIT_DELAY", 1.0), 1.0)
|
||||
session = requests.Session()
|
||||
|
||||
|
||||
@@ -23,7 +23,11 @@ from shelfmark.release_sources import (
|
||||
register_source,
|
||||
)
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_hostname, parse_size
|
||||
from shelfmark.release_sources.audiobookbay.utils import (
|
||||
normalize_hostname,
|
||||
normalize_search_punctuation,
|
||||
parse_size,
|
||||
)
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
MIN_RELEVANCE_QUERY_WORD_LENGTH = 2
|
||||
@@ -227,10 +231,12 @@ class AudiobookBaySource(ReleaseSource):
|
||||
deduped_queries[index + 1].lower(),
|
||||
)
|
||||
|
||||
# Extract query words for relevance checking
|
||||
# Extract query words for relevance checking. Both sides of the
|
||||
# comparison are punctuation-normalized: scraped titles carry the
|
||||
# typographic forms WordPress renders, queries carry the ASCII ones.
|
||||
query_words = {
|
||||
word.lower()
|
||||
for word in query_lower.split()
|
||||
for word in normalize_search_punctuation(query_lower).split()
|
||||
if len(word) > MIN_RELEVANCE_QUERY_WORD_LENGTH
|
||||
}
|
||||
|
||||
@@ -239,7 +245,7 @@ class AudiobookBaySource(ReleaseSource):
|
||||
try:
|
||||
raw_title = result["title"]
|
||||
title, author = _split_title_and_author(raw_title)
|
||||
title_for_filter = raw_title.lower()
|
||||
title_for_filter = normalize_search_punctuation(raw_title).lower()
|
||||
|
||||
# Basic relevance check: ensure title contains at least one query word
|
||||
# This filters out homepage "Latest" feed items that may leak through
|
||||
|
||||
@@ -2,6 +2,63 @@
|
||||
|
||||
import re
|
||||
|
||||
# WordPress texturizes punctuation on output only: a post stored as "The
|
||||
# Stranger's Wife" is rendered as "The Stranger’s Wife". ABB's search matches the
|
||||
# stored value, so a query carrying the typographic form matches nothing -- and
|
||||
# because ABB ANDs its search terms, one such term empties the entire result set.
|
||||
# Book metadata and phone keyboards both hand us the typographic forms, so map
|
||||
# them back before they reach a search or a title comparison.
|
||||
_ASCII_PUNCTUATION = str.maketrans(
|
||||
{
|
||||
# Single quotes
|
||||
"‘": "'", # left single quotation mark
|
||||
"’": "'", # right single quotation mark
|
||||
"‚": "'", # single low-9 quotation mark
|
||||
"‛": "'", # single high-reversed-9 quotation mark
|
||||
"′": "'", # prime
|
||||
"´": "'", # acute accent
|
||||
"`": "'", # grave accent
|
||||
# Double quotes
|
||||
"“": '"', # left double quotation mark
|
||||
"”": '"', # right double quotation mark
|
||||
"„": '"', # double low-9 quotation mark
|
||||
"‟": '"', # double high-reversed-9 quotation mark
|
||||
"″": '"', # double prime
|
||||
# Dashes
|
||||
"‐": "-", # hyphen
|
||||
"‑": "-", # non-breaking hyphen
|
||||
"‒": "-", # figure dash
|
||||
"–": "-", # en dash
|
||||
"—": "-", # em dash
|
||||
"―": "-", # horizontal bar
|
||||
"−": "-", # minus sign
|
||||
"﹘": "-", # small em dash
|
||||
"﹣": "-", # small hyphen-minus
|
||||
"-": "-", # fullwidth hyphen-minus
|
||||
# Ellipsis
|
||||
"…": "...", # horizontal ellipsis
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def normalize_search_punctuation(text: str) -> str:
|
||||
"""Replace typographic punctuation with the ASCII forms ABB stores.
|
||||
|
||||
Each character is mapped individually rather than collapsing runs, so an
|
||||
ASCII "--" is left alone: only characters ABB cannot have stored are
|
||||
rewritten.
|
||||
|
||||
Args:
|
||||
text: A search query, or a scraped title being compared against one.
|
||||
|
||||
Returns:
|
||||
The text with curly quotes, dashes and ellipses mapped to ASCII.
|
||||
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
return text.translate(_ASCII_PUNCTUATION)
|
||||
|
||||
|
||||
def normalize_hostname(raw: str | None) -> str:
|
||||
"""Normalize a user-supplied hostname for URL construction.
|
||||
|
||||
@@ -539,6 +539,98 @@ class SearchUnavailableError(SourceUnavailableError):
|
||||
"""Raised when Anna's Archive cannot be reached via any mirror/DNS."""
|
||||
|
||||
|
||||
# Markers that prove a 200 really came from Anna's Archive, and markers that mean we
|
||||
# are looking at a protection interstitial rather than the site. A page with neither
|
||||
# is a domain that answers but is not AA - seized, parked or for sale.
|
||||
#
|
||||
# Deliberately structural rather than the domain name: a parking page's whole job is
|
||||
# to display the domain it is squatting on, so "annas-archive" matches the very pages
|
||||
# this is meant to catch. These paths only exist on the real site.
|
||||
_AA_PAGE_MARKERS = (
|
||||
"/md5/",
|
||||
"aarecord",
|
||||
"anna's archive",
|
||||
"/dyn/",
|
||||
"/datasets",
|
||||
"/fast_download",
|
||||
"/slow_download",
|
||||
)
|
||||
_CHALLENGE_MARKERS = (
|
||||
"ddos-guard",
|
||||
"just a moment",
|
||||
"cloudflare",
|
||||
"checking your browser",
|
||||
"cf-browser-verification",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_aa_page(html: str) -> bool:
|
||||
"""Whether ``html`` is recognisably Anna's Archive itself."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in _AA_PAGE_MARKERS)
|
||||
|
||||
|
||||
def _looks_like_challenge_page(html: str) -> bool:
|
||||
"""Whether ``html`` is a protection interstitial rather than the site behind it."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in _CHALLENGE_MARKERS)
|
||||
|
||||
|
||||
def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[str, Tag | None]:
|
||||
"""Fetch the AA search page, retrying past mirrors that are not actually AA.
|
||||
|
||||
A parked or seized domain answers 200 with a page that has no results table and no
|
||||
"No files found." - indistinguishable from a broken search unless we check whether
|
||||
the response looks like AA at all. Those mirrors are quarantined for the session so
|
||||
later searches skip them instead of paying the timeout again.
|
||||
"""
|
||||
attempt_url = url
|
||||
for _ in range(len(network.get_available_aa_urls()) or 1):
|
||||
response = downloader.html_get_page(
|
||||
attempt_url, selector=selector, allow_bypasser_fallback=True
|
||||
)
|
||||
if not response:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
|
||||
html = _html_response_text(response)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if isinstance(table, Tag):
|
||||
return html, table
|
||||
if table is not None:
|
||||
msg = f"Expected results table tag, got {type(table).__name__}"
|
||||
raise TypeError(msg)
|
||||
if "No files found." in html:
|
||||
# A real, genuinely empty answer from a healthy mirror.
|
||||
return html, None
|
||||
if _looks_like_challenge_page(html):
|
||||
# The bypass did not actually clear the protection - the interstitial is
|
||||
# what came back. Rotating is pointless (every mirror shares the same
|
||||
# protection) and reporting it as an empty result is worse: the user is
|
||||
# told their query found nothing when the search never ran.
|
||||
msg = (
|
||||
"Anna's Archive answered with an unsolved protection challenge. "
|
||||
"Check that the bypasser is reachable and working."
|
||||
)
|
||||
raise SearchUnavailableError(msg)
|
||||
if _looks_like_aa_page(html):
|
||||
# A real AA response in a shape the caller should report as drift.
|
||||
# Not the mirror's fault.
|
||||
return html, None
|
||||
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
fatal=True, reason="responded without an Anna's Archive page"
|
||||
)
|
||||
if action not in ("mirror", "dns") or not new_base:
|
||||
return html, None
|
||||
attempt_url = selector.rewrite(url)
|
||||
logger.info("Retrying search on %s", new_base)
|
||||
|
||||
return "", None
|
||||
|
||||
|
||||
def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
"""Search for books matching the query.
|
||||
|
||||
@@ -601,15 +693,9 @@ def search_books(query: str, filters: SearchFilters) -> list[BrowseRecord]:
|
||||
f"{filters_query}"
|
||||
)
|
||||
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
|
||||
if not html:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
|
||||
soup = BeautifulSoup(_html_response_text(html), "html.parser")
|
||||
tbody = soup.find("table")
|
||||
|
||||
# AA gates /search behind a DDoS-Guard JS challenge, which every mirror shares. Rotating
|
||||
# to another mirror only collects another 403, so let the bypasser solve it.
|
||||
html, tbody = _fetch_search_table(url, selector)
|
||||
if tbody is None:
|
||||
if "No files found." in html:
|
||||
logger.info("No books found for query: %s", query)
|
||||
@@ -657,7 +743,8 @@ def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseR
|
||||
"""
|
||||
url = f"{network.get_aa_base_url()}/md5/{book_id}"
|
||||
selector = network.AAMirrorSelector()
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=False)
|
||||
# Same challenge as search: the detail page is gated on every mirror, so bypass it.
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True)
|
||||
|
||||
if not html:
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
@@ -885,6 +972,9 @@ def _parse_book_info_page(
|
||||
if fetch_download_count:
|
||||
try:
|
||||
summary_url = f"{network.get_aa_base_url()}/dyn/md5/summary/{book_id}"
|
||||
# Unlike search and the detail page above, this one stays off the bypasser: a
|
||||
# download count is decoration on the details modal, not worth holding the
|
||||
# modal open for a browser solve. If it is gated, drop it and move on.
|
||||
summary_response = downloader.html_get_page(
|
||||
summary_url, selector=network.AAMirrorSelector(), allow_bypasser_fallback=False
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -18,11 +19,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# All recognized formats for parsing IRC result lines.
|
||||
# This comprehensive list is used to identify file extensions in results.
|
||||
# User-configured formats are used separately for filtering.
|
||||
ALL_RECOGNIZED_FORMATS = {
|
||||
# Ebook formats
|
||||
# Ebook formats recognized in IRC result lines.
|
||||
EBOOK_FORMATS = (
|
||||
"epub",
|
||||
"mobi",
|
||||
"azw3",
|
||||
@@ -41,19 +39,17 @@ ALL_RECOGNIZED_FORMATS = {
|
||||
"cbz",
|
||||
"cdr",
|
||||
"jpg",
|
||||
"rar",
|
||||
"zip",
|
||||
# Audiobook formats
|
||||
"m4b",
|
||||
"mp3",
|
||||
"m4a",
|
||||
"flac",
|
||||
"ogg",
|
||||
"wma",
|
||||
"aac",
|
||||
"wav",
|
||||
"opus",
|
||||
}
|
||||
)
|
||||
|
||||
# All recognized formats for parsing IRC result lines.
|
||||
# This comprehensive list is used to identify file extensions in results.
|
||||
# User-configured formats are used separately for filtering.
|
||||
# Ordered longest-first so that scanning a line matches "azw3" before "azw" and "docx"
|
||||
# before "doc". It used to be a set, which made the winning format for a line naming more
|
||||
# than one extension depend on set iteration order, and therefore vary between restarts.
|
||||
ALL_RECOGNIZED_FORMATS = tuple(
|
||||
sorted({*EBOOK_FORMATS, *ARCHIVE_FORMATS, *AUDIOBOOK_FORMATS}, key=len, reverse=True)
|
||||
)
|
||||
|
||||
|
||||
def _normalize_config_formats(raw_formats: object) -> set[str]:
|
||||
@@ -84,13 +80,22 @@ def _get_supported_formats(content_type: str | None = None) -> set[str]:
|
||||
|
||||
# Regex to parse result lines
|
||||
# Format: !Server Author - Title.format ::INFO:: size
|
||||
#
|
||||
# The extension is matched against the known formats rather than a bare \w+. A bare \w+
|
||||
# happily matched the decimal point in the size, so a line with no file extension parsed
|
||||
# as format="5mb" out of "::INFO:: 620.5MB" - taking the title and size down with it, and
|
||||
# leaving the result to be discarded by every format filter downstream. Restricting the
|
||||
# alternation makes such a line fall through to SIMPLE_RESULT_REGEX and come back as
|
||||
# "unknown", which is what the rest of the parser already expects.
|
||||
_FORMAT_ALTERNATION = "|".join(re.escape(fmt) for fmt in ALL_RECOGNIZED_FORMATS)
|
||||
RESULT_LINE_REGEX = re.compile(
|
||||
r"^!(\S+)\s+" # !ServerName
|
||||
r"(.+?)\s+-\s+" # Author Name -
|
||||
r"(.+?)\.(\w+)" # Title.format
|
||||
rf"(.+?)\.({_FORMAT_ALTERNATION})\b" # Title.format
|
||||
r"(?:\s+::INFO::\s*(.+?))?" # Optional ::INFO:: metadata
|
||||
r"(?:\s+::HASH::\s*(\S+))?" # Optional ::HASH::
|
||||
r"\s*$"
|
||||
r"\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Simpler fallback pattern
|
||||
@@ -187,18 +192,54 @@ def parse_result_line(line: str) -> SearchResult | None:
|
||||
return None
|
||||
|
||||
|
||||
# Words that mark an archive as holding an audiobook rather than an ebook. Multi-file
|
||||
# audiobooks ship as .rar/.zip, so for those the extension says nothing about the content
|
||||
# and the release name is the only evidence there is.
|
||||
_AUDIOBOOK_MARKER_REGEX = re.compile(
|
||||
r"\b(?:audio ?books?|unabridged|abridged|narrat(?:ed|or)|audible|\d+ ?kbps|"
|
||||
+ "|".join(re.escape(fmt) for fmt in AUDIOBOOK_FORMATS)
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_AUDIOBOOK_FORMAT_SET = frozenset(AUDIOBOOK_FORMATS)
|
||||
_EBOOK_FORMAT_SET = frozenset(EBOOK_FORMATS)
|
||||
|
||||
|
||||
def detect_content_type(result: SearchResult) -> str:
|
||||
"""Classify a parsed result as an audiobook or an ebook.
|
||||
|
||||
Extension alone is not enough. It settles the plain cases, but the common audiobook
|
||||
release is a .rar or .zip of MP3s, which is indistinguishable by extension from an
|
||||
ebook archive - so for containers (and for lines with no usable extension) the
|
||||
release name decides.
|
||||
"""
|
||||
if result.format in _AUDIOBOOK_FORMAT_SET:
|
||||
return "audiobook"
|
||||
if result.format in _EBOOK_FORMAT_SET:
|
||||
return "ebook"
|
||||
return "audiobook" if _AUDIOBOOK_MARKER_REGEX.search(result.full_line) else "ebook"
|
||||
|
||||
|
||||
def parse_results_file(content: str, content_type: str | None = None) -> list[SearchResult]:
|
||||
"""Parse a search results file into SearchResult objects."""
|
||||
results = []
|
||||
supported = _get_supported_formats(content_type)
|
||||
requested = "audiobook" if check_audiobook(content_type) else "ebook"
|
||||
|
||||
for line in content.splitlines():
|
||||
result = parse_result_line(line)
|
||||
if result and (result.format in supported or result.format == "unknown"):
|
||||
# Filter to user's configured formats
|
||||
if not result:
|
||||
continue
|
||||
# Classify first, then apply the user's format filter within that bucket. Doing it
|
||||
# the other way round is what lost audiobooks entirely: an audiobook .rar matched
|
||||
# neither the ebook nor the audiobook format list, so it fell out of both.
|
||||
if detect_content_type(result) != requested:
|
||||
continue
|
||||
if result.format in supported or result.format == "unknown":
|
||||
results.append(result)
|
||||
|
||||
logger.info("Parsed %s results from search file", len(results))
|
||||
logger.info("Parsed %s %s results from search file", len(results), requested)
|
||||
return results
|
||||
|
||||
|
||||
|
||||
@@ -102,10 +102,13 @@ def irc_settings() -> list[SettingsField]:
|
||||
key="audiobook_heading",
|
||||
title="Audiobooks",
|
||||
description=(
|
||||
"Some networks index audiobooks in a separate channel from ebooks "
|
||||
"(for example #ebooks for ebooks and #bookz for audiobooks). "
|
||||
"Configure that channel here to search it for audiobook requests. "
|
||||
"Leave these blank to search the main channel above for both."
|
||||
"Most networks index audiobooks in the same channel as ebooks, so leaving "
|
||||
"these blank is the right setting for almost everyone. On irc.irchighway.net "
|
||||
"the audiobooks are in #ebooks and #bookz is effectively inactive — pointing "
|
||||
"this at an empty channel just returns no results. Only fill these in when "
|
||||
"your network really does index audiobooks elsewhere (Undernet's #bookz, for "
|
||||
"example). Audiobooks are usually posted as archives, so keep ZIP and RAR "
|
||||
"enabled under Supported Audiobook Formats or the releases are filtered out."
|
||||
),
|
||||
),
|
||||
TextField(
|
||||
@@ -113,8 +116,9 @@ def irc_settings() -> list[SettingsField]:
|
||||
label="Audiobook channel",
|
||||
placeholder="e.g. bookz",
|
||||
description=(
|
||||
"Optional. Channel name (without the # prefix) to use for audiobook "
|
||||
"searches. Leave blank to use the main channel above for audiobooks too."
|
||||
"Optional. Channel name (without the # prefix) for networks that index "
|
||||
"audiobooks separately, such as Undernet's bookz. Leave blank (the usual "
|
||||
"setting) to search the main channel above for audiobooks too."
|
||||
),
|
||||
required=False,
|
||||
env_supported=True,
|
||||
|
||||
@@ -240,11 +240,11 @@ class IRCReleaseSource(ReleaseSource):
|
||||
nick = _config_text("IRC_NICK")
|
||||
search_bot = _config_text("IRC_SEARCH_BOT")
|
||||
|
||||
# Audiobooks may be indexed in a separate channel from ebooks on some networks
|
||||
# (e.g. #ebooks for ebooks, #bookz for audiobooks). When an audiobook channel is
|
||||
# configured and an audiobook was requested, route the search there (with its own
|
||||
# search bot if set). Otherwise fall back to the main channel/bot, which keeps the
|
||||
# single-channel networks that index both formats working unchanged.
|
||||
# A few networks index audiobooks in a separate channel from ebooks (Undernet's
|
||||
# #bookz, say). When an audiobook channel is configured and an audiobook was
|
||||
# requested, route the search there (with its own search bot if set). Otherwise
|
||||
# fall back to the main channel/bot — that is the common case, since most networks
|
||||
# (irchighway included) serve both formats from the one channel.
|
||||
if is_audiobook(content_type):
|
||||
audiobook_channel = _config_text("IRC_AUDIOBOOK_CHANNEL")
|
||||
if audiobook_channel:
|
||||
|
||||
@@ -14,10 +14,6 @@ from shelfmark.release_sources.prowlarr.torznab import parse_torznab_xml
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Newznab standard book category IDs
|
||||
NEWZNAB_BOOKS = 7000
|
||||
NEWZNAB_AUDIOBOOKS = 3030
|
||||
|
||||
|
||||
class NewznabClient:
|
||||
"""Client for any Newznab-compatible indexer API."""
|
||||
|
||||
@@ -8,6 +8,7 @@ from shelfmark.core.settings_registry import (
|
||||
HeadingField,
|
||||
PasswordField,
|
||||
SettingsField,
|
||||
TagListField,
|
||||
TextField,
|
||||
register_settings,
|
||||
)
|
||||
@@ -86,6 +87,30 @@ def newznab_config_settings() -> list[SettingsField]:
|
||||
callback=_test_newznab_connection,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
TagListField(
|
||||
key="NEWZNAB_EBOOK_CATEGORIES",
|
||||
label="Ebook Categories",
|
||||
description=(
|
||||
"Newznab category IDs searched for ebooks. Most indexers use the standard 7000, "
|
||||
"but some use custom IDs. Leave empty to use 7000."
|
||||
),
|
||||
placeholder="7000",
|
||||
default=["7000"],
|
||||
normalize_urls=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
TagListField(
|
||||
key="NEWZNAB_AUDIOBOOK_CATEGORIES",
|
||||
label="Audiobook Categories",
|
||||
description=(
|
||||
"Newznab category IDs searched for audiobooks. Most indexers use the standard "
|
||||
"3030, but some use custom IDs. Leave empty to use 3030."
|
||||
),
|
||||
placeholder="3030",
|
||||
default=["3030"],
|
||||
normalize_urls=False,
|
||||
show_when={"field": "NEWZNAB_ENABLED", "value": True},
|
||||
),
|
||||
CheckboxField(
|
||||
key="NEWZNAB_AUTO_EXPAND",
|
||||
label="Auto-expand search on no results",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
@@ -39,15 +40,93 @@ from shelfmark.release_sources.prowlarr.source import (
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Newznab category IDs
|
||||
_AUDIOBOOK_CATS = [3030]
|
||||
_BOOK_CATS = [7000]
|
||||
# Standard Newznab category IDs, used when the indexer's categories aren't configured.
|
||||
_DEFAULT_AUDIOBOOK_CATS = [3030]
|
||||
_DEFAULT_BOOK_CATS = [7000]
|
||||
|
||||
# Reuse the same timeout constant as Prowlarr.
|
||||
NEWZNAB_SEARCH_TIMEOUT_SECONDS = _SEARCH_TIMEOUT
|
||||
|
||||
|
||||
def _newznab_result_to_release(result: dict, content_type: str = "ebook") -> Release:
|
||||
def _parse_category_ids(raw: object) -> list[int]:
|
||||
"""Parse a configured category setting into Newznab category IDs.
|
||||
|
||||
Accepts a list of values or a comma/whitespace separated string. Entries that
|
||||
aren't positive integers are skipped, and duplicates are dropped.
|
||||
"""
|
||||
if raw is None:
|
||||
return []
|
||||
|
||||
values = list(raw) if isinstance(raw, (list, tuple)) else [raw]
|
||||
|
||||
category_ids: list[int] = []
|
||||
for value in values:
|
||||
for token in re.split(r"[,\s]+", str(value).strip()):
|
||||
if not token:
|
||||
continue
|
||||
try:
|
||||
category_id = int(token)
|
||||
except ValueError:
|
||||
logger.warning("Newznab: ignoring invalid category ID '%s'", token)
|
||||
continue
|
||||
if category_id > 0 and category_id not in category_ids:
|
||||
category_ids.append(category_id)
|
||||
|
||||
return category_ids
|
||||
|
||||
|
||||
def _configured_categories(content_type: str) -> list[int]:
|
||||
"""Return the categories to search for a content type, falling back to defaults."""
|
||||
if content_type == "audiobook":
|
||||
key, defaults = "NEWZNAB_AUDIOBOOK_CATEGORIES", _DEFAULT_AUDIOBOOK_CATS
|
||||
else:
|
||||
key, defaults = "NEWZNAB_EBOOK_CATEGORIES", _DEFAULT_BOOK_CATS
|
||||
|
||||
return _parse_category_ids(config.get(key, None)) or list(defaults)
|
||||
|
||||
|
||||
def _result_category_ids(categories: object) -> set[int]:
|
||||
"""Extract numeric category IDs from a result's categories field."""
|
||||
if not isinstance(categories, (list, tuple)):
|
||||
return set()
|
||||
|
||||
category_ids: set[int] = set()
|
||||
for cat in categories:
|
||||
raw = cat.get("id") if isinstance(cat, dict) else cat
|
||||
try:
|
||||
category_ids.add(int(raw)) # type: ignore[arg-type]
|
||||
except TypeError, ValueError:
|
||||
continue
|
||||
return category_ids
|
||||
|
||||
|
||||
def _resolve_content_type(
|
||||
categories: object,
|
||||
content_type: str,
|
||||
searched_categories: list[int] | None,
|
||||
) -> str:
|
||||
"""Resolve a result's content type, honouring custom indexer categories.
|
||||
|
||||
Indexers using non-standard IDs (e.g. 7100 for ebooks) fall outside the standard
|
||||
ranges, so trust the searched content type when the result carries a category we
|
||||
explicitly asked for.
|
||||
"""
|
||||
category_list = list(categories) if isinstance(categories, (list, tuple)) else []
|
||||
detected = _detect_content_type_from_categories(category_list, content_type)
|
||||
if (
|
||||
detected == "other"
|
||||
and searched_categories
|
||||
and _result_category_ids(category_list) & set(searched_categories)
|
||||
):
|
||||
return "audiobook" if content_type == "audiobook" else "book"
|
||||
return detected
|
||||
|
||||
|
||||
def _newznab_result_to_release(
|
||||
result: dict,
|
||||
content_type: str = "ebook",
|
||||
searched_categories: list[int] | None = None,
|
||||
) -> Release:
|
||||
"""Convert a parsed Newznab XML result dict to a Release object."""
|
||||
raw_title = result.get("title", "Unknown")
|
||||
size_bytes = result.get("size")
|
||||
@@ -125,7 +204,7 @@ def _newznab_result_to_release(result: dict, content_type: str = "ebook") -> Rel
|
||||
indexer=indexer,
|
||||
seeders=seeders if is_torrent else None,
|
||||
peers=peers_display,
|
||||
content_type=_detect_content_type_from_categories(categories, content_type),
|
||||
content_type=_resolve_content_type(categories, content_type, searched_categories),
|
||||
extra={
|
||||
"publish_date": result.get("publishDate"),
|
||||
"categories": categories,
|
||||
@@ -230,12 +309,7 @@ class NewznabSource(ReleaseSource):
|
||||
return []
|
||||
|
||||
# Category selection — omit categories when expanding search
|
||||
if expand_search:
|
||||
categories = None
|
||||
elif content_type == "audiobook":
|
||||
categories = [3030]
|
||||
else:
|
||||
categories = [7000]
|
||||
categories = None if expand_search else _configured_categories(content_type)
|
||||
|
||||
auto_expand = config.get("NEWZNAB_AUTO_EXPAND", False)
|
||||
deadline = time.monotonic() + NEWZNAB_SEARCH_TIMEOUT_SECONDS
|
||||
@@ -283,7 +357,7 @@ class NewznabSource(ReleaseSource):
|
||||
logger.exception("Newznab search failed")
|
||||
return []
|
||||
|
||||
results = [_newznab_result_to_release(r, content_type) for r in all_results]
|
||||
results = [_newznab_result_to_release(r, content_type, categories) for r in all_results]
|
||||
|
||||
if results:
|
||||
nzb_count = sum(1 for r in results if r.protocol == ReleaseProtocol.NZB)
|
||||
|
||||
@@ -16,6 +16,7 @@ from shelfmark.core.languages import normalize_language
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.request_helpers import normalize_optional_text
|
||||
from shelfmark.core.search_plan import ReleaseSearchVariant
|
||||
from shelfmark.core.utils import AUDIOBOOK_FORMATS as CORE_AUDIOBOOK_FORMATS
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources import (
|
||||
ColumnAlign,
|
||||
@@ -222,7 +223,7 @@ EBOOK_FORMATS = [
|
||||
]
|
||||
|
||||
# Common audiobook formats
|
||||
AUDIOBOOK_FORMATS = ["m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "opus"]
|
||||
AUDIOBOOK_FORMATS = list(CORE_AUDIOBOOK_FORMATS)
|
||||
|
||||
# Combined list for format detection (audiobook formats first for priority)
|
||||
ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS
|
||||
|
||||
Generated
+164
-164
@@ -19,9 +19,9 @@
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.4",
|
||||
"@vitejs/plugin-react": "^6.0.5",
|
||||
"knip": "^6.32.0",
|
||||
"oxfmt": "^0.62.0",
|
||||
"oxlint": "^1.77.0",
|
||||
"knip": "^6.32.1",
|
||||
"oxfmt": "^0.63.0",
|
||||
"oxlint": "^1.78.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"typescript": "^7.0.2",
|
||||
@@ -846,9 +846,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@oxfmt/binding-android-arm-eabi": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.62.0.tgz",
|
||||
"integrity": "sha512-pdsv0C4gPjJ8H1+sd8u0BDx+yLACTL+rgeMIOL1ln4ihSnhw8CWXtYWgvcSkyTfgGBIzFKab+d8rx9Xl4en/Kw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.63.0.tgz",
|
||||
"integrity": "sha512-YmRth4ZPGgEXcgmkhvANbC9uD67dxmSobW7DQuyt5tOBOKvPnIpk5SVHBj88E+7wMNRI2FhqaDbOhQFBix+b8A==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -863,9 +863,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-android-arm64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.62.0.tgz",
|
||||
"integrity": "sha512-WC3YQ7uS/KtDrjmqwBviwFKe9qeoi+eXx8aX1z/ffG23Md75myjrJaQqTuJvdOLPoa4EYTjDWH0dHXfwulCVog==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.63.0.tgz",
|
||||
"integrity": "sha512-icbahX8X2X3sRamOMecvdYeZXWjPDazRDIfvWfy7Ca1nc/ZDT2Y9k5Nt7s46EqFd7NQPdgk+CM3/SgIT5LPCaQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -880,9 +880,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-darwin-arm64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.62.0.tgz",
|
||||
"integrity": "sha512-GM8Yf3LjjaR1I8PD0SfeoIlwhsh9GvSF+cQ8sf624Yxnjsyumn95aFzYfKJVefblfDIiOAnZ7QVm2sa21Er/0Q==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.63.0.tgz",
|
||||
"integrity": "sha512-WV+Ze5v5gI2qoj8jpAovt8KBTW8pjEz/AiMXXjeTQS+Bmf/MmZXTS40S8xNPDszX+W8WDv2Bbk6qKrMTtUGu1A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -897,9 +897,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-darwin-x64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.62.0.tgz",
|
||||
"integrity": "sha512-d5THp7F8bCxLqNogEXDORRsQD6dosf3EyFtnXfBer6v+8tGdcWIjoDX9WaXrrF/26zOmL8qHpPTKCEvpBDmZkQ==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.63.0.tgz",
|
||||
"integrity": "sha512-CJGSBdDxXOWIpoFXHpverimCvz084KA7L483rqJ44c3jDtzv6d4qOSoR/V9ywSHfV+Ks1lwIj2P49BFhunLNAA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -914,9 +914,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-freebsd-x64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.62.0.tgz",
|
||||
"integrity": "sha512-1DnrtXGZooOZ0fHgAXZUaDQzBVh1CM2MNW4oBXyQ2aWKvCHjyljvT9fgBkOM0fEOb96X5eqtcfJ0YUVt9jj66g==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.63.0.tgz",
|
||||
"integrity": "sha512-BDfKY+KhL2078cgswBBFQPAYuxCy93bS/iC5frdSeSbTLcGrR6VC2hsuPTanoJmg84+wSyWl0wWC1eR+uTnkRg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -931,9 +931,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm-gnueabihf": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.62.0.tgz",
|
||||
"integrity": "sha512-4pQDHOYRH+Huqe0StIaWyvk2CVl/aTaqSrbZpA3/pLS2xH24ME7lBgYprhQF2fRkHBzhGGGKliwxFsDdHwx59g==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.63.0.tgz",
|
||||
"integrity": "sha512-Ov1cQEXT4mj7cojAokWSS1eoxkoyvbDfAbxNsGIKY2o36kvdAaFzPxRN6NxFRk9fD72B8oCoTTX/NuYTUWlpsg==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -948,9 +948,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm-musleabihf": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.62.0.tgz",
|
||||
"integrity": "sha512-X0jAaZJFMCVKhB6YyWVTQ/wN2DLsBcZKSMqTS76bF6riT+XZdtg2FPEdjDvdVbunO9cG+tWiVaEs4Zs38lxYog==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.63.0.tgz",
|
||||
"integrity": "sha512-0LE7ro3+6L79jcMANycAZfRaC7zxr9YZ2+vEL5uMD9QlEep+rS/r1kSJsnuLl991NXJZD60euh0PC1GHrR20vw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -965,9 +965,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-682Z8T5s8T5ATArYtsejKvbIfd8LEAXyyDkKkoZVq8HND7Vx8TYLlrDjDSeYfodMeVwHOgkj13lJYR8cj6vUSg==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-izPk+2Z4gjuZK32Fqh5qXoMpT/2NXzLh++ob57HiEiVSQZ1iYXu8EKMzb+K5AvWyIEXhdDIt7ADjGGtFhkT9Bw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -985,9 +985,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-arm64-musl": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.62.0.tgz",
|
||||
"integrity": "sha512-lk25fAl7KWaLWVJcW0CHEXB7QlQZtx5eDkjpaGMK0hzXTjUe0Wmlu8IKuFHoviSOcEJedRTs4VE/506VqGxGew==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.63.0.tgz",
|
||||
"integrity": "sha512-alPmbOuWXFXiSo+lOtv6X71C7SYMEDW2WVvywOvf9BwKgEhSNGhMTLeFVSjKUMCamcjbbgVdsWF8GN1uy8xshg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1005,9 +1005,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-ppc64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-SFyNqHQLwySceWNLhiSldx7wPXRAzP0L0WcW9GegP3uWrpZGJiZlQO85NbHAFPEfxR9PhZ9qSnZryEh7+v+4Gw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-BdzCPvolJc4AWZ+YMzgUDJcDzbQWrFjYuqBHoNHNqP1aCaluQRJNs4k3vNU5IG7vTpjf9zeD73D7MFM1TecZpg==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -1025,9 +1025,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-riscv64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-KYj55C1ywJfHo6+aKDuEmUtVEdJALsC5GwayDGsI6FGz2GxFqNr/mA8nxVsNbJzm7sE5MRqTQ9ziImSzhYXysA==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-7sIgfLzqtNKSkMGsGVyRpHwpjNezRg2XONvUOheFZs95TSZpM0JAuPpA8KrQFsWc4wPU95roX2O69JgH8igOgw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1045,9 +1045,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-riscv64-musl": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.62.0.tgz",
|
||||
"integrity": "sha512-BhZDNo5GOU5nC378RhD0/XpvaEBHsH3HLgJp8YZX3A0InC7oivzA63HsRmiXFLtLSHAstEVrDf6fbC7Rs8Jh/A==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.63.0.tgz",
|
||||
"integrity": "sha512-9Tcg0y0WcVa6Mm9AgcgFMseDS+VkFJZpKZ8We9SpDY4gg5jewSwln+0sO04QLcTS1BtfDl9MwR+NfID8L7PUTg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1065,9 +1065,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-s390x-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-UyAFmyHkgSgUJ/wOM4p3U8AC2yAFvRH5PNBs7TnK0fObTT/XSWcdr/lAzPSWaekHaZFaMeFZyk9n93Joq3J93A==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-qWKC1pEOpx1qYhXaugPhHUeXwSfqEOk2wJH2LqVXGPV5iQYfdAZdt+d2XDiX4DTSWA2QDMUcFB+wEORh3Xn/sA==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -1085,9 +1085,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-x64-gnu": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.62.0.tgz",
|
||||
"integrity": "sha512-1iYMP0leytWazFubD/WnINJuIrzRPuoL1aWEJdlGezEzDbTxcd29R4r8IUzP2oWeKst5V02uMJgR2NILlPlG6w==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.63.0.tgz",
|
||||
"integrity": "sha512-S9wXYOiGSqYGS4Fx/TFsY+xDd/7dE5s+rUgbA4TsHiVF9e8J3ZcKmP7dsP/7iqLI9Wz7Ic7TzEr3mdthRCTdrA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1105,9 +1105,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-linux-x64-musl": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.62.0.tgz",
|
||||
"integrity": "sha512-4rA/URtJSTVNVAQz6Q8wf7SaRvOXVy+TizriT9hs/Y1XhLR/R+92uWKRQG8yFWRAIEBbFHJ6WevQcl/G9SXEfw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.63.0.tgz",
|
||||
"integrity": "sha512-5eGyTJuMZNwBSHCivXt8Yuta6GeTYksOPXRk2MIhajiyFGQx7bjaHIwY+ZusAoFHhT157A9x6sktLjYo9D5oMQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1125,9 +1125,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-openharmony-arm64": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.62.0.tgz",
|
||||
"integrity": "sha512-mSZuFHU2ar1KLUjXpI2QBQcJ1VsOB3mOCgQXuXCpKs19dgh4u+OaovNfrWDfiJb+ihJ2+f7YFcaO9bS2dlTCXA==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.63.0.tgz",
|
||||
"integrity": "sha512-Rz7hx+Dv3DoW/S6pwVAyjfFXp7/trdQ1zg+vNmsdsdDNlUccugp4XNqambSuEAeP0DaG9k72AtNyfDXCEg0AGw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1142,9 +1142,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-win32-arm64-msvc": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.62.0.tgz",
|
||||
"integrity": "sha512-OfwuhkcjDlqC4EgDojtiV9mzpLqeB9KqTOWPOjLEYBVdDCVSxqW3qzp/xcIxsbtI0UgGCnKvAqYKyY25kf5JZw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.63.0.tgz",
|
||||
"integrity": "sha512-T/IuizKN9mr4Xw6YYnptkXRNdLkyIlUZ7c8zfTOBpoytZyJ1BAsMUvsMDEx0X4YvSMpaivm+DR8112rQfzC25g==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1159,9 +1159,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-win32-ia32-msvc": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.62.0.tgz",
|
||||
"integrity": "sha512-P9uDDNFRzghO3X8QAzhkjKhK7JvtABsVn8UYtFX7uor12IAnwNt8nNIctvfWj1JkQU/kE+fmLRPiw7XlrIHsZw==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.63.0.tgz",
|
||||
"integrity": "sha512-XjrO5FJ5Wl9vsAxtCP1G/eaeT6y1K2s9CICUHGE42cEjou32/J6S+B1KnrOAboj6E7uhJnwPbRSvznWcxNdA0g==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -1176,9 +1176,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxfmt/binding-win32-x64-msvc": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.62.0.tgz",
|
||||
"integrity": "sha512-dlI5SY7XYQCiCBafntWagCR6HcAJB/NpsLtdlPx8x08+Osz8Ok1HHz1GZuusegCe/VoJ6pAnF5a4pd5OZAq7qQ==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.63.0.tgz",
|
||||
"integrity": "sha512-sgsHCQy432OTQH4Ikk3tZptp3GqwnhwUDuY0loBH41zyHWfMZY9v8Dy78wsnSofHejvFozZGgJgBB1A0LQRwMQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1277,9 +1277,9 @@
|
||||
]
|
||||
},
|
||||
"node_modules/@oxlint/binding-android-arm-eabi": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.77.0.tgz",
|
||||
"integrity": "sha512-E06sKWS6PiI6HRxS1wyQg22HvApt01hI7fV+T3wUk3OSbaaP4a3hYGY/MIQDmASqCiRjBdpRQYkgMkqH82cWmQ==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.78.0.tgz",
|
||||
"integrity": "sha512-Bu819lmAfZMUHErrpe0cEWj3iaefuUODHSU8+UbXy67V/r7/7f4K3FL0NmbD85E+wiFLDYuhP8Zlv0XnVeXshw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1294,9 +1294,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-android-arm64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.77.0.tgz",
|
||||
"integrity": "sha512-NvsKz0KZxTp9cYWPLf+FXaSZwB3oO3peAjtukpOMBgse2vhQSoIIVqeO1yR0lEo/UcdZIDL18uq+kL0LzQ0ytA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.78.0.tgz",
|
||||
"integrity": "sha512-CDfxZgB61B7buRdY2FJoAYYPPXCZ1EoC1LKscnC5dg3kjobdxiconvAvvN1BmHyW4PyFT3jRLDag/BY/roSNBQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1311,9 +1311,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-darwin-arm64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.77.0.tgz",
|
||||
"integrity": "sha512-bgjTn6nW4bQCFBvSvuHCpDD+sONvmpo4lGI4PxzMt1quBA+xYxhczk6RiCn3GZ9gY8uhaBbwhj9MdKGfu6T9DA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.78.0.tgz",
|
||||
"integrity": "sha512-2Y2U9Ahrz+OO0Ej88f9SJYq51/jUBp1Mc7iZu0ukrbeeZ3gpRGfzIFnoqfHDY96xr0GEfNrPUBFEy0nN5aD7HA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1328,9 +1328,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-darwin-x64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.77.0.tgz",
|
||||
"integrity": "sha512-aotaIttH1R6j1Rwhx0M0htgeZyGtVQqYNTVEYMN/UcgHPquGA6kmk9OyuDc3a2GKUQBC+3C3GVQCcrRPMYqAFA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.78.0.tgz",
|
||||
"integrity": "sha512-rpych6eJq6m9jDRypTEaPD1xysaEW5h9+xuxhGK/QhOg+/xaqPZrCrTNoIl/f3nEjuJeCEmstNDlrE9rJi/3/g==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1345,9 +1345,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-freebsd-x64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.77.0.tgz",
|
||||
"integrity": "sha512-nNx/wta7ksRAdYvq+l4AWjXkLxEXHALhENxjj2cYbQAIR4ybaA5L+hCbE63HOmft5czQ6ks+hb8vmEAnn7YGPg==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.78.0.tgz",
|
||||
"integrity": "sha512-IcMGrQT3QizkOESUJd5et+rOhVqSkNDfNik1cvrKDqIbzqx9KMtRswpFgkCuNTSwylCFLKhGUu8KmqY1ZnC0Dg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1362,9 +1362,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm-gnueabihf": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.77.0.tgz",
|
||||
"integrity": "sha512-tMLLjM7xXtzXisVCzkOTXNCy9bZVId2wteNwjohlFDR/jY6WagpEDA1c1wu4xRc20Hojaxj+V6DSR7gbKxijWA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.78.0.tgz",
|
||||
"integrity": "sha512-/uLdoJ0IXE6vo/0f0LKjinQAp+re+VMaCWaNT8ENIv2EOCkSsc8SGaflXAuW0Jua2dq5+GLVWm1NQK7P3UFSNQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1379,9 +1379,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm-musleabihf": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.77.0.tgz",
|
||||
"integrity": "sha512-MiAFDFaqR0tmHTAyo0YDcZ5hyLREdYw/RQhc2R3cbT+8O3tB+zqPM2th9TTQ+Uo3jn/embS+DO+HyX9ztCPkOQ==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.78.0.tgz",
|
||||
"integrity": "sha512-7xi4Wb/O8NRJhLoUXmDJMUVpNYvB5kefdhFU1Jb8rtae4QoXlTiLwI14X4YvAXVZLNZChP8m5qO9SQAlWQTbkQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1396,9 +1396,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-/xqQ3B16i1T4cyt/9Mn+4CpzhUXoBXp7kVpIwzOXNFLj5JmK1bIjsbSnX296Gg8A/o7oDtKWikFgBx0SLwztkw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-4hFW0+fVXa3OIh1Y4A5SPkmvI4wuuBSrCVKzOyE7PTjhc7yEqZ1pmvEEeS5Lj/MaqvegFxXyF33N+6jkehxdyg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1416,9 +1416,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-arm64-musl": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.77.0.tgz",
|
||||
"integrity": "sha512-LSbwuRKiNCenPDcbARqAZ5RfBy7gmj7vOvfJRLeCDU3gFtSxWbhv/+VTlaUqzUhNj1gFLHB8h7ALnxa/Az6z6g==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.78.0.tgz",
|
||||
"integrity": "sha512-oC0mvsgBJjlMijSDEhx9KuvR9zYeHXceA9MjbuXB1F8NSR78Yj2unOBrstEvTVaq+pko+kuue6DajC00eqvTdg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1436,9 +1436,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-ppc64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-QWdcH31mXEUe5Nq1s0CfCpceaKjIo9uZtwDjAuL681g1axf+5x8xrg/eXWaw//4NCxYZ4V4e5Hu5tvdR+pTBlg==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-XAllT5SUZS+ohjuZ3/5S0cwe0r7eboiuigeStCZ5DXRYx/2KVM2UvQXvAfyzXEimtQjAB7cDQ2YxDe2Zl2WNQQ==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -1456,9 +1456,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-riscv64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-GnOfYgJxbcElOiPZaDFDl406ONddwvOWk2jvAAAEjwAl4GofNoHF+/HHUIBYa6bFCArlcGPi0XjC4cU1pkgF/Q==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-trucMER/0QtecoXvc1y/UVqE3kwJipDwrx4oHfj+nNm3dq2zjP44WT0CfHNDPM3G1DXIkx/gY6lAD21NSCZVhA==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1476,9 +1476,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-riscv64-musl": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.77.0.tgz",
|
||||
"integrity": "sha512-AyEMTUCf0xY+hHF+IxqXFQIX0yQOIR8ykpY0lJNOw9xYqOzUX8dyZfRvlG0RfXwuQn2eonf/8NrMmDSZJjdqsA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.78.0.tgz",
|
||||
"integrity": "sha512-cm3O4F/HQbdzOUX5mKHqG5KDL6E5w0pnlZ+fbBy2rmLryPOowkuLagFHTopQsEIpjcaZoPOrL+BmmAytAG9HFg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1496,9 +1496,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-s390x-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-sPLzEcNvxd/oyVQ5oZo92CiHkFkpBeRop13E/P3TPY+hZfXHKCOWKI70TE2RYwMKFJDc20EMjH16L7NZICtKTw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-33wRf6HqGNsybJ3qX4cGaQN2ODPxNmc1rMa0mrTmx3eFq1VzOnvQooi9bIGVYakW8a/wmqVx1mgsUm8R2xfTiw==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -1516,9 +1516,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-x64-gnu": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.77.0.tgz",
|
||||
"integrity": "sha512-1Oh2ssH2L7lwyvkdSqaMUfsGfwU2Wfvew+obBUYjRVqhpBcUpwnsPSEr1IzVi9XqkuY10geiLsNKecqaZC34Dw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.78.0.tgz",
|
||||
"integrity": "sha512-rRdISSYegj6VganMZ9tjRjijowfHJ09IZU01i0toBAqr6n5LEtwHq2IeS4FjW2RoskOHlb6efB26H5izYb3GEQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1536,9 +1536,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-linux-x64-musl": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.77.0.tgz",
|
||||
"integrity": "sha512-0j/2wRgNGO+Qj/M1uu/p57h/hFTTWWcfie0ufkbabeus2s5+/QqkCflnMOwLLN5m2GsNeWp4xdl4cPa4n7QCOQ==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.78.0.tgz",
|
||||
"integrity": "sha512-GmsP4rW0xTL6u5CVdcDsaN5Fbc7hBc382Wmar1kttbnwSEviM+rSINKOMQ+UQ6iH+AGwC+8gaAiwu134Tgh6Lg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1556,9 +1556,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-openharmony-arm64": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.77.0.tgz",
|
||||
"integrity": "sha512-BJ/j54qS0usEnyDkLYURMj2iiD9h5Cyy+ppzeMSXBGRXaGRNWnj1Mw14NqWMR5E/PzdgB30OOCCzLzbRoduafw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.78.0.tgz",
|
||||
"integrity": "sha512-sy9yeYuADc8a+n4TLBayzMCZiHPW78DcIFVpOXTmdKHWQeM9xe5uzkqIIZmi326D5hY9XVwacipEB1p7tQjPAg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1573,9 +1573,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-win32-arm64-msvc": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.77.0.tgz",
|
||||
"integrity": "sha512-Yh8w+g2Lpx7StrvtYkoz9JJvXjB9wxgFChFNb85nrXm/wj/XTwGWS1hve9+900HL7llrntYB3YP+y32E3tRqzA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.78.0.tgz",
|
||||
"integrity": "sha512-rjc2hF1KfMi8fZj1X/m3AmnHbdsF3rL0v6KQg0Uc880Yb2khjz+3U14sfdZ7jWTpRnN1m1NQa/TT7uU9lJWPrA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1590,9 +1590,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-win32-ia32-msvc": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.77.0.tgz",
|
||||
"integrity": "sha512-zja5b7+6a7UsRFgAQSrnax5vrzliEyNPLCjfXONu/vTWswaIVZGFajJZptaeRvPE4LghtFdAzVFlexTm7MVTGA==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.78.0.tgz",
|
||||
"integrity": "sha512-zcuXFVrEFHIafRfkCQT8w/Xe41o07ozl/vwHq7p94vB29xVzsB0sZGYORU1jhcYKv3Lr0J3HbJ2T4fHH5rWmvA==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -1607,9 +1607,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxlint/binding-win32-x64-msvc": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.77.0.tgz",
|
||||
"integrity": "sha512-+teyvPDZ2RjUvo+SuCqS/UhaJl1QtdW5fWT5NJTV61V5MIuIS90Db9LixmtEGvXixyttiK62P96MSu3UlpviBw==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.78.0.tgz",
|
||||
"integrity": "sha512-Sb5ocmLSuYeOuXd+CFOToGKp/gjXUEWDnvIGwhnh8aq8wY4TMmEnKnvbogSW7RdMZv77JSARduS7/gv+khYEjA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2979,9 +2979,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/knip": {
|
||||
"version": "6.32.0",
|
||||
"resolved": "https://registry.npmjs.org/knip/-/knip-6.32.0.tgz",
|
||||
"integrity": "sha512-KDX9OmmOFmlvmxTkrx6Z0GHISMut+pXMSKR8eg84bovaxJKx2NdQD4JYCXveSbvieRe107W6vCD2xCpmz0qBYA==",
|
||||
"version": "6.32.1",
|
||||
"resolved": "https://registry.npmjs.org/knip/-/knip-6.32.1.tgz",
|
||||
"integrity": "sha512-mIiIHMTJVUgSlz0mxEgPt7wg8DmfbCp1Txqab3WpbMCJF7YHvHtC9jeAHHXfISMl72N8WzhyG71SQlaqCOGZtg==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -3390,9 +3390,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/oxfmt": {
|
||||
"version": "0.62.0",
|
||||
"resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.62.0.tgz",
|
||||
"integrity": "sha512-vxgGHTmnDU9j4CX7dDBLzxgmHxfda/yPcgJkGCMUSCwRmz+euo/V08xXLNgXTeqAB9Fhf3Pe2nO1RNKLCVgphQ==",
|
||||
"version": "0.63.0",
|
||||
"resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.63.0.tgz",
|
||||
"integrity": "sha512-kgdDwv35wvVf6554U2Ab8Jnd0zTM+TsEQWwaB70RAjK3gICFAFGO+2Hd3Be27GMoXj3XRL9IKSNRVl7KBQL6iw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -3408,25 +3408,25 @@
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oxfmt/binding-android-arm-eabi": "0.62.0",
|
||||
"@oxfmt/binding-android-arm64": "0.62.0",
|
||||
"@oxfmt/binding-darwin-arm64": "0.62.0",
|
||||
"@oxfmt/binding-darwin-x64": "0.62.0",
|
||||
"@oxfmt/binding-freebsd-x64": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm-gnueabihf": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm-musleabihf": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-arm64-musl": "0.62.0",
|
||||
"@oxfmt/binding-linux-ppc64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-riscv64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-riscv64-musl": "0.62.0",
|
||||
"@oxfmt/binding-linux-s390x-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-x64-gnu": "0.62.0",
|
||||
"@oxfmt/binding-linux-x64-musl": "0.62.0",
|
||||
"@oxfmt/binding-openharmony-arm64": "0.62.0",
|
||||
"@oxfmt/binding-win32-arm64-msvc": "0.62.0",
|
||||
"@oxfmt/binding-win32-ia32-msvc": "0.62.0",
|
||||
"@oxfmt/binding-win32-x64-msvc": "0.62.0"
|
||||
"@oxfmt/binding-android-arm-eabi": "0.63.0",
|
||||
"@oxfmt/binding-android-arm64": "0.63.0",
|
||||
"@oxfmt/binding-darwin-arm64": "0.63.0",
|
||||
"@oxfmt/binding-darwin-x64": "0.63.0",
|
||||
"@oxfmt/binding-freebsd-x64": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm-gnueabihf": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm-musleabihf": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-arm64-musl": "0.63.0",
|
||||
"@oxfmt/binding-linux-ppc64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-riscv64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-riscv64-musl": "0.63.0",
|
||||
"@oxfmt/binding-linux-s390x-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-x64-gnu": "0.63.0",
|
||||
"@oxfmt/binding-linux-x64-musl": "0.63.0",
|
||||
"@oxfmt/binding-openharmony-arm64": "0.63.0",
|
||||
"@oxfmt/binding-win32-arm64-msvc": "0.63.0",
|
||||
"@oxfmt/binding-win32-ia32-msvc": "0.63.0",
|
||||
"@oxfmt/binding-win32-x64-msvc": "0.63.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"svelte": "^5.0.0",
|
||||
@@ -3442,9 +3442,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/oxlint": {
|
||||
"version": "1.77.0",
|
||||
"resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.77.0.tgz",
|
||||
"integrity": "sha512-qnGh8XJHaQ0dprrDXNQZgS0FgjI6v+V3+X8DwmaV++5Aamy6jGKfDdQ1TUvhUxtmKFAbEf4/WeO5QZX+5WSngg==",
|
||||
"version": "1.78.0",
|
||||
"resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.78.0.tgz",
|
||||
"integrity": "sha512-QgQePuxIqKOzo1KSjG2EnITEeWvWnKAm77eq8nrMtf6AGoA+zyGc4PFYtDNJSD25g/ibOwfQ851hZ4/SPkMVoA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
@@ -3457,25 +3457,25 @@
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oxlint/binding-android-arm-eabi": "1.77.0",
|
||||
"@oxlint/binding-android-arm64": "1.77.0",
|
||||
"@oxlint/binding-darwin-arm64": "1.77.0",
|
||||
"@oxlint/binding-darwin-x64": "1.77.0",
|
||||
"@oxlint/binding-freebsd-x64": "1.77.0",
|
||||
"@oxlint/binding-linux-arm-gnueabihf": "1.77.0",
|
||||
"@oxlint/binding-linux-arm-musleabihf": "1.77.0",
|
||||
"@oxlint/binding-linux-arm64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-arm64-musl": "1.77.0",
|
||||
"@oxlint/binding-linux-ppc64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-riscv64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-riscv64-musl": "1.77.0",
|
||||
"@oxlint/binding-linux-s390x-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-x64-gnu": "1.77.0",
|
||||
"@oxlint/binding-linux-x64-musl": "1.77.0",
|
||||
"@oxlint/binding-openharmony-arm64": "1.77.0",
|
||||
"@oxlint/binding-win32-arm64-msvc": "1.77.0",
|
||||
"@oxlint/binding-win32-ia32-msvc": "1.77.0",
|
||||
"@oxlint/binding-win32-x64-msvc": "1.77.0"
|
||||
"@oxlint/binding-android-arm-eabi": "1.78.0",
|
||||
"@oxlint/binding-android-arm64": "1.78.0",
|
||||
"@oxlint/binding-darwin-arm64": "1.78.0",
|
||||
"@oxlint/binding-darwin-x64": "1.78.0",
|
||||
"@oxlint/binding-freebsd-x64": "1.78.0",
|
||||
"@oxlint/binding-linux-arm-gnueabihf": "1.78.0",
|
||||
"@oxlint/binding-linux-arm-musleabihf": "1.78.0",
|
||||
"@oxlint/binding-linux-arm64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-arm64-musl": "1.78.0",
|
||||
"@oxlint/binding-linux-ppc64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-riscv64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-riscv64-musl": "1.78.0",
|
||||
"@oxlint/binding-linux-s390x-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-x64-gnu": "1.78.0",
|
||||
"@oxlint/binding-linux-x64-musl": "1.78.0",
|
||||
"@oxlint/binding-openharmony-arm64": "1.78.0",
|
||||
"@oxlint/binding-win32-arm64-msvc": "1.78.0",
|
||||
"@oxlint/binding-win32-ia32-msvc": "1.78.0",
|
||||
"@oxlint/binding-win32-x64-msvc": "1.78.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"oxlint-tsgolint": ">=7.0.2001",
|
||||
|
||||
@@ -28,9 +28,9 @@
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.4",
|
||||
"@vitejs/plugin-react": "^6.0.5",
|
||||
"knip": "^6.32.0",
|
||||
"oxfmt": "^0.62.0",
|
||||
"oxlint": "^1.77.0",
|
||||
"knip": "^6.32.1",
|
||||
"oxfmt": "^0.63.0",
|
||||
"oxlint": "^1.78.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"typescript": "^7.0.2",
|
||||
|
||||
@@ -144,6 +144,11 @@ const mapApiErrorToActionResult = (error: unknown): ActionResult | null => {
|
||||
// Default request timeout in milliseconds (30 seconds)
|
||||
const DEFAULT_TIMEOUT_MS = 30000;
|
||||
|
||||
// Release searches can be long-running: a source behind Cloudflare/DDoS-Guard has
|
||||
// to spin up the bypasser and solve the challenge before any results come back,
|
||||
// which routinely takes well over the default timeout.
|
||||
const SEARCH_TIMEOUT_MS = 180000;
|
||||
|
||||
// Utility function for JSON fetch with credentials and timeout
|
||||
async function fetchJSON<T>(
|
||||
url: string,
|
||||
@@ -235,6 +240,8 @@ export const searchBooks = async (query: string): Promise<Book[]> => {
|
||||
if (!query) return [];
|
||||
const response = await fetchJSON<ReleasesResponse>(
|
||||
`${API_BASE}/releases?source=direct_download&${query}`,
|
||||
{},
|
||||
SEARCH_TIMEOUT_MS,
|
||||
);
|
||||
return response.releases.map(transformReleaseToDirectBook);
|
||||
};
|
||||
|
||||
@@ -317,6 +317,44 @@ class TestSearchAudiobookbay:
|
||||
assert "s=%22test+query%22" in requested_url
|
||||
assert "cat=undefined%2Cundefined" in requested_url
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_normalizes_curly_apostrophe(self, mock_config_get, mock_html_get):
|
||||
"""Test curly apostrophes are searched as the ASCII form ABB stores."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
|
||||
|
||||
scraper.search_audiobookbay(
|
||||
"the stranger’s wife",
|
||||
max_pages=1,
|
||||
hostname="audiobookbay.lu",
|
||||
)
|
||||
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert "s=the+stranger%27s+wife" in requested_url
|
||||
assert "%E2%80%99" not in requested_url
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_percent_encodes_reserved_characters(
|
||||
self, mock_config_get, mock_html_get
|
||||
):
|
||||
"""Test reserved characters cannot break out of the search parameter."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
|
||||
|
||||
scraper.search_audiobookbay(
|
||||
"sense & sensibility 100% c++",
|
||||
max_pages=1,
|
||||
hostname="audiobookbay.lu",
|
||||
)
|
||||
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert "s=sense+%26+sensibility+100%25+c%2B%2B" in requested_url
|
||||
# The only surviving '&' introduces the legacy category parameter.
|
||||
assert requested_url.count("&") == 1
|
||||
assert requested_url.endswith("&cat=undefined%2Cundefined")
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_always_uses_legacy_category_query(
|
||||
|
||||
@@ -306,6 +306,42 @@ class TestAudiobookBaySource:
|
||||
assert len(results) == 1
|
||||
assert results[0].title == "Test Book by Test Author"
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
|
||||
def test_search_relevance_filtering_spans_typographic_punctuation(self, mock_search):
|
||||
"""Test an ASCII query still matches the typographic title ABB renders."""
|
||||
mock_search.return_value = [
|
||||
{
|
||||
"title": "The Stranger’s Wife — Anna‑Lou Weatherley",
|
||||
"link": "https://audiobookbay.lu/abss/the-strangers-wife/",
|
||||
"format": "M4B",
|
||||
"size": "259 MB",
|
||||
"language": "English",
|
||||
},
|
||||
]
|
||||
|
||||
source = AudiobookBaySource()
|
||||
book = BookMetadata(
|
||||
provider="test",
|
||||
provider_id="123",
|
||||
title="Stranger's",
|
||||
authors=["Anna-Lou Weatherley"],
|
||||
)
|
||||
# Every query word carries punctuation, so the result survives only when
|
||||
# both sides of the comparison are normalized.
|
||||
plan = ReleaseSearchPlan(
|
||||
languages=["en"],
|
||||
isbn_candidates=[],
|
||||
author="",
|
||||
title_variants=[ReleaseSearchVariant(title="Stranger's", author="")],
|
||||
grouped_title_variants=[],
|
||||
)
|
||||
|
||||
results = source.search(book, plan, content_type="audiobook")
|
||||
|
||||
assert len(results) == 1
|
||||
# The release keeps the title as ABB rendered it; only matching normalizes.
|
||||
assert results[0].title == "The Stranger’s Wife — Anna‑Lou Weatherley"
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
|
||||
def test_search_result_mapping(self, mock_search):
|
||||
"""Test conversion of scraper results to Release objects."""
|
||||
|
||||
@@ -2,7 +2,52 @@
|
||||
Tests for AudiobookBay utility functions.
|
||||
"""
|
||||
|
||||
from shelfmark.release_sources.audiobookbay.utils import parse_size
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation, parse_size
|
||||
|
||||
|
||||
class TestNormalizeSearchPunctuation:
|
||||
"""Tests for the normalize_search_punctuation function."""
|
||||
|
||||
def test_curly_apostrophe_becomes_ascii(self):
|
||||
"""ABB matches the stored ASCII apostrophe, not the rendered curly one."""
|
||||
assert normalize_search_punctuation("The Stranger’s Wife") == "The Stranger's Wife"
|
||||
|
||||
def test_all_single_quote_variants(self):
|
||||
"""Every single-quote lookalike collapses to the ASCII apostrophe."""
|
||||
for variant in ("‘", "’", "‚", "‛", "′", "´", "`"):
|
||||
assert normalize_search_punctuation(f"don{variant}t") == "don't"
|
||||
|
||||
def test_all_double_quote_variants(self):
|
||||
"""Every double-quote lookalike collapses to the ASCII double quote."""
|
||||
for variant in ("“", "”", "„", "‟", "″"):
|
||||
assert normalize_search_punctuation(f"{variant}quoted{variant}") == '"quoted"'
|
||||
|
||||
def test_all_dash_variants(self):
|
||||
"""Every dash lookalike collapses to the ASCII hyphen."""
|
||||
for variant in ("‐", "‑", "‒", "–", "—", "―", "−", "﹘", "﹣", "-"):
|
||||
assert normalize_search_punctuation(f"anna{variant}lou") == "anna-lou"
|
||||
|
||||
def test_ellipsis_expands_to_three_dots(self):
|
||||
"""WordPress renders '...' as a single ellipsis character."""
|
||||
assert normalize_search_punctuation("And Then…") == "And Then..."
|
||||
|
||||
def test_ascii_query_is_unchanged(self):
|
||||
"""A query that is already ASCII passes through untouched."""
|
||||
assert normalize_search_punctuation("The Stranger's Wife") == "The Stranger's Wife"
|
||||
|
||||
def test_ascii_dash_runs_are_not_collapsed(self):
|
||||
"""Only characters ABB cannot have stored are rewritten."""
|
||||
assert normalize_search_punctuation("Book -- Subtitle") == "Book -- Subtitle"
|
||||
|
||||
def test_other_punctuation_is_preserved(self):
|
||||
"""Colons and commas carry search signal and are left alone."""
|
||||
assert normalize_search_punctuation("Weatherley: Book 3, Part 1") == (
|
||||
"Weatherley: Book 3, Part 1"
|
||||
)
|
||||
|
||||
def test_empty_query(self):
|
||||
"""An empty query is returned as-is."""
|
||||
assert normalize_search_punctuation("") == ""
|
||||
|
||||
|
||||
class TestParseSize:
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""DDoS-Guard cookie reuse between requests.
|
||||
|
||||
Anna's Archive issues nine cookies after a solve, and they are not equivalent:
|
||||
|
||||
__ddg1_/__ddg2_/__ddgid_ ~1 year clearance
|
||||
__ddgmark_ ~1 day
|
||||
__ddg5_ session
|
||||
__ddg8_/__ddg9_/__ddg10_ ~40 min one check: token, CLIENT IP, TIMESTAMP
|
||||
|
||||
Replaying the last three is what produces the ?check=1 redirect loop. They describe a
|
||||
single check, so once the timestamp ages out - or the egress IP changes, routine
|
||||
behind a VPN - DDoS-Guard stops recognising the caller and re-arms the challenge on
|
||||
every request. Storing an expired cookie and sending it forever has the same effect.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.cookie_store as cs
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_cookie_store(monkeypatch):
|
||||
monkeypatch.setattr(cs, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cs, "_cf_user_agents", {})
|
||||
|
||||
|
||||
class _Cookie:
|
||||
"""Stand-in for the CDP cookie objects the bypasser extracts."""
|
||||
|
||||
def __init__(self, name, value="v", expires=None, domain="annas-archive.gl"):
|
||||
self.name = name
|
||||
self.value = value
|
||||
self.expires = expires
|
||||
self.domain = domain
|
||||
self.path = "/"
|
||||
self.secure = True
|
||||
|
||||
|
||||
def _store(cookies, url="https://annas-archive.gl/search"):
|
||||
cs.store_extracted_cookies(url=url, cookies=cookies, user_agent="UA/1.0")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Per-check cookies must not be persisted for replay
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_per_check_cookies_are_not_stored():
|
||||
"""The IP/timestamp trio describes one check and must not outlive it."""
|
||||
_store(
|
||||
[
|
||||
_Cookie("__ddg1_", "clearance"),
|
||||
_Cookie("__ddg2_", "clearance2"),
|
||||
_Cookie("__ddg8_", "opaque"),
|
||||
_Cookie("__ddg9_", "203.0.113.7"),
|
||||
_Cookie("__ddg10_", "1786826304"),
|
||||
_Cookie("ddg_last_challenge", "1786826304"),
|
||||
]
|
||||
)
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(stored) == {"__ddg1_", "__ddg2_"}
|
||||
for ephemeral in ("__ddg8_", "__ddg9_", "__ddg10_", "ddg_last_challenge"):
|
||||
assert ephemeral not in stored
|
||||
|
||||
|
||||
def test_clearance_cookies_survive():
|
||||
_store([_Cookie("__ddg1_", "a"), _Cookie("__ddg2_", "b"), _Cookie("__ddgid_", "c")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"__ddg1_": "a", "__ddg2_": "b", "__ddgid_": "c"}
|
||||
|
||||
|
||||
def test_cloudflare_cookies_are_unaffected():
|
||||
_store([_Cookie("cf_clearance", "token"), _Cookie("__cf_bm", "bm")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"cf_clearance": "token", "__cf_bm": "bm"}
|
||||
|
||||
|
||||
def test_per_check_cookies_are_excluded_even_for_full_session_domains(monkeypatch):
|
||||
"""extract_all exists for Z-Library sessions; it must not resurrect the trio."""
|
||||
monkeypatch.setattr(cs, "_get_full_cookie_domains", lambda: {"annas-archive.gl"})
|
||||
_store([_Cookie("sessionid", "s"), _Cookie("__ddg9_", "203.0.113.7")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert "sessionid" in stored
|
||||
assert "__ddg9_" not in stored
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Expiry must be honoured for every cookie, not only cf_clearance
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_expired_ddg_cookies_are_dropped():
|
||||
"""The old code only expiry-checked cf_clearance, so DDoS-Guard domains - which
|
||||
have none - replayed dead cookies forever."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "live"), _Cookie("__ddgmark_", "dead", expires=past)])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_all_cookies_expired_returns_empty_so_caller_re_solves():
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "dead", expires=past)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
assert cs.has_valid_cf_cookies("annas-archive.gl") is False
|
||||
|
||||
|
||||
def test_unexpired_cookies_are_kept():
|
||||
future = int(time.time()) + 3600
|
||||
_store([_Cookie("__ddg1_", "live", expires=future)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_session_cookies_never_expire():
|
||||
"""expires<=0 means a session cookie, not an already-expired one."""
|
||||
_store([_Cookie("__ddg5_", "s", expires=0), _Cookie("__ddg1_", "a", expires=None)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg5_": "s", "__ddg1_": "a"}
|
||||
|
||||
|
||||
def test_expired_cf_clearance_still_drops_the_whole_domain():
|
||||
"""Pre-existing Cloudflare behaviour must not regress."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("cf_clearance", "dead", expires=past), _Cookie("__cf_bm", "bm")])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_expired_cookies_are_pruned_from_the_store():
|
||||
"""A dropped cookie must not linger and be re-evaluated on every request."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "live"), _Cookie("__ddgmark_", "dead", expires=past)])
|
||||
|
||||
ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(cs._cf_cookies["annas-archive.gl"]) == {"__ddg1_"}
|
||||
|
||||
|
||||
def test_solve_that_yields_only_per_check_cookies_stores_nothing():
|
||||
"""No clearance means no reuse - the caller must go back to the bypasser rather
|
||||
than believe it holds a valid session."""
|
||||
_store([_Cookie("__ddg9_", "203.0.113.7"), _Cookie("__ddg10_", "1786826304")])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Rejected cookies are discarded, never retried forever
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _Resp:
|
||||
def __init__(self, status_code, text="page"):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
|
||||
def _seed(monkeypatch):
|
||||
_store([_Cookie("__ddg1_", "clearance"), _Cookie("__ddg2_", "c2")])
|
||||
monkeypatch.setattr(ib, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(ib, "get_ssl_verify", lambda _url: True)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
|
||||
def test_rejected_cached_cookies_are_discarded(monkeypatch):
|
||||
"""A 403 while presenting cookies proves they are dead - keep them and every
|
||||
later request re-presents a known-rejected cookie."""
|
||||
_seed(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
assert (
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl") is None
|
||||
)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_redirect_loop_on_cached_cookies_discards_them(monkeypatch):
|
||||
"""DDoS-Guard answers dead clearance with an endless ?check=1 bounce, which
|
||||
surfaces as an exception rather than a status code."""
|
||||
_seed(monkeypatch)
|
||||
|
||||
def boom(*_a, **_k):
|
||||
raise ib.requests.exceptions.TooManyRedirects("Exceeded 30 redirects")
|
||||
|
||||
monkeypatch.setattr(ib.requests, "get", boom)
|
||||
|
||||
assert (
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl") is None
|
||||
)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_working_cookies_are_kept(monkeypatch):
|
||||
_seed(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(200, "the page"))
|
||||
|
||||
result = ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl")
|
||||
|
||||
assert result == "the page"
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") != {}
|
||||
|
||||
|
||||
def test_failure_only_clears_the_failing_host(monkeypatch):
|
||||
"""clear_cf_cookies('') means every host - a blank hostname must not wipe
|
||||
clearance for sites that are working fine."""
|
||||
_seed(monkeypatch)
|
||||
_store([_Cookie("__ddg1_", "other")], url="https://other-site.test/x")
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl")
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
assert ib.get_cf_cookies_for_domain("other-site.test") == {"__ddg1_": "other"}
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Tests for the external bypasser flow."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
@@ -55,6 +57,165 @@ def test_fetch_via_bypasser_posts_expected_payload_and_uses_ssl_verify(monkeypat
|
||||
]
|
||||
|
||||
|
||||
def _stub_solution(monkeypatch, external_bypasser, solution: dict) -> None:
|
||||
"""Answer one bypass with `solution`, with config and SSL stubbed out."""
|
||||
|
||||
def fake_get(key, default=""):
|
||||
values = {
|
||||
"EXT_BYPASSER_URL": "https://bypass.example",
|
||||
"EXT_BYPASSER_PATH": "/v1",
|
||||
"EXT_BYPASSER_TIMEOUT": 60000,
|
||||
}
|
||||
return values.get(key, default)
|
||||
|
||||
monkeypatch.setattr(external_bypasser.config, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
external_bypasser.requests,
|
||||
"post",
|
||||
lambda *_a, **_k: _FakeResponse({"status": "ok", "solution": solution}),
|
||||
)
|
||||
monkeypatch.setattr(external_bypasser, "get_ssl_verify", lambda _url: False)
|
||||
|
||||
|
||||
def test_solved_clearance_is_stored_for_reuse(monkeypatch):
|
||||
"""A solve costs tens of seconds of real browser; its clearance must be kept.
|
||||
|
||||
Without this every request paid a 403 plus a full solve, and the file download -
|
||||
which the solver cannot proxy - presented no clearance at all.
|
||||
"""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"userAgent": "Mozilla/5.0 (solver)",
|
||||
"cookies": [
|
||||
{"name": "__ddg1_", "value": "clearance", "domain": ".annas-archive.gl"},
|
||||
{"name": "__ddg2_", "value": "c2", "domain": ".annas-archive.gl"},
|
||||
# Per-check cookies: kept out of the store, same as the internal path.
|
||||
{"name": "__ddg9_", "value": "203.0.113.7", "domain": ".annas-archive.gl"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {
|
||||
"__ddg1_": "clearance",
|
||||
"__ddg2_": "c2",
|
||||
}
|
||||
# Cloudflare ties clearance to the solving UA, so replaying one without the other fails.
|
||||
assert cookie_store.get_cf_user_agent_for_domain("annas-archive.gl") == "Mozilla/5.0 (solver)"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "shape"),
|
||||
[
|
||||
# Byparr drives Playwright/camoufox, whose cookies spell it "expires".
|
||||
("expires", "playwright"),
|
||||
# FlareSolverr assigns driver.get_cookies() - the WebDriver cookie object,
|
||||
# which spells it "expiry". Reading only "expires" made every FlareSolverr
|
||||
# cookie immortal, so dead clearance was replayed forever.
|
||||
("expiry", "webdriver"),
|
||||
],
|
||||
)
|
||||
def test_expired_solution_cookie_is_not_replayed(monkeypatch, field, shape):
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"cookies": [{"name": "__ddg1_", "value": "dead", field: int(time.time()) - 60}],
|
||||
},
|
||||
)
|
||||
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}, (
|
||||
f"a dead {shape} cookie was kept for replay"
|
||||
)
|
||||
|
||||
|
||||
def test_solution_cookie_expiry_is_coerced_not_trusted(monkeypatch):
|
||||
"""The solver is not ours; a stringified expiry must be read, not raised on."""
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"cookies": [
|
||||
{"name": "__ddg1_", "value": "live", "expires": str(int(time.time()) + 3600)},
|
||||
{"name": "__ddg2_", "value": "dead", "expires": str(int(time.time()) - 60)},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_storing_clearance_can_never_discard_the_solved_page(monkeypatch):
|
||||
"""A solve costs ~30s; a surprise in the cookie shape must not throw it away.
|
||||
|
||||
The store call sits inside the request try/except, whose handler returns None -
|
||||
so without its own guard a raising store turned a good page into a failed fetch
|
||||
and sent the caller round for up to MAX_RETRY more solves.
|
||||
"""
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{"response": "<html>ok</html>", "cookies": [{"name": "__ddg1_", "value": "v"}]},
|
||||
)
|
||||
|
||||
def boom(*_args, **_kwargs):
|
||||
raise TypeError("unexpected cookie shape")
|
||||
|
||||
monkeypatch.setattr(external_bypasser, "store_extracted_cookies", boom)
|
||||
|
||||
assert (
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
== "<html>ok</html>"
|
||||
)
|
||||
|
||||
|
||||
def test_solution_without_cookies_is_still_returned(monkeypatch):
|
||||
"""A solver that returns no cookie list must not break the page fetch."""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(monkeypatch, external_bypasser, {"response": "<html>ok</html>"})
|
||||
|
||||
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_get_bypassed_page_retries_and_rotates_selector_between_attempts(monkeypatch):
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -113,7 +115,9 @@ def test_extract_cookies_from_cdp_keeps_full_session_cookies_for_configured_zlib
|
||||
async def evaluate(self, _expr):
|
||||
return "TestUA/1.0"
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "_get_full_cookie_domains", lambda: {"z-lib.fm"})
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_get_full_cookie_domains", lambda: {"z-lib.fm"})
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
asyncio.run(
|
||||
@@ -165,12 +169,14 @@ def test_extract_cookies_from_cdp_normalizes_session_expiry():
|
||||
)
|
||||
)
|
||||
|
||||
stored = internal_bypasser._cf_cookies.get("example.com", {})
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
stored = cookie_store._cf_cookies.get("example.com", {})
|
||||
assert stored["cf_clearance"]["expiry"] is None
|
||||
assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {"cf_clearance": "abc"}
|
||||
|
||||
# Verify fallback to "expires" key for expiry checks
|
||||
internal_bypasser._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10
|
||||
cookie_store._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10
|
||||
assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {}
|
||||
|
||||
|
||||
@@ -374,7 +380,9 @@ def test_try_with_cached_cookies_returns_none_on_request_exception(monkeypatch):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
internal_bypasser._cf_cookies["example.com"] = {
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
cookie_store._cf_cookies["example.com"] = {
|
||||
"cf_clearance": {
|
||||
"value": "abc",
|
||||
"domain": "example.com",
|
||||
@@ -482,3 +490,239 @@ def test_run_bypass_in_current_process_bounds_its_wait(monkeypatch):
|
||||
|
||||
assert result == "html"
|
||||
assert observed["timeout"] == internal_bypasser._IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def _write_fake_proc_entry(proc_root, pid: int, pgid: int, argv: list[str]) -> None:
|
||||
"""Create a /proc-shaped entry for a fake process."""
|
||||
entry = proc_root / str(pid)
|
||||
entry.mkdir()
|
||||
(entry / "cmdline").write_bytes(b"\0".join(arg.encode() for arg in argv) + b"\0")
|
||||
# pid (comm) state ppid pgrp ... - comm is parenthesised and may contain spaces.
|
||||
(entry / "stat").write_text(f"{pid} (some (odd) name) S 1 {pgid} {pgid} 0 -1 4194304 0 0")
|
||||
|
||||
|
||||
def test_cleanup_only_kills_own_and_abandoned_browser_sessions(monkeypatch, tmp_path):
|
||||
"""Regression test for issue #1231: the sweep used a container-wide `pkill -f chrome`,
|
||||
so every worker that started a bypass killed the browsers the other workers were
|
||||
still driving. Only our own process group and groups whose leader is gone are ours."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
proc_root = tmp_path / "proc"
|
||||
proc_root.mkdir()
|
||||
_write_fake_proc_entry(proc_root, 1000, 1000, ["python", "-m", "shelfmark.bypass"])
|
||||
_write_fake_proc_entry(proc_root, 1001, 1000, ["/usr/bin/chromium", "--headless"])
|
||||
_write_fake_proc_entry(proc_root, 1002, 1000, ["Xvfb", ":99"])
|
||||
# Live sibling session: another worker is solving a challenge with these right now.
|
||||
_write_fake_proc_entry(proc_root, 2000, 2000, ["python", "-m", "shelfmark.bypass"])
|
||||
_write_fake_proc_entry(proc_root, 2001, 2000, ["/usr/bin/chromium", "--headless"])
|
||||
# Abandoned session: its leader (pid 3000) is gone, so its browser really is an orphan.
|
||||
_write_fake_proc_entry(proc_root, 3001, 3000, ["/usr/bin/chromium", "--headless"])
|
||||
|
||||
killed: list[int] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.env, "DOCKERMODE", True)
|
||||
monkeypatch.setattr(internal_bypasser, "_PROC_ROOT", proc_root)
|
||||
monkeypatch.setattr(internal_bypasser.os, "getpid", lambda: 1000)
|
||||
monkeypatch.setattr(internal_bypasser.os, "getpgrp", lambda: 1000)
|
||||
monkeypatch.setattr(internal_bypasser.os, "kill", lambda pid, _sig: killed.append(pid))
|
||||
monkeypatch.setattr(internal_bypasser.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert internal_bypasser._cleanup_orphan_processes() == 3
|
||||
assert sorted(killed) == [1001, 1002, 3001]
|
||||
|
||||
|
||||
def test_cleanup_is_skipped_without_proc(monkeypatch, tmp_path):
|
||||
"""Without /proc there is no way to tell sessions apart, so kill nothing."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.env, "DOCKERMODE", True)
|
||||
monkeypatch.setattr(internal_bypasser, "_PROC_ROOT", tmp_path / "missing")
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.os, "kill", lambda *_args: pytest.fail("must not kill anything")
|
||||
)
|
||||
|
||||
assert internal_bypasser._cleanup_orphan_processes() == 0
|
||||
|
||||
|
||||
class _FakeHelperStdin:
|
||||
"""The request pipe: a write is how the helper receives one request."""
|
||||
|
||||
def __init__(self, process):
|
||||
self._process = process
|
||||
self.closed = False
|
||||
|
||||
def write(self, data):
|
||||
self._process.serve(data)
|
||||
|
||||
def flush(self):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeHelperProcess:
|
||||
"""Stand-in for the bypass helper subprocess.
|
||||
|
||||
The helper serves one request per line of stdin and answers by writing the result file
|
||||
the request named, so that is what this fakes: a write produces an answer.
|
||||
"""
|
||||
|
||||
def __init__(self, *_args, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.pid = 4242
|
||||
self.returncode = None
|
||||
self.answers = True
|
||||
self.killed = False
|
||||
self.waited = False
|
||||
self.stdin = _FakeHelperStdin(self)
|
||||
self.requests: list[dict] = []
|
||||
|
||||
def serve(self, payload):
|
||||
request = json.loads(payload)
|
||||
self.requests.append(request)
|
||||
if not self.answers:
|
||||
return
|
||||
result = {"ok": True, "html": "<html>solved</html>", "cookies": {}, "user_agents": {}}
|
||||
Path(request["result_path"]).write_text(json.dumps(result), encoding="utf-8")
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def kill(self):
|
||||
self.killed = True
|
||||
self.returncode = -9
|
||||
|
||||
def wait(self, timeout=None):
|
||||
self.waited = True
|
||||
if self.returncode is None:
|
||||
self.returncode = 0
|
||||
return self.returncode
|
||||
|
||||
|
||||
def _patch_helper_subprocess(monkeypatch, internal_bypasser, process, killed_groups):
|
||||
monkeypatch.setattr(internal_bypasser.subprocess, "Popen", lambda *a, **kw: process(*a, **kw))
|
||||
monkeypatch.setattr(internal_bypasser.network, "get_dns_config", dict)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.os, "killpg", lambda pgid, _sig: killed_groups.append(pgid)
|
||||
)
|
||||
# A fresh helper per test: the module-level one is shared, and a process parked by one
|
||||
# test would be handed to the next.
|
||||
helper = internal_bypasser._BypassHelper()
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 0.0)
|
||||
monkeypatch.setattr(internal_bypasser, "_BYPASS_HELPER", helper)
|
||||
return helper
|
||||
|
||||
|
||||
def test_helper_runs_in_its_own_session_and_is_torn_down(monkeypatch):
|
||||
"""Regression test for issue #1231: the helper's Chrome and Xvfb must belong to the
|
||||
helper's own process group, and the whole group must die with it - otherwise the
|
||||
leftovers break the next worker's browser and can only be cleared by a sweep broad
|
||||
enough to kill a concurrent worker's browser too.
|
||||
|
||||
The helper outlives a single request, so the teardown happens when it is dropped rather
|
||||
than after every solve. Each bypass still closes its own browser, so what survives in
|
||||
between is the process, not a Chrome.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
helper = _patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
|
||||
assert internal_bypasser._get_via_subprocess("https://example.com", 1) == "<html>solved</html>"
|
||||
assert processes[0].kwargs["start_new_session"] is True
|
||||
assert killed_groups == [], "the helper was torn down after a single request"
|
||||
|
||||
helper._discard()
|
||||
|
||||
assert killed_groups == [processes[0].pid]
|
||||
|
||||
|
||||
def test_helper_serves_a_second_request_without_respawning(monkeypatch):
|
||||
"""The interpreter start and imports are paid once, not per protected request."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
_patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
|
||||
internal_bypasser._get_via_subprocess("https://example.com/one", 1)
|
||||
internal_bypasser._get_via_subprocess("https://example.com/two", 1)
|
||||
|
||||
assert len(processes) == 1
|
||||
assert [request["url"] for request in processes[0].requests] == [
|
||||
"https://example.com/one",
|
||||
"https://example.com/two",
|
||||
]
|
||||
|
||||
|
||||
def test_helper_timeout_kills_the_whole_session(monkeypatch):
|
||||
"""A timed-out solve must not leave a live browser behind for the next worker."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
process.answers = False # accepts the request, never writes a result
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
_patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
monkeypatch.setattr(internal_bypasser, "_BYPASS_SUBPROCESS_TIMEOUT_SECONDS", 0.1)
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
internal_bypasser._get_via_subprocess("https://example.com", 1)
|
||||
|
||||
assert killed_groups == [processes[0].pid]
|
||||
assert processes[0].killed is True
|
||||
|
||||
|
||||
def test_helper_takes_the_browser_down_when_its_parent_dies(monkeypatch):
|
||||
"""Cleanup only reclaims process groups whose leader is gone (#1231), so an orphaned
|
||||
helper must not sit there holding a browser no later bypass is allowed to touch."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
terminated: list[str] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.os, "getppid", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser, "_terminate_own_session", lambda: terminated.append("terminated")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.time, "sleep", lambda _seconds: pytest.fail("should not wait")
|
||||
)
|
||||
|
||||
internal_bypasser._watch_parent_process(999, interval=0.0)
|
||||
|
||||
assert terminated == ["terminated"]
|
||||
|
||||
|
||||
def test_helper_watchdog_waits_while_its_parent_is_alive(monkeypatch):
|
||||
"""The watchdog must only fire on a changed ppid, not on every poll."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
ppids = iter([999, 999, 1])
|
||||
sleeps: list[float] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.os, "getppid", lambda: next(ppids))
|
||||
monkeypatch.setattr(internal_bypasser, "_terminate_own_session", lambda: None)
|
||||
monkeypatch.setattr(internal_bypasser.time, "sleep", sleeps.append)
|
||||
|
||||
internal_bypasser._watch_parent_process(999, interval=0.5)
|
||||
|
||||
assert sleeps == [0.5, 0.5]
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
"""Tests for keeping the bypass helper process alive between requests.
|
||||
|
||||
The browser is deliberately not kept: every bypass starts and closes its own Chrome. What
|
||||
survives is the helper process, whose interpreter start and imports are pure overhead.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _FakeStdin:
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
self.written: list[str] = []
|
||||
|
||||
def write(self, data: str) -> None:
|
||||
if self.closed:
|
||||
raise BrokenPipeError("stdin is closed")
|
||||
self.written.append(data)
|
||||
|
||||
def flush(self) -> None:
|
||||
return None
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeProc:
|
||||
"""Enough of subprocess.Popen for the helper's process bookkeeping."""
|
||||
|
||||
_next_pid = 90001
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.stdin = _FakeStdin()
|
||||
self.returncode: int | None = None
|
||||
self.waited = False
|
||||
# A pid nothing may actually be signalled by: _terminate_helper_session is patched
|
||||
# out in these tests, and a stray killpg on a live pid would take out the test run.
|
||||
type(self)._next_pid += 1
|
||||
self.pid = type(self)._next_pid
|
||||
|
||||
def poll(self) -> int | None:
|
||||
return self.returncode
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
self.waited = True
|
||||
if self.returncode is None:
|
||||
self.returncode = 0
|
||||
return self.returncode
|
||||
|
||||
def kill(self) -> None:
|
||||
self.returncode = -9
|
||||
|
||||
|
||||
def _helper_with_fake_spawn(monkeypatch, procs: list[_FakeProc], terminated=None):
|
||||
"""Build a helper that hands out fake processes and never arms a real timer."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
def _spawn(_self) -> _FakeProc:
|
||||
proc = _FakeProc()
|
||||
procs.append(proc)
|
||||
return proc
|
||||
|
||||
def _terminate(proc) -> None:
|
||||
if terminated is not None:
|
||||
terminated.append(proc)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_spawn", _spawn)
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 0.0)
|
||||
monkeypatch.setattr(internal_bypasser, "_terminate_helper_session", _terminate)
|
||||
return internal_bypasser._BypassHelper()
|
||||
|
||||
|
||||
def _answered_payload(tmp_path, name: str = "result.json") -> dict:
|
||||
"""A request whose result file already exists, so the helper resolves immediately."""
|
||||
result_path = tmp_path / name
|
||||
result_path.write_text(json.dumps({"ok": True, "html": "<html/>"}), encoding="utf-8")
|
||||
return {"url": "https://example.com", "retry": 1, "result_path": str(result_path)}
|
||||
|
||||
|
||||
def test_helper_serves_consecutive_requests_from_one_process(monkeypatch, tmp_path):
|
||||
"""The point of the whole thing: request two and three must not re-pay the spawn."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
for i in range(3):
|
||||
result = helper.run(_answered_payload(tmp_path, f"r{i}.json"), timeout=5, cancel_flag=None)
|
||||
assert result["ok"] is True
|
||||
|
||||
assert len(procs) == 1, "each request spawned its own helper"
|
||||
assert len(procs[0].stdin.written) == 3
|
||||
assert all(line.endswith("\n") for line in procs[0].stdin.written), (
|
||||
"requests must be newline-delimited or the helper's loop cannot split them"
|
||||
)
|
||||
|
||||
|
||||
def test_helper_respawns_after_the_previous_one_died(monkeypatch, tmp_path):
|
||||
"""A helper can be reaped while idle; the next request must not fail on it."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path, "a.json"), timeout=5, cancel_flag=None)
|
||||
procs[0].returncode = 1 # died between requests
|
||||
|
||||
result = helper.run(_answered_payload(tmp_path, "b.json"), timeout=5, cancel_flag=None)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert len(procs) == 2
|
||||
|
||||
|
||||
def test_helper_retries_once_when_the_pipe_breaks_on_write(monkeypatch, tmp_path):
|
||||
"""poll() can still say alive when the far end is already gone."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path, "a.json"), timeout=5, cancel_flag=None)
|
||||
procs[0].stdin.closed = True # pipe gone, but poll() still reports running
|
||||
|
||||
result = helper.run(_answered_payload(tmp_path, "b.json"), timeout=5, cancel_flag=None)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert len(procs) == 2
|
||||
|
||||
|
||||
def test_helper_reports_a_helper_that_exits_without_answering(monkeypatch, tmp_path):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
payload = {
|
||||
"url": "https://example.com",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / "never-written.json"),
|
||||
}
|
||||
|
||||
def _die_on_write(_self, proc, _line) -> None:
|
||||
proc.returncode = 3
|
||||
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_write", _die_on_write)
|
||||
|
||||
with pytest.raises(RuntimeError, match="exited without a result"):
|
||||
helper.run(payload, timeout=5, cancel_flag=None)
|
||||
|
||||
|
||||
def test_helper_times_out_and_discards_the_wedged_process(monkeypatch, tmp_path):
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
payload = {
|
||||
"url": "https://example.com",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / "never-written.json"),
|
||||
}
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
helper.run(payload, timeout=0.05, cancel_flag=None)
|
||||
|
||||
assert helper._proc is None, "a wedged helper must not be handed to the next request"
|
||||
|
||||
|
||||
def test_idle_reaper_rearms_when_work_arrived_while_it_waited(monkeypatch, tmp_path):
|
||||
"""The timer fires on its own thread and can lose the race against a new request."""
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
|
||||
rearmed: list[bool] = []
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 3600.0)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser._BypassHelper, "_arm_idle_timer", lambda _self: rearmed.append(True)
|
||||
)
|
||||
helper._last_used = time.monotonic()
|
||||
|
||||
helper._reap_if_idle()
|
||||
|
||||
assert rearmed == [True]
|
||||
assert helper._proc is not None, "helper was killed despite recent work"
|
||||
|
||||
|
||||
def test_idle_reaper_closes_a_genuinely_idle_helper(monkeypatch, tmp_path):
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 60.0)
|
||||
helper._last_used = time.monotonic() - 120
|
||||
|
||||
helper._reap_if_idle()
|
||||
|
||||
assert helper._proc is None
|
||||
assert procs[0].stdin.closed
|
||||
|
||||
|
||||
def test_discard_tears_down_the_whole_session(monkeypatch, tmp_path):
|
||||
"""Dropping the helper must reach its browser tree, not just the helper itself.
|
||||
|
||||
The helper is a session leader (start_new_session), so a Chrome left behind by one
|
||||
killed mid-bypass would keep a process group alive that the cleanup sweep is then not
|
||||
allowed to reclaim - the leak #1231 was about.
|
||||
"""
|
||||
procs: list[_FakeProc] = []
|
||||
terminated: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs, terminated)
|
||||
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
helper._discard()
|
||||
|
||||
assert terminated == [procs[0]]
|
||||
|
||||
|
||||
def test_helper_asks_before_it_kills(monkeypatch, tmp_path):
|
||||
"""An idle helper should get to exit on its own; the kill is the fallback."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
helper._discard()
|
||||
|
||||
assert procs[0].stdin.closed, "stdin must be closed to end the helper's request loop"
|
||||
assert procs[0].returncode == 0, "an idle helper should have exited on its own"
|
||||
|
||||
|
||||
def _bypass_with_recorded_driver(monkeypatch, get_impl):
|
||||
"""Wire up a bypass whose browser creation and closing are observable."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
driver = object()
|
||||
closed: list[object] = []
|
||||
|
||||
async def _create(_url):
|
||||
return driver
|
||||
|
||||
async def _close(drv):
|
||||
closed.append(drv)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "_create_cdp_browser", _create)
|
||||
monkeypatch.setattr(internal_bypasser, "_get", get_impl)
|
||||
monkeypatch.setattr(internal_bypasser, "_close_cdp_driver", _close)
|
||||
return driver, closed
|
||||
|
||||
|
||||
def test_successful_bypass_closes_its_browser(monkeypatch):
|
||||
"""A living helper must not accumulate browsers: each bypass ends with Chrome gone."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
return "<html>ok</html>"
|
||||
|
||||
driver, closed = _bypass_with_recorded_driver(monkeypatch, _get)
|
||||
|
||||
result = internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert closed == [driver]
|
||||
|
||||
|
||||
def test_failed_bypass_closes_its_browser(monkeypatch):
|
||||
"""The same has to hold when the bypass raises on its way out."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
raise internal_bypasser.BypassCancelledError("cancelled")
|
||||
|
||||
driver, closed = _bypass_with_recorded_driver(monkeypatch, _get)
|
||||
|
||||
with pytest.raises(internal_bypasser.BypassCancelledError):
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert closed == [driver]
|
||||
|
||||
|
||||
def test_child_process_serves_every_line_it_is_given(monkeypatch, tmp_path):
|
||||
"""One helper, several requests: the loop is what saves the repeated process start."""
|
||||
import io
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
urls: list[str] = []
|
||||
|
||||
def _fake_get(url, retry=None, cancel_flag=None):
|
||||
urls.append(url)
|
||||
return f"<html>{url}</html>"
|
||||
|
||||
requests = [
|
||||
{"url": "https://example.com/one", "retry": 1, "result_path": str(tmp_path / "1.json")},
|
||||
{"url": "https://example.com/two", "retry": 1, "result_path": str(tmp_path / "2.json")},
|
||||
]
|
||||
stdin = io.StringIO("\n".join(json.dumps(request) for request in requests) + "\n")
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "get", _fake_get)
|
||||
monkeypatch.setattr(internal_bypasser.sys, "stdin", stdin)
|
||||
|
||||
assert internal_bypasser._run_child_process() == 0
|
||||
assert urls == ["https://example.com/one", "https://example.com/two"]
|
||||
|
||||
for index, request in enumerate(requests, start=1):
|
||||
result = json.loads((tmp_path / f"{index}.json").read_text(encoding="utf-8"))
|
||||
assert result["ok"] is True
|
||||
assert result["html"] == f"<html>{request['url']}</html>"
|
||||
|
||||
|
||||
def test_child_process_keeps_serving_after_a_failed_request(monkeypatch, tmp_path):
|
||||
"""One failing URL must not take the helper - and everything queued - down."""
|
||||
import io
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
def _fake_get(url, retry=None, cancel_flag=None):
|
||||
if url.endswith("boom"):
|
||||
raise RuntimeError("bypass exploded")
|
||||
return "<html>ok</html>"
|
||||
|
||||
requests = [
|
||||
{"url": "https://example.com/boom", "retry": 1, "result_path": str(tmp_path / "1.json")},
|
||||
{"url": "https://example.com/fine", "retry": 1, "result_path": str(tmp_path / "2.json")},
|
||||
]
|
||||
stdin = io.StringIO("\n".join(json.dumps(request) for request in requests) + "\n")
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "get", _fake_get)
|
||||
monkeypatch.setattr(internal_bypasser.sys, "stdin", stdin)
|
||||
|
||||
assert internal_bypasser._run_child_process() == 0
|
||||
|
||||
failed = json.loads((tmp_path / "1.json").read_text(encoding="utf-8"))
|
||||
assert failed["ok"] is False
|
||||
assert failed["error"] == "bypass exploded"
|
||||
|
||||
served = json.loads((tmp_path / "2.json").read_text(encoding="utf-8"))
|
||||
assert served["ok"] is True
|
||||
|
||||
|
||||
def test_result_file_becomes_visible_only_when_complete(tmp_path):
|
||||
"""The parent treats the file's existence as the answer, so no partial writes."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
result_path = tmp_path / "result.json"
|
||||
internal_bypasser._publish_result(result_path, {"ok": True, "html": "<html/>"})
|
||||
|
||||
assert json.loads(result_path.read_text(encoding="utf-8"))["ok"] is True
|
||||
assert list(tmp_path.iterdir()) == [result_path], "temporary file was left behind"
|
||||
|
||||
|
||||
def test_child_bypass_runs_on_the_long_lived_worker_loop(monkeypatch):
|
||||
"""A helper serving many requests must not build and close a loop per bypass.
|
||||
|
||||
asyncio.run() owns the loop for one call and closes it on the way out, which is why the
|
||||
child goes through the worker unconditionally: one loop for the process's lifetime.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setenv("SHELFMARK_INTERNAL_BYPASSER_CHILD", "1")
|
||||
|
||||
loops: list[asyncio.AbstractEventLoop] = []
|
||||
|
||||
async def _record_loop(_url, _driver, _cancel=None):
|
||||
loops.append(asyncio.get_running_loop())
|
||||
return "<html>ok</html>"
|
||||
|
||||
_bypass_with_recorded_driver(monkeypatch, _record_loop)
|
||||
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert len(loops) == 2
|
||||
assert loops[0] is loops[1], "second bypass ran on a different loop than the first"
|
||||
assert not loops[0].is_closed()
|
||||
|
||||
|
||||
def test_child_bypass_carries_its_own_deadline(monkeypatch):
|
||||
"""The child bounds itself, rather than relying only on the parent's deadline."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setenv("SHELFMARK_INTERNAL_BYPASSER_CHILD", "1")
|
||||
|
||||
timeouts: list[float | None] = []
|
||||
real_run = internal_bypasser._CDP_WORKER.run
|
||||
|
||||
def _record_timeout(coro, timeout=None):
|
||||
timeouts.append(timeout)
|
||||
return real_run(coro, timeout=timeout)
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
return "<html>ok</html>"
|
||||
|
||||
_bypass_with_recorded_driver(monkeypatch, _get)
|
||||
monkeypatch.setattr(internal_bypasser._CDP_WORKER, "run", _record_timeout)
|
||||
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert timeouts == [internal_bypasser._CHILD_BYPASS_TIMEOUT_SECONDS]
|
||||
|
||||
|
||||
def test_child_deadline_leaves_the_parent_room_to_hear_the_answer(monkeypatch):
|
||||
"""If the parent gave up first it could only kill the helper, losing a warm process."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
# The child's worst case is its deadline plus the grace it is given to close the
|
||||
# browser after that deadline cancels the bypass, and all of it has to fit inside the
|
||||
# parent's wait - otherwise the parent gives up first and kills a helper that was
|
||||
# about to answer.
|
||||
assert (
|
||||
internal_bypasser._CHILD_BYPASS_TIMEOUT_SECONDS
|
||||
+ internal_bypasser._CDP_UNWIND_GRACE_SECONDS
|
||||
< internal_bypasser._BYPASS_SUBPROCESS_TIMEOUT_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def test_in_process_bypass_keeps_the_parents_budget(monkeypatch):
|
||||
"""Non-Docker installs run in-process, where there is no helper to outlive anything."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.delenv("SHELFMARK_INTERNAL_BYPASSER_CHILD", raising=False)
|
||||
|
||||
timeouts: list[float | None] = []
|
||||
real_run = internal_bypasser._CDP_WORKER.run
|
||||
|
||||
def _record_timeout(coro, timeout=None):
|
||||
timeouts.append(timeout)
|
||||
return real_run(coro, timeout=timeout)
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
return "<html>ok</html>"
|
||||
|
||||
_bypass_with_recorded_driver(monkeypatch, _get)
|
||||
monkeypatch.setattr(internal_bypasser._CDP_WORKER, "run", _record_timeout)
|
||||
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert timeouts == [internal_bypasser._IN_PROCESS_BYPASS_TIMEOUT_SECONDS]
|
||||
|
||||
|
||||
def test_timed_out_bypass_finishes_unwinding_before_the_call_returns():
|
||||
"""A helper serving the next request must not race the browser teardown of the last.
|
||||
|
||||
The deadline cancels the bypass, but cancelling from the calling thread only schedules
|
||||
that - it returns while `finally: await _close_cdp_driver(driver)` is still running.
|
||||
In a helper that now outlives the request, the next bypass would open its Chrome on the
|
||||
same loop while the abandoned one was still closing its own, sharing the DISPLAY
|
||||
globals and one process group.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
events: list[str] = []
|
||||
|
||||
async def _wedged():
|
||||
try:
|
||||
await asyncio.sleep(30)
|
||||
finally:
|
||||
# Teardown that yields, the way closing websockets and Chrome does.
|
||||
await asyncio.sleep(0.05)
|
||||
events.append("browser closed")
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
internal_bypasser._CDP_WORKER.run(_wedged(), timeout=0.1)
|
||||
|
||||
assert events == ["browser closed"], "run() returned before the bypass had unwound"
|
||||
|
||||
|
||||
def test_unwind_that_wedges_does_not_hold_the_caller_forever(monkeypatch):
|
||||
"""The grace is a bound, not a promise: cleanup can hang on a dead browser too."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "_CDP_UNWIND_GRACE_SECONDS", 0.1)
|
||||
|
||||
async def _wedged_on_both_ends():
|
||||
try:
|
||||
await asyncio.sleep(30)
|
||||
finally:
|
||||
await asyncio.sleep(30)
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
internal_bypasser._CDP_WORKER.run(_wedged_on_both_ends(), timeout=0.1)
|
||||
|
||||
|
||||
def test_cancelling_does_not_wait_out_the_shutdown_grace(monkeypatch, tmp_path):
|
||||
"""The grace only helps a helper that can still read its stdin.
|
||||
|
||||
One dropped mid-bypass is blocked inside the solve and will never reach its read loop,
|
||||
so waiting it out cannot end in anything but the kill - while the user who asked to
|
||||
cancel, and every bypass queued behind them on LOCKED, waits for it.
|
||||
"""
|
||||
import threading
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
cancel_flag = threading.Event()
|
||||
cancel_flag.set()
|
||||
payload = {
|
||||
"url": "https://example.com",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / "never-written.json"),
|
||||
}
|
||||
|
||||
with pytest.raises(internal_bypasser.BypassCancelledError):
|
||||
helper.run(payload, timeout=5, cancel_flag=cancel_flag)
|
||||
|
||||
assert not procs[0].waited, "a helper wedged mid-bypass was given the full exit grace"
|
||||
assert procs[0].stdin.closed
|
||||
|
||||
|
||||
def test_idle_helper_still_gets_its_grace(monkeypatch, tmp_path):
|
||||
"""The reaper drops a helper that *is* in its read loop, and that one gets to exit."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
helper._discard()
|
||||
|
||||
assert procs[0].waited, "an idle helper should be asked to exit before being killed"
|
||||
|
||||
|
||||
def test_failed_request_leaves_no_result_files_behind(monkeypatch, tmp_path):
|
||||
"""Result paths are unique per request, so anything left is left for good."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
result_path = tmp_path / "result.json"
|
||||
# A helper killed part-way through _publish_result leaves the staging file.
|
||||
(tmp_path / "result.json.part").write_text('{"ok": tr', encoding="utf-8")
|
||||
payload = {"url": "https://example.com", "retry": 1, "result_path": str(result_path)}
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
helper.run(payload, timeout=0.05, cancel_flag=None)
|
||||
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_child_does_not_export_cookies_left_by_an_earlier_request(monkeypatch, tmp_path):
|
||||
"""The parent owns the store; a warm helper must not push its own history back over it.
|
||||
|
||||
http.py purges a host's clearance the moment that host challenges a request carrying
|
||||
it. A helper that kept its store across requests would still be holding the purged
|
||||
cookies, and the next solve - for some entirely different host - would export them and
|
||||
the parent would merge them straight back in.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
def _solve_host(url, retry=None, cancel_flag=None):
|
||||
# A solve fills the store for the host it solved, which is all it should report.
|
||||
host = url.rsplit("/", 1)[-1]
|
||||
internal_bypasser.import_store({host: {"cf_clearance": "fresh"}}, {host: "UA"})
|
||||
return "<html>ok</html>"
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "get", _solve_host)
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
|
||||
for index, host in enumerate(("first.example", "second.example")):
|
||||
internal_bypasser._handle_child_request(
|
||||
json.dumps(
|
||||
{
|
||||
"url": f"https://example.com/{host}",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / f"{index}.json"),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
second = json.loads((tmp_path / "1.json").read_text(encoding="utf-8"))
|
||||
assert list(second["cookies"]) == ["second.example"], (
|
||||
"the helper exported clearance won by an earlier request"
|
||||
)
|
||||
assert list(second["user_agents"]) == ["second.example"]
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
|
||||
|
||||
def _record_dns_calls(monkeypatch):
|
||||
"""Stand in for the network module: report a resolver state, record changes to it.
|
||||
|
||||
A helper starts on system DNS, which is what the parent reports as "auto".
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
calls: list[tuple] = []
|
||||
state = {"provider": "auto", "servers": [], "doh_enabled": False}
|
||||
|
||||
def _set(provider, servers=None, use_doh=None):
|
||||
calls.append((provider, servers, use_doh))
|
||||
state.update({"provider": provider, "servers": servers or [], "doh_enabled": bool(use_doh)})
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.network, "set_dns_provider", _set)
|
||||
monkeypatch.setattr(internal_bypasser.network, "get_dns_config", lambda: dict(state))
|
||||
return internal_bypasser, calls
|
||||
|
||||
|
||||
def test_helper_follows_the_parent_back_to_auto_dns(monkeypatch):
|
||||
"""A user flipping CUSTOM_DNS back to auto applies live - the helper has to hear it.
|
||||
|
||||
The old early-return on "auto" was correct only because a fresh helper had never been
|
||||
told anything else. One that outlives the request has, and would go on resolving AA
|
||||
through a resolver the parent has already abandoned.
|
||||
"""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "cloudflare", "servers": [], "doh_enabled": True}
|
||||
)
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "auto", "servers": [], "doh_enabled": False}
|
||||
)
|
||||
|
||||
assert calls == [("cloudflare", None, True), ("auto", None, False)]
|
||||
|
||||
|
||||
def test_helper_does_not_reinitialize_dns_for_an_unchanged_config(monkeypatch):
|
||||
"""set_dns_provider() rebuilds resolvers; every request would pay for it otherwise."""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
for _ in range(3):
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "quad9", "servers": [], "doh_enabled": True}
|
||||
)
|
||||
|
||||
assert calls == [("quad9", None, True)]
|
||||
|
||||
|
||||
def test_fresh_helper_leaves_auto_dns_alone(monkeypatch):
|
||||
"""A helper starts on system DNS, which is what the parent reports as auto."""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "auto", "servers": [], "doh_enabled": False}
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_failed_dns_apply_is_retried_on_the_next_request(monkeypatch):
|
||||
"""A provider that did not land leaves the resolver where it was, so the next request
|
||||
sees the same mismatch and tries again."""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
def _explode(provider, servers=None, use_doh=None):
|
||||
calls.append((provider, servers, use_doh))
|
||||
msg = "resolver unreachable"
|
||||
raise OSError(msg)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.network, "set_dns_provider", _explode)
|
||||
|
||||
config = {"provider": "google", "servers": [], "doh_enabled": True}
|
||||
internal_bypasser._apply_parent_dns_config(config)
|
||||
internal_bypasser._apply_parent_dns_config(config)
|
||||
|
||||
assert calls == [("google", None, True), ("google", None, True)]
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Tests for widening the audiobook format list on existing installs.
|
||||
|
||||
`initialize_default_configs()` only writes field defaults when a tab has no config file
|
||||
yet, so widening the default alone would have reached fresh installs only - exactly not
|
||||
the installs already carrying the narrow m4b/mp3 list that loses FLAC/OPUS releases.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.config.migrations import migrate_audiobook_formats
|
||||
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
|
||||
|
||||
WIDENED = [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def migrate():
|
||||
"""Run the migration over an in-memory config, returning the resulting config."""
|
||||
|
||||
def run(config: dict | None) -> dict:
|
||||
stored = {} if config is None else dict(config)
|
||||
saved: dict = {}
|
||||
|
||||
def save(values: dict) -> None:
|
||||
saved.update(values)
|
||||
stored.update(values)
|
||||
|
||||
migrate_audiobook_formats(
|
||||
load_general_config=lambda: stored,
|
||||
save_general_config=save,
|
||||
widened_formats=WIDENED,
|
||||
logger=logging.getLogger("test"),
|
||||
)
|
||||
return stored
|
||||
|
||||
return run
|
||||
|
||||
|
||||
def test_legacy_default_is_widened(migrate):
|
||||
result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"]})
|
||||
|
||||
assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED
|
||||
|
||||
|
||||
def test_legacy_default_is_widened_regardless_of_order_or_case(migrate):
|
||||
result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["MP3", " m4b "]})
|
||||
|
||||
assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED
|
||||
|
||||
|
||||
def test_other_settings_are_preserved(migrate):
|
||||
result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"], "SUPPORTED_FORMATS": ["epub"]})
|
||||
|
||||
assert result["SUPPORTED_FORMATS"] == ["epub"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"customized",
|
||||
[
|
||||
["m4b"], # deliberately narrowed - re-enabling formats would override the choice
|
||||
["mp3", "flac"],
|
||||
["m4b", "mp3", "zip"],
|
||||
[],
|
||||
],
|
||||
)
|
||||
def test_customized_lists_are_left_alone(migrate, customized):
|
||||
result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": customized})
|
||||
|
||||
assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == customized
|
||||
|
||||
|
||||
def test_absent_key_is_left_alone(migrate):
|
||||
"""Nothing persisted means the field default already applies - don't write one."""
|
||||
result = migrate({"SUPPORTED_FORMATS": ["epub"]})
|
||||
|
||||
assert "SUPPORTED_AUDIOBOOK_FORMATS" not in result
|
||||
|
||||
|
||||
def test_unexpected_type_is_left_alone(migrate):
|
||||
result = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": "m4b,mp3"})
|
||||
|
||||
assert result["SUPPORTED_AUDIOBOOK_FORMATS"] == "m4b,mp3"
|
||||
|
||||
|
||||
def test_migration_is_idempotent(migrate):
|
||||
once = migrate({"SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"]})
|
||||
twice = migrate(once)
|
||||
|
||||
assert twice["SUPPORTED_AUDIOBOOK_FORMATS"] == WIDENED
|
||||
@@ -0,0 +1,38 @@
|
||||
"""The audiobook format list must stay in agreement across every layer that gates on it.
|
||||
|
||||
These lists were maintained by hand in four places and drifted: the settings UI offered
|
||||
only m4b/mp3/m4a, so FLAC/OPUS/OGG could never be enabled even though the parsers
|
||||
recognized them, the sorter ranked them, and archive extraction knew them. The result was
|
||||
a FLAC audiobook that was invisible in search and rejected after download. They now all
|
||||
derive from `shelfmark.core.utils.AUDIOBOOK_FORMATS`; this test fails if one drifts again.
|
||||
"""
|
||||
|
||||
from shelfmark.config.settings import _AUDIOBOOK_FORMAT_OPTIONS
|
||||
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
|
||||
from shelfmark.download.archive import ALL_AUDIO_EXTENSIONS
|
||||
from shelfmark.release_sources.irc import parser
|
||||
from shelfmark.release_sources.prowlarr.source import AUDIOBOOK_FORMATS as PROWLARR_FORMATS
|
||||
|
||||
|
||||
def test_archive_extraction_knows_every_audiobook_format():
|
||||
assert ALL_AUDIO_EXTENSIONS == {f".{fmt}" for fmt in AUDIOBOOK_FORMATS}
|
||||
|
||||
|
||||
def test_prowlarr_knows_every_audiobook_format():
|
||||
assert PROWLARR_FORMATS == list(AUDIOBOOK_FORMATS)
|
||||
|
||||
|
||||
def test_irc_parser_knows_every_audiobook_format():
|
||||
assert set(AUDIOBOOK_FORMATS) <= set(parser.ALL_RECOGNIZED_FORMATS)
|
||||
|
||||
|
||||
def test_every_audiobook_format_is_selectable_in_settings():
|
||||
"""The settings list is the only one a user's config can be built from."""
|
||||
selectable = {option["value"] for option in _AUDIOBOOK_FORMAT_OPTIONS}
|
||||
|
||||
assert selectable == {*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS}
|
||||
|
||||
|
||||
def test_audiobook_and_ebook_formats_do_not_overlap():
|
||||
"""Overlap would make content-type classification by extension ambiguous."""
|
||||
assert not set(AUDIOBOOK_FORMATS) & set(parser.EBOOK_FORMATS)
|
||||
@@ -95,6 +95,64 @@ def test_upsert_updates_existing_cwa_user_by_username_before_email(user_db):
|
||||
assert user["role"] == "admin"
|
||||
|
||||
|
||||
def test_upsert_renames_existing_cwa_user_matched_by_email(user_db):
|
||||
cwa_user = user_db.create_user(
|
||||
username="old_reader",
|
||||
email="reader@example.com",
|
||||
role="user",
|
||||
auth_source="cwa",
|
||||
)
|
||||
|
||||
user, action = upsert_cwa_user(
|
||||
user_db,
|
||||
cwa_username="renamed_reader",
|
||||
cwa_email="reader@example.com",
|
||||
role="user",
|
||||
)
|
||||
|
||||
assert action == "updated"
|
||||
assert user["id"] == cwa_user["id"]
|
||||
assert user["username"] == "renamed_reader"
|
||||
assert user_db.get_user(username="old_reader") is None
|
||||
|
||||
|
||||
def test_upsert_uses_stable_alias_when_renamed_cwa_username_is_taken(user_db):
|
||||
cwa_user = user_db.create_user(
|
||||
username="old_reader",
|
||||
email="reader@example.com",
|
||||
role="user",
|
||||
auth_source="cwa",
|
||||
)
|
||||
local_user = user_db.create_user(
|
||||
username="renamed_reader",
|
||||
email="local@example.com",
|
||||
role="user",
|
||||
auth_source="builtin",
|
||||
)
|
||||
|
||||
first, first_action = upsert_cwa_user(
|
||||
user_db,
|
||||
cwa_username="renamed_reader",
|
||||
cwa_email="reader@example.com",
|
||||
role="admin",
|
||||
)
|
||||
second, second_action = upsert_cwa_user(
|
||||
user_db,
|
||||
cwa_username="renamed_reader",
|
||||
cwa_email="reader@example.com",
|
||||
role="admin",
|
||||
)
|
||||
|
||||
assert first_action == second_action == "updated"
|
||||
assert first["id"] == second["id"] == cwa_user["id"]
|
||||
assert first["username"] == second["username"] == "renamed_reader__cwa"
|
||||
assert first["role"] == second["role"] == "admin"
|
||||
local_after = user_db.get_user(user_id=local_user["id"])
|
||||
assert local_after is not None
|
||||
assert local_after["username"] == "renamed_reader"
|
||||
assert local_after["email"] == "local@example.com"
|
||||
|
||||
|
||||
def test_sync_prunes_cwa_users_missing_from_source(user_db):
|
||||
active_cwa = user_db.create_user(
|
||||
username="active_cwa",
|
||||
|
||||
@@ -568,11 +568,13 @@ class TestUserCRUD:
|
||||
user = user_db.create_user(username="john", role="user")
|
||||
user_db.update_user(
|
||||
user["id"],
|
||||
username="jane",
|
||||
role="admin",
|
||||
email="new@example.com",
|
||||
auth_source="proxy",
|
||||
)
|
||||
updated = user_db.get_user(user_id=user["id"])
|
||||
assert updated["username"] == "jane"
|
||||
assert updated["role"] == "admin"
|
||||
assert updated["email"] == "new@example.com"
|
||||
assert updated["auth_source"] == "proxy"
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""A mirror that answers 200 with a non-AA page is quarantined, not reported as empty.
|
||||
|
||||
Seized and for-sale domains keep serving 200. Without a look at *what* came back, a
|
||||
parking page is indistinguishable from a broken search, so the mirror stays in
|
||||
rotation and every later search pays for it again.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from bs4 import Tag
|
||||
|
||||
PARKED_PAGE = """<!doctype html><html><head><title>annas-archive.li</title></head>
|
||||
<body><h1>This domain is for sale</h1><p>Inquire now. Buy this domain.</p></body></html>"""
|
||||
|
||||
AA_RESULTS_PAGE = """<!doctype html><html><body><main><table><tbody>
|
||||
<tr><td><a href="/md5/abc123"><img src="/c.jpg"></a></td><td><span>Dune</span></td></tr>
|
||||
</tbody></table></main></body></html>"""
|
||||
|
||||
AA_EMPTY_PAGE = """<!doctype html><html><body><main>
|
||||
<div>No files found.</div><a href="https://annas-archive.gl/about">about</a>
|
||||
</main></body></html>"""
|
||||
|
||||
DDOS_GUARD_PAGE = """<!doctype html><html><head><title>DDoS-Guard</title></head>
|
||||
<body><div id="ddos-guard">Checking your browser</div></body></html>"""
|
||||
|
||||
|
||||
class _Selector:
|
||||
def __init__(self, bases: list[str]) -> None:
|
||||
self._bases = bases
|
||||
self._index = 0
|
||||
self.current_base = bases[0]
|
||||
self.quarantined: list[str] = []
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
for base in self._bases:
|
||||
if url.startswith(base):
|
||||
return url.replace(base, self.current_base, 1)
|
||||
return url
|
||||
|
||||
def next_mirror_or_rotate_dns(self, *, fatal: bool = False, reason: str = ""):
|
||||
if fatal:
|
||||
self.quarantined.append(self.current_base)
|
||||
self._index += 1
|
||||
if self._index >= len(self._bases):
|
||||
return None, "exhausted"
|
||||
self.current_base = self._bases[self._index]
|
||||
return self.current_base, "mirror"
|
||||
|
||||
|
||||
def _patch_pages(monkeypatch, pages: list[str]):
|
||||
"""Serve `pages` in order, recording the URL each call was made against."""
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_get(url, **_kwargs):
|
||||
calls.append(url)
|
||||
return pages[len(calls) - 1] if len(calls) <= len(pages) else ""
|
||||
|
||||
monkeypatch.setattr(dd.downloader, "html_get_page", fake_get)
|
||||
monkeypatch.setattr(dd.network, "get_available_aa_urls", lambda: ["a", "b", "c"])
|
||||
return dd, calls
|
||||
|
||||
|
||||
def test_parked_mirror_is_quarantined_and_search_retries_next_mirror(monkeypatch):
|
||||
dd, calls = _patch_pages(monkeypatch, [PARKED_PAGE, AA_RESULTS_PAGE])
|
||||
selector = _Selector(["https://parked.test", "https://real.test"])
|
||||
|
||||
html, table = dd._fetch_search_table("https://parked.test/search?q=dune", selector)
|
||||
|
||||
assert selector.quarantined == ["https://parked.test"]
|
||||
assert isinstance(table, Tag)
|
||||
assert "Dune" in html
|
||||
# The retry went to the live mirror, not back to the parked one.
|
||||
assert calls[1].startswith("https://real.test")
|
||||
|
||||
|
||||
def test_genuinely_empty_aa_result_does_not_quarantine(monkeypatch):
|
||||
"""'No files found.' is a real answer from a healthy mirror."""
|
||||
dd, _calls = _patch_pages(monkeypatch, [AA_EMPTY_PAGE])
|
||||
selector = _Selector(["https://real.test", "https://other.test"])
|
||||
|
||||
html, table = dd._fetch_search_table("https://real.test/search?q=zzz", selector)
|
||||
|
||||
assert selector.quarantined == []
|
||||
assert table is None
|
||||
assert "No files found." in html
|
||||
|
||||
|
||||
def test_challenge_page_is_reported_not_passed_off_as_an_empty_result(monkeypatch):
|
||||
"""An unsolved interstitial means the search never ran.
|
||||
|
||||
The mirror is alive and holds our clearance, so it must not be quarantined - but
|
||||
returning it as "no table" made the caller tell the user their query found nothing.
|
||||
"""
|
||||
dd, _calls = _patch_pages(monkeypatch, [DDOS_GUARD_PAGE])
|
||||
selector = _Selector(["https://real.test", "https://other.test"])
|
||||
|
||||
with pytest.raises(dd.SearchUnavailableError, match="protection challenge"):
|
||||
dd._fetch_search_table("https://real.test/search?q=dune", selector)
|
||||
|
||||
assert selector.quarantined == []
|
||||
|
||||
|
||||
def test_unreachable_mirror_raises_search_unavailable(monkeypatch):
|
||||
dd, _calls = _patch_pages(monkeypatch, [""])
|
||||
selector = _Selector(["https://real.test"])
|
||||
|
||||
try:
|
||||
dd._fetch_search_table("https://real.test/search?q=dune", selector)
|
||||
except dd.SearchUnavailableError:
|
||||
return
|
||||
raise AssertionError("expected SearchUnavailableError")
|
||||
@@ -0,0 +1,143 @@
|
||||
"""RFC 8484 wireformat codec.
|
||||
|
||||
Quad9 and OpenDNS reject the JSON API that Cloudflare and Google popularised, so
|
||||
these providers only work through wireformat. Quad9 additionally requires HTTP/2
|
||||
(section 5.2) and answers HTTP/1.1 with 505.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import struct
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.download import doh_wireformat as wf
|
||||
|
||||
|
||||
def _decode_param(param: str) -> bytes:
|
||||
padding = "=" * (-len(param) % 4)
|
||||
return base64.urlsafe_b64decode(param + padding)
|
||||
|
||||
|
||||
def _build_response(
|
||||
*, qname: str = "example.com", answers: list[tuple[int, bytes]], qtype: int = wf.TYPE_A
|
||||
) -> bytes:
|
||||
"""Assemble a response whose answer names are compression pointers to the question."""
|
||||
question = b""
|
||||
for label in qname.split("."):
|
||||
question += bytes([len(label)]) + label.encode()
|
||||
question += b"\x00" + struct.pack(">HH", qtype, 1)
|
||||
|
||||
body = b""
|
||||
for rtype, rdata in answers:
|
||||
body += b"\xc0\x0c" # pointer to offset 12 (the question name)
|
||||
body += struct.pack(">HHIH", rtype, 1, 300, len(rdata)) + rdata
|
||||
|
||||
header = struct.pack(">HHHHHH", 0, 0x8180, 1, len(answers), 0, 0)
|
||||
return header + question + body
|
||||
|
||||
|
||||
def test_encode_query_is_a_well_formed_dns_message():
|
||||
raw = _decode_param(wf.encode_query_param("example.com", wf.TYPE_A))
|
||||
|
||||
msg_id, flags, qdcount, ancount, _ns, _ar = struct.unpack_from(">HHHHHH", raw, 0)
|
||||
assert msg_id == 0 # RFC 8484 section 4.1: zero for cacheability
|
||||
assert flags == 0x0100 # recursion desired
|
||||
assert (qdcount, ancount) == (1, 0)
|
||||
assert raw[12:] == b"\x07example\x03com\x00" + struct.pack(">HH", wf.TYPE_A, 1)
|
||||
|
||||
|
||||
def test_encode_query_param_is_unpadded_base64url():
|
||||
param = wf.encode_query_param("example.com", wf.TYPE_A)
|
||||
assert "=" not in param
|
||||
assert "+" not in param and "/" not in param
|
||||
|
||||
|
||||
def test_encode_query_strips_trailing_dot():
|
||||
assert _decode_param(wf.encode_query_param("example.com.", wf.TYPE_A)) == _decode_param(
|
||||
wf.encode_query_param("example.com", wf.TYPE_A)
|
||||
)
|
||||
|
||||
|
||||
def test_encode_query_rejects_empty_hostname():
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.encode_query("", wf.TYPE_A)
|
||||
|
||||
|
||||
def test_encode_query_rejects_oversized_label():
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.encode_query("a" * 64 + ".com", wf.TYPE_A)
|
||||
|
||||
|
||||
def test_decode_a_records():
|
||||
response = _build_response(answers=[(wf.TYPE_A, bytes([93, 184, 216, 34]))])
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == ["93.184.216.34"]
|
||||
|
||||
|
||||
def test_decode_multiple_a_records_preserves_order():
|
||||
response = _build_response(
|
||||
answers=[(wf.TYPE_A, bytes([1, 1, 1, 1])), (wf.TYPE_A, bytes([8, 8, 8, 8]))]
|
||||
)
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == ["1.1.1.1", "8.8.8.8"]
|
||||
|
||||
|
||||
def test_decode_skips_cname_records_in_the_chain():
|
||||
"""Answers routinely lead with a CNAME; only the requested type is an address."""
|
||||
cname = b"\x03www\x07example\x03com\x00"
|
||||
response = _build_response(answers=[(5, cname), (wf.TYPE_A, bytes([93, 184, 216, 34]))])
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == ["93.184.216.34"]
|
||||
|
||||
|
||||
def test_decode_aaaa_compresses_zero_run():
|
||||
# 2606:4700:0:0:0:0:6810:84e5 -> the middle zero run collapses to "::"
|
||||
rdata = struct.pack(">8H", 0x2606, 0x4700, 0, 0, 0, 0, 0x6810, 0x84E5)
|
||||
response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA)
|
||||
assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2606:4700::6810:84e5"]
|
||||
|
||||
|
||||
def test_decode_aaaa_collapses_only_the_longest_zero_run():
|
||||
# 2001:0:0:1:0:0:0:1 - the second, longer run is the one that collapses.
|
||||
rdata = struct.pack(">8H", 0x2001, 0, 0, 1, 0, 0, 0, 1)
|
||||
response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA)
|
||||
assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2001:0:0:1::1"]
|
||||
|
||||
|
||||
def test_decode_aaaa_without_zero_run():
|
||||
rdata = struct.pack(">8H", 0x2001, 0x0DB8, 1, 2, 3, 4, 5, 6)
|
||||
response = _build_response(answers=[(wf.TYPE_AAAA, rdata)], qtype=wf.TYPE_AAAA)
|
||||
assert wf.decode_answer(response, wf.TYPE_AAAA) == ["2001:db8:1:2:3:4:5:6"]
|
||||
|
||||
|
||||
def test_decode_nxdomain_returns_empty_not_an_error():
|
||||
"""An empty answer is a valid response; the caller falls back rather than retrying."""
|
||||
response = _build_response(answers=[])
|
||||
assert wf.decode_answer(response, wf.TYPE_A) == []
|
||||
|
||||
|
||||
def test_decode_rejects_truncated_header():
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.decode_answer(b"\x00\x01", wf.TYPE_A)
|
||||
|
||||
|
||||
def test_decode_rejects_truncated_record():
|
||||
response = _build_response(answers=[(wf.TYPE_A, bytes([1, 2, 3, 4]))])
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.decode_answer(response[:-2], wf.TYPE_A)
|
||||
|
||||
|
||||
def test_decode_does_not_hang_on_a_malicious_name():
|
||||
"""A self-referential name must not loop forever."""
|
||||
header = struct.pack(">HHHHHH", 0, 0x8180, 1, 0, 0, 0)
|
||||
# A run of maximum-length labels that never terminates.
|
||||
body = (b"\x3f" + b"a" * 63) * 8
|
||||
with pytest.raises(wf.WireformatError):
|
||||
wf.decode_answer(header + body, wf.TYPE_A)
|
||||
|
||||
|
||||
def test_wireformat_providers_are_flagged(monkeypatch):
|
||||
"""The provider table and the resolver must agree on who needs wireformat."""
|
||||
import shelfmark.download.network as network
|
||||
|
||||
for name, servers, url in network.DNS_PROVIDERS:
|
||||
resolver = network.DoHResolver(url, "x.invalid", servers[0])
|
||||
expected = name in ("quad9", "opendns")
|
||||
assert resolver.use_wireformat is expected, f"{name} wireformat flag wrong"
|
||||
@@ -27,6 +27,7 @@ class _DummySelector:
|
||||
self._index = 0
|
||||
self.current_base = bases[0]
|
||||
self.attempts_this_dns = 0
|
||||
self.quarantined: list[tuple[str, str]] = []
|
||||
|
||||
def rewrite(self, url: str) -> str:
|
||||
for base in self._bases:
|
||||
@@ -34,7 +35,11 @@ class _DummySelector:
|
||||
return url.replace(base, self.current_base, 1)
|
||||
return url
|
||||
|
||||
def next_mirror_or_rotate_dns(self, allow_dns: bool = True) -> tuple[str | None, str]:
|
||||
def next_mirror_or_rotate_dns(
|
||||
self, allow_dns: bool = True, *, fatal: bool = False, reason: str = ""
|
||||
) -> tuple[str | None, str]:
|
||||
if fatal:
|
||||
self.quarantined.append((self.current_base, reason))
|
||||
self.attempts_this_dns += 1
|
||||
self._index = (self._index + 1) % len(self._bases)
|
||||
self.current_base = self._bases[self._index]
|
||||
@@ -149,3 +154,50 @@ def test_html_get_page_locked_aa_does_not_fail_over_on_cross_host_redirect(monke
|
||||
|
||||
assert html == ""
|
||||
assert calls == ["https://annas-archive.li/search?q=test"]
|
||||
|
||||
|
||||
def test_html_get_page_echoes_cookies_across_same_host_redirects(monkeypatch):
|
||||
"""DDoS-Guard's ?check=1 probe is cleared by echoing the Set-Cookie it issues.
|
||||
|
||||
Without this the __ddg* cookie is dropped on every hop, the server re-issues the same
|
||||
redirect, and the request dies with TooManyRedirects.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: False)
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(http.network, "get_aa_base_url", lambda: "https://annas-archive.li")
|
||||
monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: True)
|
||||
|
||||
sent_cookies: list[dict[str, str]] = []
|
||||
|
||||
def fake_get(url: str, **kwargs):
|
||||
sent_cookies.append(dict(kwargs["cookies"]))
|
||||
if url == "https://annas-archive.li/search?q=test":
|
||||
response = _FakeResponse(302, headers={"Location": "/search?q=test&check=1"}, url=url)
|
||||
response.cookies = {"__ddg2_": "probe"}
|
||||
return response
|
||||
if url == "https://annas-archive.li/search?q=test&check=1":
|
||||
# The probe only clears if the cookie comes back on this hop.
|
||||
if kwargs["cookies"].get("__ddg2_") != "probe":
|
||||
response = _FakeResponse(
|
||||
302, headers={"Location": "/search?q=test&check=1"}, url=url
|
||||
)
|
||||
response.cookies = {"__ddg2_": "probe"}
|
||||
return response
|
||||
return _FakeResponse(200, text="RESULTS", url=url)
|
||||
raise AssertionError(f"Unexpected URL: {url}")
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", fake_get)
|
||||
|
||||
selector = _DummySelector(["https://annas-archive.li"])
|
||||
html = http.html_get_page(
|
||||
"https://annas-archive.li/search?q=test",
|
||||
selector=selector,
|
||||
retry=1,
|
||||
allow_bypasser_fallback=False,
|
||||
)
|
||||
|
||||
assert html == "RESULTS"
|
||||
assert sent_cookies == [{}, {"__ddg2_": "probe"}]
|
||||
|
||||
@@ -11,6 +11,134 @@ class _FakeResponse:
|
||||
self.url = url
|
||||
|
||||
|
||||
def test_external_bypasser_clearance_is_presented_on_the_next_request(monkeypatch):
|
||||
"""Clearance is read from the shared store whichever bypasser filled it.
|
||||
|
||||
Guards the regression where the external path returned {} unconditionally: every
|
||||
request re-paid a 403 plus a full solve, and a download - which the solver cannot
|
||||
proxy - presented no clearance at all.
|
||||
"""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: True)
|
||||
|
||||
cookie_store.store_extracted_cookies(
|
||||
url="https://annas-archive.gl/search",
|
||||
cookies=[{"name": "__ddg1_", "value": "clearance"}],
|
||||
user_agent="Mozilla/5.0 (solver)",
|
||||
)
|
||||
|
||||
headers: dict[str, str] = {}
|
||||
cookies = http._apply_cf_bypass("https://annas-archive.gl/md5/abc", headers)
|
||||
|
||||
assert cookies == {"__ddg1_": "clearance"}
|
||||
assert headers["User-Agent"] == "Mozilla/5.0 (solver)"
|
||||
|
||||
|
||||
def test_external_bypasser_solve_is_reused_instead_of_re_solved(monkeypatch):
|
||||
"""One solve should clear the following requests, not just the one that paid for it.
|
||||
|
||||
A solve is tens of seconds of real browser, so re-running it per request is what
|
||||
made direct download unusable behind an external bypasser.
|
||||
"""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: True)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
class _Cleared:
|
||||
is_redirect = False
|
||||
status_code = 200
|
||||
cookies: dict[str, str] = {}
|
||||
text = "<table>results</table>"
|
||||
url = "https://annas-archive.gl/search?q=dune"
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def gated_get(url: str, **kwargs):
|
||||
if kwargs.get("cookies", {}).get("cf_clearance") != "token":
|
||||
error = requests.exceptions.HTTPError("forbidden")
|
||||
error.response = _FakeResponse(403, url=url)
|
||||
raise error
|
||||
return _Cleared()
|
||||
|
||||
solves: list[str] = []
|
||||
|
||||
def fake_solve(url: str, *_args, **_kwargs):
|
||||
solves.append(url)
|
||||
cookie_store.store_extracted_cookies(
|
||||
url=url,
|
||||
cookies=[{"name": "cf_clearance", "value": "token"}],
|
||||
user_agent="Mozilla/5.0 (solver)",
|
||||
)
|
||||
return "<table>results</table>"
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", gated_get)
|
||||
monkeypatch.setattr(http, "get_bypassed_page", fake_solve)
|
||||
|
||||
url = "https://annas-archive.gl/search?q=dune"
|
||||
first = http.html_get_page(url, retry=2, allow_bypasser_fallback=True, success_delay=0)
|
||||
second = http.html_get_page(url, retry=2, allow_bypasser_fallback=True, success_delay=0)
|
||||
|
||||
assert first == "<table>results</table>"
|
||||
assert second == "<table>results</table>"
|
||||
# The second request rode the stored clearance instead of paying for another solve.
|
||||
assert solves == [url]
|
||||
|
||||
|
||||
def test_403_with_a_concurrently_won_clearance_still_reaches_the_bypasser(monkeypatch):
|
||||
"""The last attempt must hand off, not `continue` into the end of the loop.
|
||||
|
||||
Another worker's solve can land between our request and its 403, which used to
|
||||
send this branch back round the retry loop - but on the final attempt (and
|
||||
MAX_RETRY=1 is the supported setting) `continue` just ends it, abandoning the
|
||||
request without ever offering the URL to the bypasser.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
|
||||
# A concurrent solve has filled the store, but this request went out before it did.
|
||||
monkeypatch.setattr(http, "get_cf_cookies_for_domain", lambda _hostname: {"__ddg1_": "fresh"})
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
def gated(url: str, **_kwargs):
|
||||
error = requests.exceptions.HTTPError("forbidden")
|
||||
error.response = _FakeResponse(403, url=url)
|
||||
raise error
|
||||
|
||||
bypassed: list[str] = []
|
||||
monkeypatch.setattr(http.requests, "get", gated)
|
||||
monkeypatch.setattr(
|
||||
http,
|
||||
"get_bypassed_page",
|
||||
lambda url, *_a, **_k: bypassed.append(url) or "<table>results</table>",
|
||||
)
|
||||
|
||||
url = "https://annas-archive.gl/search?q=dune"
|
||||
html = http.html_get_page(url, retry=1, allow_bypasser_fallback=True, success_delay=0)
|
||||
|
||||
assert html == "<table>results</table>"
|
||||
assert bypassed == [url]
|
||||
|
||||
|
||||
def test_html_get_page_ignores_status_callback_failure(monkeypatch):
|
||||
"""A raising status_callback must not break the bypass it was reporting on."""
|
||||
import shelfmark.download.http as http
|
||||
@@ -138,6 +266,104 @@ def test_html_get_page_returns_empty_on_bypass_cancellation(monkeypatch):
|
||||
assert html == ""
|
||||
|
||||
|
||||
def test_challenged_search_switches_to_bypasser(monkeypatch):
|
||||
"""AA gates /search behind DDoS-Guard; the 403 must reach the bypasser, not a 503.
|
||||
|
||||
Guards the regression where search passed allow_bypasser_fallback=False, so a
|
||||
challenge on every mirror surfaced as "mirrors are blocked" with no solve attempted.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
|
||||
monkeypatch.setattr(http, "get_cf_cookies_for_domain", lambda _hostname: {})
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
def ddos_guarded(url: str, **_kwargs):
|
||||
error = requests.exceptions.HTTPError("forbidden")
|
||||
error.response = _FakeResponse(403, url=url)
|
||||
raise error
|
||||
|
||||
bypassed: list[str] = []
|
||||
monkeypatch.setattr(http.requests, "get", ddos_guarded)
|
||||
monkeypatch.setattr(
|
||||
http,
|
||||
"get_bypassed_page",
|
||||
lambda url, *_a, **_k: bypassed.append(url) or "<table>results</table>",
|
||||
)
|
||||
|
||||
html = http.html_get_page(
|
||||
"https://annas-archive.gl/search?q=dune",
|
||||
retry=10,
|
||||
allow_bypasser_fallback=True,
|
||||
success_delay=0,
|
||||
)
|
||||
|
||||
assert html == "<table>results</table>"
|
||||
assert bypassed == ["https://annas-archive.gl/search?q=dune"]
|
||||
|
||||
|
||||
def test_redirect_loop_purges_stale_cookies_and_switches_to_bypasser(monkeypatch):
|
||||
"""A stale clearance cookie turns the gate into a `?check=1` redirect loop.
|
||||
|
||||
Guards the regression where TooManyRedirects carried no status code, so the
|
||||
403-only rescue never fired and every retry re-sent the dead cookie.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
stale = {"__ddg8_": "stale"}
|
||||
cleared: list[str] = []
|
||||
|
||||
def fake_clear(domain: str) -> None:
|
||||
cleared.append(domain)
|
||||
stale.clear()
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: False)
|
||||
monkeypatch.setattr(http.cookie_store, "clear_cf_cookies", fake_clear)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: dict(stale))
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: True)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
sent_cookies: list[dict[str, str]] = []
|
||||
|
||||
def check_redirect(url: str, **kwargs):
|
||||
sent_cookies.append(kwargs["cookies"])
|
||||
response = _FakeResponse(302, url=url)
|
||||
response.is_redirect = True
|
||||
response.headers = {"Location": f"{url}&check=1"}
|
||||
return response
|
||||
|
||||
bypassed: list[str] = []
|
||||
monkeypatch.setattr(http.requests, "get", check_redirect)
|
||||
monkeypatch.setattr(
|
||||
http,
|
||||
"get_bypassed_page",
|
||||
lambda url, *_a, **_k: bypassed.append(url) or "<table>results</table>",
|
||||
)
|
||||
|
||||
html = http.html_get_page(
|
||||
"https://annas-archive.gl/search?q=dune",
|
||||
retry=10,
|
||||
allow_bypasser_fallback=True,
|
||||
success_delay=0,
|
||||
)
|
||||
|
||||
assert html == "<table>results</table>"
|
||||
assert cleared == ["annas-archive.gl"]
|
||||
assert len(bypassed) == 1
|
||||
# Escaped on the first exception, not retried with the dead cookie.
|
||||
assert sent_cookies[0] == {"__ddg8_": "stale"}
|
||||
|
||||
|
||||
def test_download_url_ignores_zlib_cookie_refresh_failure(monkeypatch):
|
||||
import shelfmark.download.http as http
|
||||
|
||||
@@ -183,3 +409,138 @@ def test_get_bypassed_page_uses_external_bypasser_when_enabled(monkeypatch):
|
||||
|
||||
assert http.get_bypassed_page("https://example.com", selector, cancel_flag) == "EXT"
|
||||
assert calls == [("https://example.com", selector, cancel_flag)]
|
||||
|
||||
|
||||
def test_redirect_loop_gives_up_immediately_when_bypasser_not_allowed(monkeypatch):
|
||||
"""A loop the bypasser may not rescue must fail fast, not burn the retry budget.
|
||||
|
||||
Guards the regression where the unrescued loop raised TooManyRedirects into the
|
||||
retry path: that error is not retryable and carries no status, so every attempt
|
||||
re-ran the full 6-redirect loop for ~60 requests to AA before giving up.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: True)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
requested: list[str] = []
|
||||
|
||||
def check_redirect(url: str, **_kwargs):
|
||||
requested.append(url)
|
||||
response = _FakeResponse(302, url=url)
|
||||
response.is_redirect = True
|
||||
response.headers = {"Location": f"{url}&check=1"}
|
||||
return response
|
||||
|
||||
def unreachable_bypasser(*_args, **_kwargs):
|
||||
msg = "bypasser must not run when allow_bypasser_fallback is False"
|
||||
raise AssertionError(msg)
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", check_redirect)
|
||||
monkeypatch.setattr(http, "get_bypassed_page", unreachable_bypasser)
|
||||
|
||||
html = http.html_get_page(
|
||||
"https://annas-archive.gl/dyn/md5/summary/abc",
|
||||
retry=10,
|
||||
allow_bypasser_fallback=False,
|
||||
success_delay=0,
|
||||
)
|
||||
|
||||
assert html == ""
|
||||
# One pass through the redirect cap, not one pass per retry attempt.
|
||||
assert len(requested) == http._MAX_REDIRECTS + 1
|
||||
|
||||
|
||||
def test_html_get_page_redirect_loop_purges_cookies_and_bypasses(monkeypatch):
|
||||
"""A redirect loop is the challenge served against stale cookies, not a retryable error.
|
||||
|
||||
TooManyRedirects carries no status code, so without an explicit branch it falls through
|
||||
to the generic retry path and repeats the identical failure for the whole retry budget.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: False)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 330.0)
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "get_aa_base_url", lambda: "https://annas-archive.li")
|
||||
monkeypatch.setattr(http.network, "is_aa_auto_mode", lambda: False)
|
||||
|
||||
cleared: list[str] = []
|
||||
|
||||
monkeypatch.setattr(http.cookie_store, "clear_cf_cookies", cleared.append)
|
||||
monkeypatch.setattr(
|
||||
http.cookie_store, "get_cf_cookies_for_domain", lambda _domain: {"__ddg2_": "stale"}
|
||||
)
|
||||
monkeypatch.setattr(http.cookie_store, "get_cf_user_agent_for_domain", lambda _domain: None)
|
||||
monkeypatch.setattr(http, "get_bypassed_page", lambda *_args, **_kwargs: "SOLVED")
|
||||
|
||||
class _FakeRedirect:
|
||||
"""A 302 that always points at the same ?check=1 URL, cookies unchanged."""
|
||||
|
||||
is_redirect = True
|
||||
status_code = 302
|
||||
cookies = {"__ddg2_": "stale"}
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
self.url = url
|
||||
self.headers = {"Location": "https://annas-archive.li/search?q=test&check=1"}
|
||||
|
||||
hits: list[str] = []
|
||||
|
||||
def fake_get(url: str, **kwargs):
|
||||
hits.append(url)
|
||||
# Stale cookies: the server keeps re-issuing the same ?check=1 redirect.
|
||||
return _FakeRedirect(url)
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", fake_get)
|
||||
|
||||
html = http.html_get_page(
|
||||
"https://annas-archive.li/search?q=test",
|
||||
retry=2,
|
||||
success_delay=0,
|
||||
)
|
||||
|
||||
assert html == "SOLVED"
|
||||
assert cleared == ["annas-archive.li"]
|
||||
# The loop is cut short: no second attempt spent repeating the same redirects.
|
||||
assert len(hits) == http._MAX_REDIRECTS + 1
|
||||
|
||||
|
||||
def test_html_get_page_redirect_loop_on_non_aa_host_is_left_alone(monkeypatch):
|
||||
"""Only hosts whose redirects we follow manually get the challenge treatment.
|
||||
|
||||
Elsewhere requests follows redirects itself, so a loop is an ordinary misconfiguration -
|
||||
purging that host's cookies and forcing a bypass would be the wrong response.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: False)
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
|
||||
bypassed: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
http, "get_bypassed_page", lambda url, *_args, **_kwargs: bypassed.append(url) or "SOLVED"
|
||||
)
|
||||
|
||||
def fake_get(_url: str, **_kwargs):
|
||||
raise requests.exceptions.TooManyRedirects("Exceeded 30 redirects.")
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", fake_get)
|
||||
|
||||
html = http.html_get_page("https://example.com/loop", retry=2, success_delay=0)
|
||||
|
||||
assert html == ""
|
||||
assert bypassed == []
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Tests for handing a 503 that carries a browser challenge to the bypasser."""
|
||||
|
||||
import requests
|
||||
|
||||
_CHALLENGE_HTML = (
|
||||
"<html><head><title>Checking your browser before accessing z-lib.gd</title>"
|
||||
"<script src='/.well-known/ddos-guard/check.js'></script></head>"
|
||||
"<body>Please wait...</body></html>"
|
||||
)
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
"""Minimal stand-in for requests.Response covering what html_get_page touches."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
*,
|
||||
url: str = "https://z-lib.gd/md5/abc",
|
||||
text: str = "",
|
||||
cookies: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self.url = url
|
||||
self.text = text
|
||||
self.cookies = cookies or {}
|
||||
self.headers = {"Content-Type": "text/html;charset=utf-8"}
|
||||
self.is_redirect = False
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status_code >= 400:
|
||||
error = requests.exceptions.HTTPError(f"{self.status_code} Error")
|
||||
error.response = self
|
||||
raise error
|
||||
|
||||
|
||||
def _neutralize_network(monkeypatch, http):
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
|
||||
def test_503_challenge_is_handed_to_the_bypasser(monkeypatch):
|
||||
"""The reissued-cookie 503 from #1233 reaches the bypasser instead of retrying."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
_neutralize_network(monkeypatch, http)
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
|
||||
attempts: list[dict[str, str]] = []
|
||||
bypassed: list[str] = []
|
||||
|
||||
def fake_get(_url: str, **kwargs):
|
||||
attempts.append(dict(kwargs["cookies"]))
|
||||
# Hit 1 issues the cookie; every later hit re-serves the challenge unchanged,
|
||||
# which is what leaves the handshake with nothing to echo back.
|
||||
if len(attempts) == 1:
|
||||
return _FakeResponse(503, cookies={"bsrv": "1"})
|
||||
return _FakeResponse(503, text=_CHALLENGE_HTML, cookies={"bsrv": "1"})
|
||||
|
||||
def fake_bypass(url: str, _selector=None, _cancel_flag=None):
|
||||
bypassed.append(url)
|
||||
return "<html>real page</html>"
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", fake_get)
|
||||
monkeypatch.setattr(http, "get_bypassed_page", fake_bypass)
|
||||
|
||||
html = http.html_get_page("https://z-lib.gd/md5/abc", retry=10, success_delay=0)
|
||||
|
||||
assert html == "<html>real page</html>"
|
||||
assert bypassed == ["https://z-lib.gd/md5/abc"]
|
||||
# The handshake still gets its echo; the challenge ends the loop on the second hit
|
||||
# rather than burning all ten attempts.
|
||||
assert attempts == [{}, {"bsrv": "1"}]
|
||||
|
||||
|
||||
def test_plain_503_still_retries_without_bypassing(monkeypatch):
|
||||
"""An overloaded origin has no challenge marker, so its retry path is untouched."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
_neutralize_network(monkeypatch, http)
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
|
||||
attempts: list[dict[str, str]] = []
|
||||
bypassed: list[str] = []
|
||||
|
||||
def fake_get(_url: str, **kwargs):
|
||||
attempts.append(dict(kwargs["cookies"]))
|
||||
return _FakeResponse(503, text="<html><body>Service Unavailable</body></html>")
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or ""
|
||||
)
|
||||
|
||||
html = http.html_get_page("https://z-lib.gd/md5/abc", retry=3, success_delay=0)
|
||||
|
||||
assert html == ""
|
||||
assert bypassed == []
|
||||
assert attempts == [{}, {}, {}]
|
||||
|
||||
|
||||
def test_503_challenge_respects_disabled_bypasser_fallback(monkeypatch):
|
||||
"""Best-effort fetches must not stall on a minutes-long solve."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
_neutralize_network(monkeypatch, http)
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
|
||||
bypassed: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
http.requests,
|
||||
"get",
|
||||
lambda _url, **_kwargs: _FakeResponse(503, text=_CHALLENGE_HTML),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or ""
|
||||
)
|
||||
|
||||
html = http.html_get_page(
|
||||
"https://z-lib.gd/md5/abc", retry=1, success_delay=0, allow_bypasser_fallback=False
|
||||
)
|
||||
|
||||
assert html == ""
|
||||
assert bypassed == []
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Tests for AA mirror quarantine: dead mirrors leave the rotation, live ones stay.
|
||||
|
||||
The distinction these guard is the whole point of the feature. A mirror that answers
|
||||
403 (DDoS-Guard) is alive and holds our bypass clearance, so rotating off it makes the
|
||||
next search solve a fresh challenge on a domain we have no cookie for. A mirror that
|
||||
NXDOMAINs, refuses the connection, or answers 200 with a parking page is not a mirror
|
||||
at all and must never be tried again this session.
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def _fresh_network(monkeypatch, urls: list[str], *, auto: bool = True):
|
||||
import shelfmark.download.network as network
|
||||
|
||||
monkeypatch.setattr(network, "_initialized", True)
|
||||
monkeypatch.setattr(network, "_aa_urls", list(urls))
|
||||
monkeypatch.setattr(network, "_aa_base_url", urls[0])
|
||||
monkeypatch.setattr(network, "_current_aa_url_index", 0)
|
||||
monkeypatch.setattr(network, "_dead_aa_urls", set())
|
||||
monkeypatch.setattr(network, "_save_state", lambda **kwargs: None)
|
||||
monkeypatch.setattr(network, "is_aa_auto_mode", lambda: auto)
|
||||
return network
|
||||
|
||||
|
||||
MIRRORS = ["https://aa-one.test", "https://aa-two.test", "https://aa-three.test"]
|
||||
|
||||
|
||||
def test_quarantined_mirror_leaves_the_available_list(monkeypatch):
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True
|
||||
assert network.get_available_aa_urls() == ["https://aa-one.test", "https://aa-three.test"]
|
||||
assert network.get_dead_aa_urls() == {"https://aa-two.test"}
|
||||
|
||||
|
||||
def test_quarantine_accepts_a_full_request_url(monkeypatch):
|
||||
"""Callers hold the failing request URL, not the bare mirror base."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-two.test/search?q=dune", "parked") is True
|
||||
assert "https://aa-two.test" in network.get_dead_aa_urls()
|
||||
|
||||
|
||||
def test_quarantine_is_idempotent(monkeypatch):
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is False
|
||||
assert network.get_available_aa_urls() == ["https://aa-one.test", "https://aa-three.test"]
|
||||
|
||||
|
||||
def test_last_surviving_mirror_is_never_quarantined(monkeypatch):
|
||||
"""Misclassification must not leave the app with nowhere to search."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
|
||||
assert network.mark_aa_url_dead("https://aa-one.test", "NXDOMAIN") is True
|
||||
assert network.mark_aa_url_dead("https://aa-two.test", "NXDOMAIN") is True
|
||||
assert network.mark_aa_url_dead("https://aa-three.test", "NXDOMAIN") is False
|
||||
assert network.get_available_aa_urls() == ["https://aa-three.test"]
|
||||
|
||||
|
||||
def test_selector_skips_quarantined_mirror_when_rotating(monkeypatch):
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
selector = network.AAMirrorSelector()
|
||||
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(fatal=True, reason="NXDOMAIN")
|
||||
|
||||
assert action == "mirror"
|
||||
# Landed on the next live mirror, not skipped past it onto the third.
|
||||
assert new_base == "https://aa-two.test"
|
||||
assert "https://aa-one.test" in network.get_dead_aa_urls()
|
||||
assert selector.rewrite("https://aa-one.test/search") == "https://aa-two.test/search"
|
||||
|
||||
|
||||
def test_non_fatal_rotation_keeps_the_mirror(monkeypatch):
|
||||
"""A 5xx or a challenge rotates but must not burn the mirror."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
selector = network.AAMirrorSelector()
|
||||
|
||||
selector.next_mirror_or_rotate_dns()
|
||||
|
||||
assert network.get_dead_aa_urls() == set()
|
||||
assert network.get_available_aa_urls() == MIRRORS
|
||||
|
||||
|
||||
def test_dns_reset_does_not_resurrect_quarantined_mirrors(monkeypatch):
|
||||
"""A new DNS provider cannot revive a parked domain, so it stays skipped."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
monkeypatch.setattr(network, "rotate_dns_provider", lambda: True)
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
network.mark_aa_url_dead("https://aa-one.test", "parked")
|
||||
|
||||
assert network.rotate_dns_and_reset_aa() is True
|
||||
assert network.get_aa_base_url() == "https://aa-two.test"
|
||||
|
||||
|
||||
def test_editing_the_mirror_list_clears_quarantine(monkeypatch):
|
||||
"""Quarantine decisions were made about a list the user has now changed."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-two.test", "parked")
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: [*MIRRORS, "https://aa-four.test"])
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"})
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert network.get_dead_aa_urls() == set()
|
||||
|
||||
|
||||
def test_reinit_with_an_unchanged_list_keeps_quarantine(monkeypatch):
|
||||
"""Re-init happens constantly (settings sync, DNS rotation, helper startup).
|
||||
|
||||
Clearing quarantine on every one of those resurrects a parked mirror mid-session,
|
||||
which is exactly the bug this guards: the mirror gets re-elected and the next
|
||||
search pays for it again.
|
||||
"""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-two.test", "parked")
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS))
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"})
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert network.get_dead_aa_urls() == {"https://aa-two.test"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Failure classification
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _http_error(status: int) -> requests.exceptions.HTTPError:
|
||||
response = requests.Response()
|
||||
response.status_code = status
|
||||
return requests.exceptions.HTTPError(response=response)
|
||||
|
||||
|
||||
def test_dns_failure_is_fatal_for_the_mirror():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
exc = requests.exceptions.ConnectionError(
|
||||
"HTTPSConnectionPool(host='aa.test', port=443): Max retries exceeded "
|
||||
"(Caused by NameResolutionError(\"Failed to resolve 'aa.test'\"))"
|
||||
)
|
||||
assert http._fatal_mirror_reason(exc) == "DNS does not resolve"
|
||||
|
||||
|
||||
def test_connection_refused_is_fatal_for_the_mirror():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
exc = requests.exceptions.ConnectionError("Connection refused")
|
||||
assert http._fatal_mirror_reason(exc) == "connection refused"
|
||||
|
||||
|
||||
def test_gone_and_legal_block_are_fatal():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
assert http._fatal_mirror_reason(_http_error(410)) == "HTTP 410"
|
||||
assert http._fatal_mirror_reason(_http_error(451)) == "HTTP 451"
|
||||
|
||||
|
||||
def test_timeout_is_not_fatal():
|
||||
"""A slow mirror is still a mirror - and may hold our bypass clearance."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
assert http._fatal_mirror_reason(requests.exceptions.ConnectTimeout("timed out")) is None
|
||||
assert http._fatal_mirror_reason(requests.exceptions.ReadTimeout("timed out")) is None
|
||||
|
||||
|
||||
def test_challenge_and_server_errors_are_not_fatal():
|
||||
import shelfmark.download.http as http
|
||||
|
||||
for status in (403, 429, 500, 502, 503):
|
||||
assert http._fatal_mirror_reason(_http_error(status)) is None
|
||||
|
||||
|
||||
def test_startup_probe_skips_quarantined_mirrors(monkeypatch):
|
||||
"""Re-init must not re-probe (or re-elect) a mirror already known to be dead.
|
||||
|
||||
A parking page answers 200, so an unfiltered probe elects it every single time
|
||||
the app re-initialises - one wasted request per re-init, forever.
|
||||
"""
|
||||
import requests
|
||||
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-one.test", "parked")
|
||||
probed: list[str] = []
|
||||
|
||||
def fake_get(url, **_kwargs):
|
||||
probed.append(url)
|
||||
response = requests.Response()
|
||||
response.status_code = 200
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(network.requests, "get", fake_get)
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS))
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {})
|
||||
monkeypatch.setattr(network, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(network, "get_ssl_verify", lambda _url: True)
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert "https://aa-one.test" not in probed
|
||||
assert network.get_aa_base_url() == "https://aa-two.test"
|
||||
|
||||
|
||||
def test_startup_probe_does_not_restore_a_quarantined_mirror(monkeypatch):
|
||||
"""Saved state can name a mirror that has since been quarantined."""
|
||||
network = _fresh_network(monkeypatch, MIRRORS)
|
||||
network.mark_aa_url_dead("https://aa-one.test", "parked")
|
||||
monkeypatch.setattr(network, "_build_aa_urls", lambda: list(MIRRORS))
|
||||
monkeypatch.setattr(network, "_get_configured_aa_url", lambda: "auto")
|
||||
monkeypatch.setattr(network, "state", {"aa_base_url": "https://aa-one.test"})
|
||||
monkeypatch.setattr(network, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(network, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(network.requests, "get", lambda *a, **k: (_ for _ in ()).throw(OSError()))
|
||||
|
||||
network._initialize_aa_state()
|
||||
|
||||
assert network.get_aa_base_url() != "https://aa-one.test"
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Boot-time search warm-up.
|
||||
|
||||
The warm-up exists to move the cold DDoS-Guard solve off the user's first search. It
|
||||
is an optimisation, so the load-bearing property is that it can never affect startup:
|
||||
a source that is down, misconfigured or raising must leave the app running.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def warmup(monkeypatch):
|
||||
import shelfmark.download.warmup as warmup_module
|
||||
|
||||
monkeypatch.setattr(warmup_module, "_warmup_thread", None)
|
||||
return warmup_module
|
||||
|
||||
|
||||
def _patch_config(monkeypatch, warmup, values: dict):
|
||||
def fake_get(key, default=None):
|
||||
return values.get(key, default)
|
||||
|
||||
monkeypatch.setattr(warmup.config, "get", fake_get)
|
||||
|
||||
|
||||
def test_disabled_by_setting(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": False})
|
||||
assert warmup.is_enabled() is False
|
||||
assert warmup.start() is False
|
||||
|
||||
|
||||
def test_disabled_by_string_false(monkeypatch, warmup):
|
||||
"""Deployment ENV arrives as a string, not a bool."""
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_ENABLED": "false"})
|
||||
assert warmup.is_enabled() is False
|
||||
|
||||
|
||||
def test_skipped_when_direct_download_is_off(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {"DIRECT_DOWNLOAD_ENABLED": False})
|
||||
assert warmup.is_enabled() is False
|
||||
|
||||
|
||||
def test_enabled_by_default(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
assert warmup.is_enabled() is True
|
||||
|
||||
|
||||
def test_query_defaults_and_is_configurable(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
assert warmup.warmup_query() == "The Great Gatsby"
|
||||
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": "Dune"})
|
||||
assert warmup.warmup_query() == "Dune"
|
||||
|
||||
# A blank override must not send an empty query at the source.
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": " "})
|
||||
assert warmup.warmup_query() == "The Great Gatsby"
|
||||
|
||||
|
||||
def test_skipped_when_no_mirrors_configured(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: False)
|
||||
|
||||
called: list[str] = []
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(dd, "search_books", lambda q, f: called.append(q))
|
||||
|
||||
assert warmup.run_warmup() is False
|
||||
assert called == []
|
||||
|
||||
|
||||
def test_successful_warmup_reports_true(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
|
||||
seen: list[str] = []
|
||||
|
||||
def fake_search(query, _filters):
|
||||
seen.append(query)
|
||||
return ["a", "b"]
|
||||
|
||||
monkeypatch.setattr(dd, "search_books", fake_search)
|
||||
|
||||
assert warmup.run_warmup() is True
|
||||
assert seen == ["The Great Gatsby"]
|
||||
|
||||
|
||||
def test_empty_results_are_not_an_error(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
|
||||
monkeypatch.setattr(dd, "search_books", lambda q, f: [])
|
||||
|
||||
assert warmup.run_warmup() is False
|
||||
|
||||
|
||||
def test_search_failure_is_swallowed(monkeypatch, warmup):
|
||||
"""A source that is down at boot must not propagate out of the warm-up."""
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
import shelfmark.core.mirrors as mirrors
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True)
|
||||
|
||||
def boom(_query, _filters):
|
||||
msg = "mirrors are blocked"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
monkeypatch.setattr(dd, "search_books", boom)
|
||||
|
||||
assert warmup.run_warmup() is False
|
||||
|
||||
|
||||
def test_start_schedules_a_daemon_thread_and_is_idempotent(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
|
||||
assert warmup.start(delay_seconds=30) is True
|
||||
thread = warmup._warmup_thread
|
||||
assert thread is not None
|
||||
assert thread.daemon is True
|
||||
|
||||
# A second call must not stack up another timer.
|
||||
assert warmup.start(delay_seconds=30) is False
|
||||
assert warmup._warmup_thread is thread
|
||||
|
||||
thread.cancel()
|
||||
|
||||
|
||||
def test_start_does_not_run_the_search_inline(monkeypatch, warmup):
|
||||
"""Startup must not block on a search that can take a minute."""
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
ran: list[bool] = []
|
||||
monkeypatch.setattr(warmup, "run_warmup", lambda: ran.append(True))
|
||||
|
||||
warmup.start(delay_seconds=30)
|
||||
assert ran == []
|
||||
|
||||
if warmup._warmup_thread:
|
||||
warmup._warmup_thread.cancel()
|
||||
|
||||
|
||||
def test_env_var_can_disable_the_warmup(monkeypatch, warmup):
|
||||
"""SEARCH_WARMUP_ENABLED is not in the settings registry, so config.get never
|
||||
sees it - the documented off-switch only works if os.environ is consulted."""
|
||||
_patch_config(monkeypatch, warmup, {}) # config knows nothing about the key
|
||||
monkeypatch.setenv("SEARCH_WARMUP_ENABLED", "false")
|
||||
|
||||
assert warmup.is_enabled() is False
|
||||
assert warmup.start() is False
|
||||
|
||||
|
||||
def test_env_var_can_set_the_query(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
monkeypatch.setenv("SEARCH_WARMUP_QUERY", "Moby Dick")
|
||||
|
||||
assert warmup.warmup_query() == "Moby Dick"
|
||||
|
||||
|
||||
def test_env_var_absent_falls_back_to_config(monkeypatch, warmup):
|
||||
monkeypatch.delenv("SEARCH_WARMUP_QUERY", raising=False)
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": "From Config"})
|
||||
|
||||
assert warmup.warmup_query() == "From Config"
|
||||
@@ -87,12 +87,14 @@ end-to-end (`docker compose up` + suite + teardown) and passes.
|
||||
| `full` | **real Chrome solves Cloudflare** + DoH + **real qBittorrent** download → /books (Moby-Dick) | 1,4,5 + DoH | **#284 #1030** #386 #1040 #214 | ✅ 6 passed |
|
||||
| *(every profile)* | boots healthy under PUID/PGID, no perm errors | 6 entrypoint | #171 #447 #801 | ✅ |
|
||||
|
||||
> **The bypasser is download-time, not search-time.** Running the stack revealed
|
||||
> that shelfmark fetches AA search/detail with `allow_bypasser_fallback=False`, so a
|
||||
> search behind Cloudflare returns 503 **regardless** of the bypasser; the bypasser
|
||||
> (internal Chrome or external FlareSolverr) only runs during a file *download*
|
||||
> (`use_bypasser=True`). The bypasser profiles therefore assert a *clean*
|
||||
> CF-gated-search failure, while the **`full` profile exercises the real end-to-end
|
||||
> **The bypasser now runs for search too.** AA search/detail used to be fetched with
|
||||
> `allow_bypasser_fallback=False`, so a search behind Cloudflare returned 503
|
||||
> regardless of the bypasser — which left search dead when AA put DDoS-Guard in front
|
||||
> of `/search`. Both now pass `allow_bypasser_fallback=True`, so a 403 switches
|
||||
> straight to the bypasser rather than rotating to another mirror behind the same
|
||||
> gate. The bypasser profiles therefore assert that a gated search *recovers* when the
|
||||
> external bypasser is available and still fails cleanly when it is off, while the
|
||||
> **`full` profile exercises the real end-to-end
|
||||
> CF solve**: AA search/detail are reachable, but the AA slow-download link points
|
||||
> at the gate, so downloading Moby-Dick forces the in-image headless Chromium to
|
||||
> detect the challenge, solve it (`_bypass_method_cdp_solve`), and fetch the file —
|
||||
|
||||
@@ -1,20 +1,18 @@
|
||||
"""Cluster 1 — Cloudflare bypasser wiring + clean-failure behavior.
|
||||
"""Cluster 1 — Cloudflare bypasser wiring + gated-search behavior.
|
||||
|
||||
Reality discovered by running the stack: shelfmark's AA *search* and *detail*
|
||||
fetches use ``html_get_page(allow_bypasser_fallback=False)``, so a search behind a
|
||||
Cloudflare gate returns 503 **regardless** of the bypasser. The bypasser (internal
|
||||
Chrome or external FlareSolverr) is a *download-time* mechanism
|
||||
(``html_get_page(use_bypasser=True)``); it never runs for search.
|
||||
AA search/detail used to be fetched with ``allow_bypasser_fallback=False``, so a
|
||||
search behind a Cloudflare gate returned 503 no matter how the bypasser was
|
||||
configured — which left search dead when AA put DDoS-Guard in front of ``/search``.
|
||||
Both now pass ``allow_bypasser_fallback=True``: a 403 switches straight to the
|
||||
bypasser instead of rotating to another mirror behind the same gate.
|
||||
|
||||
So these tests assert what is actually true and host-observable:
|
||||
* the external bypasser is configured from env, and
|
||||
* a CF-gated search fails *cleanly* (a 503 the client can act on, not a hang or
|
||||
a crash) — both with the bypasser on (it isn't used for search) and off.
|
||||
So these tests assert:
|
||||
* the external bypasser is configured from env,
|
||||
* a CF-gated search *recovers* once the external bypasser is available, and
|
||||
* with the bypasser off it still fails *cleanly* (a 503 the client can act on,
|
||||
not a hang or a crash) — the negative control that the gate is not ignored.
|
||||
|
||||
Exercising shelfmark's *use* of the bypasser end-to-end (a real CF solve during a
|
||||
download) needs the AA slow-download HTML flow mocked — see the README roadmap.
|
||||
The bypass *mechanism* itself is verified to work: the mock FlareSolverr solves
|
||||
the gate (manually confirmed; see README). Guards: #284 #226 #202 #1030 #410 #369.
|
||||
Guards: #284 #226 #202 #1030 #410 #369.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -70,10 +68,26 @@ def test_external_bypasser_is_configured(client) -> None:
|
||||
|
||||
|
||||
@pytest.mark.profiles("bypasser-external")
|
||||
def test_cf_gated_search_fails_cleanly_with_external_bypasser(client) -> None:
|
||||
"""Even with the external bypasser configured, a CF-gated *search* yields no
|
||||
releases (the bypasser is download-time) — but it must fail cleanly."""
|
||||
assert _cf_gated_search_has_no_releases(client)
|
||||
def test_cf_gated_search_recovers_via_external_bypasser(client) -> None:
|
||||
"""A CF-gated search is solved through the bypasser instead of returning 503.
|
||||
|
||||
The mock FlareSolverr refetches with the clearance cookie, which the cloudflare
|
||||
role then passes through to the AA origin — so the 403 that used to end the
|
||||
search now turns into real results.
|
||||
"""
|
||||
resp = client.get(
|
||||
"/api/releases",
|
||||
params={"source": "direct_download", "query": "Mistborn"},
|
||||
timeout=120,
|
||||
)
|
||||
assert resp.status_code == 200, (
|
||||
f"CF-gated search did not recover through the external bypasser: "
|
||||
f"{resp.status_code} {resp.text[:200]}"
|
||||
)
|
||||
assert client.releases_from(resp), (
|
||||
"external bypasser was configured but the gated search returned no releases — "
|
||||
"the 403 did not switch search over to the bypasser"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.profiles("bypasser-disabled")
|
||||
|
||||
@@ -1,5 +1,96 @@
|
||||
import pytest
|
||||
|
||||
from shelfmark.core.utils import ARCHIVE_FORMATS, AUDIOBOOK_FORMATS
|
||||
from shelfmark.release_sources.irc import parser
|
||||
|
||||
# What a stock install actually filters with, so these tests fail if the defaults regress.
|
||||
_DEFAULT_CONFIG = {
|
||||
"SUPPORTED_FORMATS": ["epub", "mobi", "azw3", "fb2", "djvu", "cbz", "cbr"],
|
||||
"SUPPORTED_AUDIOBOOK_FORMATS": [*AUDIOBOOK_FORMATS, *ARCHIVE_FORMATS],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def default_formats(monkeypatch):
|
||||
"""Filter with the shipped default format lists."""
|
||||
monkeypatch.setattr(
|
||||
parser.config, "get", lambda key, default=None: _DEFAULT_CONFIG.get(key, default)
|
||||
)
|
||||
|
||||
|
||||
def test_audiobook_archives_are_found_with_default_settings(default_formats):
|
||||
"""Regression for #1129: audiobooks ship as .rar/.zip and were dropped by both buckets.
|
||||
|
||||
A single "@search" answers with one file holding every format. These are the lines an
|
||||
audiobook actually occupies in it - the extension is a container, and the release name
|
||||
is the only thing saying what is inside.
|
||||
"""
|
||||
content = "\n".join(
|
||||
[
|
||||
"!Oatmeal Andy Weir - Project Hail Mary (Audiobook) [MP3 64kbps].rar ::INFO:: 620.5MB",
|
||||
"!DV8 Andy Weir - Project Hail Mary - Audiobook.zip ::INFO:: 700MB",
|
||||
"!Horla Andy Weir - Project Hail Mary [Unabridged].m4b ::INFO:: 850.1MB",
|
||||
]
|
||||
)
|
||||
|
||||
results = parser.parse_results_file(content, content_type="audiobook")
|
||||
|
||||
assert [result.format for result in results] == ["rar", "zip", "m4b"]
|
||||
|
||||
|
||||
def test_ebook_archive_does_not_leak_into_audiobook_results(default_formats):
|
||||
"""An ebook .rar must not be offered as an audiobook just because it is an archive."""
|
||||
content = "!bald Andy Weir - Project Hail Mary (retail).rar ::INFO:: 2.1MB"
|
||||
|
||||
assert parser.parse_results_file(content, content_type="audiobook") == []
|
||||
|
||||
|
||||
def test_flac_audiobook_is_reachable_with_default_settings(default_formats):
|
||||
"""FLAC was recognized by the parser and ranked by the sorter, but never selectable."""
|
||||
content = "!Ook Andy Weir - Project Hail Mary.flac ::INFO:: 1.1GB"
|
||||
|
||||
results = parser.parse_results_file(content, content_type="audiobook")
|
||||
|
||||
assert [result.format for result in results] == ["flac"]
|
||||
|
||||
|
||||
def test_decimal_size_is_not_mistaken_for_a_file_extension():
|
||||
"""A line with no extension used to parse as format="5mb" out of "::INFO:: 620.5MB"."""
|
||||
line = "!Ook Andy Weir - Project Hail Mary (2021) Audiobook ::INFO:: 620.5MB"
|
||||
|
||||
result = parser.parse_result_line(line)
|
||||
|
||||
assert result.format == "unknown"
|
||||
assert result.title == "Project Hail Mary (2021) Audiobook"
|
||||
assert result.size == "620.5MB"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("line", "expected"),
|
||||
[
|
||||
("!s A - T.epub ::INFO:: 1MB", "ebook"),
|
||||
("!s A - T.mp3 ::INFO:: 1MB", "audiobook"),
|
||||
("!s A - T.flac ::INFO:: 1MB", "audiobook"),
|
||||
# Archives carry no format information, so the name has to decide.
|
||||
("!s A - T (Audiobook).rar ::INFO:: 1MB", "audiobook"),
|
||||
("!s A - T [Unabridged].zip ::INFO:: 1MB", "audiobook"),
|
||||
("!s A - T (Narrated by Someone).rar ::INFO:: 1MB", "audiobook"),
|
||||
("!s A - T [64kbps].zip ::INFO:: 1MB", "audiobook"),
|
||||
("!s A - T (retail).rar ::INFO:: 1MB", "ebook"),
|
||||
("!s A - T.zip ::INFO:: 1MB", "ebook"),
|
||||
],
|
||||
)
|
||||
def test_detect_content_type(line, expected):
|
||||
assert parser.detect_content_type(parser.parse_result_line(line)) == expected
|
||||
|
||||
|
||||
def test_recognized_formats_order_is_deterministic():
|
||||
"""This was a set, so which format won for a multi-extension line varied per restart."""
|
||||
assert parser.ALL_RECOGNIZED_FORMATS == tuple(parser.ALL_RECOGNIZED_FORMATS)
|
||||
# Longest-first, so ".azw3" cannot be truncated to "azw" (nor ".docx" to "doc").
|
||||
lengths = [len(fmt) for fmt in parser.ALL_RECOGNIZED_FORMATS]
|
||||
assert lengths == sorted(lengths, reverse=True)
|
||||
|
||||
|
||||
def test_parse_results_file_uses_audiobook_format_settings(monkeypatch):
|
||||
values = {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import pytest
|
||||
|
||||
from shelfmark.metadata_providers import hardcover
|
||||
from shelfmark.metadata_providers.hardcover import (
|
||||
HardcoverProvider,
|
||||
_test_hardcover_connection,
|
||||
)
|
||||
|
||||
# Hardcover replaced its ~500 char JWTs with short opaque tokens.
|
||||
PAT = "hc_pat_" + "a" * 32
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_config_writes(monkeypatch):
|
||||
"""Keep the connection test from touching the on-disk provider config."""
|
||||
monkeypatch.setattr(hardcover, "_save_connected_user", lambda user_id, username: None)
|
||||
|
||||
|
||||
class TestHardcoverApiKey:
|
||||
def test_personal_access_token_is_accepted(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
HardcoverProvider,
|
||||
"_execute_query",
|
||||
lambda self, query, variables: {"me": [{"id": 1, "username": "alex"}]},
|
||||
)
|
||||
|
||||
result = _test_hardcover_connection({"HARDCOVER_API_KEY": PAT})
|
||||
|
||||
assert result == {"success": True, "message": "Connected as: alex"}
|
||||
|
||||
def test_short_key_without_the_prefix_is_rejected(self):
|
||||
result = _test_hardcover_connection({"HARDCOVER_API_KEY": "eyJhbGciOiJIUzI1NiJ9.short"})
|
||||
|
||||
assert result["success"] is False
|
||||
assert "too short" in result["message"]
|
||||
|
||||
def test_short_prefixed_key_still_reaches_the_api(self, monkeypatch):
|
||||
"""A key wearing the hc_pat_ prefix is Hardcover's to accept or reject."""
|
||||
monkeypatch.setattr(
|
||||
HardcoverProvider,
|
||||
"_execute_query",
|
||||
lambda self, query, variables: None,
|
||||
)
|
||||
|
||||
result = _test_hardcover_connection({"HARDCOVER_API_KEY": "hc_pat_ab"})
|
||||
|
||||
assert result == {"success": False, "message": "API request failed - check your API key"}
|
||||
|
||||
@pytest.mark.parametrize("pasted", [f"Bearer {PAT}", f"bearer {PAT}", f" {PAT} "])
|
||||
def test_pasted_auth_header_noise_is_stripped(self, pasted):
|
||||
provider = HardcoverProvider(api_key=pasted)
|
||||
|
||||
assert provider.session.headers["Authorization"] == f"Bearer {PAT}"
|
||||
@@ -1,4 +1,8 @@
|
||||
from shelfmark.metadata_providers.hardcover import HardcoverProvider
|
||||
from shelfmark.metadata_providers.hardcover import (
|
||||
TITLE_SUGGESTION_FIELDS,
|
||||
TITLE_SUGGESTION_WEIGHTS,
|
||||
HardcoverProvider,
|
||||
)
|
||||
|
||||
|
||||
class TestHardcoverFieldOptions:
|
||||
@@ -130,8 +134,10 @@ class TestHardcoverFieldOptions:
|
||||
"limit": 7,
|
||||
"page": 1,
|
||||
"sort": "_text_match:desc,users_count:desc",
|
||||
"fields": "title,alternative_titles",
|
||||
"weights": "5,2",
|
||||
# Hardcover rejects a Book search that narrows to fewer fields than its
|
||||
# preset expects, so the typeahead sends the full list and leans on weights.
|
||||
"fields": TITLE_SUGGESTION_FIELDS,
|
||||
"weights": TITLE_SUGGESTION_WEIGHTS,
|
||||
}
|
||||
|
||||
def test_get_search_field_options_skips_short_text_queries(self):
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Guards on the shape of Hardcover's `fields`/`weights` search parameters.
|
||||
|
||||
Hardcover turns `fields` into Typesense's `query_by` but keeps `num_typos` and
|
||||
`query_by_weights` as fixed-length presets per query_type. A field list of the
|
||||
wrong length is not searched loosely -- the whole search is rejected with a null
|
||||
results body, which used to surface as "0 results". These tests pin the counts
|
||||
so a narrower field list cannot silently ship again.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.metadata_providers.hardcover import (
|
||||
AUTHOR_SUGGESTION_FIELDS,
|
||||
AUTHOR_SUGGESTION_WEIGHTS,
|
||||
BOOK_SEARCH_FIELD_COUNT,
|
||||
BOOK_SEARCH_FIELDS,
|
||||
BOOK_TITLE_AUTHOR_WEIGHTS,
|
||||
BOOK_TITLE_WEIGHTS,
|
||||
SERIES_SEARCH_FIELDS,
|
||||
SERIES_SEARCH_WEIGHTS,
|
||||
TITLE_SUGGESTION_FIELDS,
|
||||
TITLE_SUGGESTION_WEIGHTS,
|
||||
HardcoverProvider,
|
||||
)
|
||||
|
||||
|
||||
def _count(value: str) -> int:
|
||||
return len([part for part in value.split(",") if part.strip()])
|
||||
|
||||
|
||||
class TestBookSearchFieldCounts:
|
||||
def test_book_field_list_matches_hardcovers_preset_length(self):
|
||||
assert _count(BOOK_SEARCH_FIELDS) == BOOK_SEARCH_FIELD_COUNT
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("label", "weights"),
|
||||
[
|
||||
("title", BOOK_TITLE_WEIGHTS),
|
||||
("title+author", BOOK_TITLE_AUTHOR_WEIGHTS),
|
||||
("title typeahead", TITLE_SUGGESTION_WEIGHTS),
|
||||
],
|
||||
)
|
||||
def test_book_weights_line_up_with_the_field_list(self, label, weights):
|
||||
assert _count(weights) == BOOK_SEARCH_FIELD_COUNT, label
|
||||
|
||||
def test_title_typeahead_uses_the_full_book_field_list(self):
|
||||
assert TITLE_SUGGESTION_FIELDS == BOOK_SEARCH_FIELDS
|
||||
|
||||
|
||||
class TestNonBookSearchFieldCounts:
|
||||
@pytest.mark.parametrize(
|
||||
("fields", "weights"),
|
||||
[
|
||||
(AUTHOR_SUGGESTION_FIELDS, AUTHOR_SUGGESTION_WEIGHTS),
|
||||
(SERIES_SEARCH_FIELDS, SERIES_SEARCH_WEIGHTS),
|
||||
],
|
||||
)
|
||||
def test_weights_line_up_with_their_field_list(self, fields, weights):
|
||||
assert _count(fields) == _count(weights)
|
||||
|
||||
|
||||
class TestBuildSearchParams:
|
||||
@pytest.mark.parametrize(
|
||||
("author", "title", "series"),
|
||||
[
|
||||
("", "Dune", ""),
|
||||
("Herbert", "Dune", ""),
|
||||
("Herbert", "", ""),
|
||||
("", "", ""),
|
||||
],
|
||||
)
|
||||
def test_every_branch_sends_a_usable_field_weight_pair(self, author, title, series):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
|
||||
_query, fields, weights = provider._build_search_params("dune", author, title, series)
|
||||
|
||||
if fields is None:
|
||||
# No override: Hardcover applies its own preset, so weights must be absent too.
|
||||
assert weights is None
|
||||
return
|
||||
assert _count(fields) == BOOK_SEARCH_FIELD_COUNT
|
||||
assert _count(weights) == BOOK_SEARCH_FIELD_COUNT
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -6,9 +7,10 @@ from shelfmark.metadata_providers import MetadataSearchOptions
|
||||
from shelfmark.metadata_providers.hardcover import HardcoverProvider
|
||||
|
||||
# Hardcover answers a rejected search with HTTP 200, no GraphQL errors, and a
|
||||
# null results body. A search that genuinely matched nothing still returns a
|
||||
# results object with found: 0.
|
||||
REJECTED = {"search": {"results": None}}
|
||||
# null results body; the reason shows up in the sibling error field. A search
|
||||
# that genuinely matched nothing still returns a results object with found: 0.
|
||||
REJECTED = {"search": {"error": "Parameter `sort_by` is malformed.", "results": None}}
|
||||
REJECTED_SILENTLY = {"search": {"results": None}}
|
||||
EMPTY = {"search": {"results": {"hits": [], "found": 0}}}
|
||||
ONE_HIT = {"search": {"results": {"hits": [{"document": {"id": 7, "title": "Dune"}}], "found": 1}}}
|
||||
|
||||
@@ -19,6 +21,29 @@ def _reset_sort_fallback(monkeypatch):
|
||||
monkeypatch.setattr("shelfmark.metadata_providers.hardcover._sort_fallback_until", 0.0)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hardcover_logs():
|
||||
"""Collect Hardcover log messages.
|
||||
|
||||
The provider's logger is built outside the standard hierarchy, so its
|
||||
records never reach the root handler that caplog installs.
|
||||
"""
|
||||
from shelfmark.metadata_providers import hardcover
|
||||
|
||||
messages: list[str] = []
|
||||
|
||||
class _Capture(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
messages.append(record.getMessage())
|
||||
|
||||
handler = _Capture()
|
||||
hardcover.logger.addHandler(handler)
|
||||
try:
|
||||
yield messages
|
||||
finally:
|
||||
hardcover.logger.removeHandler(handler)
|
||||
|
||||
|
||||
def _reject_sorted(calls: list[dict[str, Any]], *, success=ONE_HIT):
|
||||
"""Build an _execute_query stand-in that rejects any request carrying a sort."""
|
||||
|
||||
@@ -38,7 +63,9 @@ class TestHardcoverSortFallback:
|
||||
result = provider._execute_search_query("query", {"query": "dune", "sort": "relevance"})
|
||||
|
||||
assert result == ONE_HIT
|
||||
assert [call["sort"] for call in calls] == ["relevance", ""]
|
||||
# The retry drops sort entirely -- an empty sort is a value Hardcover can reject too.
|
||||
assert [call.get("sort") for call in calls] == ["relevance", None]
|
||||
assert "sort" not in calls[1]
|
||||
|
||||
def test_treats_an_empty_result_set_as_success(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
@@ -64,6 +91,24 @@ class TestHardcoverSortFallback:
|
||||
assert result is None
|
||||
assert len(calls) == 2
|
||||
|
||||
def test_keeps_sorting_when_the_sort_was_not_the_culprit(self, monkeypatch):
|
||||
"""A rejection that survives dropping the sort must not disable sorting globally."""
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
calls: list[dict[str, Any]] = []
|
||||
monkeypatch.setattr(
|
||||
provider, "_execute_query", lambda query, variables: calls.append(variables) or REJECTED
|
||||
)
|
||||
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": "rating:desc"})
|
||||
provider._execute_search_query("query", {"query": "hyperion", "sort": "rating:desc"})
|
||||
|
||||
assert [call.get("sort") for call in calls] == [
|
||||
"rating:desc",
|
||||
None,
|
||||
"rating:desc",
|
||||
None,
|
||||
]
|
||||
|
||||
def test_reports_failure_for_an_unsorted_rejection(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
calls: list[dict[str, Any]] = []
|
||||
@@ -76,6 +121,24 @@ class TestHardcoverSortFallback:
|
||||
assert result is None
|
||||
assert len(calls) == 1
|
||||
|
||||
def test_logs_the_reason_hardcover_gave(self, monkeypatch, hardcover_logs):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
monkeypatch.setattr(provider, "_execute_query", lambda query, variables: REJECTED)
|
||||
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": ""})
|
||||
|
||||
assert any("Parameter `sort_by` is malformed." in message for message in hardcover_logs)
|
||||
|
||||
def test_falls_back_to_a_placeholder_when_hardcover_says_nothing(
|
||||
self, monkeypatch, hardcover_logs
|
||||
):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
monkeypatch.setattr(provider, "_execute_query", lambda query, variables: REJECTED_SILENTLY)
|
||||
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": ""})
|
||||
|
||||
assert any("no error message" in message for message in hardcover_logs)
|
||||
|
||||
def test_skips_the_doomed_request_on_later_searches(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
calls: list[dict[str, Any]] = []
|
||||
@@ -84,7 +147,7 @@ class TestHardcoverSortFallback:
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": "relevance"})
|
||||
provider._execute_search_query("query", {"query": "hyperion", "sort": "relevance"})
|
||||
|
||||
assert [call["sort"] for call in calls] == ["relevance", "", ""]
|
||||
assert [call.get("sort") for call in calls] == ["relevance", None, None]
|
||||
|
||||
def test_search_returns_results_despite_a_rejected_sort(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
@@ -97,7 +160,7 @@ class TestHardcoverSortFallback:
|
||||
|
||||
assert result.total_found == 1
|
||||
assert [book.title for book in result.books] == ["Dune"]
|
||||
assert [call["sort"] for call in calls] == ["_text_match:desc,users_count:desc", ""]
|
||||
assert [call.get("sort") for call in calls] == ["_text_match:desc,users_count:desc", None]
|
||||
|
||||
|
||||
class TestSearchPayloadRejection:
|
||||
@@ -105,9 +168,19 @@ class TestSearchPayloadRejection:
|
||||
from shelfmark.metadata_providers.hardcover import _search_payload_rejected
|
||||
|
||||
assert _search_payload_rejected(REJECTED) is True
|
||||
assert _search_payload_rejected(REJECTED_SILENTLY) is True
|
||||
assert _search_payload_rejected(EMPTY) is False
|
||||
assert _search_payload_rejected(ONE_HIT) is False
|
||||
assert _search_payload_rejected(None) is False
|
||||
assert _search_payload_rejected({}) is False
|
||||
# Non-search payloads (list lookups, book fetches) must pass through.
|
||||
assert _search_payload_rejected({"series": [{"id": 1}]}) is False
|
||||
|
||||
def test_reads_the_error_hardcover_attached(self):
|
||||
from shelfmark.metadata_providers.hardcover import _search_rejection_reason
|
||||
|
||||
assert _search_rejection_reason(REJECTED) == "Parameter `sort_by` is malformed."
|
||||
assert _search_rejection_reason(REJECTED_SILENTLY) == ""
|
||||
assert _search_rejection_reason(EMPTY) == ""
|
||||
assert _search_rejection_reason(None) == ""
|
||||
assert _search_rejection_reason({"search": {"error": None, "results": None}}) == ""
|
||||
|
||||
@@ -10,6 +10,7 @@ from shelfmark.release_sources import ReleaseProtocol
|
||||
from shelfmark.release_sources.newznab.source import (
|
||||
NewznabSource,
|
||||
_newznab_result_to_release,
|
||||
_parse_category_ids,
|
||||
)
|
||||
|
||||
# ── fixtures / helpers ─────────────────────────────────────────────────────────
|
||||
@@ -100,6 +101,18 @@ class TestResultToRelease:
|
||||
r = _newznab_result_to_release(_make_result(categories=[]), "audiobook")
|
||||
assert r.content_type == "audiobook"
|
||||
|
||||
def test_custom_searched_category_treated_as_book(self):
|
||||
r = _newznab_result_to_release(_make_result(categories=[8010]), "ebook", [8010])
|
||||
assert r.content_type == "book"
|
||||
|
||||
def test_custom_searched_category_treated_as_audiobook(self):
|
||||
r = _newznab_result_to_release(_make_result(categories=[{"id": 3040}]), "audiobook", [3040])
|
||||
assert r.content_type == "audiobook"
|
||||
|
||||
def test_unsearched_out_of_range_category_stays_other(self):
|
||||
r = _newznab_result_to_release(_make_result(categories=[2000]), "ebook", [8010])
|
||||
assert r.content_type == "other"
|
||||
|
||||
def test_freeleech_flag_detected_via_download_volume(self):
|
||||
r = _newznab_result_to_release(_make_result(downloadVolumeFactor=0.0))
|
||||
assert r.extra["freeleech"] is True
|
||||
@@ -209,6 +222,31 @@ class TestIsAvailable:
|
||||
assert NewznabSource().is_available() is False
|
||||
|
||||
|
||||
# ── category parsing ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseCategoryIds:
|
||||
def test_parses_list_of_strings(self):
|
||||
assert _parse_category_ids(["7100", "7120"]) == [7100, 7120]
|
||||
|
||||
def test_parses_comma_separated_string(self):
|
||||
assert _parse_category_ids("7100, 7120") == [7100, 7120]
|
||||
|
||||
def test_parses_comma_separated_entry_inside_list(self):
|
||||
assert _parse_category_ids(["7100,7120", "8010"]) == [7100, 7120, 8010]
|
||||
|
||||
def test_drops_duplicates_and_keeps_order(self):
|
||||
assert _parse_category_ids(["7120", "7100", "7120"]) == [7120, 7100]
|
||||
|
||||
def test_skips_non_numeric_and_non_positive_values(self):
|
||||
assert _parse_category_ids(["books", "0", "-7000", "7100"]) == [7100]
|
||||
|
||||
def test_returns_empty_for_unset_or_blank(self):
|
||||
assert _parse_category_ids(None) == []
|
||||
assert _parse_category_ids([]) == []
|
||||
assert _parse_category_ids(" ") == []
|
||||
|
||||
|
||||
# ── NewznabSource.search ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -276,6 +314,44 @@ class TestSearch:
|
||||
_, kwargs = client.search.call_args
|
||||
assert kwargs["categories"] == [3030]
|
||||
|
||||
def test_searches_with_configured_ebook_categories(self, monkeypatch):
|
||||
client = MagicMock()
|
||||
client.search.return_value = []
|
||||
src = self._patched_source(
|
||||
monkeypatch, client, {"NEWZNAB_EBOOK_CATEGORIES": ["7100", "7120"]}
|
||||
)
|
||||
book = _make_book()
|
||||
src.search(book, _make_plan(book), content_type="ebook")
|
||||
_, kwargs = client.search.call_args
|
||||
assert kwargs["categories"] == [7100, 7120]
|
||||
|
||||
def test_searches_with_configured_audiobook_categories(self, monkeypatch):
|
||||
client = MagicMock()
|
||||
client.search.return_value = []
|
||||
src = self._patched_source(monkeypatch, client, {"NEWZNAB_AUDIOBOOK_CATEGORIES": "3040"})
|
||||
book = _make_book()
|
||||
src.search(book, _make_plan(book), content_type="audiobook")
|
||||
_, kwargs = client.search.call_args
|
||||
assert kwargs["categories"] == [3040]
|
||||
|
||||
def test_falls_back_to_default_when_categories_cleared(self, monkeypatch):
|
||||
client = MagicMock()
|
||||
client.search.return_value = []
|
||||
src = self._patched_source(monkeypatch, client, {"NEWZNAB_EBOOK_CATEGORIES": []})
|
||||
book = _make_book()
|
||||
src.search(book, _make_plan(book), content_type="ebook")
|
||||
_, kwargs = client.search.call_args
|
||||
assert kwargs["categories"] == [7000]
|
||||
|
||||
def test_ebook_categories_do_not_affect_audiobook_search(self, monkeypatch):
|
||||
client = MagicMock()
|
||||
client.search.return_value = []
|
||||
src = self._patched_source(monkeypatch, client, {"NEWZNAB_EBOOK_CATEGORIES": ["7100"]})
|
||||
book = _make_book()
|
||||
src.search(book, _make_plan(book), content_type="audiobook")
|
||||
_, kwargs = client.search.call_args
|
||||
assert kwargs["categories"] == [3030]
|
||||
|
||||
def test_expand_search_removes_categories(self, monkeypatch):
|
||||
client = MagicMock()
|
||||
client.search.return_value = []
|
||||
|
||||
@@ -727,6 +727,62 @@ class TestProwlarrHandlerSeedCriteria:
|
||||
assert call_kwargs["ratio_limit"] == 1.25
|
||||
|
||||
|
||||
class TestProwlarrHandlerContentType:
|
||||
"""Regression tests for issue #1235 — content type must reach the client."""
|
||||
|
||||
def test_download_passes_content_type_to_client(self):
|
||||
"""rTorrent picks its audiobook label from content_type, not category."""
|
||||
mock_client = MagicMock()
|
||||
mock_client.name = "rtorrent"
|
||||
mock_client.find_existing.return_value = None
|
||||
mock_client.add_download.return_value = "download_id"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"shelfmark.release_sources.prowlarr.handler.get_release",
|
||||
return_value={
|
||||
"protocol": "torrent",
|
||||
"title": "Test Release",
|
||||
"magnetUrl": "magnet:?xt=urn:btih:abc123",
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"shelfmark.release_sources.prowlarr.handler.get_client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"shelfmark.release_sources.prowlarr.handler.remove_release",
|
||||
),
|
||||
patch("shelfmark.release_sources.prowlarr.handler.config.get", return_value=True),
|
||||
patch.object(
|
||||
ProwlarrHandler,
|
||||
"_poll_and_complete",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
handler = ProwlarrHandler()
|
||||
task = DownloadTask(
|
||||
task_id="content-type-pass-through",
|
||||
source="prowlarr",
|
||||
title="Test Audiobook",
|
||||
content_type="audiobook",
|
||||
)
|
||||
cancel_flag = Event()
|
||||
recorder = ProgressRecorder()
|
||||
|
||||
handler.download(
|
||||
task=task,
|
||||
cancel_flag=cancel_flag,
|
||||
progress_callback=recorder.progress_callback,
|
||||
status_callback=recorder.status_callback,
|
||||
)
|
||||
|
||||
call_kwargs = mock_client.add_download.call_args.kwargs
|
||||
assert call_kwargs["content_type"] == "audiobook"
|
||||
# rTorrent gets no category, so content_type is its only audiobook signal.
|
||||
assert call_kwargs["category"] is None
|
||||
|
||||
|
||||
class TestProwlarrHandlerExistingDownload:
|
||||
"""Tests for handling existing downloads."""
|
||||
|
||||
|
||||
@@ -563,6 +563,36 @@ class TestRTorrentClientAudiobookLabel:
|
||||
assert "d.custom1.set=books" in args[2]
|
||||
assert "d.custom1.set=audiobooks" not in args[2]
|
||||
|
||||
def test_uses_audiobook_label_for_compound_content_type(self, monkeypatch):
|
||||
"""Issue #1235 — an exact "audiobook" match missed forms like "book (audiobook)"."""
|
||||
config_values = {
|
||||
"RTORRENT_URL": "http://localhost:8080/RPC2",
|
||||
"RTORRENT_LABEL": "books",
|
||||
"RTORRENT_AUDIOBOOK_LABEL": "audiobooks",
|
||||
"RTORRENT_DOWNLOAD_DIR": "/downloads",
|
||||
}
|
||||
mock_rpc, mock_xmlrpc, mock_torrent_info = self._make_client(monkeypatch, config_values)
|
||||
|
||||
with patch.dict("sys.modules", {"xmlrpc.client": mock_xmlrpc}):
|
||||
with patch(
|
||||
"shelfmark.download.clients.torrent_utils.extract_torrent_info",
|
||||
return_value=mock_torrent_info,
|
||||
):
|
||||
if "shelfmark.download.clients.rtorrent" in sys.modules:
|
||||
del sys.modules["shelfmark.download.clients.rtorrent"]
|
||||
from shelfmark.download.clients.rtorrent import RTorrentClient
|
||||
|
||||
client = RTorrentClient()
|
||||
client.add_download(
|
||||
"magnet:?xt=urn:btih:abc123",
|
||||
"Test Audiobook",
|
||||
content_type="Book (Audiobook)",
|
||||
)
|
||||
|
||||
args = mock_rpc.load.start.call_args[0]
|
||||
assert "d.custom1.set=audiobooks" in args[2]
|
||||
assert "d.custom1.set=books" not in args[2]
|
||||
|
||||
|
||||
class TestRTorrentClientGetStatus:
|
||||
"""Tests for RTorrentClient.get_status()."""
|
||||
|
||||
@@ -2,6 +2,18 @@ version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.14"
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "idna" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "apprise"
|
||||
version = "1.12.0"
|
||||
@@ -44,14 +56,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "basedpyright"
|
||||
version = "1.39.9"
|
||||
version = "1.39.10"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nodejs-wheel-binaries" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7a/4b/c1f4e211e50389304d6af32b9280e026a7133e3ad59bbdf8f7a3250f8bee/basedpyright-1.39.9.tar.gz", hash = "sha256:32cbea5fc8273e89df3db20daea56cb7286e419ccdfdc479c64759d2dc071901", size = 24412216, upload-time = "2026-06-27T02:19:49.834Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/68/43/ad2999f3b09eb2b1e59931d88fac0f7bcc9c17fc18c903268779bd10cc97/basedpyright-1.39.10.tar.gz", hash = "sha256:c8eaf5302f3265e275c7df4fba194d7afa7c1cb53fbfd448e90098360aca2c2e", size = 24740347, upload-time = "2026-08-13T17:09:02.51Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/d4/e1fa108710d0498a18c77b1e13897f31eab47c69aa8cfe2d2a4df746541e/basedpyright-1.39.9-py3-none-any.whl", hash = "sha256:6b0837b9eba972c71895167ab9b127e6afdbc17abc92312e3f8d15ca82a5611c", size = 13374276, upload-time = "2026-06-27T02:19:54.431Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/2a/a224054d75a58786c482f63b8ff2a09fc3362268bd1ebd0a61fb3f982153/basedpyright-1.39.10-py3-none-any.whl", hash = "sha256:cbd75d83c0be841329bcfef2d2f1182f152a6d975b8eb199e75cf5b8e9a3de78", size = 13482322, upload-time = "2026-08-13T17:08:59.074Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -110,35 +122,61 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "cffi"
|
||||
version = "2.0.0"
|
||||
version = "2.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pycparser", marker = "implementation_name != 'PyPy'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -410,7 +448,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "gevent"
|
||||
version = "26.7.0"
|
||||
version = "26.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" },
|
||||
@@ -418,24 +456,26 @@ dependencies = [
|
||||
{ name = "zope-event" },
|
||||
{ name = "zope-interface" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b8/eb/5f2db8013f1a4a6df2c23201f384a066f13ff5764a9f62a608c8a50ac8cc/gevent-26.8.0.tar.gz", hash = "sha256:96039f41bbde6dcd72559e5ffbd408a04f46774b47d991d4cf032da8fa79e5a0", size = 6625998, upload-time = "2026-08-10T18:02:28.038Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/bb/ab60d496cbdc0293ebbd6c2070b34da0632bd7a2ca20163c17e18d2d2dc9/gevent-26.7.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0e0e3bf7ae0f82dbc5c6be26b4781e86c97f1e28d516b7a9746ac8b04bcc6948", size = 2992503, upload-time = "2026-07-22T16:24:36.503Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/35/75f27c06a82a5b22600aaccbd9567d89bb4091be43e96c02981f10aff23d/gevent-26.7.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:740050b53048207b080a1e183a377c47809ad0b7b7b0cd7eab0dea1045f7e480", size = 1809173, upload-time = "2026-07-22T18:11:30.724Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/5f/a6b32b4db3fa76bd8a070f0f46f5306123bf6336e7a0ca0cd2f9b99473df/gevent-26.7.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:67983607eb6c7bafa362c5c43b69a27145b936c34a3d6441ed42413d62fae0a6", size = 1906630, upload-time = "2026-07-22T18:10:45.836Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/87/832495d8fcc05ff7432f038b7c4decbd5632425a2cd5da2ce73cb2d800c4/gevent-26.7.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:475848518d708e07d1987c3d94cb8ff53e2b3a69df32e39feda2779cafe400b0", size = 1855278, upload-time = "2026-07-22T18:29:11.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/8b/2f36c0fa389fa2b7ceb5a8972b0e7da7bc770f9135315cf4246c607ca5fc/gevent-26.7.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f8ed457dd616bfe6682569f92730f9ab45aafb1aeca5e80eb2f6b9a2ce26d11", size = 2136155, upload-time = "2026-07-22T16:48:33.865Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b5/98/09f2cfaa23dbce48e3271e95b0d003f93acece6b5cfd40f4cebe3850d79b/gevent-26.7.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15373c68cf1fa14114bec2f09b16e2c65374bd5309e897e0a28740b09ce329e0", size = 1822108, upload-time = "2026-07-22T18:07:15.397Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/d8/05a294165c17569f04284ad3c889684c8780544885b4cdf77b1432947d0c/gevent-26.7.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:73f3d53f2f390369e290c933b75bd87f1f2261f2f2f2175aa667c43ee3049bad", size = 2162814, upload-time = "2026-07-22T17:02:18.066Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/89/58a545c4eda33e106d6887a0387adc2249abc14c779e3eb88bbfdf3768d6/gevent-26.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:f11b558d544ad2249029ba023cd6519ec3a0eee54a3d027e6515c1eaa322422a", size = 1706971, upload-time = "2026-07-22T16:27:02.263Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/cd/413f293e54961e5c89c54235370e3603ec0f561e7ace8357980410efbf78/gevent-26.7.0-cp314-cp314-win_arm64.whl", hash = "sha256:3871f4ca59ec2328c3ef638a0fe01a28a825443a133368dc78eb5ceadcad7609", size = 1585078, upload-time = "2026-07-22T16:30:48.145Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/3a/47f29f632aaa38aa12410f57f1732fc50bfd4d4006d2e7e022ce731cabc9/gevent-26.7.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3e3d6e20a94239ad353b776e72b8ce18c35dbe4e98c279aef3932651553d8404", size = 2996208, upload-time = "2026-07-22T16:23:23.859Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/b3/4620f1ce81ecec9890229806c73f07dd022e40f76a552bd430391e7316c4/gevent-26.7.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ddbd3cc76b9bc69df651a216c2a62fc6415ad463b3ac9c6cbbbb8b7b8224af17", size = 1811545, upload-time = "2026-07-22T18:11:32.224Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/fd/d285212ffd5585d511299e13e61d76262def8e826e9f20c92cb85df406f2/gevent-26.7.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:01ceab7e608dc1b9859d9511a0a29d7ce2e7d909ab19fddc860e70a2ed5b10ce", size = 1910418, upload-time = "2026-07-22T18:10:47.709Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/e0/c5d666e6065652918cfb6e6a3cf8f721d0e152c57cec217ad81792a1323b/gevent-26.7.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:2e6c917b2b8baeb6080797a6b25e35e1fd784319a05bb92b87c53546e5578eb2", size = 1857891, upload-time = "2026-07-22T18:29:13.13Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/67/e945ed458fa98b34572876bfd0d35fe4fa3f1159f43660b71d982b7cb63e/gevent-26.7.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:df75a1748b26030f2f7f10042cc45640b22954d9d0dc6b4b6f0dbe0b6751a2d4", size = 2138121, upload-time = "2026-07-22T16:48:35.358Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/99/55/622468fa1a3c4cf51f20e14813f5cc1592fe6e44a42ccca9195a0b18c769/gevent-26.7.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ee1b389587e5d5c1eb19d0455b5b4d7a0fb5c5287af4e226ec66d9dfd2548107", size = 1825114, upload-time = "2026-07-22T18:07:16.667Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/7a/151a2afcacf487ca25faf8b1bdd6c5b4ace2f7c1e6b4eaffe0a5e6e1df61/gevent-26.7.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c2918641ba756f46aa01ab9dd82d6dfceec403c77c2787298746b411dcf0288e", size = 2165990, upload-time = "2026-07-22T17:02:19.817Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/61/cb/1bed6675f6cba42bfe23c38eacf482e08dfaa7af251cb0cddfcbb084b757/gevent-26.8.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:e0f6a96cd5f9ad8f1f91d5d56d3e5534e15682b4a0abecd8396a18b836296426", size = 3006150, upload-time = "2026-08-10T16:56:41.378Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/f5/bb4f3272d419bc785ce6a39440f4d157a6f7c154fef93c77444579ee5f43/gevent-26.8.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:113d12a2d4276047980e491cc5856388954642eb555479c6962f52fb181eb1df", size = 1819363, upload-time = "2026-08-10T17:58:52.056Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/c3/351a5c6890804e39a109c7df8aa8f17f953d2f0acf8215e040670a0e573a/gevent-26.8.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:3ea7d0ff714ac9c634bfaaa3cfb25a5fcf7a272df47cad2261642323b16a3266", size = 1916853, upload-time = "2026-08-10T17:48:12.677Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/81/a37201ced7b96eeba4554a1e656d040ad808f254a9f3d7b94f8ac4cfa82a/gevent-26.8.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:1bee3c0cb1aa2cee3369de46f8d952d10310166d7b4a744ae64a32ee1e14a1f5", size = 1865982, upload-time = "2026-08-10T17:54:30.173Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/6d/e70113648ee1041070e6190389796414dd70b8cb86ff5ec8a9762284cb1a/gevent-26.8.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:64bc5c3302b02f9ad012243173e13705711dbbcc7590443d974f2961154d7077", size = 2147279, upload-time = "2026-08-10T17:19:24.24Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/10/1af5f1910d5534bba338d954e77f401e167189c7af8b621104307e2e7e16/gevent-26.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e544cba5810ec64056d78f0d3c79ceeafc91ac612b4e9008134645bc437dd885", size = 1832674, upload-time = "2026-08-10T17:42:41.002Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/23/bc09e7f2a0dc699269d5dd03a4860afa80f8d29c1f28a18859ebd06d4ab1/gevent-26.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3445b3a8a51fcb7b881485b1089f7d64ca057aa921a78f146d702853650f958d", size = 2173988, upload-time = "2026-08-10T17:23:52.475Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/21/44ebdb32eb5050c367c4ccf6d43627bf875847ef65d8455beff52044846c/gevent-26.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:4fe56814f6d36d8fdfe0254909388fa58ccb61e7945d2f031a2dbd81f90ced4c", size = 1716724, upload-time = "2026-08-10T17:01:37.23Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/71/6708a3aae223a326b648a01beb5244caa562d75f6515528a8241260763f3/gevent-26.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:5914bab0ebe7fbd6077d703b61b2e74afec0436487f29b8154302c61f4d58502", size = 1594414, upload-time = "2026-08-10T17:00:51.867Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/18/42a8129bdfe7285f35be21e6737a135462cb723a30952f06cb2203ecc8df/gevent-26.8.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:9afd9fbccbcea0b803d554b9e7bac75382e919aa07a5cb98d04edeba10c6cf77", size = 3009619, upload-time = "2026-08-10T16:59:31.344Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/ff/f540c92d4c6ff3af5e60e990c7f417d1419a5a2aee5f5047612e708221c7/gevent-26.8.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:cd225e954d57e7d8a994a8d152f0d76609c73fa701a3e27293cc46675b2dce77", size = 1821950, upload-time = "2026-08-10T17:58:53.313Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/eb/b40a8b9df1d4955cae3f600db416d6a8abb73f5f31d8a315cf2acf225462/gevent-26.8.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:8cb1402c0c7bdbd6d772fd5eb700b98b31ce0d8f613f68823f32cce5ec956d8c", size = 1921096, upload-time = "2026-08-10T17:48:14.477Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/e4/59fc824207ac7f63ac83af7a7bed2707d6768488f1ae2c6c85794d614570/gevent-26.8.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:242d5e3622a39236f57a4e740c5480e86b6bc15c3a790e997b8cc99bee34ef27", size = 1868863, upload-time = "2026-08-10T17:54:31.624Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/8b/cb001b1906ce78c63a252ad5baaebba745ca819bd7239ff60417fbc0d24a/gevent-26.8.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:b13173a992de43e92d15c6ac318f8ae019cf08eec15a78f99a0d4a917837052a", size = 2149188, upload-time = "2026-08-10T17:19:25.356Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/c6/3085b52ec0ecc8173de40fca1f5faf8d02761288e1712ffb34628cfab214/gevent-26.8.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:f431af3f2737ae01cf1c5a303f193cc2a8f726521e997ea8baf2d8567c6fe07c", size = 1835672, upload-time = "2026-08-10T17:42:42.296Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/99/ee5722f7d51ee4bd09396629802ae90c9e0bebef0e7938a4edf08eb92749/gevent-26.8.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1d8b76e525d7e301db83d8124a27700c55ad23754f3efbbf15c7051d6fa19853", size = 2177315, upload-time = "2026-08-10T17:23:53.854Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/cb/c2fea129f29ca114dc63f53178337845ba63532e07979c10fd70635a6dde/gevent-26.8.0-cp315-cp315-win_amd64.whl", hash = "sha256:b16931069d0044a23a566d16dc5787122e858fa9d591c81500b6e6ad2148cfec", size = 1716882, upload-time = "2026-08-10T17:00:09.172Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/f4/b4de17304026cbd6386d427bc5ece10ec65015e9bfd66e14672350ee5dfb/gevent-26.8.0-cp315-cp315-win_arm64.whl", hash = "sha256:42b8ed34f7aff517fac448ab57084e5e39d6a84a2b6c2b709223d5751c85e657", size = 1594517, upload-time = "2026-08-10T17:00:47.046Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -518,6 +558,70 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "4.4.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "hpack" },
|
||||
{ name = "hyperframe" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hpack"
|
||||
version = "4.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpcore"
|
||||
version = "1.0.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "h11" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpx"
|
||||
version = "0.28.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
{ name = "certifi" },
|
||||
{ name = "httpcore" },
|
||||
{ name = "idna" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
http2 = [
|
||||
{ name = "h2" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyperframe"
|
||||
version = "6.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.18"
|
||||
@@ -751,11 +855,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "platformdirs"
|
||||
version = "4.11.0"
|
||||
version = "4.11.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b8/d7/e7bfbc86e9f99ff7807e24de7703f032e9c9ba80bb355cf26e0e9bc5a75e/platformdirs-4.11.3.tar.gz", hash = "sha256:66a73d38a849810252df809a3d8bcbda8e26f6c189920e7535ad608a48dbb5ab", size = 33050, upload-time = "2026-08-13T22:43:27.52Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/a9/c34aebedd3a4c9afe5101b1b8713710b3fec18087c8a36c35d2f909861bd/platformdirs-4.11.3-py3-none-any.whl", hash = "sha256:5ed065d443751de711da036041a7a214122efc4a4de393b3f4137ba5576540e7", size = 23491, upload-time = "2026-08-13T22:43:26.121Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -769,26 +873,26 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "prek"
|
||||
version = "0.4.12"
|
||||
version = "0.4.13"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fc/5c/cb6e63f7e5a58a5313ddb70409174f4dc004e4b0910b8a8d3f59b2225a95/prek-0.4.12.tar.gz", hash = "sha256:04beeba7f40437cd2f36804b84101bd7f3c9fb40b52da46a25604642ab2bfb09", size = 519080, upload-time = "2026-08-03T11:28:33.147Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d4/79/19f47eeb4d6092d36f94f47a056e7ae7a421d60220c772e8513483521b63/prek-0.4.13.tar.gz", hash = "sha256:9bf3dce400ef38a281836e4fe6429aa5f1690848be77cd97cb53c93769db4681", size = 533200, upload-time = "2026-08-10T08:54:15.477Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/23/5811a3161e072e5f93e4da01af611ee30c32922507b8ab4d9873df6affd3/prek-0.4.12-py3-none-linux_armv6l.whl", hash = "sha256:cd92000b051e433f26340821cf1cc8e6e3960f1275f3d516ca01f05905abba64", size = 5793226, upload-time = "2026-08-03T11:28:09.534Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/88/8607845d94eb1482e1bd335dadf098618f077a15775f7e98de99669052b4/prek-0.4.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5904fe6c6ab26e7d8792a3c7f1e3fc8d94fcfb63ad33b247c35f004b62cb6275", size = 6132269, upload-time = "2026-08-03T11:28:11.147Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/28/571d79ba457fbd9ecf40ae879c91952e12f5fa475306218c91139b86db7a/prek-0.4.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:df3eff1db9c24dc293010a07bc7a0ae0c541d55af828f5586405dedc28c4920d", size = 5614964, upload-time = "2026-08-03T11:28:12.983Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/a9/3f5cb79a73c764a8ac38d5bcd51e0df57239856eca7949b09bdac4338bf3/prek-0.4.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:c7733b44ca772ea32ec6a8bee669d0358bdf45873e79767afed196065084f31c", size = 5941047, upload-time = "2026-08-03T11:28:14.45Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/00/1dfed0ef8af10c5c32aa903486dccd33d2df171f3d945a037c5692f10760/prek-0.4.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:87f170cf1ffd6e3a196f947b83dff1f6c2cd68635f8d49740278bebe7b682262", size = 5707994, upload-time = "2026-08-03T11:28:15.914Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c0/bd/5f388f6cbdc0445b850e7c1a160d0be67fcef8bf221e3c8141a1feccef17/prek-0.4.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57dad513831f060cf73808df8edec29d46ec311435aa69f21c80edebf23dc5e1", size = 6133784, upload-time = "2026-08-03T11:28:17.184Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/47/342091a987bf68a74acec6d226a40ce7d51faf0019aa4126cc7bc952f8a7/prek-0.4.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b204844abc7ded983471f576ae8dc13b99e9b8d022e4d4b46176c6654769c9d8", size = 6901589, upload-time = "2026-08-03T11:28:18.545Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/8a/3ef7bdc3c3441649ebc040b9e164a13163e1e5fabae23e7bbb901992f3de/prek-0.4.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43b0a5a9d3f2f77871fdcb7893bfc5c8fe7e44f4e603ce6e4712bfec96b2d6f2", size = 6342189, upload-time = "2026-08-03T11:28:20Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/da/6277908442301b1b92a2879f6b04aaa03accb900f80e42776fc28b8197ef/prek-0.4.12-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0d188e572c306cc44b96e1bae5647e25b7bd311113f3f3f4a67320c257ee64a3", size = 5951250, upload-time = "2026-08-03T11:28:21.339Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/68/bff51a7332837edb1ecbe017325adb7fafd69b9c7828ddc81a1334b884af/prek-0.4.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:986f52d104b7066190f0f32aebe3467710356de265e9bfd892101ba99371db4d", size = 5804147, upload-time = "2026-08-03T11:28:22.656Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/de/b7f544971072ed7814125145dfeb1f7c15cce6b78ccea65a96298ff37838/prek-0.4.12-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:13e34d9e09bafcbf1f25a01cf86985e2c5e486591d3f45b2786ba3de82e5153a", size = 5680104, upload-time = "2026-08-03T11:28:24.271Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/94/95942bcc20a6a91ec2989aa30fdeb00ad095be736ec48b4bbcf0376166b1/prek-0.4.12-py3-none-musllinux_1_1_i686.whl", hash = "sha256:3d0208370da73e8b5bc97f2492dc3975f8dd2c22f4bf6e1f2cf3342503764b52", size = 5975030, upload-time = "2026-08-03T11:28:25.683Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/6d/26e6497198d81cf9aa82495400aef46adea8df3e4a4efc5f00e3b6ab3292/prek-0.4.12-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b1005f42920111bec1403c25e8f2f12ec7af0be06686cc3b8dcf85429af908a8", size = 6458532, upload-time = "2026-08-03T11:28:27.121Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/02/ee140c2eb4701bd194db429d84630733492be94897d5f72b61d6f11e6619/prek-0.4.12-py3-none-win32.whl", hash = "sha256:afee229488dcceaea282288e4d7096a93da5a8b85649d9ef506dbdbcd78f38a7", size = 5502213, upload-time = "2026-08-03T11:28:28.691Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/7b/744cff84def48c1ce38c0b4f643a3553c66976c5bb7869ab7317044870e4/prek-0.4.12-py3-none-win_amd64.whl", hash = "sha256:fdd27bad8adafea8fe77606950ca09200d59296a47ab131cfb88718d460949d7", size = 5868065, upload-time = "2026-08-03T11:28:30.377Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/1d/e2c0fc222904ef73df1739b11a83edc29e38bc4bc61259f2ca6d2f15abb0/prek-0.4.12-py3-none-win_arm64.whl", hash = "sha256:45e34a24fba4a4e4568682477158591698efc2375b8d1d418ae424691c4bd01b", size = 5632819, upload-time = "2026-08-03T11:28:31.743Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/12/661dc1c63c322000580dffa8503d28d633cb5d4c3181e662cbb86eded12e/prek-0.4.13-py3-none-linux_armv6l.whl", hash = "sha256:6a313f5f041b2fcbd33bceb6b6e11ee9b8621c8c6ad8ef13787a6d90541b624d", size = 5801508, upload-time = "2026-08-10T08:53:52.18Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/4a/c50597a45d08b22e5704a5d4d5c03269a581b46cf1f060861f9ba96cdade/prek-0.4.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:40436cd7247d2a2fc036ef07d7efcd828acf1aab9d648a8a3f21475e3ad3f789", size = 6141612, upload-time = "2026-08-10T08:53:53.86Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/82/93/abc084bbd76bb6c34efbf7db441392f17b7bfc50a54c4d53d3e37c7ad6e0/prek-0.4.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:019a33b477b7b949fb6dcd6cb33ed494c4a28117e53c370c637ec0aba60211aa", size = 5625418, upload-time = "2026-08-10T08:53:55.39Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/4d/a310ff9adcb4b822d935eb55f0f5cdcfe4c18a16610bc5e220a11215dc38/prek-0.4.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:5a2851b6e60912e73be1bf2cf61fab5b314add15ba7bcf230f860282bdbaf16b", size = 5942132, upload-time = "2026-08-10T08:53:56.789Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/d8/811230ff285000abdc0fb98b56b9ef2d23b2dc530cf2da86fe1665f844ba/prek-0.4.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9ba9b94bd3f47b5f94a4ae504c44f9529cb3258ebaf577a5a6070ea0c6a853d6", size = 5710181, upload-time = "2026-08-10T08:53:58.278Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/84/01f7163cc4267daba6b01cfac726327c6d848d2bb7349bd2fd877d88f744/prek-0.4.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33fc9e0d435cb9e650ec108f840febe7a94fd266c77149860091cead212dc82d", size = 6157705, upload-time = "2026-08-10T08:53:59.586Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/e3/061a0e9ebd7064edf70bb783afa575d045ac0cc9035c2143b76401ebcaf5/prek-0.4.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91066d8978eab83c111e7c34ee3046a003819d2df15ef7dc2bddeb83dce62bc0", size = 6898016, upload-time = "2026-08-10T08:54:00.927Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/68/5e8e2b9ff6a30b46bf35ca3b50e4d164c26406d9e7457d5b4633c6e1fbc0/prek-0.4.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85646fcc940f30bd946d63b6bbd1000d946994c88e154c5bdde7773a2c358dcf", size = 6365056, upload-time = "2026-08-10T08:54:02.704Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/23/afe543b69fb7f35016645eb4b766191e814cbcfa5545d695a2a7dd9b712e/prek-0.4.13-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:961859c3ddddb8e10367afcf93742da36fa213eda080f168fc1dcbe33e0b004d", size = 5952174, upload-time = "2026-08-10T08:54:04.079Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/6e/9e651ce51aa0b03244277f5e0660cf1b946a270cb62635a8a100389a67a2/prek-0.4.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ccf4fcbfc686ad6b589f2908ed6012a315091c45163e1f4420648b9669651a9e", size = 5761875, upload-time = "2026-08-10T08:54:05.378Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/64/e70e18734d93df272476d2f1b30d92c71d41ee7141070f1dd13cfbb2eff8/prek-0.4.13-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:eb4b7edfe00ca73e58d7e26fb871abddc5854e8d21067a202e43418fb7f98ef2", size = 5685365, upload-time = "2026-08-10T08:54:06.729Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/78/686fd5d3f12249368a0fdd8f7705e3ba540402a6157ef0b888e48734ff83/prek-0.4.13-py3-none-musllinux_1_1_i686.whl", hash = "sha256:c29449443f89da1647331742984b7046a55084759ad642373a9cc0a973494339", size = 5999013, upload-time = "2026-08-10T08:54:08.183Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/aa/93ffac2460b44f6182dbbb3194db976d18332edcb8208c4a796f8f9955f8/prek-0.4.13-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:7a04d5ac901819b115ecd0bf79cee091f0034323767abcc4a53626eeb96c3be0", size = 6486759, upload-time = "2026-08-10T08:54:09.775Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/3c/9f12c0d469ea345c249d5b0a027a1f2bf1ca05041d7785c34c455a9b605c/prek-0.4.13-py3-none-win32.whl", hash = "sha256:6d1bdcc1699ae18270f9bac9c4b4d29c6f1512b7a067e17ce5e220c30636f88c", size = 5515046, upload-time = "2026-08-10T08:54:11.456Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/3c/ef9ec67c560e60525d6a49b48a2a60434906f25ec2efa5e010fe2d42bbfa/prek-0.4.13-py3-none-win_amd64.whl", hash = "sha256:2d8fd796ed7944154fbee6d5a6d2490b9d4f14ce3626b1a0c9ca698455b25d9b", size = 5894683, upload-time = "2026-08-10T08:54:12.88Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/ce/8fe8fdf8154108a552d5578400da44144697ed11e5bd71d5a4980d7e202e/prek-0.4.13-py3-none-win_arm64.whl", hash = "sha256:65d6811b0220444bcdf539e157d7d5cb8edab01a5ed89534c70d73d675266ecb", size = 5650616, upload-time = "2026-08-10T08:54:14.21Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1139,16 +1243,16 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "qbittorrent-api"
|
||||
version = "2026.8.0"
|
||||
version = "2026.8.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
{ name = "requests" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/eb/d2/4a6d6ae6aaca1ba83039d27f28e5bf30bacef27ff4eb635c729d2d582363/qbittorrent_api-2026.8.0.tar.gz", hash = "sha256:4127a9c1d5c7ad9cd4e8de16f83daeea2e8b2ed962f99f284d238f927479391a", size = 1430678, upload-time = "2026-08-01T21:05:55.88Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/04/00/b7f41dfe8af6c3991f7182f9c14c46851379b8e66ac74c819f0952e9923d/qbittorrent_api-2026.8.1.tar.gz", hash = "sha256:9642c4528baee67216eab87022ce297048ac96df2f8c9f1cf920de078eb50b62", size = 1436760, upload-time = "2026-08-16T20:30:28.35Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/81/b456688c20e75cae5964700b58712630930b9dfaf32ce7c6a8881658d0d0/qbittorrent_api-2026.8.0-py3-none-any.whl", hash = "sha256:03b3db598d351194fd3c65f1d3e53843f65f85bef1d5077791efaa2e26869d2b", size = 68196, upload-time = "2026-08-01T21:05:54.376Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/60/77c7b63675f12ba492dee9f8aabab0fc61ced5d2131a83bd7e92dcc335c1/qbittorrent_api-2026.8.1-py3-none-any.whl", hash = "sha256:c17260e416ec5832c3ed7aff46fb0e2b47e70b90de182e89c34133b9d4ce5a6e", size = 71970, upload-time = "2026-08-16T20:30:26.67Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1217,27 +1321,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.16.2"
|
||||
version = "0.16.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1268,7 +1372,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "seleniumbase"
|
||||
version = "4.51.11"
|
||||
version = "4.51.12"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
@@ -1332,18 +1436,18 @@ dependencies = [
|
||||
{ name = "wheel" },
|
||||
{ name = "wsproto" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/17/d9/0e54caf802c04f2f61dc034ae65d96defde37dc6d1462f6a8554ba3c3aab/seleniumbase-4.51.11.tar.gz", hash = "sha256:42eef7baa910df5771d7196a17562466073fee92a8ce52c5d5c3e53d55516793", size = 671742, upload-time = "2026-08-07T04:55:05.673Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8a/6d/b852196360200892bc0efb8daeb5ad2278e09eec5b6aaee9f35fdf7b6860/seleniumbase-4.51.12.tar.gz", hash = "sha256:a914610bbe5c9084bce045243e3134987064bbe6e1f2725fe369ce4dbce313d1", size = 672929, upload-time = "2026-08-10T19:28:37.957Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/44/4f/70bb412ddfdc63efae4c30c30acab8f543675199e0514666eecb3b6c4523/seleniumbase-4.51.11-py3-none-any.whl", hash = "sha256:9286930104e255b2a4b28f0136210c120178f0e534434ea28743dfbef825b076", size = 677503, upload-time = "2026-08-07T04:55:02.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/93/15/199f8ed68290e553a6bc3c10920817d41a006ab3d9aaac06b980256df966/seleniumbase-4.51.12-py3-none-any.whl", hash = "sha256:64ea9fa52b8c1ac8d1b8ccbb3dcf7daa02c1cb1ffdd7e1f358677e36cbbc071c", size = 677511, upload-time = "2026-08-10T19:28:34.24Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "setuptools"
|
||||
version = "83.0.0"
|
||||
version = "84.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1363,6 +1467,7 @@ dependencies = [
|
||||
{ name = "gevent" },
|
||||
{ name = "gevent-websocket" },
|
||||
{ name = "gunicorn" },
|
||||
{ name = "httpx", extra = ["http2"] },
|
||||
{ name = "psutil" },
|
||||
{ name = "python-socketio" },
|
||||
{ name = "qbittorrent-api" },
|
||||
@@ -1405,15 +1510,16 @@ requires-dist = [
|
||||
{ name = "gevent" },
|
||||
{ name = "gevent-websocket" },
|
||||
{ name = "gunicorn" },
|
||||
{ name = "httpx", extras = ["http2"], specifier = ">=0.28.1" },
|
||||
{ name = "psutil" },
|
||||
{ name = "pyautogui", marker = "extra == 'browser'" },
|
||||
{ name = "python-socketio" },
|
||||
{ name = "python-xlib", marker = "extra == 'browser'" },
|
||||
{ name = "pyvirtualdisplay", marker = "extra == 'browser'" },
|
||||
{ name = "qbittorrent-api", specifier = ">=2026.8.0" },
|
||||
{ name = "qbittorrent-api", specifier = ">=2026.8.1" },
|
||||
{ name = "rarfile" },
|
||||
{ name = "requests", extras = ["socks"] },
|
||||
{ name = "seleniumbase", marker = "extra == 'browser'", specifier = "==4.51.11" },
|
||||
{ name = "seleniumbase", marker = "extra == 'browser'", specifier = "==4.51.12" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "transmission-rpc" },
|
||||
]
|
||||
@@ -1421,12 +1527,12 @@ provides-extras = ["browser"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
{ name = "basedpyright", specifier = ">=1.39.9" },
|
||||
{ name = "basedpyright", specifier = ">=1.39.10" },
|
||||
{ name = "prek" },
|
||||
{ name = "pytest" },
|
||||
{ name = "pytest-cov" },
|
||||
{ name = "pytest-xdist", specifier = ">=3.8.0" },
|
||||
{ name = "ruff", specifier = "==0.16.2" },
|
||||
{ name = "ruff", specifier = "==0.16.3" },
|
||||
{ name = "vulture", specifier = ">=2.14" },
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user