mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-09-24 19:30:30 +01:00
Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 CaliBrain
|
2baaf179b4 | ||
 dependabot[bot]
|
1e45add4d5 | ||
|
dependabot[bot]
|
cb3f6fee82 | ||
|
dependabot[bot]
|
0dc13c1ca4 | ||
|
dependabot[bot]
|
d1f8527089 | ||
|
Calibrain
|
a52c436d6a | ||
|
CaliBrain
|
95e34670f7 | ||
|
CaliBrain
|
7d56624ab6 | ||
 
|
f4421ff189 | ||
|
CaliBrain
|
e7007865a4 | ||
|
CaliBrain
|
5b3df2a463 | ||
|
CaliBrain
|
5247ec6124 | ||
|
CaliBrain
|
bd21ec1257 | ||
  helgehelge123andhelgehelge123
|
7b9c416df8 | ||
|
CaliBrain
|
646b531669 | ||
|
dependabot[bot]
|
eafb965662 | ||
|
CaliBrain
|
7193036626 | ||
|
CaliBrain
|
12d554a92f | ||
|
CaliBrain
|
fae6140c6a | ||
|
dependabot[bot]
|
63133097e4 | ||
|
dependabot[bot]
|
82aeee387e | ||
|
CaliBrain
|
4cd1091d16 | ||
|
CaliBrain
|
651096ed7b | ||
|
CaliBrain
|
ebb833a82c | ||
|
CaliBrain
|
b7093f4594 |
@@ -96,7 +96,7 @@ jobs:
|
||||
type=ref,event=tag
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Build and push ${{ matrix.target }} Docker image
|
||||
id: push
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Create legacy aliases
|
||||
run: |
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: "0.11.3"
|
||||
python-version: "3.14"
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Build shelfmark-lite image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
|
||||
@@ -25,14 +25,14 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v3
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Install uv and Python
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
python-version: "3.14"
|
||||
enable-cache: true
|
||||
|
||||
+9
-4
@@ -28,7 +28,7 @@ RUN npm run build
|
||||
# than copied into the image. A COPY here would land ~24 MB in a `base` layer that
|
||||
# every published image inherits, and a later `rm` cannot take it back out again --
|
||||
# a RUN adds a layer, it does not rewrite the one underneath.
|
||||
FROM ghcr.io/astral-sh/uv:0.11.3@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 AS uv
|
||||
FROM ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1 AS uv
|
||||
|
||||
# Use python-slim as the base image
|
||||
FROM python:3.14.7-slim@sha256:ce40764625a4ff50df3548277632e7f96c4e77fe75fa848aae9885476e7df5a4 AS base
|
||||
@@ -152,9 +152,14 @@ RUN mkdir -p \
|
||||
EXPOSE ${FLASK_PORT}
|
||||
|
||||
# Add healthcheck for container status
|
||||
# Uses /api/health which doesn't require authentication
|
||||
HEALTHCHECK --interval=60s --timeout=60s --start-period=60s --retries=3 \
|
||||
CMD curl -s http://localhost:${FLASK_PORT}/api/health > /dev/null || exit 1
|
||||
# Uses /api/health which doesn't require authentication.
|
||||
# curl needs -f so an HTTP error status fails the probe instead of passing it:
|
||||
# plain `curl -s` exits 0 on a 500, which reported a broken app as healthy.
|
||||
# timeout stays well under interval so a hung probe cannot occupy a whole cycle.
|
||||
# --start-interval matches the daemon default (5s), made explicit so startup
|
||||
# probing does not depend on that default staying put.
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=90s --start-interval=5s --retries=3 \
|
||||
CMD curl -fsS http://localhost:${FLASK_PORT}/api/health > /dev/null || exit 1
|
||||
|
||||
# Use dumb-init as the entrypoint to handle signals properly
|
||||
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
||||
|
||||
@@ -704,7 +704,7 @@ Choose how downloaded audiobook files are named and organized.
|
||||
|
||||
- **Type:** string (choice)
|
||||
- **Default:** `rename`
|
||||
- **Options:** `none` (None), `rename` (Rename Only), `organize` (Rename and Organize)
|
||||
- **Options:** `none` (None), `rename` (Rename Only), `organize` (Rename and Organize), `rename_and_group` (Rename and Group)
|
||||
|
||||
#### `TEMPLATE_AUDIOBOOK_RENAME`
|
||||
|
||||
@@ -1223,6 +1223,7 @@ How long to cache individual book details. Default: 600 (10 minutes). Max: 60480
|
||||
| `PROWLARR_URL` | Base URL of your Prowlarr instance | string | _none_ |
|
||||
| `PROWLARR_API_KEY` | Found in Prowlarr: Settings > General > API Key | string (secret) | _none_ |
|
||||
| `PROWLARR_INDEXERS` | Select which indexers to search. 📚 = has book categories. Leave empty to search all. | string (comma-separated) | _empty list_ |
|
||||
| `PROWLARR_INDEXER_TIMEOUT` | How long to wait for a single indexer to answer a search. Indexers behind FlareSolverr can need 90 seconds or more while a cold Cloudflare challenge is solved; raise this if searches come back empty and the Prowlarr log shows the search still running. | number | `90` |
|
||||
| `PROWLARR_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
|
||||
| `PROWLARR_COLLAPSE_DUPLICATES` | Collapse a release that several indexer entries returned down to a single row, keeping the entry with the best Prowlarr priority. Turn this off to see every entry that carried it, which is what makes results from filter-specific entries (freeleech and the like) visible. | boolean | `true` |
|
||||
| `PROWLARR_USE_SEED_PREFERENCES` | Apply per-indexer seed time and ratio preferences from Prowlarr when sending torrents to the download client | boolean | `false` |
|
||||
@@ -1268,6 +1269,16 @@ Select which indexers to search. 📚 = has book categories. Leave empty to sear
|
||||
- **Type:** string (comma-separated)
|
||||
- **Default:** _empty list_
|
||||
|
||||
#### `PROWLARR_INDEXER_TIMEOUT`
|
||||
|
||||
**Indexer Search Timeout (seconds)**
|
||||
|
||||
How long to wait for a single indexer to answer a search. Indexers behind FlareSolverr can need 90 seconds or more while a cold Cloudflare challenge is solved; raise this if searches come back empty and the Prowlarr log shows the search still running.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `90`
|
||||
- **Constraints:** min: 5, max: 300
|
||||
|
||||
#### `PROWLARR_AUTO_EXPAND`
|
||||
|
||||
**Auto-expand search on no results**
|
||||
@@ -1977,7 +1988,7 @@ Move deletes the job from your usenet client after import; Copy keeps it in the
|
||||
| Variable | Description | Type | Default |
|
||||
|----------|-------------|------|---------|
|
||||
| `HARDCOVER_ENABLED` | Enable Hardcover as a metadata provider for book searches | boolean | `false` |
|
||||
| `HARDCOVER_API_KEY` | Get your API key from hardcover.app/account/api | string (secret) | _none_ |
|
||||
| `HARDCOVER_API_KEY` | Get your API key from hardcover.app/account/api (starts with hc_pat_) | string (secret) | _none_ |
|
||||
| `HARDCOVER_DEFAULT_SORT` | Default sort order for Hardcover search results. | string (choice) | `relevance` |
|
||||
| `HARDCOVER_EXCLUDE_COMPILATIONS` | Filter out compilations, anthologies, and omnibus editions from search results | boolean | `false` |
|
||||
| `HARDCOVER_EXCLUDE_UNRELEASED` | Filter out books with a release year in the future | boolean | `false` |
|
||||
@@ -1999,7 +2010,7 @@ Enable Hardcover as a metadata provider for book searches
|
||||
|
||||
**API Key**
|
||||
|
||||
Get your API key from hardcover.app/account/api
|
||||
Get your API key from hardcover.app/account/api (starts with hc_pat_)
|
||||
|
||||
- **Type:** string (secret)
|
||||
- **Default:** _none_
|
||||
@@ -2304,6 +2315,7 @@ Override destination based on content type metadata.
|
||||
| `EXT_BYPASSER_URL` | URL of the external bypasser service (e.g., FlareSolverr). | string | `http://flaresolverr:8191` |
|
||||
| `EXT_BYPASSER_PATH` | API path for the external bypasser. | string | `/v1` |
|
||||
| `EXT_BYPASSER_TIMEOUT` | Timeout for external bypasser requests in milliseconds. | number | `60000` |
|
||||
| `BYPASS_BROWSER_IDLE_TIMEOUT` | How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner. | number | `180` |
|
||||
|
||||
<details>
|
||||
<summary>Detailed descriptions</summary>
|
||||
@@ -2359,6 +2371,17 @@ Timeout for external bypasser requests in milliseconds.
|
||||
- **Requires restart:** Yes
|
||||
- **Constraints:** min: 10000, max: 300000
|
||||
|
||||
#### `BYPASS_BROWSER_IDLE_TIMEOUT`
|
||||
|
||||
**Bypasser Idle Timeout (seconds)**
|
||||
|
||||
How long the bypass helper process may sit unused before it is shut down. Higher keeps more searches fast, lower frees memory sooner.
|
||||
|
||||
- **Type:** number
|
||||
- **Default:** `180`
|
||||
- **Requires restart:** Yes
|
||||
- **Constraints:** min: 30, max: 3600
|
||||
|
||||
</details>
|
||||
|
||||
### Direct Download: Mirrors
|
||||
|
||||
+6
-6
@@ -19,31 +19,31 @@ dependencies = [
|
||||
"psutil",
|
||||
"emoji",
|
||||
"rarfile",
|
||||
"qbittorrent-api>=2026.8.0",
|
||||
"qbittorrent-api>=2026.8.1",
|
||||
"transmission-rpc",
|
||||
"authlib>=1.7.2,<1.8",
|
||||
"apprise>=1.12.0",
|
||||
"apprise>=1.13.0",
|
||||
# HTTP/2 client for RFC 8484 DoH: quad9 rejects HTTP/1.1 outright (505), which
|
||||
# requests cannot speak. See shelfmark/download/doh_wireformat.py.
|
||||
"httpx[http2]>=0.27",
|
||||
"httpx[http2]>=0.28.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
browser = [
|
||||
"pyvirtualdisplay",
|
||||
"pyautogui",
|
||||
"seleniumbase==4.51.12",
|
||||
"seleniumbase==4.52.1",
|
||||
"python-xlib",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"basedpyright>=1.39.9",
|
||||
"basedpyright>=1.39.10",
|
||||
"prek",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pytest-xdist>=3.8.0",
|
||||
"ruff==0.16.2",
|
||||
"ruff==0.16.4",
|
||||
"vulture>=2.14",
|
||||
]
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<img src="src/frontend/public/logo.png" alt="Shelfmark" width="200">
|
||||
|
||||
> [!NOTE]
|
||||
> This project is in a stable state as of May 2026 but is not under active maintenance.
|
||||
> Shelfmark is feature stable and maintained on a best-effort basis. Bug fixes, security updates, and small quality-of-life improvements are still shipped, and pull requests are reviewed — including new features. There is no roadmap for new features for now.
|
||||
|
||||
Shelfmark is a self-hosted web interface for searching and requesting books and audiobooks across multiple sources. Bring your own sources, metadata providers, and download clients to build a single hub for your digital library. Supports multiple users with a built-in request system, so you can share your instance with others and let them browse and request books on their own.
|
||||
|
||||
@@ -95,6 +95,30 @@ volumes:
|
||||
- Aggregates releases from multiple configured sources
|
||||
- Full audiobook support
|
||||
|
||||
### Hardcover API Key
|
||||
|
||||
Hardcover powers metadata search in Universal mode. Create a token at
|
||||
[hardcover.app/account/api](https://hardcover.app/account/api) — current keys start with `hc_pat_`
|
||||
and are far shorter than the JWTs Hardcover issued before August 2026.
|
||||
|
||||
Tick these seven scopes on the token screen:
|
||||
|
||||
| Scope | Used for |
|
||||
|-------|----------|
|
||||
| `read:catalog` | Metadata search, plus book, edition, author and series lookups |
|
||||
| `read:library` | Your reading status and shelf counts |
|
||||
| `read:lists` | Your lists and the books on them |
|
||||
| `read:me:content` | Test Connection and the "Connected as" label |
|
||||
| `read:users` | Usernames shown alongside lists |
|
||||
| `write:library` | Setting a book's reading status from Shelfmark |
|
||||
| `write:lists` | Adding and removing books from lists, including auto-remove on download |
|
||||
|
||||
The two `write:` scopes matter only if you set reading status from Shelfmark or leave
|
||||
**Auto-Remove from List on Download** enabled (it is on by default) — without them those actions
|
||||
fail silently. Everything else Hardcover offers (journal, goals, reviews, prompts, notifications,
|
||||
account) can stay unticked. The `all` scope works too, but it grants full account access including
|
||||
deletion, so prefer the list above.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Environment variables work for initial setup and Docker deployments. They serve as defaults that can be overridden in the web interface.
|
||||
@@ -238,9 +262,11 @@ These are non-goals, not missing features.
|
||||
|
||||
## Contributing
|
||||
|
||||
Shelfmark's core feature set is complete. Development focuses on stability, bug fixes, quality-of-life improvements, and refining the search experience. Contributions in these areas are welcome, please file issues or submit pull requests on GitHub.
|
||||
Shelfmark's core feature set is complete.
|
||||
|
||||
Feature requests that fall outside the project scope (library integration, automation, collection management) will be closed. If you're unsure whether something fits, open a discussion first.
|
||||
Pull requests are welcome and all of them get reviewed, new features included. If you want a feature, the fastest path is to send a PR for it rather than to file a request.
|
||||
|
||||
Feature requests that fall outside the project scope (library integration, automation, collection management) will be closed, and PRs implementing them won't be merged. If you're unsure whether something fits, open a discussion first.
|
||||
|
||||
## Health Monitoring
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Challenge-page detection shared by the bypassers and the HTTP retry path.
|
||||
|
||||
Kept out of `internal_bypasser` so the HTTP layer can recognise an interstitial
|
||||
without importing SeleniumBase: that module is imported lazily precisely because its
|
||||
browser dependencies are optional, and external-bypasser setups run without them.
|
||||
"""
|
||||
|
||||
# Matched against lowercased text, so every entry must be lowercase.
|
||||
CLOUDFLARE_INDICATORS = [
|
||||
"just a moment",
|
||||
"verify you are human",
|
||||
"verifying you are human",
|
||||
"cloudflare.com/products/turnstile",
|
||||
]
|
||||
|
||||
DDOS_GUARD_INDICATORS = [
|
||||
"ddos-guard",
|
||||
"ddos guard",
|
||||
"checking your browser before accessing",
|
||||
"complete the manual check to continue",
|
||||
"could not verify your browser automatically",
|
||||
]
|
||||
|
||||
# Markers that exist only in raw markup: the bypassers scan rendered innerText, where
|
||||
# a script src or a <title> never appears. The title match is scoped to the tag on
|
||||
# purpose - hosts word the rest of that sentence differently, and matching "checking
|
||||
# your browser" as free text would trip on any page that merely discusses a challenge.
|
||||
_RAW_HTML_MARKERS = (
|
||||
"<title>checking your browser",
|
||||
"/cdn-cgi/challenge-platform",
|
||||
"/.well-known/ddos-guard/",
|
||||
)
|
||||
|
||||
# An interstitial is a few KB of markup. Past that it is a real page that happens to
|
||||
# mention a marker - a protected site links its own DDoS-Guard endpoints on every page.
|
||||
MAX_CHALLENGE_HTML_CHARS = 64 * 1024
|
||||
|
||||
|
||||
def challenge_marker(html: str) -> str | None:
|
||||
"""Return the marker proving `html` is an unsolved challenge page, or None.
|
||||
|
||||
Only meaningful for a response that already carries a challenge status: the
|
||||
markers appear on protected sites' real pages too, so the status is what
|
||||
separates "blocked" from "served".
|
||||
"""
|
||||
if not html or len(html) > MAX_CHALLENGE_HTML_CHARS:
|
||||
return None
|
||||
lowered = html.lower()
|
||||
for marker in (*_RAW_HTML_MARKERS, *DDOS_GUARD_INDICATORS, *CLOUDFLARE_INDICATORS):
|
||||
if marker in lowered:
|
||||
return marker
|
||||
return None
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Clearance cookies won by a bypass, shared by every bypasser implementation.
|
||||
|
||||
Kept in its own module rather than inside a bypasser because both of them feed it and
|
||||
both read from it. The internal bypasser cannot host it: it imports seleniumbase at
|
||||
module scope, which is exactly the dependency an external-bypasser deployment is
|
||||
entitled not to have installed.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Cookie storage - shared with requests library for Cloudflare bypass
|
||||
# Nested mapping of domain to cookie name to cookie metadata.
|
||||
_cf_cookies: dict[str, dict] = {}
|
||||
_cf_cookies_lock = threading.Lock()
|
||||
|
||||
# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
|
||||
_cf_user_agents: dict[str, str] = {}
|
||||
|
||||
# Protection cookie names we care about (Cloudflare and DDoS-Guard)
|
||||
CF_COOKIE_NAMES = {"cf_clearance", "__cf_bm", "cf_chl_2", "cf_chl_prog"}
|
||||
DDG_COOKIE_NAMES = {
|
||||
"__ddg1_",
|
||||
"__ddg2_",
|
||||
"__ddg5_",
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"__ddgid_",
|
||||
"__ddgmark_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
# DDoS-Guard cookies that describe *one* check rather than granting clearance, and so
|
||||
# must never be replayed on a later request. Observed live on Anna's Archive:
|
||||
#
|
||||
# __ddg9_ the client IP address
|
||||
# __ddg10_ the unix timestamp the check was issued
|
||||
# __ddg8_ an opaque token issued with them, same ~40 minute expiry
|
||||
#
|
||||
# Clearance itself lives in __ddg1_/__ddg2_/__ddgid_ (roughly a year) and __ddg5_.
|
||||
# Replaying the trio is actively harmful: once the timestamp ages out - or the egress
|
||||
# IP changes, which happens routinely behind a VPN - the values no longer describe the
|
||||
# caller, DDoS-Guard re-arms its check and answers every request with a ?check=1
|
||||
# redirect. That is the redirect loop, and it is self-inflicted. Dropping them simply
|
||||
# lets DDoS-Guard issue a fresh set, exactly as it does for a browser.
|
||||
DDG_EPHEMERAL_COOKIE_NAMES = {
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
|
||||
def _get_base_domain(domain: str) -> str:
|
||||
"""Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
|
||||
return ".".join(domain.split(".")[-2:]) if "." in domain else domain
|
||||
|
||||
|
||||
def _get_full_cookie_domains() -> set[str]:
|
||||
"""Return mirror domains that need full-session cookie extraction."""
|
||||
from shelfmark.core.mirrors import get_zlib_cookie_domains
|
||||
|
||||
return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()}
|
||||
|
||||
|
||||
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
|
||||
"""Determine if a cookie should be extracted based on its name."""
|
||||
# Checked before extract_all: a per-check token is wrong to replay for every
|
||||
# domain, including the full-session ones.
|
||||
if name in DDG_EPHEMERAL_COOKIE_NAMES:
|
||||
return False
|
||||
if extract_all:
|
||||
return True
|
||||
is_cf = name in CF_COOKIE_NAMES or name.startswith("cf_")
|
||||
is_ddg = name in DDG_COOKIE_NAMES or name.startswith("__ddg")
|
||||
return is_cf or is_ddg
|
||||
|
||||
|
||||
def _cookie_field(cookie: Any, name: str) -> Any:
|
||||
"""Read one field from a cookie in either shape we are handed.
|
||||
|
||||
The internal bypasser extracts CDP cookie objects; an external bypasser returns
|
||||
the same fields as JSON objects, so the difference is attribute versus key access.
|
||||
"""
|
||||
if isinstance(cookie, Mapping):
|
||||
return cookie.get(name)
|
||||
return getattr(cookie, name, None)
|
||||
|
||||
|
||||
def _cookie_expiry(cookie: Any) -> float | None:
|
||||
"""A cookie's absolute expiry, or None when it is a session cookie.
|
||||
|
||||
The two spellings are not interchangeable and both reach this store. CDP and
|
||||
Playwright cookies carry `expires`; the WebDriver cookie object - what a
|
||||
Selenium-based solver such as FlareSolverr returns - carries `expiry`. Reading
|
||||
only one silently turns every cookie from the other into a never-expiring one,
|
||||
which is exactly how dead clearance ends up replayed forever (see
|
||||
get_cf_cookies_for_domain).
|
||||
|
||||
The value is coerced rather than trusted: it arrives as JSON from a service we
|
||||
do not control, and a string here used to raise straight out of the store.
|
||||
"""
|
||||
for field in ("expires", "expiry"):
|
||||
raw = _cookie_field(cookie, field)
|
||||
if raw is None:
|
||||
continue
|
||||
try:
|
||||
expiry = float(raw)
|
||||
except TypeError, ValueError:
|
||||
logger.debug("Unreadable cookie expiry %r; treating as a session cookie", raw)
|
||||
return None
|
||||
# <= 0 is how both shapes spell "session cookie", not "expired in 1970".
|
||||
return expiry if expiry > 0 else None
|
||||
return None
|
||||
|
||||
|
||||
def store_extracted_cookies(
|
||||
*,
|
||||
url: str,
|
||||
cookies: list[Any],
|
||||
user_agent: str | None = None,
|
||||
) -> None:
|
||||
"""Store filtered bypass cookies (and optional UA) for a URL domain."""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.hostname or ""
|
||||
if not domain:
|
||||
return
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
extract_all = base_domain in _get_full_cookie_domains()
|
||||
|
||||
cookies_found: dict[str, dict[str, Any]] = {}
|
||||
for cookie in cookies:
|
||||
name = _cookie_field(cookie, "name") or ""
|
||||
if not _should_extract_cookie(name, extract_all=extract_all):
|
||||
continue
|
||||
secure = _cookie_field(cookie, "secure")
|
||||
cookies_found[name] = {
|
||||
"value": _cookie_field(cookie, "value") or "",
|
||||
"domain": _cookie_field(cookie, "domain") or domain,
|
||||
"path": _cookie_field(cookie, "path") or "/",
|
||||
"expiry": _cookie_expiry(cookie),
|
||||
"secure": True if secure is None else bool(secure),
|
||||
"httpOnly": True,
|
||||
}
|
||||
|
||||
if not cookies_found:
|
||||
return
|
||||
|
||||
with _cf_cookies_lock:
|
||||
_cf_cookies[base_domain] = cookies_found
|
||||
if user_agent:
|
||||
_cf_user_agents[base_domain] = user_agent
|
||||
logger.debug("Stored UA for %s: %s...", base_domain, str(user_agent)[:60])
|
||||
else:
|
||||
logger.debug("No UA captured for %s", base_domain)
|
||||
|
||||
cookie_type = "all" if extract_all else "protection"
|
||||
logger.debug("Extracted %s %s cookies for %s", len(cookies_found), cookie_type, base_domain)
|
||||
|
||||
|
||||
def _is_cookie_expired(cookie: dict[str, Any]) -> bool:
|
||||
"""Whether a stored cookie's expiry has passed. Session cookies never expire here."""
|
||||
expiry = cookie.get("expiry")
|
||||
if expiry is None:
|
||||
expiry = cookie.get("expires")
|
||||
if not expiry or expiry <= 0:
|
||||
return False
|
||||
return time.time() > expiry
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get stored cookies for a domain. Returns empty dict if none available."""
|
||||
if not domain:
|
||||
return {}
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
|
||||
with _cf_cookies_lock:
|
||||
cookies = _cf_cookies.get(base_domain, {})
|
||||
if not cookies:
|
||||
return {}
|
||||
|
||||
cf_clearance = cookies.get("cf_clearance", {})
|
||||
if cf_clearance and _is_cookie_expired(cf_clearance):
|
||||
logger.debug("CF cookies expired for %s", base_domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
return {}
|
||||
|
||||
# Expiry applies to every cookie, not just Cloudflare's. DDoS-Guard domains
|
||||
# have no cf_clearance, so the check above never fired for them and dead
|
||||
# cookies were replayed indefinitely - the server answers those with a
|
||||
# challenge, which is indistinguishable from having sent nothing at all.
|
||||
live = {name: c for name, c in cookies.items() if not _is_cookie_expired(c)}
|
||||
if len(live) != len(cookies):
|
||||
expired = sorted(set(cookies) - set(live))
|
||||
logger.debug("Dropping expired cookies for %s: %s", base_domain, expired)
|
||||
if live:
|
||||
_cf_cookies[base_domain] = live
|
||||
else:
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
|
||||
return {name: c["value"] for name, c in live.items()}
|
||||
|
||||
|
||||
def has_valid_cf_cookies(domain: str) -> bool:
|
||||
"""Check if we have valid Cloudflare cookies for a domain."""
|
||||
return bool(get_cf_cookies_for_domain(domain))
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get the User-Agent that was used during bypass for a domain."""
|
||||
if not domain:
|
||||
return None
|
||||
with _cf_cookies_lock:
|
||||
return _cf_user_agents.get(_get_base_domain(domain))
|
||||
|
||||
|
||||
def export_store() -> tuple[dict[str, dict], dict[str, str]]:
|
||||
"""Snapshot the whole store, for handing to another process.
|
||||
|
||||
The internal bypasser's Docker helper solves in a subprocess, so the clearance it
|
||||
wins has to be serialized back to the parent or the solve is lost with the child.
|
||||
"""
|
||||
with _cf_cookies_lock:
|
||||
return (
|
||||
{domain: dict(cookies) for domain, cookies in _cf_cookies.items()},
|
||||
dict(_cf_user_agents),
|
||||
)
|
||||
|
||||
|
||||
def import_store(cookies: object, user_agents: object) -> None:
|
||||
"""Merge a snapshot produced by :func:`export_store` into this process's store."""
|
||||
with _cf_cookies_lock:
|
||||
if isinstance(cookies, dict):
|
||||
_cf_cookies.update(cookies)
|
||||
if isinstance(user_agents, dict):
|
||||
_cf_user_agents.update(
|
||||
{str(domain): str(agent) for domain, agent in user_agents.items()}
|
||||
)
|
||||
|
||||
|
||||
def clear_cf_cookies(domain: str | None = None) -> None:
|
||||
"""Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
|
||||
with _cf_cookies_lock:
|
||||
if domain:
|
||||
base_domain = _get_base_domain(domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
_cf_user_agents.pop(base_domain, None)
|
||||
else:
|
||||
_cf_cookies.clear()
|
||||
_cf_user_agents.clear()
|
||||
@@ -2,17 +2,19 @@
|
||||
|
||||
import random
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass.cookie_store import store_extracted_cookies
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
from threading import Event
|
||||
|
||||
from shelfmark.download import network
|
||||
@@ -63,6 +65,31 @@ def max_duration_seconds() -> float:
|
||||
return MAX_RETRY * read_timeout + backoff_total
|
||||
|
||||
|
||||
def _store_solution_clearance(target_url: str, solution: Mapping[str, Any]) -> None:
|
||||
"""Keep the clearance the solver won, so later requests do not re-solve.
|
||||
|
||||
A solve is the expensive part of an external bypass - tens of seconds of real
|
||||
browser - and FlareSolverr-compatible services hand back the cookies and the
|
||||
User-Agent that earned it. Dropping them meant every single request paid a 403
|
||||
plus a full solve, and a file download (which the solver cannot proxy, being
|
||||
binary) never presented clearance at all.
|
||||
|
||||
The UA matters as much as the cookies: Cloudflare ties cf_clearance to the UA
|
||||
that solved the challenge, so replaying the cookie under our own UA is rejected.
|
||||
"""
|
||||
cookies = solution.get("cookies") or []
|
||||
if not isinstance(cookies, list):
|
||||
logger.debug("External bypasser returned no usable cookie list for '%s'", target_url)
|
||||
return
|
||||
|
||||
user_agent = solution.get("userAgent")
|
||||
store_extracted_cookies(
|
||||
url=target_url,
|
||||
cookies=cookies,
|
||||
user_agent=user_agent if isinstance(user_agent, str) else None,
|
||||
)
|
||||
|
||||
|
||||
def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
"""Make a single request to the external bypasser service. Returns HTML or None."""
|
||||
raw_bypasser_url = _coerce_config_str(
|
||||
@@ -116,6 +143,15 @@ def _fetch_via_bypasser(target_url: str) -> str | None:
|
||||
logger.warning("External bypasser returned empty response for '%s'", target_url)
|
||||
return None
|
||||
|
||||
try:
|
||||
_store_solution_clearance(target_url, solution)
|
||||
except AttributeError, KeyError, TypeError, ValueError:
|
||||
# Storing clearance is an optimisation; the page is the product. The
|
||||
# solution JSON comes from a service we do not control, so a surprise in
|
||||
# its cookie shape must not discard HTML that already cost a ~30s solve
|
||||
# and send the caller round for up to MAX_RETRY more of them.
|
||||
logger.debug("Could not store bypass clearance for '%s'", target_url, exc_info=True)
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning(
|
||||
"External bypasser timed out for '%s' (connect: %ss, read: %.0fs)",
|
||||
|
||||
@@ -5,7 +5,6 @@ import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import stat
|
||||
@@ -28,6 +27,15 @@ from seleniumbase import cdp_driver
|
||||
from seleniumbase.undetected.cdp_driver.connection import ProtocolException
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass.challenge import CLOUDFLARE_INDICATORS, DDOS_GUARD_INDICATORS
|
||||
from shelfmark.bypass.cookie_store import (
|
||||
clear_cf_cookies,
|
||||
export_store,
|
||||
get_cf_cookies_for_domain,
|
||||
get_cf_user_agent_for_domain,
|
||||
import_store,
|
||||
store_extracted_cookies,
|
||||
)
|
||||
from shelfmark.bypass.fingerprint import get_screen_size
|
||||
from shelfmark.config import env
|
||||
from shelfmark.config.env import LOG_DIR
|
||||
@@ -50,26 +58,31 @@ _LOADING_BODY_LENGTH_MAX = 50
|
||||
_PAGE_BODY_PREVIEW_CHARS = 500
|
||||
_BROWSER_START_TIMEOUT_SECONDS = 45.0
|
||||
_BYPASS_SUBPROCESS_TIMEOUT_SECONDS = 420.0
|
||||
# Same budget as the Docker helper process, applied to the in-process CDP path so both
|
||||
# branches of get() are bounded the same way.
|
||||
_IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS
|
||||
# How long a cancelled bypass may take to close its browser before the calling thread
|
||||
# stops waiting for it. Counted on top of the bypass deadline, so every budget below is
|
||||
# set to leave room for it.
|
||||
_CDP_UNWIND_GRACE_SECONDS = 15.0
|
||||
# Same wall-clock budget as the Docker helper process, applied to the in-process CDP path
|
||||
# so both branches of get() are bounded the same way: the deadline plus the unwind grace
|
||||
# comes to _BYPASS_SUBPROCESS_TIMEOUT_SECONDS either way.
|
||||
_IN_PROCESS_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS - _CDP_UNWIND_GRACE_SECONDS
|
||||
_BYPASS_CHILD_ENV = "SHELFMARK_INTERNAL_BYPASSER_CHILD"
|
||||
|
||||
# Challenge detection indicators
|
||||
CLOUDFLARE_INDICATORS = [
|
||||
"just a moment",
|
||||
"verify you are human",
|
||||
"verifying you are human",
|
||||
"cloudflare.com/products/turnstile",
|
||||
]
|
||||
|
||||
DDOS_GUARD_INDICATORS = [
|
||||
"ddos-guard",
|
||||
"ddos guard",
|
||||
"checking your browser before accessing",
|
||||
"complete the manual check to continue",
|
||||
"could not verify your browser automatically",
|
||||
]
|
||||
# The helper bounds each bypass below the parent's deadline, so it is the side that gives
|
||||
# up first: it still gets to report the timeout and close its browser, and stays available
|
||||
# for the next request. A parent that hit its deadline first could only kill the helper,
|
||||
# throwing away a process the next request would have to start again. The 30s covers the
|
||||
# unwind grace as well, so a helper that times out and closes its browser as slowly as it
|
||||
# is allowed to still answers with 15s to spare.
|
||||
_CHILD_BYPASS_TIMEOUT_SECONDS = _BYPASS_SUBPROCESS_TIMEOUT_SECONDS - 30.0
|
||||
# The helper publishes its answer by writing the result file the request named, so the
|
||||
# parent waits by watching for that file rather than by reading a stream it would have to
|
||||
# demultiplex from the helper's log output.
|
||||
_HELPER_RESULT_POLL_SECONDS = 0.05
|
||||
# Closing the helper's stdin asks it to shut down; this is how long it may take to finish
|
||||
# what it is doing and exit before its session is killed instead.
|
||||
_HELPER_SHUTDOWN_GRACE_SECONDS = 15.0
|
||||
_HELPER_IDLE_TIMEOUT_DEFAULT = 180.0
|
||||
_PARENT_WATCHDOG_INTERVAL_SECONDS = 5.0
|
||||
|
||||
|
||||
class _DisplayState(TypedDict):
|
||||
@@ -90,8 +103,8 @@ DISPLAY: _DisplayState = {
|
||||
"ffmpeg_output": None,
|
||||
}
|
||||
LOCKED = threading.Lock()
|
||||
_PGREP_PATH = shutil.which("pgrep")
|
||||
_PKILL_PATH = shutil.which("pkill")
|
||||
_PROC_ROOT = Path("/proc")
|
||||
_BROWSER_PROCESS_PATTERNS = ("chrome", "chromium", "Xvfb", "ffmpeg")
|
||||
_RNG = random.SystemRandom()
|
||||
|
||||
_CDP_OPERATION_ERRORS = (
|
||||
@@ -214,14 +227,33 @@ class _CdpWorker:
|
||||
msg = "CDP worker loop failed to start"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
@staticmethod
|
||||
async def _bounded(coro: Any, timeout: float | None) -> Any:
|
||||
"""Run the coroutine under its deadline, on the loop that owns it.
|
||||
|
||||
The deadline has to be enforced from inside the loop rather than by the calling
|
||||
thread: asyncio.wait_for() cancels the bypass and then *waits for it to unwind*,
|
||||
so `finally: await _close_cdp_driver(driver)` has finished by the time this
|
||||
raises. Cancelling from outside returns the moment the cancellation is scheduled,
|
||||
which in a helper serving many requests let the abandoned bypass close its browser
|
||||
while the next one was already opening its own - on the same loop, sharing the
|
||||
DISPLAY globals and one process group.
|
||||
"""
|
||||
if timeout is None:
|
||||
return await coro
|
||||
return await asyncio.wait_for(coro, timeout)
|
||||
|
||||
def run(self, coro: Any, timeout: float | None = None) -> Any:
|
||||
self.start()
|
||||
if not self._loop or self._loop.is_closed():
|
||||
msg = "CDP worker loop not available"
|
||||
raise RuntimeError(msg)
|
||||
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
|
||||
future = asyncio.run_coroutine_threadsafe(self._bounded(coro, timeout), self._loop)
|
||||
# Backstop for an unwind that wedges too: _close_cdp_driver awaits websockets that
|
||||
# a dead browser may never answer, and _bounded cannot outlive its own cleanup.
|
||||
wait_for = None if timeout is None else timeout + _CDP_UNWIND_GRACE_SECONDS
|
||||
try:
|
||||
return future.result(timeout=timeout)
|
||||
return future.result(timeout=wait_for)
|
||||
except TimeoutError:
|
||||
# Otherwise the coroutine keeps running in the worker loop after we stop
|
||||
# waiting, holding the browser and racing the next bypass.
|
||||
@@ -231,96 +263,6 @@ class _CdpWorker:
|
||||
|
||||
_CDP_WORKER = _CdpWorker()
|
||||
|
||||
# Cookie storage - shared with requests library for Cloudflare bypass
|
||||
# Nested mapping of domain to cookie name to cookie metadata.
|
||||
_cf_cookies: dict[str, dict] = {}
|
||||
_cf_cookies_lock = threading.Lock()
|
||||
|
||||
# User-Agent storage - Cloudflare ties cf_clearance to the UA that solved the challenge
|
||||
_cf_user_agents: dict[str, str] = {}
|
||||
|
||||
# Protection cookie names we care about (Cloudflare and DDoS-Guard)
|
||||
CF_COOKIE_NAMES = {"cf_clearance", "__cf_bm", "cf_chl_2", "cf_chl_prog"}
|
||||
DDG_COOKIE_NAMES = {
|
||||
"__ddg1_",
|
||||
"__ddg2_",
|
||||
"__ddg5_",
|
||||
"__ddg8_",
|
||||
"__ddg9_",
|
||||
"__ddg10_",
|
||||
"__ddgid_",
|
||||
"__ddgmark_",
|
||||
"ddg_last_challenge",
|
||||
}
|
||||
|
||||
|
||||
def _get_base_domain(domain: str) -> str:
|
||||
"""Extract base domain from hostname (e.g., 'www.example.com' -> 'example.com')."""
|
||||
return ".".join(domain.split(".")[-2:]) if "." in domain else domain
|
||||
|
||||
|
||||
def _get_full_cookie_domains() -> set[str]:
|
||||
"""Return mirror domains that need full-session cookie extraction."""
|
||||
from shelfmark.core.mirrors import get_zlib_cookie_domains
|
||||
|
||||
return {_get_base_domain(domain) for domain in get_zlib_cookie_domains()}
|
||||
|
||||
|
||||
def _should_extract_cookie(name: str, *, extract_all: bool) -> bool:
|
||||
"""Determine if a cookie should be extracted based on its name."""
|
||||
if extract_all:
|
||||
return True
|
||||
is_cf = name in CF_COOKIE_NAMES or name.startswith("cf_")
|
||||
is_ddg = name in DDG_COOKIE_NAMES or name.startswith("__ddg")
|
||||
return is_cf or is_ddg
|
||||
|
||||
|
||||
def _store_extracted_cookies(
|
||||
*,
|
||||
url: str,
|
||||
cookies: list[Any],
|
||||
user_agent: str | None = None,
|
||||
) -> None:
|
||||
"""Store filtered bypass cookies (and optional UA) for a URL domain."""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.hostname or ""
|
||||
if not domain:
|
||||
return
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
extract_all = base_domain in _get_full_cookie_domains()
|
||||
|
||||
cookies_found: dict[str, dict[str, Any]] = {}
|
||||
for cookie in cookies:
|
||||
name = getattr(cookie, "name", "") or ""
|
||||
if not _should_extract_cookie(name, extract_all=extract_all):
|
||||
continue
|
||||
expires = getattr(cookie, "expires", None)
|
||||
if expires is not None and expires <= 0:
|
||||
expires = None
|
||||
cookies_found[name] = {
|
||||
"value": getattr(cookie, "value", ""),
|
||||
"domain": getattr(cookie, "domain", None) or domain,
|
||||
"path": getattr(cookie, "path", None) or "/",
|
||||
"expiry": expires,
|
||||
"secure": bool(getattr(cookie, "secure", True)),
|
||||
"httpOnly": True,
|
||||
}
|
||||
|
||||
if not cookies_found:
|
||||
return
|
||||
|
||||
with _cf_cookies_lock:
|
||||
_cf_cookies[base_domain] = cookies_found
|
||||
if user_agent:
|
||||
_cf_user_agents[base_domain] = user_agent
|
||||
logger.debug("Stored UA for %s: %s...", base_domain, str(user_agent)[:60])
|
||||
else:
|
||||
logger.debug("No UA captured for %s", base_domain)
|
||||
|
||||
cookie_type = "all" if extract_all else "protection"
|
||||
logger.debug("Extracted %s %s cookies for %s", len(cookies_found), cookie_type, base_domain)
|
||||
|
||||
|
||||
async def _extract_cookies_from_cdp(driver: Any, page: Any, url: str) -> None:
|
||||
"""Extract cookies from a CDP browser after successful bypass."""
|
||||
@@ -336,117 +278,116 @@ async def _extract_cookies_from_cdp(driver: Any, page: Any, url: str) -> None:
|
||||
except _CDP_OPERATION_ERRORS:
|
||||
user_agent = None
|
||||
|
||||
_store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
|
||||
store_extracted_cookies(url=url, cookies=all_cookies, user_agent=user_agent)
|
||||
|
||||
except _CDP_OPERATION_ERRORS as e:
|
||||
logger.debug("Failed to extract cookies: %s", e)
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get stored cookies for a domain. Returns empty dict if none available."""
|
||||
if not domain:
|
||||
return {}
|
||||
|
||||
base_domain = _get_base_domain(domain)
|
||||
|
||||
with _cf_cookies_lock:
|
||||
cookies = _cf_cookies.get(base_domain, {})
|
||||
if not cookies:
|
||||
return {}
|
||||
|
||||
cf_clearance = cookies.get("cf_clearance", {})
|
||||
if cf_clearance:
|
||||
expiry = cf_clearance.get("expiry")
|
||||
if expiry is None:
|
||||
expiry = cf_clearance.get("expires")
|
||||
if expiry and expiry > 0 and time.time() > expiry:
|
||||
logger.debug("CF cookies expired for %s", base_domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
return {}
|
||||
|
||||
return {name: c["value"] for name, c in cookies.items()}
|
||||
def _read_process_cmdline(proc_dir: Path) -> str:
|
||||
"""Return a process's full command line, or "" when it cannot be read."""
|
||||
try:
|
||||
raw = (proc_dir / "cmdline").read_bytes()
|
||||
except OSError:
|
||||
return ""
|
||||
return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def has_valid_cf_cookies(domain: str) -> bool:
|
||||
"""Check if we have valid Cloudflare cookies for a domain."""
|
||||
return bool(get_cf_cookies_for_domain(domain))
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get the User-Agent that was used during bypass for a domain."""
|
||||
if not domain:
|
||||
def _read_process_pgid(proc_dir: Path) -> int | None:
|
||||
"""Return a process's group id from /proc/<pid>/stat, or None when unreadable."""
|
||||
try:
|
||||
stat_line = (proc_dir / "stat").read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return None
|
||||
# Field 2 (comm) is parenthesised and may itself contain spaces and parens, so the
|
||||
# fields are only unambiguous after the last ')': state, ppid, pgrp, ...
|
||||
fields = stat_line.rpartition(")")[2].split()
|
||||
pgrp_index = 2
|
||||
if len(fields) <= pgrp_index:
|
||||
return None
|
||||
try:
|
||||
return int(fields[pgrp_index])
|
||||
except ValueError:
|
||||
return None
|
||||
with _cf_cookies_lock:
|
||||
return _cf_user_agents.get(_get_base_domain(domain))
|
||||
|
||||
|
||||
def clear_cf_cookies(domain: str | None = None) -> None:
|
||||
"""Clear stored Cloudflare cookies and User-Agent. If domain is None, clear all."""
|
||||
with _cf_cookies_lock:
|
||||
if domain:
|
||||
base_domain = _get_base_domain(domain)
|
||||
_cf_cookies.pop(base_domain, None)
|
||||
_cf_user_agents.pop(base_domain, None)
|
||||
else:
|
||||
_cf_cookies.clear()
|
||||
_cf_user_agents.clear()
|
||||
def _find_browser_processes() -> list[tuple[int, int, str]]:
|
||||
"""Return (pid, pgid, cmdline) for every browser-ish process visible in /proc."""
|
||||
found: list[tuple[int, int, str]] = []
|
||||
try:
|
||||
entries = list(_PROC_ROOT.iterdir())
|
||||
except OSError as e:
|
||||
logger.debug("Could not list %s: %s", _PROC_ROOT, e)
|
||||
return found
|
||||
|
||||
for entry in entries:
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
cmdline = _read_process_cmdline(entry)
|
||||
if not cmdline or not any(name in cmdline for name in _BROWSER_PROCESS_PATTERNS):
|
||||
continue
|
||||
pgid = _read_process_pgid(entry)
|
||||
if pgid is None:
|
||||
continue
|
||||
found.append((int(entry.name), pgid, cmdline))
|
||||
return found
|
||||
|
||||
|
||||
def _kill_process(pid: int, cmdline: str) -> bool:
|
||||
"""SIGKILL one process, reporting whether it was actually signalled."""
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except OSError as e:
|
||||
logger.warning("Failed to kill pid %s: %s", pid, e)
|
||||
return False
|
||||
logger.debug("Killed leftover process %s: %s", pid, cmdline[:120])
|
||||
return True
|
||||
|
||||
|
||||
def _cleanup_orphan_processes() -> int:
|
||||
"""Kill orphan Chrome/Xvfb/ffmpeg processes. Only runs in Docker mode."""
|
||||
"""Kill leftover Chrome/Xvfb/ffmpeg processes. Only runs in Docker mode.
|
||||
|
||||
Scoped to this bypass session's process group plus groups whose leader has died.
|
||||
A container-wide sweep (the old `pkill -9 -f chrome`) also matched the browsers a
|
||||
concurrently running bypass was still driving, so with MAX_CONCURRENT_DOWNLOADS > 1
|
||||
every worker that started a solve killed the others' browsers (#1231).
|
||||
"""
|
||||
if not env.DOCKERMODE:
|
||||
return 0
|
||||
|
||||
_stop_ffmpeg_recording()
|
||||
|
||||
processes_to_kill = ["chrome", "chromium", "Xvfb", "ffmpeg"]
|
||||
total_killed = 0
|
||||
|
||||
logger.debug("Checking for orphan processes...")
|
||||
logger.debug("Checking for leftover browser processes...")
|
||||
logger.log_resource_usage()
|
||||
|
||||
if _PGREP_PATH is None or _PKILL_PATH is None:
|
||||
logger.warning("Skipping orphan-process cleanup because pgrep/pkill are unavailable")
|
||||
if not _PROC_ROOT.is_dir():
|
||||
logger.warning("Skipping browser-process cleanup because %s is unavailable", _PROC_ROOT)
|
||||
return 0
|
||||
|
||||
for proc_name in processes_to_kill:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[_PGREP_PATH, "-f", proc_name],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
continue
|
||||
own_pid = os.getpid()
|
||||
own_pgid = os.getpgrp()
|
||||
total_killed = 0
|
||||
|
||||
pids = result.stdout.strip().split("\n")
|
||||
count = len(pids)
|
||||
logger.info("Found %s orphan %s process(es), killing...", count, proc_name)
|
||||
|
||||
kill_result = subprocess.run(
|
||||
[_PKILL_PATH, "-9", "-f", proc_name],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=5,
|
||||
)
|
||||
if kill_result.returncode == 0:
|
||||
total_killed += count
|
||||
else:
|
||||
logger.warning("pkill for %s returned %s", proc_name, kill_result.returncode)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Timeout while checking for %s processes", proc_name)
|
||||
except _SUBPROCESS_OPERATION_ERRORS as e:
|
||||
logger.debug("Error checking for %s processes: %s", proc_name, e)
|
||||
for pid, pgid, cmdline in _find_browser_processes():
|
||||
if pid == own_pid:
|
||||
continue
|
||||
# Another live process group means another bypass session: its browsers are in
|
||||
# use, not orphans. Only our own group and groups whose leader is gone (a helper
|
||||
# that died or was killed, leaving its browser behind) are ours to clean up.
|
||||
if pgid != own_pgid and (_PROC_ROOT / str(pgid)).exists():
|
||||
logger.debug("Leaving pid %s to its live bypass session (pgid %s)", pid, pgid)
|
||||
continue
|
||||
if _kill_process(pid, cmdline):
|
||||
total_killed += 1
|
||||
|
||||
if total_killed > 0:
|
||||
time.sleep(1)
|
||||
logger.info("Cleaned up %s orphan process(es)", total_killed)
|
||||
logger.info("Cleaned up %s leftover browser process(es)", total_killed)
|
||||
logger.log_resource_usage()
|
||||
else:
|
||||
logger.debug("No orphan processes found")
|
||||
logger.debug("No leftover browser processes found")
|
||||
|
||||
return total_killed
|
||||
|
||||
@@ -903,27 +844,25 @@ def _run_bypass_in_current_process(url: str, retry: int, cancel_flag: Event | No
|
||||
if driver:
|
||||
await _close_cdp_driver(driver)
|
||||
|
||||
if os.environ.get(_BYPASS_CHILD_ENV) == "1":
|
||||
return asyncio.run(_run_bypass())
|
||||
# Bound the wait: this path runs in-process (non-Docker installs), holds the module-wide
|
||||
# LOCKED for its whole duration, and neither page.get() nor page.wait() has a timeout of
|
||||
# its own. Without a deadline here a single wedged CDP session blocks every subsequent
|
||||
# bypass in the process forever.
|
||||
return _CDP_WORKER.run(_run_bypass(), timeout=_IN_PROCESS_BYPASS_TIMEOUT_SECONDS)
|
||||
# Bound the wait: this holds the module-wide LOCKED for its whole duration, and neither
|
||||
# page.get() nor page.wait() has a timeout of its own. Without a deadline here a single
|
||||
# wedged CDP session blocks every subsequent bypass in the process forever.
|
||||
#
|
||||
# The helper goes through the worker too, rather than asyncio.run: that owns a loop for
|
||||
# one call and closes it on the way out, so a helper serving many requests would build
|
||||
# and tear down a loop per bypass and would carry no deadline of its own. The worker's
|
||||
# loop lives in a thread, outlives any single bypass, and cancels the coroutine when the
|
||||
# deadline passes.
|
||||
timeout = (
|
||||
_CHILD_BYPASS_TIMEOUT_SECONDS
|
||||
if os.environ.get(_BYPASS_CHILD_ENV) == "1"
|
||||
else _IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
)
|
||||
return _CDP_WORKER.run(_run_bypass(), timeout=timeout)
|
||||
|
||||
|
||||
def _store_child_bypass_state(payload: dict[str, Any]) -> None:
|
||||
cookies = payload.get("cookies")
|
||||
if isinstance(cookies, dict):
|
||||
with _cf_cookies_lock:
|
||||
_cf_cookies.update(cookies)
|
||||
|
||||
user_agents = payload.get("user_agents")
|
||||
if isinstance(user_agents, dict):
|
||||
with _cf_cookies_lock:
|
||||
_cf_user_agents.update(
|
||||
{str(domain): str(agent) for domain, agent in user_agents.items()}
|
||||
)
|
||||
import_store(payload.get("cookies"), payload.get("user_agents"))
|
||||
|
||||
|
||||
def _prepare_child_browser_env(env_vars: dict[str, str]) -> dict[str, str]:
|
||||
@@ -946,6 +885,228 @@ def _prepare_child_browser_env(env_vars: dict[str, str]) -> dict[str, str]:
|
||||
return env_vars
|
||||
|
||||
|
||||
def _terminate_helper_session(proc: subprocess.Popen[str]) -> None:
|
||||
"""Kill the bypass helper and every process it spawned.
|
||||
|
||||
start_new_session makes the helper a session leader, so its pid doubles as the
|
||||
process-group id of the browser tree underneath it and one killpg reaches all of it.
|
||||
"""
|
||||
if hasattr(os, "killpg"):
|
||||
with suppress(OSError):
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
with suppress(OSError):
|
||||
proc.kill()
|
||||
with suppress(OSError, subprocess.SubprocessError):
|
||||
proc.wait(timeout=5)
|
||||
|
||||
|
||||
def _part_path(result_path: Path) -> Path:
|
||||
"""Where the helper stages a result before renaming it into place."""
|
||||
return result_path.with_name(result_path.name + ".part")
|
||||
|
||||
|
||||
class _BypassHelper:
|
||||
"""The helper subprocess that runs the bypasses, kept alive across them.
|
||||
|
||||
Spawning it costs about 4.5 seconds of interpreter start and imports before any work
|
||||
begins, paid on every protected request - and a single search issues several. What it
|
||||
keeps is the process, not the browser: each bypass still starts and closes its own
|
||||
Chrome, so nothing accumulates between requests.
|
||||
|
||||
Protocol: one JSON request per line on stdin, answered by writing the result file that
|
||||
request named. stdout and stderr stay attached to the parent's, so helper logs keep
|
||||
showing up in `docker logs` as before.
|
||||
|
||||
Only one request is ever in flight - get() serializes every bypass behind LOCKED. The
|
||||
lock here is for the idle reaper, which runs on a timer thread.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._proc: subprocess.Popen[str] | None = None
|
||||
self._last_used = 0.0
|
||||
self._idle_timer: threading.Timer | None = None
|
||||
|
||||
def _idle_timeout(self) -> float:
|
||||
return _coerce_non_negative_float(
|
||||
app_config.get("BYPASS_BROWSER_IDLE_TIMEOUT", _HELPER_IDLE_TIMEOUT_DEFAULT),
|
||||
_HELPER_IDLE_TIMEOUT_DEFAULT,
|
||||
)
|
||||
|
||||
def _spawn(self) -> subprocess.Popen[str]:
|
||||
env_vars = os.environ.copy()
|
||||
env_vars[_BYPASS_CHILD_ENV] = "1"
|
||||
env_vars = _prepare_child_browser_env(env_vars)
|
||||
return subprocess.Popen(
|
||||
[sys.executable, "-m", "shelfmark.bypass.internal_bypasser"],
|
||||
stdin=subprocess.PIPE,
|
||||
text=True,
|
||||
env=env_vars,
|
||||
# Give the helper its own session: Chrome, Xvfb and ffmpeg inherit its process
|
||||
# group, which is what lets the cleanup sweep tell this helper's browsers apart
|
||||
# from a concurrent worker's (#1231) and lets us kill the whole tree below.
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
def _running(self) -> subprocess.Popen[str] | None:
|
||||
proc = self._proc
|
||||
if proc is None:
|
||||
return None
|
||||
if proc.poll() is not None or proc.stdin is None or proc.stdin.closed:
|
||||
return None
|
||||
return proc
|
||||
|
||||
def _ensure_running(self) -> subprocess.Popen[str]:
|
||||
proc = self._running()
|
||||
if proc is not None:
|
||||
return proc
|
||||
if self._proc is not None:
|
||||
logger.info("Bypass helper exited (code %s), starting a new one", self._proc.returncode)
|
||||
self._discard()
|
||||
self._proc = self._spawn()
|
||||
return self._proc
|
||||
|
||||
def _discard(self, *, wait_for_exit: bool = True) -> None:
|
||||
"""Stop the helper and forget it.
|
||||
|
||||
`wait_for_exit` belongs to a helper that could still act on the closed pipe: an
|
||||
idle one is sitting in its stdin read, notices EOF and exits on its own. A helper
|
||||
dropped mid-bypass is blocked inside the solve and will not return to that read,
|
||||
so the grace cannot end in anything but the kill below - and the caller waiting it
|
||||
out is a user cancelling a download, holding LOCKED while every other bypass in
|
||||
the worker queues behind them.
|
||||
"""
|
||||
proc = self._proc
|
||||
self._proc = None
|
||||
if proc is None:
|
||||
return
|
||||
|
||||
# Closing stdin ends the helper's request loop, so an idle helper gets to exit on
|
||||
# its own. One mid-bypass cannot answer, and is killed below.
|
||||
with suppress(OSError):
|
||||
if proc.stdin is not None and not proc.stdin.closed:
|
||||
proc.stdin.close()
|
||||
if wait_for_exit:
|
||||
try:
|
||||
proc.wait(timeout=_HELPER_SHUTDOWN_GRACE_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Bypass helper did not exit on request, killing its session")
|
||||
|
||||
# Tear the session down either way: a helper killed mid-bypass leaves its Chrome
|
||||
# and Xvfb running, and those leftovers are what made the next worker's browser
|
||||
# fail to start. Harmless once it has already exited.
|
||||
_terminate_helper_session(proc)
|
||||
|
||||
def _cancel_idle_timer(self) -> None:
|
||||
if self._idle_timer is not None:
|
||||
self._idle_timer.cancel()
|
||||
self._idle_timer = None
|
||||
|
||||
def _arm_idle_timer(self) -> None:
|
||||
self._cancel_idle_timer()
|
||||
timeout = self._idle_timeout()
|
||||
if self._proc is None or timeout <= 0:
|
||||
return
|
||||
timer = threading.Timer(timeout, self._reap_if_idle)
|
||||
timer.daemon = True
|
||||
self._idle_timer = timer
|
||||
timer.start()
|
||||
|
||||
def _reap_if_idle(self) -> None:
|
||||
with self._lock:
|
||||
if self._proc is None:
|
||||
return
|
||||
idle_for = time.monotonic() - self._last_used
|
||||
timeout = self._idle_timeout()
|
||||
if idle_for < timeout:
|
||||
# A bypass started while this timer was waiting for the lock.
|
||||
self._arm_idle_timer()
|
||||
return
|
||||
logger.info("Closing idle bypass helper after %.0fs without work", idle_for)
|
||||
self._discard()
|
||||
|
||||
def run(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
timeout: float,
|
||||
cancel_flag: Event | None,
|
||||
) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
self._cancel_idle_timer()
|
||||
try:
|
||||
return self._exchange(payload, timeout, cancel_flag)
|
||||
finally:
|
||||
self._last_used = time.monotonic()
|
||||
self._arm_idle_timer()
|
||||
|
||||
def _exchange(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
timeout: float,
|
||||
cancel_flag: Event | None,
|
||||
) -> dict[str, Any]:
|
||||
request_line = json.dumps(payload) + "\n"
|
||||
proc = self._ensure_running()
|
||||
try:
|
||||
self._write(proc, request_line)
|
||||
except OSError as exc:
|
||||
# A live helper can die between the liveness check and the write, so one retry
|
||||
# on a fresh process. A fresh one failing here is a real failure.
|
||||
logger.info("Bypass helper closed its pipe (%s), retrying on a new one", exc)
|
||||
# Nothing to ask of a helper we cannot write to: its read end is already gone.
|
||||
self._discard(wait_for_exit=False)
|
||||
proc = self._ensure_running()
|
||||
self._write(proc, request_line)
|
||||
|
||||
return self._await_result(proc, Path(str(payload["result_path"])), timeout, cancel_flag)
|
||||
|
||||
def _write(self, proc: subprocess.Popen[str], request_line: str) -> None:
|
||||
if proc.stdin is None:
|
||||
msg = "Bypass helper has no stdin pipe"
|
||||
raise OSError(msg)
|
||||
proc.stdin.write(request_line)
|
||||
proc.stdin.flush()
|
||||
|
||||
def _await_result(
|
||||
self,
|
||||
proc: subprocess.Popen[str],
|
||||
result_path: Path,
|
||||
timeout: float,
|
||||
cancel_flag: Event | None,
|
||||
) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout
|
||||
try:
|
||||
while not result_path.exists():
|
||||
if proc.poll() is not None:
|
||||
returncode = proc.returncode
|
||||
self._discard(wait_for_exit=False)
|
||||
msg = f"Internal bypasser helper exited without a result (code {returncode})"
|
||||
raise RuntimeError(msg)
|
||||
if cancel_flag is not None and cancel_flag.is_set():
|
||||
# The helper is mid-bypass and cannot be told to stop, so it goes.
|
||||
self._discard(wait_for_exit=False)
|
||||
_check_cancellation(cancel_flag, "Bypass cancelled while waiting for helper")
|
||||
if time.monotonic() >= deadline:
|
||||
self._discard(wait_for_exit=False)
|
||||
msg = "Internal bypasser helper process timed out"
|
||||
raise TimeoutError(msg)
|
||||
time.sleep(_HELPER_RESULT_POLL_SECONDS)
|
||||
|
||||
return json.loads(result_path.read_text(encoding="utf-8"))
|
||||
finally:
|
||||
# Every way out of here is final for this request: either the answer has been
|
||||
# read, or the helper that would have written it has just been killed. Nothing
|
||||
# will write these paths afterwards and nothing will come looking for them, so
|
||||
# they are cleaned on the failure paths too - otherwise every cancelled
|
||||
# download and every wedged solve leaves one behind for the container's life.
|
||||
for path in (result_path, _part_path(result_path)):
|
||||
with suppress(OSError):
|
||||
path.unlink()
|
||||
|
||||
|
||||
_BYPASS_HELPER = _BypassHelper()
|
||||
|
||||
|
||||
def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None) -> str:
|
||||
"""Run the browser bypass in a helper process isolated from gunicorn/gevent."""
|
||||
_check_cancellation(cancel_flag, "Bypass cancelled before helper process")
|
||||
@@ -956,39 +1117,15 @@ def _get_via_subprocess(url: str, retry: int, cancel_flag: Event | None = None)
|
||||
# freshly spawned helper would otherwise pre-resolve AA hostnames against the system
|
||||
# resolver - which may be blocked or hijacked by the user's ISP. Pass the parent's
|
||||
# active DNS config so the helper mirrors it (e.g. DoH) when building Chrome's host
|
||||
# resolver rules.
|
||||
# resolver rules. Sent with every request, not just at spawn, because a helper outlives
|
||||
# changes the parent makes to its DNS provider.
|
||||
payload = {
|
||||
"url": url,
|
||||
"retry": retry,
|
||||
"result_path": str(result_path),
|
||||
"dns_config": network.get_dns_config(),
|
||||
}
|
||||
env_vars = os.environ.copy()
|
||||
env_vars[_BYPASS_CHILD_ENV] = "1"
|
||||
env_vars = _prepare_child_browser_env(env_vars)
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-m", "shelfmark.bypass.internal_bypasser"],
|
||||
stdin=subprocess.PIPE,
|
||||
text=True,
|
||||
env=env_vars,
|
||||
)
|
||||
try:
|
||||
proc.communicate(json.dumps(payload), timeout=_BYPASS_SUBPROCESS_TIMEOUT_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
msg = "Internal bypasser helper process timed out"
|
||||
raise TimeoutError(msg) from None
|
||||
|
||||
try:
|
||||
result = json.loads(result_path.read_text())
|
||||
except FileNotFoundError as exc:
|
||||
msg = f"Internal bypasser helper exited without a result (code {proc.returncode})"
|
||||
raise RuntimeError(msg) from exc
|
||||
finally:
|
||||
with suppress(OSError):
|
||||
result_path.unlink()
|
||||
result = _BYPASS_HELPER.run(payload, _BYPASS_SUBPROCESS_TIMEOUT_SECONDS, cancel_flag)
|
||||
|
||||
if not isinstance(result, dict):
|
||||
msg = "Internal bypasser helper returned an invalid result"
|
||||
@@ -1263,9 +1400,23 @@ def _try_with_cached_cookies(url: str, hostname: str) -> str | None:
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
logger.debug("Cached cookies worked, skipped Chrome bypass")
|
||||
return response.text
|
||||
logger.debug(
|
||||
"Cached cookies rejected (%s) for %s; discarding them",
|
||||
response.status_code,
|
||||
url,
|
||||
)
|
||||
except _REQUEST_OPERATION_ERRORS as exc:
|
||||
# A redirect loop lands here too: DDoS-Guard answers a dead clearance cookie
|
||||
# with an endless ?check=1 bounce rather than a status we can read.
|
||||
logger.debug("Cached cookie retry failed for %s: %s", url, exc)
|
||||
|
||||
# Reached only when the cached cookies did not produce a page, so they are no
|
||||
# longer clearance. Dropping them now means the imminent Chrome solve starts from
|
||||
# a clean slate and later requests cannot re-present the same rejected cookie.
|
||||
# Guarded because clear_cf_cookies("") means "every host", which would wipe
|
||||
# clearance for sites that are working fine.
|
||||
if hostname:
|
||||
clear_cf_cookies(hostname)
|
||||
return None
|
||||
|
||||
|
||||
@@ -1311,33 +1462,88 @@ def get_bypassed_page(
|
||||
return response_html
|
||||
|
||||
|
||||
def _dns_fingerprint(dns_config: dict[str, Any]) -> tuple[str, tuple[str, ...], bool]:
|
||||
"""Reduce a DNS config to what has to match for two of them to be the same one."""
|
||||
provider = str(dns_config.get("provider") or "").strip().lower()
|
||||
servers = dns_config.get("servers") if provider == "manual" else None
|
||||
server_list = tuple(str(server) for server in servers) if isinstance(servers, list) else ()
|
||||
return (provider, server_list, bool(dns_config.get("doh_enabled")))
|
||||
|
||||
|
||||
def _apply_parent_dns_config(dns_config: dict[str, Any]) -> None:
|
||||
"""Mirror the parent process's active DNS provider in this helper subprocess.
|
||||
|
||||
DNS state is in-memory only, so a fresh helper defaults to system DNS and would
|
||||
pre-resolve AA hostnames (for Chrome's --host-resolver-rules) against a resolver
|
||||
that may be blocked/hijacked. Re-applying the parent's provider keeps the helper on
|
||||
the same DoH/custom resolver the parent already validated.
|
||||
DNS state is in-memory only, so a helper left to itself would pre-resolve AA hostnames
|
||||
(for Chrome's --host-resolver-rules) against a resolver that may be blocked or
|
||||
hijacked. Re-applying the parent's provider keeps the helper on the same DoH/custom
|
||||
resolver the parent already validated.
|
||||
|
||||
Compared against what this process is *actually* resolving through, rather than
|
||||
against the last config it happened to be handed. The helper now outlives the request,
|
||||
so it has to be able to travel back to auto as well as away from it - which a user
|
||||
flipping CUSTOM_DNS in settings does live, without a restart - and asking the network
|
||||
module what it is doing beats keeping a second, drifting copy of that answer here.
|
||||
"""
|
||||
provider = str(dns_config.get("provider") or "").strip().lower()
|
||||
# "auto" means the parent has not rotated off system DNS yet, so the helper's own
|
||||
# default initialization already matches it - nothing to override.
|
||||
if not provider or provider == "auto":
|
||||
wanted = _dns_fingerprint(dns_config)
|
||||
provider, servers, use_doh = wanted
|
||||
if not provider:
|
||||
return
|
||||
manual_servers = dns_config.get("servers") if provider == "manual" else None
|
||||
# set_dns_provider() rebuilds the resolvers, so it is worth doing only on a real change.
|
||||
if wanted == _dns_fingerprint(network.get_dns_config()):
|
||||
return
|
||||
|
||||
try:
|
||||
network.set_dns_provider(
|
||||
provider,
|
||||
manual_servers,
|
||||
use_doh=bool(dns_config.get("doh_enabled")),
|
||||
)
|
||||
network.set_dns_provider(provider, list(servers) or None, use_doh=use_doh)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
logger.warning("Could not apply parent DNS config (%s): %s", provider, exc)
|
||||
|
||||
|
||||
def _run_child_process() -> int:
|
||||
"""CLI entrypoint used by the Docker helper subprocess."""
|
||||
request = json.loads(sys.stdin.read() or "{}")
|
||||
def _terminate_own_session() -> None:
|
||||
"""SIGKILL this process and every process it spawned, browser included."""
|
||||
if hasattr(os, "killpg") and os.getpgrp() == os.getpid():
|
||||
with suppress(OSError):
|
||||
os.killpg(os.getpgrp(), signal.SIGKILL)
|
||||
# A thread cannot end the process any other way; sys.exit would only end itself.
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def _watch_parent_process(original_ppid: int, interval: float) -> None:
|
||||
"""Take the browser down with us once the app process that spawned us is gone.
|
||||
|
||||
Cleanup only reclaims process groups whose leader has died, so a helper that outlives
|
||||
its parent (worker restart, OOM kill) would sit there holding a browser that no later
|
||||
bypass is allowed to touch.
|
||||
"""
|
||||
while os.getppid() == original_ppid:
|
||||
time.sleep(interval)
|
||||
logger.warning("Bypass helper lost its parent process; taking the browser down")
|
||||
_terminate_own_session()
|
||||
|
||||
|
||||
def _start_parent_watchdog() -> None:
|
||||
"""Watch the spawning process in the background for the life of this helper."""
|
||||
threading.Thread(
|
||||
target=_watch_parent_process,
|
||||
args=(os.getppid(), _PARENT_WATCHDOG_INTERVAL_SECONDS),
|
||||
daemon=True,
|
||||
name="BypassParentWatchdog",
|
||||
).start()
|
||||
|
||||
|
||||
def _publish_result(result_path: Path, payload: dict[str, Any]) -> None:
|
||||
"""Write the result file atomically.
|
||||
|
||||
The parent decides the request is answered the moment this path exists, so it must
|
||||
never observe a half-written file. Rename within the same directory is atomic.
|
||||
"""
|
||||
tmp_path = _part_path(result_path)
|
||||
tmp_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
tmp_path.replace(result_path)
|
||||
|
||||
|
||||
def _handle_child_request(request_line: str) -> int:
|
||||
"""Answer one request from the parent."""
|
||||
request = json.loads(request_line or "{}")
|
||||
result_path = Path(str(request["result_path"]))
|
||||
url = str(request["url"])
|
||||
retry = _coerce_positive_int(
|
||||
@@ -1348,15 +1554,25 @@ def _run_child_process() -> int:
|
||||
if isinstance(dns_config, dict):
|
||||
_apply_parent_dns_config(dns_config)
|
||||
|
||||
# The parent owns the cookie store; this process only solves. Starting each request
|
||||
# from an empty store is what a helper spawned per request gave for free, and losing
|
||||
# it is what let clearance the parent had deliberately purged for some *other* host
|
||||
# survive here and get merged back over the parent's copy by the export below - the
|
||||
# dead-cookie resurrection that http.py's _redirect_loop_handoff purges to avoid.
|
||||
# Nothing is lost by dropping it: get() below re-checks cached cookies, and the
|
||||
# parent already ran that same check against a store that is a superset of this one.
|
||||
clear_cf_cookies()
|
||||
|
||||
try:
|
||||
html = get(url, retry=retry)
|
||||
cookies, user_agents = export_store()
|
||||
payload = {
|
||||
"ok": True,
|
||||
"html": html,
|
||||
"cookies": _cf_cookies,
|
||||
"user_agents": _cf_user_agents,
|
||||
"cookies": cookies,
|
||||
"user_agents": user_agents,
|
||||
}
|
||||
result_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
_publish_result(result_path, payload)
|
||||
except Exception as exc: # noqa: BLE001 - helper boundary must serialize failures.
|
||||
payload = {
|
||||
"ok": False,
|
||||
@@ -1364,10 +1580,30 @@ def _run_child_process() -> int:
|
||||
"error": str(exc),
|
||||
"traceback": traceback.format_exc(),
|
||||
}
|
||||
result_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
_publish_result(result_path, payload)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _run_child_process() -> int:
|
||||
"""CLI entrypoint used by the Docker helper subprocess.
|
||||
|
||||
Serves one request per line of stdin until the parent closes the pipe, so a burst of
|
||||
protected requests - a single search is several - pays the interpreter start and imports
|
||||
once instead of per request. Each bypass still gets its own browser, closed before the
|
||||
answer is published.
|
||||
"""
|
||||
exit_code = 0
|
||||
for line in sys.stdin:
|
||||
request_line = line.strip()
|
||||
if not request_line:
|
||||
continue
|
||||
exit_code = _handle_child_request(request_line)
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Started here rather than in _run_child_process() so it only ever watches a real
|
||||
# spawned helper, never a test or an embedded call.
|
||||
_start_parent_watchdog()
|
||||
raise SystemExit(_run_child_process())
|
||||
|
||||
@@ -766,7 +766,10 @@ def _on_save_downloads(values: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
# Audiobooks are always folder output.
|
||||
if effective.get("FILE_ORGANIZATION_AUDIOBOOK", "rename") == "rename":
|
||||
if effective.get("FILE_ORGANIZATION_AUDIOBOOK", "rename") in {
|
||||
"rename",
|
||||
"rename_and_group",
|
||||
}:
|
||||
template = effective.get("TEMPLATE_AUDIOBOOK_RENAME", "")
|
||||
if _contains_path_separators(template):
|
||||
return {
|
||||
@@ -1294,6 +1297,11 @@ def download_settings() -> list[SettingsField]:
|
||||
"label": "Rename and Organize",
|
||||
"description": "Create folders and rename files using a template. Recommended for Audiobookshelf. Do not use with ingest folders.",
|
||||
},
|
||||
{
|
||||
"value": "rename_and_group",
|
||||
"label": "Rename and Group",
|
||||
"description": "Rename single-file downloads; keep multi-file downloads grouped in their source folder. Do not use with ingest folders.",
|
||||
},
|
||||
],
|
||||
default="rename",
|
||||
universal_only=True,
|
||||
@@ -1312,7 +1320,10 @@ def download_settings() -> list[SettingsField]:
|
||||
),
|
||||
default="{Author} - {Title}",
|
||||
placeholder="{Author} - {Title}{ - Part }{PartNumber}",
|
||||
show_when={"field": "FILE_ORGANIZATION_AUDIOBOOK", "value": "rename"},
|
||||
show_when={
|
||||
"field": "FILE_ORGANIZATION_AUDIOBOOK",
|
||||
"value": ["rename", "rename_and_group"],
|
||||
},
|
||||
universal_only=True,
|
||||
),
|
||||
# Organize mode template - folders allowed
|
||||
@@ -1655,6 +1666,19 @@ def cloudflare_bypass_settings() -> list[SettingsField]:
|
||||
requires_restart=True,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": True},
|
||||
),
|
||||
NumberField(
|
||||
key="BYPASS_BROWSER_IDLE_TIMEOUT",
|
||||
label="Bypasser Idle Timeout (seconds)",
|
||||
description=(
|
||||
"How long the bypass helper process may sit unused before it is shut down. "
|
||||
"Higher keeps more searches fast, lower frees memory sooner."
|
||||
),
|
||||
default=180,
|
||||
min_value=30,
|
||||
max_value=3600,
|
||||
requires_restart=True,
|
||||
show_when={"field": "USING_EXTERNAL_BYPASSER", "value": False},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -26,6 +26,11 @@ from shelfmark.download.clients import (
|
||||
register_client,
|
||||
)
|
||||
from shelfmark.download.clients._coercion import config_text
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
DebridMagnet,
|
||||
DebridUpload,
|
||||
resolve_debrid_upload,
|
||||
)
|
||||
from shelfmark.download.http import download_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
|
||||
@@ -202,41 +207,19 @@ class AllDebridClient(DownloadClient):
|
||||
expected_hash: str | None = None,
|
||||
**kwargs: object,
|
||||
) -> str:
|
||||
"""Upload a magnet link to AllDebrid and return the magnet ID."""
|
||||
"""Send a torrent to AllDebrid and return the magnet ID.
|
||||
|
||||
Accepts a magnet link, a .torrent URL, or an indexer proxy URL; anything
|
||||
that is not already a magnet is resolved first, since an HTTP URL posted
|
||||
as a magnet is rejected rather than downloaded (#1250).
|
||||
"""
|
||||
if not self._api_key:
|
||||
msg = "AllDebrid API key is not configured"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
magnet_link = url
|
||||
if not magnet_link.startswith("magnet:") and expected_hash:
|
||||
magnet_link = f"magnet:?xt=urn:btih:{expected_hash}"
|
||||
|
||||
api_url = f"{_API_BASE}/magnet/upload"
|
||||
try:
|
||||
resp = requests.post(
|
||||
api_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnets[]": magnet_link},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(api_url),
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("status") != "success":
|
||||
code = data.get("error", {}).get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid upload failed: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
magnets = data.get("data", {}).get("magnets", [])
|
||||
if not magnets:
|
||||
msg = "No magnet returned from AllDebrid"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
info = magnets[0]
|
||||
if info.get("error"):
|
||||
code = info["error"].get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid magnet error: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
upload = resolve_debrid_upload(url, expected_hash=expected_hash)
|
||||
info = self._send_torrent(upload)
|
||||
|
||||
magnet_id = str(info.get("id", ""))
|
||||
if not magnet_id:
|
||||
@@ -262,12 +245,65 @@ class AllDebridClient(DownloadClient):
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Failed to upload magnet to AllDebrid")
|
||||
logger.exception("Failed to add torrent to AllDebrid")
|
||||
raise
|
||||
|
||||
else:
|
||||
return magnet_id
|
||||
|
||||
def _send_torrent(self, upload: DebridUpload) -> dict[str, Any]:
|
||||
"""Hand the torrent to AllDebrid, as a magnet or as a file upload.
|
||||
|
||||
Both endpoints answer with the same envelope and the same per-entry
|
||||
error shape, differing only in which key holds the entries.
|
||||
"""
|
||||
if isinstance(upload, DebridMagnet):
|
||||
api_url = f"{_API_BASE}/magnet/upload"
|
||||
entries_key = "magnets"
|
||||
resp = requests.post(
|
||||
api_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnets[]": upload.magnet_url},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(api_url),
|
||||
)
|
||||
else:
|
||||
api_url = f"{_API_BASE}/magnet/upload/file"
|
||||
entries_key = "files"
|
||||
resp = requests.post(
|
||||
api_url,
|
||||
headers=self._auth_headers(),
|
||||
files={
|
||||
"files[]": (
|
||||
"release.torrent",
|
||||
upload.torrent_data,
|
||||
"application/x-bittorrent",
|
||||
)
|
||||
},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(api_url),
|
||||
)
|
||||
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("status") != "success":
|
||||
code = data.get("error", {}).get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid upload failed: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
entries = data.get("data", {}).get(entries_key, [])
|
||||
if not entries:
|
||||
msg = "AllDebrid accepted the upload but returned no torrent"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
info = entries[0]
|
||||
if info.get("error"):
|
||||
code = info["error"].get("code", "UNKNOWN")
|
||||
msg = f"AllDebrid rejected the torrent: {code}"
|
||||
_raise_runtime_error(msg)
|
||||
|
||||
return info
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""Poll AllDebrid for magnet status and drive the download."""
|
||||
state = self._ensure_state(download_id)
|
||||
|
||||
@@ -843,6 +843,9 @@ class ExternalClientHandler(DownloadHandler, ABC):
|
||||
expected_hash=request.expected_hash,
|
||||
seeding_time_limit=request.seeding_time_limit,
|
||||
ratio_limit=request.ratio_limit,
|
||||
# rTorrent has no category concept, so its audiobook label
|
||||
# can only be chosen from the content type (#1235).
|
||||
content_type=task.content_type,
|
||||
)
|
||||
except Exception as e:
|
||||
if not refresh_attempted:
|
||||
|
||||
@@ -24,6 +24,11 @@ from shelfmark.download.clients import (
|
||||
register_client,
|
||||
)
|
||||
from shelfmark.download.clients._coercion import config_text
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
DebridMagnet,
|
||||
DebridUpload,
|
||||
resolve_debrid_upload,
|
||||
)
|
||||
from shelfmark.download.http import download_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
|
||||
@@ -173,26 +178,19 @@ class RealDebridClient(DownloadClient):
|
||||
expected_hash: str | None = None,
|
||||
**kwargs: object,
|
||||
) -> str:
|
||||
"""Upload a magnet link to Real-Debrid and select all files."""
|
||||
"""Send a torrent to Real-Debrid and select all files.
|
||||
|
||||
Accepts a magnet link, a .torrent URL, or an indexer proxy URL; anything
|
||||
that is not already a magnet is resolved first, because Real-Debrid
|
||||
answers a non-magnet body on addMagnet with a bare 404 (#1250).
|
||||
"""
|
||||
if not self._api_key:
|
||||
msg = "Real-Debrid API key is not configured"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
magnet_link = url
|
||||
if not magnet_link.startswith("magnet:") and expected_hash:
|
||||
magnet_link = f"magnet:?xt=urn:btih:{expected_hash}"
|
||||
|
||||
add_url = f"{_API_BASE}/torrents/addMagnet"
|
||||
try:
|
||||
resp = requests.post(
|
||||
add_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnet": magnet_link},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(add_url),
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
upload = resolve_debrid_upload(url, expected_hash=expected_hash)
|
||||
data = self._send_torrent(upload)
|
||||
|
||||
torrent_id = str(data.get("id", ""))
|
||||
if not torrent_id:
|
||||
@@ -229,12 +227,41 @@ class RealDebridClient(DownloadClient):
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Failed to upload magnet to Real-Debrid")
|
||||
logger.exception("Failed to add torrent to Real-Debrid")
|
||||
raise
|
||||
|
||||
else:
|
||||
return torrent_id
|
||||
|
||||
def _send_torrent(self, upload: DebridUpload) -> dict[str, Any]:
|
||||
"""Hand the torrent to Real-Debrid, as a magnet or as a file upload."""
|
||||
if isinstance(upload, DebridMagnet):
|
||||
add_url = f"{_API_BASE}/torrents/addMagnet"
|
||||
resp = requests.post(
|
||||
add_url,
|
||||
headers=self._auth_headers(),
|
||||
data={"magnet": upload.magnet_url},
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(add_url),
|
||||
)
|
||||
else:
|
||||
# addTorrent is a PUT that takes the raw file as the request body,
|
||||
# not a form field: https://api.real-debrid.com/
|
||||
add_url = f"{_API_BASE}/torrents/addTorrent"
|
||||
resp = requests.put(
|
||||
add_url,
|
||||
headers={
|
||||
**self._auth_headers(),
|
||||
"Content-Type": "application/x-bittorrent",
|
||||
},
|
||||
data=upload.torrent_data,
|
||||
timeout=_API_TIMEOUT,
|
||||
verify=get_ssl_verify(add_url),
|
||||
)
|
||||
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def get_status(self, download_id: str) -> DownloadStatus:
|
||||
"""Poll Real-Debrid for torrent status and drive the download."""
|
||||
state = self._ensure_state(download_id)
|
||||
|
||||
@@ -11,7 +11,12 @@ from urllib.parse import urlparse
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import get_hardened_xmlrpc_client
|
||||
from shelfmark.core.utils import (
|
||||
get_hardened_xmlrpc_client,
|
||||
)
|
||||
from shelfmark.core.utils import (
|
||||
is_audiobook as check_audiobook,
|
||||
)
|
||||
from shelfmark.download.clients import (
|
||||
DownloadClient,
|
||||
DownloadStatus,
|
||||
@@ -173,7 +178,8 @@ class RTorrentClient(DownloadClient):
|
||||
|
||||
commands = []
|
||||
|
||||
is_audiobook = kwargs.get("content_type") == "audiobook"
|
||||
content_type = kwargs.get("content_type")
|
||||
is_audiobook = check_audiobook(content_type if isinstance(content_type, str) else None)
|
||||
default_label = (
|
||||
self._audiobook_label if is_audiobook and self._audiobook_label else self._label
|
||||
)
|
||||
|
||||
@@ -82,6 +82,60 @@ class TorrentInfo:
|
||||
return self
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebridMagnet:
|
||||
"""A magnet link, ready to hand to a debrid service as-is."""
|
||||
|
||||
magnet_url: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebridTorrentFile:
|
||||
"""Raw .torrent bytes, for a debrid service's file-upload endpoint."""
|
||||
|
||||
torrent_data: bytes
|
||||
|
||||
|
||||
# A debrid service takes one or the other, never an indexer page or a proxy URL.
|
||||
type DebridUpload = DebridMagnet | DebridTorrentFile
|
||||
|
||||
|
||||
def resolve_debrid_upload(url: str, *, expected_hash: str | None = None) -> DebridUpload:
|
||||
"""Resolve a release download URL into a magnet link or .torrent bytes.
|
||||
|
||||
Prowlarr hands out a proxy URL, with no magnetUrl and no infoHash, for any
|
||||
indexer that only publishes torrent files - 1337x among them. Posting that
|
||||
URL to a debrid service as if it were a magnet is what produced a bare 404
|
||||
from the service instead of a download (#1250).
|
||||
|
||||
The torrent file is preferred over a synthesized `urn:btih:` magnet because
|
||||
it carries the tracker list, which is how the service finds a swarm that is
|
||||
not already cached. Fetches are shared with the rest of the add path through
|
||||
the torrent fetch cache, so resolving here costs at most one request.
|
||||
|
||||
Raises:
|
||||
ValueError: The URL resolved to neither form, so there is nothing to send.
|
||||
|
||||
"""
|
||||
if url.startswith("magnet:"):
|
||||
return DebridMagnet(magnet_url=url)
|
||||
|
||||
info = extract_torrent_info(url, expected_hash=expected_hash)
|
||||
|
||||
if info.is_magnet and info.magnet_url:
|
||||
# The download URL redirected to, or returned, a magnet link.
|
||||
return DebridMagnet(magnet_url=info.magnet_url)
|
||||
if info.torrent_data:
|
||||
return DebridTorrentFile(torrent_data=info.torrent_data)
|
||||
if info.info_hash:
|
||||
# No file to upload, but the hash alone still identifies the torrent.
|
||||
return DebridMagnet(magnet_url=f"magnet:?xt=urn:btih:{info.info_hash}")
|
||||
|
||||
reason = info.fetch_error or "no magnet link, info hash, or torrent file was available"
|
||||
msg = f"Could not resolve a torrent to send from {url[:120]} ({reason})"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def extract_torrent_info(
|
||||
url: str,
|
||||
*,
|
||||
|
||||
+150
-32
@@ -10,7 +10,8 @@ from urllib.parse import urljoin, urlparse
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
from shelfmark.bypass import BypassCancelledError
|
||||
from shelfmark.bypass import BypassCancelledError, cookie_store
|
||||
from shelfmark.bypass.challenge import challenge_marker
|
||||
from shelfmark.core.config import config as app_config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.request_helpers import coerce_bool, normalize_positive_int
|
||||
@@ -145,19 +146,13 @@ def get_bypassed_page(
|
||||
|
||||
|
||||
def get_cf_cookies_for_domain(domain: str) -> dict[str, str]:
|
||||
"""Get CF cookies - only available with internal bypasser."""
|
||||
if _is_using_external_bypasser():
|
||||
logger.debug("External bypasser in use, CF cookies not available for %s", domain)
|
||||
return {}
|
||||
return _get_internal_bypasser().get_cf_cookies_for_domain(domain)
|
||||
"""Get the clearance cookies won by whichever bypasser solved this domain."""
|
||||
return cookie_store.get_cf_cookies_for_domain(domain)
|
||||
|
||||
|
||||
def get_cf_user_agent_for_domain(domain: str) -> str | None:
|
||||
"""Get CF user agent - only available with internal bypasser."""
|
||||
if _is_using_external_bypasser():
|
||||
logger.debug("External bypasser in use, CF user agent not available for %s", domain)
|
||||
return None
|
||||
return _get_internal_bypasser().get_cf_user_agent_for_domain(domain)
|
||||
"""Get the User-Agent that solved this domain's challenge, if one is stored."""
|
||||
return cookie_store.get_cf_user_agent_for_domain(domain)
|
||||
|
||||
|
||||
def _apply_cf_bypass(url: str, headers: dict) -> dict:
|
||||
@@ -239,6 +234,22 @@ def _is_retryable_error(e: Exception) -> bool:
|
||||
_DEAD_MIRROR_CODES = (410, 451)
|
||||
|
||||
|
||||
def _response_challenge_marker(response: requests.Response) -> str | None:
|
||||
"""The challenge marker in a response body, or None if it carries no challenge.
|
||||
|
||||
Content type is checked first so a JSON or octet-stream error body is never
|
||||
decoded just to be scanned; a missing header is scanned anyway, since an
|
||||
interstitial served without one is still an interstitial.
|
||||
"""
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if content_type and "html" not in content_type.lower():
|
||||
return None
|
||||
try:
|
||||
return challenge_marker(response.text)
|
||||
except UnicodeDecodeError, ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _fatal_mirror_reason(e: Exception) -> str | None:
|
||||
"""Return why ``e`` proves the mirror is unusable, or None if it may recover.
|
||||
|
||||
@@ -322,11 +333,26 @@ def html_get_page(
|
||||
|
||||
"""
|
||||
|
||||
# Normalise before the closures below capture it: they touch selector.last_failure,
|
||||
# so it must be a concrete selector, not the Optional parameter.
|
||||
selector = selector or network.AAMirrorSelector()
|
||||
|
||||
def _result(html: str, response_url: str) -> str | tuple[str, str]:
|
||||
if include_response_url:
|
||||
return html, response_url
|
||||
return html
|
||||
|
||||
def _fail(reason: str, response_url: str) -> str | tuple[str, str]:
|
||||
"""Record why the fetch is giving up, then return the empty result.
|
||||
|
||||
Every give-up path returns an empty page, which is all the caller used to
|
||||
see. Stashing the concrete reason on the shared selector lets the caller
|
||||
surface it (see release_sources.direct_download) rather than reporting the
|
||||
same generic "network restricted or mirrors blocked" for every cause.
|
||||
"""
|
||||
selector.last_failure = reason
|
||||
return _result("", response_url)
|
||||
|
||||
def _run_bypasser(bypass_url: str) -> str | tuple[str, str]:
|
||||
"""Run the active bypasser for one URL and return its result.
|
||||
|
||||
@@ -344,7 +370,13 @@ def html_get_page(
|
||||
# bypasser that fails to load is still reported as a bypasser error.
|
||||
request_activity_grace(status_callback, _bypass_grace_seconds())
|
||||
result = get_bypassed_page(bypass_url, selector, cancel_flag)
|
||||
return _result(result or "", bypass_url)
|
||||
if result:
|
||||
return _result(result, bypass_url)
|
||||
return _fail(
|
||||
"The protection bypasser returned an empty page — the challenge was "
|
||||
"not solved. Check that FlareSolverr/the CF bypasser is reachable.",
|
||||
bypass_url,
|
||||
)
|
||||
except _BYPASSER_ERRORS as e:
|
||||
logger.warning("Bypasser error: %s: %s", type(e).__name__, e)
|
||||
# Surface the real reason. Without this the caller only sees an empty
|
||||
@@ -355,7 +387,9 @@ def html_get_page(
|
||||
status_callback("error", f"Bypass failed: {type(e).__name__}: {e}")
|
||||
except _STATUS_CALLBACK_ERRORS:
|
||||
logger.debug("Bypass error status callback failed", exc_info=True)
|
||||
return _result("", bypass_url)
|
||||
if isinstance(e, BypassCancelledError):
|
||||
return _fail("The protection bypass was cancelled.", bypass_url)
|
||||
return _fail(f"The protection bypasser failed: {type(e).__name__}: {e}", bypass_url)
|
||||
finally:
|
||||
release_activity_grace(status_callback)
|
||||
|
||||
@@ -369,39 +403,49 @@ def html_get_page(
|
||||
"""
|
||||
return allow_bypasser_fallback and _is_cf_bypass_enabled() and not use_bypasser_now
|
||||
|
||||
def _purge_clearance(target_url: str) -> None:
|
||||
"""Drop the host's stored clearance cookies.
|
||||
|
||||
Called whenever the protection answered a request that *carried* cookies:
|
||||
being challenged while presenting them proves they no longer work, so keeping
|
||||
them only guarantees the same rejection on every later request. Applies to
|
||||
either bypasser, since both fill the same store.
|
||||
"""
|
||||
hostname = urlparse(target_url).hostname or ""
|
||||
# An empty domain means "clear every host" to the store, so skip the purge
|
||||
# rather than wipe clearance for sites that are working fine.
|
||||
if hostname:
|
||||
cookie_store.clear_cf_cookies(hostname)
|
||||
|
||||
def _redirect_loop_handoff(bypass_url: str) -> str | tuple[str, str]:
|
||||
"""Drop the host's stale clearance cookies, then bypass `bypass_url`.
|
||||
|
||||
A `?check=1` loop is how DDoS-Guard answers a clearance cookie that has gone
|
||||
stale, so the dead cookie has to go before the solve — otherwise it is merged
|
||||
back over the fresh one on the next request and the loop simply resumes. Purging
|
||||
is internal-bypasser only; with an external one get_cf_cookies_for_domain()
|
||||
already returns {}.
|
||||
back over the fresh one on the next request and the loop simply resumes.
|
||||
"""
|
||||
hostname = urlparse(bypass_url).hostname or ""
|
||||
# An empty domain means "clear every host" to the bypasser, so skip the purge
|
||||
# rather than wipe clearance for sites that are working fine.
|
||||
if hostname and not _is_using_external_bypasser():
|
||||
_get_internal_bypasser().clear_cf_cookies(hostname)
|
||||
_purge_clearance(bypass_url)
|
||||
return _run_bypasser(bypass_url)
|
||||
|
||||
configured_retry = normalize_positive_int(app_config.MAX_RETRY)
|
||||
retry_limit = (
|
||||
retry if retry is not None else (configured_retry if configured_retry is not None else 1)
|
||||
)
|
||||
selector = selector or network.AAMirrorSelector()
|
||||
original_url = url
|
||||
current_url = selector.rewrite(original_url)
|
||||
use_bypasser_now = use_bypasser
|
||||
# Survives across attempts so a cookie won once is still presented on later retries.
|
||||
handshake_cookies: dict[str, str] = {}
|
||||
handshake_retries = 0
|
||||
# Last transport error seen, so the exhausted-retries path can name the real
|
||||
# cause (timeout, connection refused, DNS, ...) instead of a generic message.
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, retry_limit + 1):
|
||||
# Check for cancellation before each attempt
|
||||
if cancel_flag and cancel_flag.is_set():
|
||||
logger.info("html_get_page cancelled before attempt %s", attempt)
|
||||
return _result("", current_url)
|
||||
return _fail("The request was cancelled.", current_url)
|
||||
|
||||
cookies: dict[str, str] = {}
|
||||
try:
|
||||
@@ -453,6 +497,35 @@ def html_get_page(
|
||||
)
|
||||
continue
|
||||
|
||||
# A 503 still serving a challenge is protection, not a busy origin. The
|
||||
# handshake above has nothing left to echo back, and 503 is in
|
||||
# RETRYABLE_CODES, so without this the request spends every attempt on
|
||||
# the same wall: the bypasser is only ever reached from the 403 branch
|
||||
# and the AA redirect rescues. Gate on the body, not the status, so a
|
||||
# genuine overloaded-origin 503 keeps its retry path.
|
||||
if response.status_code == _HTTP_STATUS_SERVICE_UNAVAILABLE:
|
||||
marker = _response_challenge_marker(response)
|
||||
if marker and _bypass_handoff_allowed():
|
||||
if cookies:
|
||||
# Challenged while presenting clearance means those cookies
|
||||
# are dead; same reasoning as the 403 branch below.
|
||||
logger.debug(
|
||||
"503 challenge with cookies presented; purging: %s", current_url
|
||||
)
|
||||
_purge_clearance(current_url)
|
||||
logger.info(
|
||||
"503 challenge detected (%s); switching to bypasser: %s",
|
||||
marker,
|
||||
current_url,
|
||||
)
|
||||
return _run_bypasser(current_url)
|
||||
if marker:
|
||||
logger.debug(
|
||||
"503 challenge (%s) but no bypasser handoff available: %s",
|
||||
marker,
|
||||
current_url,
|
||||
)
|
||||
|
||||
if is_aa_url and response.is_redirect:
|
||||
location = response.headers.get("Location", "")
|
||||
if not location:
|
||||
@@ -474,7 +547,12 @@ def html_get_page(
|
||||
redirect_host,
|
||||
current_url,
|
||||
)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
f"The configured mirror {current_host} redirected to "
|
||||
f"{redirect_host}; it may be down or seized. Point MIRROR at "
|
||||
"a working host or switch to auto mode.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
new_url = _try_rotation(original_url, current_url, selector)
|
||||
if new_url:
|
||||
@@ -493,7 +571,11 @@ def html_get_page(
|
||||
redirect_host,
|
||||
current_url,
|
||||
)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
"Every Anna's Archive mirror redirected away to a dead host — "
|
||||
"all configured mirrors are unreachable.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
# Same-host redirect (relative or absolute) - follow manually.
|
||||
# DDoS-Guard gates AA /search behind a cookie probe: the 302 to
|
||||
@@ -524,7 +606,12 @@ def html_get_page(
|
||||
logger.warning(
|
||||
"Redirect loop and no bypasser available, giving up: %s", current_url
|
||||
)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
"Anna's Archive is behind a protection challenge (endless "
|
||||
"redirect loop) and no bypasser is enabled to solve it. Enable "
|
||||
"FlareSolverr/the CF bypasser.",
|
||||
current_url,
|
||||
)
|
||||
current_url = redirect_url
|
||||
continue
|
||||
|
||||
@@ -534,6 +621,7 @@ def html_get_page(
|
||||
return _result(response.text, response.url)
|
||||
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
status = _get_status_code(e)
|
||||
|
||||
# The same DDoS-Guard rescue, for the loops the manual AA follower above hands
|
||||
@@ -560,20 +648,35 @@ def html_get_page(
|
||||
current_url = new_url
|
||||
continue
|
||||
logger.warning("403 error, mirrors exhausted: %s", current_url)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
"Anna's Archive returned 403 (blocked) and all mirrors are exhausted.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
if _is_cf_bypass_enabled() and not use_bypasser_now:
|
||||
# Before switching to bypasser, check if cookies have become available
|
||||
# (another concurrent download may have completed bypass and extracted cookies)
|
||||
parsed = urlparse(current_url)
|
||||
fresh_cookies = get_cf_cookies_for_domain(parsed.hostname or "")
|
||||
if fresh_cookies and not cookies:
|
||||
# Cookies are now available - retry with cookies before using bypasser
|
||||
if fresh_cookies and not cookies and attempt < retry_limit:
|
||||
# Cookies are now available - retry with cookies before using bypasser.
|
||||
# Guarded on there being a next attempt: `continue` on the last one
|
||||
# ends the retry loop and abandons the request without ever offering
|
||||
# the URL to the bypasser, and MAX_RETRY=1 is the supported setting.
|
||||
# Same reasoning as the bypasser invocation below.
|
||||
logger.debug(
|
||||
"403 but cookies now available - retrying with cookies: %s",
|
||||
current_url,
|
||||
)
|
||||
continue
|
||||
if cookies:
|
||||
# Challenged *while presenting* clearance: those cookies are
|
||||
# dead. Without this they survive the solve and get merged back
|
||||
# over the fresh ones, so every later request re-presents a
|
||||
# known-rejected cookie and is challenged again - the stale
|
||||
# retry that never ends.
|
||||
logger.debug("403 with cookies presented; purging: %s", current_url)
|
||||
_purge_clearance(current_url)
|
||||
logger.info("403 detected; switching to bypasser: %s", current_url)
|
||||
# Invoke it here rather than setting use_bypasser_now and continuing.
|
||||
# The branch that acts on that flag runs at the top of the *next* retry
|
||||
@@ -582,12 +685,18 @@ def html_get_page(
|
||||
# Same reasoning as the redirect-loop handoffs.
|
||||
return _run_bypasser(current_url)
|
||||
logger.warning("403 error, giving up: %s", current_url)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
"Anna's Archive returned 403 (blocked) and no bypasser is enabled "
|
||||
"to solve the protection challenge.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
# 404 = Not found
|
||||
if status == _HTTP_STATUS_NOT_FOUND:
|
||||
logger.warning("404 error: %s", current_url)
|
||||
return _result("", current_url)
|
||||
return _fail(
|
||||
f"Anna's Archive returned 404 Not Found for {current_url}.", current_url
|
||||
)
|
||||
|
||||
# Try mirror/DNS rotation on retryable errors. A failure that proves the
|
||||
# mirror is unusable also drops it from this process's rotation, so the
|
||||
@@ -616,7 +725,16 @@ def html_get_page(
|
||||
else:
|
||||
logger.exception("Giving up after %s attempts: %s", retry_limit, current_url)
|
||||
|
||||
return _result("", current_url)
|
||||
if last_error is not None:
|
||||
return _fail(
|
||||
f"Could not reach Anna's Archive after {retry_limit} attempt(s): "
|
||||
f"{type(last_error).__name__}: {last_error}",
|
||||
current_url,
|
||||
)
|
||||
return _fail(
|
||||
"Could not reach Anna's Archive — all mirrors were exhausted without a usable response.",
|
||||
current_url,
|
||||
)
|
||||
|
||||
|
||||
def download_url(
|
||||
|
||||
@@ -683,8 +683,6 @@ def create_custom_getaddrinfo(
|
||||
source: str,
|
||||
provider_label: str,
|
||||
res: Sequence[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]],
|
||||
*,
|
||||
is_bypass: bool = False,
|
||||
) -> None:
|
||||
"""Emit a unified resolver log with the IPs returned.
|
||||
|
||||
@@ -692,7 +690,6 @@ def create_custom_getaddrinfo(
|
||||
source: Description of resolver source
|
||||
provider_label: Label for the DNS provider
|
||||
res: Resolution results
|
||||
is_bypass: If True, log at DEBUG level (for local/IP addresses)
|
||||
|
||||
"""
|
||||
# Skip logging entirely for localhost to reduce noise
|
||||
@@ -708,11 +705,7 @@ def create_custom_getaddrinfo(
|
||||
ip = sockaddr[0]
|
||||
if isinstance(ip, str):
|
||||
ips.append(ip)
|
||||
msg = f"Resolved {host_str} via {source} [{provider_label}]: {ips}"
|
||||
if is_bypass:
|
||||
logger.debug(msg)
|
||||
else:
|
||||
logger.info(msg)
|
||||
logger.debug("Resolved %s via %s [%s]: %s", host_str, source, provider_label, ips)
|
||||
|
||||
# Skip custom resolution for IP addresses, local addresses, or if skip check passes
|
||||
if (
|
||||
@@ -722,7 +715,7 @@ def create_custom_getaddrinfo(
|
||||
):
|
||||
# Quietly bypass custom resolution for IP/local targets
|
||||
res = original_getaddrinfo(host, port, family, socket_type, proto, flags)
|
||||
_log_results("system resolver (bypass)", "system", res, is_bypass=True)
|
||||
_log_results("system resolver (bypass)", "system", res)
|
||||
return res
|
||||
|
||||
results: list[tuple[AddressFamily, SocketKind, int, str, tuple[Any, ...]]] = []
|
||||
@@ -1493,6 +1486,11 @@ class AAMirrorSelector:
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize mirror state from the current AA configuration."""
|
||||
# Set by html_get_page at each give-up path so a caller that only sees the
|
||||
# returned empty page can still report *why* the fetch produced nothing
|
||||
# (403, 404, redirect loop, bypasser error, mirrors exhausted, ...) instead
|
||||
# of a blanket "network restricted" guess. None means "no failure recorded".
|
||||
self.last_failure: str | None = None
|
||||
self._ensure_fresh_state(reset_attempts=True)
|
||||
|
||||
def _ensure_fresh_state(self, *, reset_attempts: bool = False) -> None:
|
||||
|
||||
@@ -205,6 +205,7 @@ def process_folder_output(
|
||||
is_torrent=is_torrent,
|
||||
preserve_source=preserve_source,
|
||||
organization_mode=plan.organization_mode,
|
||||
source_root=source_path,
|
||||
)
|
||||
|
||||
if error:
|
||||
|
||||
@@ -52,7 +52,7 @@ def get_file_organization(*, is_audiobook: bool) -> str:
|
||||
"""Get the file organization mode for the content type."""
|
||||
key = "FILE_ORGANIZATION_AUDIOBOOK" if is_audiobook else "FILE_ORGANIZATION"
|
||||
mode = _config_text(core_config.config.get(key, "rename")).strip().lower()
|
||||
return mode if mode in ("none", "rename", "organize") else "rename"
|
||||
return mode if mode in ("none", "rename", "rename_and_group", "organize") else "rename"
|
||||
|
||||
|
||||
def get_template(*, is_audiobook: bool, organization_mode: str) -> str:
|
||||
|
||||
@@ -17,6 +17,7 @@ from shelfmark.core.naming import (
|
||||
sanitize_filename,
|
||||
)
|
||||
from shelfmark.core.utils import is_audiobook as check_audiobook
|
||||
from shelfmark.download.archive import is_archive
|
||||
from shelfmark.download.fs import (
|
||||
atomic_copy,
|
||||
atomic_hardlink,
|
||||
@@ -160,6 +161,24 @@ def _transfer_single_file(
|
||||
return atomic_move(source_path, dest_path, max_attempts=max_attempts), "move"
|
||||
|
||||
|
||||
def _group_folder_name(source_root: Path | None) -> str:
|
||||
"""Name the folder a grouped multi-file audiobook is transferred into.
|
||||
|
||||
A directory names the group directly. A file cannot hold several book files
|
||||
on its own, so a non-directory source that produced more than one means
|
||||
`collect_staged_files` extracted an archive: the stem is the release name and
|
||||
the suffix is packaging, which is why `Book.zip` groups into `Book/` rather
|
||||
than `Book.zip/` or, worse, not at all.
|
||||
"""
|
||||
if source_root is None:
|
||||
return ""
|
||||
if run_blocking_io(source_root.is_dir):
|
||||
return sanitize_filename(source_root.name)
|
||||
if is_archive(source_root):
|
||||
return sanitize_filename(source_root.stem)
|
||||
return ""
|
||||
|
||||
|
||||
def transfer_book_files(
|
||||
book_files: list[Path],
|
||||
destination: Path,
|
||||
@@ -169,6 +188,7 @@ def transfer_book_files(
|
||||
is_torrent: bool,
|
||||
preserve_source: bool = False,
|
||||
organization_mode: str | None = None,
|
||||
source_root: Path | None = None,
|
||||
) -> tuple[list[Path], str | None, dict[str, int]]:
|
||||
"""Transfer discovered book files into their final destination layout."""
|
||||
if not book_files:
|
||||
@@ -238,6 +258,13 @@ def transfer_book_files(
|
||||
|
||||
return final_paths, None, op_counts
|
||||
|
||||
transfer_destination = destination
|
||||
if is_audiobook and len(book_files) > 1 and organization_mode == "rename_and_group":
|
||||
source_folder = _group_folder_name(source_root)
|
||||
if source_folder:
|
||||
transfer_destination = destination / source_folder
|
||||
run_blocking_io(transfer_destination.mkdir, parents=True, exist_ok=True)
|
||||
|
||||
for book_file in book_files:
|
||||
if len(book_files) == 1 and organization_mode != "none":
|
||||
if not task.format:
|
||||
@@ -256,7 +283,7 @@ def transfer_book_files(
|
||||
else:
|
||||
filename = book_file.name
|
||||
|
||||
dest_path = destination / filename
|
||||
dest_path = transfer_destination / filename
|
||||
final_path, op = _transfer_single_file(
|
||||
book_file,
|
||||
dest_path,
|
||||
|
||||
@@ -15,6 +15,7 @@ source that is down at boot must not affect startup or health.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
||||
from shelfmark.core.config import config
|
||||
@@ -43,13 +44,27 @@ def _as_bool(value: object, *, default: bool) -> bool:
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _setting(key: str, default: object) -> object:
|
||||
"""Read a warm-up setting, preferring the deployment environment.
|
||||
|
||||
These keys are not in the settings registry, and ``config.get`` only consults the
|
||||
environment for keys it knows about - so reading config alone silently ignored
|
||||
SEARCH_WARMUP_ENABLED and always returned the default. Check os.environ first so
|
||||
the documented switches actually work.
|
||||
"""
|
||||
raw = os.environ.get(key)
|
||||
if raw is not None and raw.strip():
|
||||
return raw
|
||||
return config.get(key, default)
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""Whether the boot-time warm-up search should run."""
|
||||
if not _as_bool(config.get("SEARCH_WARMUP_ENABLED", True), default=True):
|
||||
if not _as_bool(_setting("SEARCH_WARMUP_ENABLED", True), default=True):
|
||||
return False
|
||||
# Nothing to warm if the source is off, and no challenge to pre-solve without
|
||||
# the bypasser - a plain search is fast enough not to need this.
|
||||
if not _as_bool(config.get("DIRECT_DOWNLOAD_ENABLED", True), default=True):
|
||||
if not _as_bool(_setting("DIRECT_DOWNLOAD_ENABLED", True), default=True):
|
||||
logger.debug("Search warm-up skipped: direct download disabled")
|
||||
return False
|
||||
return True
|
||||
@@ -57,7 +72,7 @@ def is_enabled() -> bool:
|
||||
|
||||
def warmup_query() -> str:
|
||||
"""The query used to warm the source."""
|
||||
raw = config.get("SEARCH_WARMUP_QUERY", _DEFAULT_QUERY)
|
||||
raw = _setting("SEARCH_WARMUP_QUERY", _DEFAULT_QUERY)
|
||||
query = str(raw).strip() if raw else ""
|
||||
return query or _DEFAULT_QUERY
|
||||
|
||||
|
||||
@@ -48,6 +48,10 @@ HARDCOVER_PAGE_SIZE = 25 # Hardcover API returns max 25 results per page
|
||||
HARDCOVER_MIN_AUTHOR_PARTS = 2
|
||||
HARDCOVER_MIN_TYPEAHEAD_QUERY_LENGTH = 2
|
||||
HARDCOVER_MAX_SERIES_OPTIONS = 7
|
||||
# Hardcover hands out short opaque tokens now ("hc_pat_...") instead of the ~500 char
|
||||
# JWTs it used to, so the length floor only applies to keys without that prefix.
|
||||
HARDCOVER_API_KEY_PREFIX = "hc_pat_"
|
||||
HARDCOVER_BEARER_PREFIX_PATTERN = re.compile(r"^bearer\s+", re.IGNORECASE)
|
||||
HARDCOVER_API_KEY_MIN_LENGTH = 100
|
||||
HARDCOVER_LIST_URL_PATTERN = re.compile(
|
||||
r"^/(?:@([\w.-]+)/)?lists?/([\w-]+)/?$",
|
||||
@@ -318,6 +322,7 @@ query SearchFieldOptions(
|
||||
fields: $fields,
|
||||
weights: $weights
|
||||
) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -536,13 +541,19 @@ SORT_MAPPING: dict[SortOrder, str] = {
|
||||
SortOrder.OLDEST: "release_year:asc",
|
||||
}
|
||||
|
||||
# Mapping from abstract search type to Hardcover fields parameter
|
||||
SEARCH_TYPE_FIELDS: dict[SearchType, str] = {
|
||||
SearchType.GENERAL: "title,isbns,series_names,author_names,alternative_titles",
|
||||
SearchType.TITLE: "title,alternative_titles",
|
||||
SearchType.AUTHOR: "author_names",
|
||||
# ISBN is handled separately via search_by_isbn()
|
||||
}
|
||||
# `fields` becomes Typesense's `query_by`, but Hardcover keeps `num_typos` and
|
||||
# `query_by_weights` as fixed-length presets per query_type. Passing a different
|
||||
# number of fields than the preset expects makes Typesense reject the whole search,
|
||||
# complaining that the number of num_typos values does not match the number of
|
||||
# query_by fields. So a Book search may only ever narrow to *these five* names --
|
||||
# a shorter list is rejected outright rather than searched, and any weights sent
|
||||
# alongside must match one-for-one.
|
||||
# Weights only bias ranking: a field weighted 0 still matches, so `fields` can no
|
||||
# longer restrict which fields a Book query looks at.
|
||||
BOOK_SEARCH_FIELDS = "title,alternative_titles,author_names,series_names,isbns"
|
||||
BOOK_SEARCH_FIELD_COUNT = 5
|
||||
BOOK_TITLE_WEIGHTS = "5,1,0,0,0"
|
||||
BOOK_TITLE_AUTHOR_WEIGHTS = "5,1,3,0,0"
|
||||
|
||||
SERIES_SEARCH_FIELDS = "name,books,author_name"
|
||||
SERIES_SEARCH_WEIGHTS = "2,1,1"
|
||||
@@ -550,22 +561,28 @@ SERIES_SEARCH_SORT = "_text_match:desc,readers_count:desc"
|
||||
AUTHOR_SUGGESTION_FIELDS = "name,name_personal,alternate_names"
|
||||
AUTHOR_SUGGESTION_WEIGHTS = "4,3,2"
|
||||
AUTHOR_SUGGESTION_SORT = "_text_match:desc,books_count:desc"
|
||||
TITLE_SUGGESTION_FIELDS = "title,alternative_titles"
|
||||
TITLE_SUGGESTION_WEIGHTS = "5,2"
|
||||
TITLE_SUGGESTION_FIELDS = BOOK_SEARCH_FIELDS
|
||||
TITLE_SUGGESTION_WEIGHTS = "5,2,0,0,0"
|
||||
TITLE_SUGGESTION_SORT = "_text_match:desc,users_count:desc"
|
||||
|
||||
# Hardcover forwards `sort` to Typesense's `sort_by` and rejects the whole search
|
||||
# if it does not like the value -- an unknown field, a bare field name with no
|
||||
# direction, more than three keys. A rejected search comes back as HTTP 200 with
|
||||
# no GraphQL errors and a null `results` body, which is otherwise indistinguishable
|
||||
# from "nothing matched". An empty sort is always accepted, so fall back to it and
|
||||
# keep the fallback sticky for a while rather than paying for a doomed request on
|
||||
# every search.
|
||||
SORT_FALLBACK = ""
|
||||
# from "nothing matched"; the reason only shows up in the sibling `error` field,
|
||||
# so every search asks for it. Dropping `sort` from the request is the one shape
|
||||
# Hardcover always accepts -- an empty string is a value like any other and has
|
||||
# been rejected too -- so retry that way and keep the fallback sticky for a while
|
||||
# rather than paying for a doomed request on every search.
|
||||
SORT_FALLBACK_TTL = 900.0
|
||||
_sort_fallback_until = 0.0
|
||||
|
||||
|
||||
def _without_sort(variables: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop `sort` entirely so Hardcover applies its own default ordering."""
|
||||
return {key: value for key, value in variables.items() if key != "sort"}
|
||||
|
||||
|
||||
def _search_payload_rejected(result: dict[str, Any] | None) -> bool:
|
||||
"""Report whether Hardcover answered a search with a null results body.
|
||||
|
||||
@@ -580,6 +597,17 @@ def _search_payload_rejected(result: dict[str, Any] | None) -> bool:
|
||||
return root["results"] is None
|
||||
|
||||
|
||||
def _search_rejection_reason(result: dict[str, Any] | None) -> str:
|
||||
"""Return Hardcover's explanation for a rejected search, if it sent one."""
|
||||
if not isinstance(result, dict):
|
||||
return ""
|
||||
root = result.get("search", result)
|
||||
if not isinstance(root, dict):
|
||||
return ""
|
||||
error = root.get("error")
|
||||
return error.strip() if isinstance(error, str) else ""
|
||||
|
||||
|
||||
def _combine_headline_description(headline: str | None, description: str | None) -> str | None:
|
||||
"""Combine headline (tagline) and description into a single description."""
|
||||
if headline and description:
|
||||
@@ -646,7 +674,7 @@ def _normalize_series_position(value: Any) -> float | None:
|
||||
def _normalize_hardcover_api_key(value: object) -> str:
|
||||
"""Normalize Hardcover API keys, stripping copied auth-header prefixes."""
|
||||
normalized_value = normalize_optional_text(value) or ""
|
||||
return normalized_value.removeprefix("Bearer ").strip()
|
||||
return HARDCOVER_BEARER_PREFIX_PATTERN.sub("", normalized_value.strip()).strip()
|
||||
|
||||
|
||||
def _normalize_search_text(value: str) -> str:
|
||||
@@ -1012,13 +1040,15 @@ class HardcoverProvider(MetadataProvider):
|
||||
"""Build search query, fields, and weights based on provided values.
|
||||
|
||||
Returns (query, fields, weights) tuple. Fields/weights are None for general search.
|
||||
A narrowed search still sends all of BOOK_SEARCH_FIELDS -- Hardcover rejects a
|
||||
shorter list outright -- and leans on the weights to rank the wanted field first.
|
||||
"""
|
||||
if author and not title and not series:
|
||||
return author, None, None
|
||||
if title and not author and not series:
|
||||
return title, "title,alternative_titles", "5,1"
|
||||
return title, BOOK_SEARCH_FIELDS, BOOK_TITLE_WEIGHTS
|
||||
if author and title and not series:
|
||||
return f"{title} {author}", "title,alternative_titles,author_names", "5,1,3"
|
||||
return f"{title} {author}", BOOK_SEARCH_FIELDS, BOOK_TITLE_AUTHOR_WEIGHTS
|
||||
return default_query, None, None
|
||||
|
||||
def _detect_list_url(self, query: str) -> tuple[str | None, str] | None:
|
||||
@@ -2384,6 +2414,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
graphql_query = """
|
||||
query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String, $fields: String, $weights: String) {
|
||||
search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort, fields: $fields, weights: $weights) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -2392,6 +2423,7 @@ class HardcoverProvider(MetadataProvider):
|
||||
graphql_query = """
|
||||
query SearchBooks($query: String!, $limit: Int!, $page: Int!, $sort: String) {
|
||||
search(query: $query, query_type: "Book", per_page: $limit, page: $page, sort: $sort) {
|
||||
error
|
||||
results
|
||||
}
|
||||
}
|
||||
@@ -2690,33 +2722,42 @@ class HardcoverProvider(MetadataProvider):
|
||||
|
||||
sort = variables.get("sort")
|
||||
if sort and time.monotonic() < _sort_fallback_until:
|
||||
variables = {**variables, "sort": SORT_FALLBACK}
|
||||
variables = _without_sort(variables)
|
||||
sort = None
|
||||
|
||||
result = self._execute_query(query, variables)
|
||||
if not _search_payload_rejected(result):
|
||||
return result
|
||||
|
||||
reason = _search_rejection_reason(result)
|
||||
if not sort:
|
||||
logger.error(
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s) and returned "
|
||||
"no result body",
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s): %s",
|
||||
variables.get("queryType", "Book"),
|
||||
variables.get("fields"),
|
||||
reason or "no error message",
|
||||
)
|
||||
return None
|
||||
|
||||
retry = self._execute_query(query, _without_sort(variables))
|
||||
if _search_payload_rejected(retry):
|
||||
# The sort was not the culprit, so leave sorting alone for other searches.
|
||||
logger.error(
|
||||
"Hardcover rejected this search (query_type=%s, fields=%s) with and without "
|
||||
"a sort order: %s",
|
||||
variables.get("queryType", "Book"),
|
||||
variables.get("fields"),
|
||||
_search_rejection_reason(retry) or reason or "no error message",
|
||||
)
|
||||
return None
|
||||
|
||||
logger.warning(
|
||||
"Hardcover rejected sort '%s'; retrying searches without a sort order for %ss",
|
||||
"Hardcover rejected sort '%s' (%s); dropping the sort order from searches for %ss",
|
||||
sort,
|
||||
reason or "no error message",
|
||||
int(SORT_FALLBACK_TTL),
|
||||
)
|
||||
_sort_fallback_until = time.monotonic() + SORT_FALLBACK_TTL
|
||||
|
||||
retry = self._execute_query(query, {**variables, "sort": SORT_FALLBACK})
|
||||
if _search_payload_rejected(retry):
|
||||
logger.error("Hardcover rejected this search even without a sort order")
|
||||
return None
|
||||
return retry
|
||||
|
||||
def _parse_search_result(self, item: dict) -> BookMetadata | None:
|
||||
@@ -2982,12 +3023,13 @@ def _test_hardcover_connection(current_values: dict[str, Any] | None = None) ->
|
||||
_save_connected_user(None, None)
|
||||
return {"success": False, "message": "API key is required"}
|
||||
|
||||
if key_len < HARDCOVER_API_KEY_MIN_LENGTH:
|
||||
is_prefixed_key = api_key.startswith(HARDCOVER_API_KEY_PREFIX)
|
||||
if not is_prefixed_key and key_len < HARDCOVER_API_KEY_MIN_LENGTH:
|
||||
return {
|
||||
"success": False,
|
||||
"message": (
|
||||
f"API key seems too short ({key_len} chars). "
|
||||
f"Expected {HARDCOVER_API_KEY_MIN_LENGTH}+ chars."
|
||||
f"API key seems too short ({key_len} chars). Expected a key starting "
|
||||
f"with {HARDCOVER_API_KEY_PREFIX} or {HARDCOVER_API_KEY_MIN_LENGTH}+ chars."
|
||||
),
|
||||
}
|
||||
|
||||
@@ -3094,7 +3136,7 @@ def hardcover_settings() -> list[SettingsField]:
|
||||
PasswordField(
|
||||
key="HARDCOVER_API_KEY",
|
||||
label="API Key",
|
||||
description="Get your API key from hardcover.app/account/api",
|
||||
description="Get your API key from hardcover.app/account/api (starts with hc_pat_)",
|
||||
required=True,
|
||||
),
|
||||
ActionButton(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
from urllib.parse import quote, quote_plus
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
@@ -10,6 +10,7 @@ from bs4 import BeautifulSoup
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.download import http as downloader
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
@@ -98,8 +99,10 @@ def _encode_search_query(query: str, *, exact_phrase: bool) -> str:
|
||||
and not (search_query.startswith('"') and search_query.endswith('"'))
|
||||
):
|
||||
search_query = f'"{search_query}"'
|
||||
# Keep ABB-friendly encoding style (spaces as '+') while percent-encoding quotes.
|
||||
return search_query.replace('"', "%22").replace(" ", "+")
|
||||
# Keep ABB's space-as-'+' style, but percent-encode everything else: a bare
|
||||
# '&' would otherwise start a new query parameter, '%' would open an invalid
|
||||
# escape, and a literal '+' would arrive as a space.
|
||||
return quote_plus(search_query)
|
||||
|
||||
|
||||
def _normalize_result_url(url: str, hostname: str) -> str:
|
||||
@@ -153,6 +156,9 @@ def search_audiobookbay(
|
||||
|
||||
"""
|
||||
results = []
|
||||
# ABB matches the stored, untexturized title, so a curly apostrophe reaching
|
||||
# the search returns nothing at all rather than merely ranking worse.
|
||||
query = normalize_search_punctuation(query)
|
||||
rate_limit_delay = _coerce_non_negative_float(config.get("ABB_RATE_LIMIT_DELAY", 1.0), 1.0)
|
||||
session = requests.Session()
|
||||
|
||||
|
||||
@@ -23,7 +23,11 @@ from shelfmark.release_sources import (
|
||||
register_source,
|
||||
)
|
||||
from shelfmark.release_sources.audiobookbay import scraper
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_hostname, parse_size
|
||||
from shelfmark.release_sources.audiobookbay.utils import (
|
||||
normalize_hostname,
|
||||
normalize_search_punctuation,
|
||||
parse_size,
|
||||
)
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
MIN_RELEVANCE_QUERY_WORD_LENGTH = 2
|
||||
@@ -227,10 +231,12 @@ class AudiobookBaySource(ReleaseSource):
|
||||
deduped_queries[index + 1].lower(),
|
||||
)
|
||||
|
||||
# Extract query words for relevance checking
|
||||
# Extract query words for relevance checking. Both sides of the
|
||||
# comparison are punctuation-normalized: scraped titles carry the
|
||||
# typographic forms WordPress renders, queries carry the ASCII ones.
|
||||
query_words = {
|
||||
word.lower()
|
||||
for word in query_lower.split()
|
||||
for word in normalize_search_punctuation(query_lower).split()
|
||||
if len(word) > MIN_RELEVANCE_QUERY_WORD_LENGTH
|
||||
}
|
||||
|
||||
@@ -239,7 +245,7 @@ class AudiobookBaySource(ReleaseSource):
|
||||
try:
|
||||
raw_title = result["title"]
|
||||
title, author = _split_title_and_author(raw_title)
|
||||
title_for_filter = raw_title.lower()
|
||||
title_for_filter = normalize_search_punctuation(raw_title).lower()
|
||||
|
||||
# Basic relevance check: ensure title contains at least one query word
|
||||
# This filters out homepage "Latest" feed items that may leak through
|
||||
|
||||
@@ -2,6 +2,63 @@
|
||||
|
||||
import re
|
||||
|
||||
# WordPress texturizes punctuation on output only: a post stored as "The
|
||||
# Stranger's Wife" is rendered as "The Stranger’s Wife". ABB's search matches the
|
||||
# stored value, so a query carrying the typographic form matches nothing -- and
|
||||
# because ABB ANDs its search terms, one such term empties the entire result set.
|
||||
# Book metadata and phone keyboards both hand us the typographic forms, so map
|
||||
# them back before they reach a search or a title comparison.
|
||||
_ASCII_PUNCTUATION = str.maketrans(
|
||||
{
|
||||
# Single quotes
|
||||
"‘": "'", # left single quotation mark
|
||||
"’": "'", # right single quotation mark
|
||||
"‚": "'", # single low-9 quotation mark
|
||||
"‛": "'", # single high-reversed-9 quotation mark
|
||||
"′": "'", # prime
|
||||
"´": "'", # acute accent
|
||||
"`": "'", # grave accent
|
||||
# Double quotes
|
||||
"“": '"', # left double quotation mark
|
||||
"”": '"', # right double quotation mark
|
||||
"„": '"', # double low-9 quotation mark
|
||||
"‟": '"', # double high-reversed-9 quotation mark
|
||||
"″": '"', # double prime
|
||||
# Dashes
|
||||
"‐": "-", # hyphen
|
||||
"‑": "-", # non-breaking hyphen
|
||||
"‒": "-", # figure dash
|
||||
"–": "-", # en dash
|
||||
"—": "-", # em dash
|
||||
"―": "-", # horizontal bar
|
||||
"−": "-", # minus sign
|
||||
"﹘": "-", # small em dash
|
||||
"﹣": "-", # small hyphen-minus
|
||||
"－": "-", # fullwidth hyphen-minus
|
||||
# Ellipsis
|
||||
"…": "...", # horizontal ellipsis
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def normalize_search_punctuation(text: str) -> str:
|
||||
"""Replace typographic punctuation with the ASCII forms ABB stores.
|
||||
|
||||
Each character is mapped individually rather than collapsing runs, so an
|
||||
ASCII "--" is left alone: only characters ABB cannot have stored are
|
||||
rewritten.
|
||||
|
||||
Args:
|
||||
text: A search query, or a scraped title being compared against one.
|
||||
|
||||
Returns:
|
||||
The text with curly quotes, dashes and ellipses mapped to ASCII.
|
||||
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
return text.translate(_ASCII_PUNCTUATION)
|
||||
|
||||
|
||||
def normalize_hostname(raw: str | None) -> str:
|
||||
"""Normalize a user-supplied hostname for URL construction.
|
||||
|
||||
@@ -565,9 +565,15 @@ _CHALLENGE_MARKERS = (
|
||||
|
||||
|
||||
def _looks_like_aa_page(html: str) -> bool:
|
||||
"""Whether ``html`` is recognisably Anna's Archive, or a challenge in front of it."""
|
||||
"""Whether ``html`` is recognisably Anna's Archive itself."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in (*_AA_PAGE_MARKERS, *_CHALLENGE_MARKERS))
|
||||
return any(marker in lowered for marker in _AA_PAGE_MARKERS)
|
||||
|
||||
|
||||
def _looks_like_challenge_page(html: str) -> bool:
|
||||
"""Whether ``html`` is a protection interstitial rather than the site behind it."""
|
||||
lowered = html.lower()
|
||||
return any(marker in lowered for marker in _CHALLENGE_MARKERS)
|
||||
|
||||
|
||||
def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[str, Tag | None]:
|
||||
@@ -584,9 +590,13 @@ def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[s
|
||||
attempt_url, selector=selector, allow_bypasser_fallback=True
|
||||
)
|
||||
if not response:
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
# Network/mirror exhaustion path bubbles up so API can notify clients.
|
||||
# html_get_page records the concrete give-up reason on the selector; fall
|
||||
# back to the generic line only if nothing was recorded.
|
||||
detail = getattr(selector, "last_failure", None) or (
|
||||
"Network restricted or mirrors are blocked."
|
||||
)
|
||||
raise SearchUnavailableError(f"Unable to reach download source. {detail}")
|
||||
|
||||
html = _html_response_text(response)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
@@ -596,9 +606,22 @@ def _fetch_search_table(url: str, selector: network.AAMirrorSelector) -> tuple[s
|
||||
if table is not None:
|
||||
msg = f"Expected results table tag, got {type(table).__name__}"
|
||||
raise TypeError(msg)
|
||||
if "No files found." in html or _looks_like_aa_page(html):
|
||||
# A real AA response - either genuinely empty, or a shape the caller
|
||||
# should report as drift. Not the mirror's fault.
|
||||
if "No files found." in html:
|
||||
# A real, genuinely empty answer from a healthy mirror.
|
||||
return html, None
|
||||
if _looks_like_challenge_page(html):
|
||||
# The bypass did not actually clear the protection - the interstitial is
|
||||
# what came back. Rotating is pointless (every mirror shares the same
|
||||
# protection) and reporting it as an empty result is worse: the user is
|
||||
# told their query found nothing when the search never ran.
|
||||
msg = (
|
||||
"Anna's Archive answered with an unsolved protection challenge. "
|
||||
"Check that the bypasser is reachable and working."
|
||||
)
|
||||
raise SearchUnavailableError(msg)
|
||||
if _looks_like_aa_page(html):
|
||||
# A real AA response in a shape the caller should report as drift.
|
||||
# Not the mirror's fault.
|
||||
return html, None
|
||||
|
||||
new_base, action = selector.next_mirror_or_rotate_dns(
|
||||
@@ -728,8 +751,10 @@ def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseR
|
||||
html = downloader.html_get_page(url, selector=selector, allow_bypasser_fallback=True)
|
||||
|
||||
if not html:
|
||||
msg = "Unable to reach download source. Network restricted or mirrors are blocked."
|
||||
raise SearchUnavailableError(msg)
|
||||
detail = getattr(selector, "last_failure", None) or (
|
||||
"Network restricted or mirrors are blocked."
|
||||
)
|
||||
raise SearchUnavailableError(f"Unable to reach download source. {detail}")
|
||||
|
||||
soup = BeautifulSoup(_html_response_text(html), "html.parser")
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any, TypedDict
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.core.config import config
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.download.network import get_ssl_verify
|
||||
@@ -18,6 +19,19 @@ logger = setup_logger(__name__)
|
||||
_HTTP_STATUS_UNAUTHORIZED = HTTPStatus.UNAUTHORIZED
|
||||
_BOOK_CATEGORY_RANGE_START = 7000
|
||||
_BOOK_CATEGORY_RANGE_END = 8000
|
||||
|
||||
# Prowlarr's own JSON endpoints (status, indexer list) read local state and answer
|
||||
# in milliseconds, so they keep a short timeout. A Torznab search is different: it
|
||||
# is Prowlarr proxying a live request to the tracker, which for a Cloudflare-fronted
|
||||
# indexer means waiting on FlareSolverr to solve a challenge. A cold challenge
|
||||
# routinely runs past a minute, so indexer searches get their own, longer budget.
|
||||
DEFAULT_INDEXER_TIMEOUT_SECONDS = 90
|
||||
MIN_INDEXER_TIMEOUT_SECONDS = 5
|
||||
MAX_INDEXER_TIMEOUT_SECONDS = 300
|
||||
|
||||
# Connecting to Prowlarr itself is a LAN hop; only the read is allowed to be slow.
|
||||
_CONNECT_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
_PROWLARR_CLIENT_ERRORS = (
|
||||
requests.exceptions.RequestException,
|
||||
OSError,
|
||||
@@ -27,6 +41,37 @@ _PROWLARR_CLIENT_ERRORS = (
|
||||
)
|
||||
|
||||
|
||||
class ProwlarrSearchError(RuntimeError):
|
||||
"""A Torznab search could not be completed.
|
||||
|
||||
Deliberately distinct from an empty result list. Reporting a failed search as
|
||||
"this indexer has nothing" is what turns a slow FlareSolverr challenge into
|
||||
"No releases found for this book" in the UI (#1249), and it also makes the
|
||||
auto-expand retry fire a second request on top of the one still running.
|
||||
"""
|
||||
|
||||
|
||||
def resolve_indexer_timeout(timeout: object = None) -> int:
|
||||
"""Resolve the per-indexer search timeout, falling back to config.
|
||||
|
||||
Out-of-range and unparsable values are clamped rather than rejected: this
|
||||
feeds an HTTP timeout, and a bad setting should not take searching down.
|
||||
"""
|
||||
if timeout is None:
|
||||
timeout = config.get("PROWLARR_INDEXER_TIMEOUT", DEFAULT_INDEXER_TIMEOUT_SECONDS)
|
||||
|
||||
resolved = coerce_int_like(timeout)
|
||||
if resolved is None:
|
||||
logger.warning(
|
||||
"Invalid PROWLARR_INDEXER_TIMEOUT %r - using %ss",
|
||||
timeout,
|
||||
DEFAULT_INDEXER_TIMEOUT_SECONDS,
|
||||
)
|
||||
return DEFAULT_INDEXER_TIMEOUT_SECONDS
|
||||
|
||||
return max(MIN_INDEXER_TIMEOUT_SECONDS, min(MAX_INDEXER_TIMEOUT_SECONDS, resolved))
|
||||
|
||||
|
||||
class IndexerSeedSettings(TypedDict, total=False):
|
||||
ratio_limit: float
|
||||
seeding_time_limit_minutes: int
|
||||
@@ -77,11 +122,23 @@ def _get_field_value(fields: object, name: str) -> object | None:
|
||||
class ProwlarrClient:
|
||||
"""Client for interacting with the Prowlarr API."""
|
||||
|
||||
def __init__(self, url: str, api_key: str, timeout: int = 30) -> None:
|
||||
"""Initialize the API client with base URL, key, and timeout."""
|
||||
def __init__(
|
||||
self, url: str, api_key: str, timeout: int = 30, indexer_timeout: int | None = None
|
||||
) -> None:
|
||||
"""Initialize the API client with base URL, key, and timeouts.
|
||||
|
||||
Args:
|
||||
url: Prowlarr base URL.
|
||||
api_key: Prowlarr API key.
|
||||
timeout: Timeout for Prowlarr's own JSON endpoints.
|
||||
indexer_timeout: Timeout for Torznab searches, which Prowlarr proxies
|
||||
out to the tracker. Defaults to PROWLARR_INDEXER_TIMEOUT.
|
||||
|
||||
"""
|
||||
self.base_url = normalize_http_url(url)
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
self.indexer_timeout = resolve_indexer_timeout(indexer_timeout)
|
||||
self._session = requests.Session()
|
||||
self._session.headers.update(
|
||||
{
|
||||
@@ -307,6 +364,12 @@ class ProwlarrClient:
|
||||
|
||||
This returns richer fields (e.g., author/booktitle, torznab tags like
|
||||
FreeLeech) than the JSON /api/v1/search endpoint.
|
||||
|
||||
Raises:
|
||||
ProwlarrSearchError: The search could not be completed. An empty list
|
||||
strictly means the indexer answered with no matches, never that
|
||||
the request timed out or errored.
|
||||
|
||||
"""
|
||||
if not query:
|
||||
return []
|
||||
@@ -329,7 +392,7 @@ class ProwlarrClient:
|
||||
response = self._session.get(
|
||||
url=url,
|
||||
params=params,
|
||||
timeout=self.timeout,
|
||||
timeout=(_CONNECT_TIMEOUT_SECONDS, self.indexer_timeout),
|
||||
headers={
|
||||
# Override the session default JSON accept header.
|
||||
"Accept": "application/rss+xml, application/xml;q=0.9, */*;q=0.8"
|
||||
@@ -347,9 +410,20 @@ class ProwlarrClient:
|
||||
for r in results:
|
||||
if r.get("indexerId") is None:
|
||||
r["indexerId"] = int(indexer_id)
|
||||
except Exception:
|
||||
except requests.exceptions.Timeout as e:
|
||||
logger.warning(
|
||||
"Prowlarr Torznab search for indexer %s timed out after %ss. An indexer "
|
||||
"behind FlareSolverr can need far longer than that on a cold Cloudflare "
|
||||
"challenge - raise PROWLARR_INDEXER_TIMEOUT if this keeps happening.",
|
||||
indexer_id,
|
||||
self.indexer_timeout,
|
||||
)
|
||||
msg = f"indexer {indexer_id} did not respond within {self.indexer_timeout}s"
|
||||
raise ProwlarrSearchError(msg) from e
|
||||
except Exception as e:
|
||||
logger.exception("Prowlarr Torznab search failed for indexer %s", indexer_id)
|
||||
return []
|
||||
msg = f"indexer {indexer_id} search failed: {e}"
|
||||
raise ProwlarrSearchError(msg) from e
|
||||
else:
|
||||
return results
|
||||
|
||||
|
||||
@@ -10,12 +10,18 @@ from shelfmark.core.settings_registry import (
|
||||
CheckboxField,
|
||||
HeadingField,
|
||||
MultiSelectField,
|
||||
NumberField,
|
||||
PasswordField,
|
||||
SettingsField,
|
||||
TextField,
|
||||
register_settings,
|
||||
)
|
||||
from shelfmark.core.utils import normalize_http_url
|
||||
from shelfmark.release_sources.prowlarr.api import (
|
||||
DEFAULT_INDEXER_TIMEOUT_SECONDS,
|
||||
MAX_INDEXER_TIMEOUT_SECONDS,
|
||||
MIN_INDEXER_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
# ==================== Dynamic Options Loaders ====================
|
||||
|
||||
@@ -183,6 +189,20 @@ def prowlarr_config_settings() -> list[SettingsField]:
|
||||
default=[],
|
||||
show_when={"field": "PROWLARR_ENABLED", "value": True},
|
||||
),
|
||||
NumberField(
|
||||
key="PROWLARR_INDEXER_TIMEOUT",
|
||||
label="Indexer Search Timeout (seconds)",
|
||||
description=(
|
||||
"How long to wait for a single indexer to answer a search. Indexers behind "
|
||||
"FlareSolverr can need 90 seconds or more while a cold Cloudflare challenge "
|
||||
"is solved; raise this if searches come back empty and the Prowlarr log "
|
||||
"shows the search still running."
|
||||
),
|
||||
default=DEFAULT_INDEXER_TIMEOUT_SECONDS,
|
||||
min_value=MIN_INDEXER_TIMEOUT_SECONDS,
|
||||
max_value=MAX_INDEXER_TIMEOUT_SECONDS,
|
||||
show_when={"field": "PROWLARR_ENABLED", "value": True},
|
||||
),
|
||||
CheckboxField(
|
||||
key="PROWLARR_AUTO_EXPAND",
|
||||
label="Auto-expand search on no results",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING, ClassVar, NoReturn
|
||||
|
||||
@@ -30,9 +31,14 @@ from shelfmark.release_sources import (
|
||||
ReleaseProtocol,
|
||||
ReleaseSource,
|
||||
SortOption,
|
||||
SourceUnavailableError,
|
||||
register_source,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, ProwlarrClient
|
||||
from shelfmark.release_sources.prowlarr.api import (
|
||||
IndexerSeedSettings,
|
||||
ProwlarrClient,
|
||||
ProwlarrSearchError,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.cache import cache_release
|
||||
from shelfmark.release_sources.prowlarr.utils import (
|
||||
build_source_id,
|
||||
@@ -50,10 +56,10 @@ _PROWLARR_SOURCE_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, Val
|
||||
# Prowlarr indexer priority is 1-50 and lower is preferred; unknown sorts last.
|
||||
_UNRANKED_INDEXER_RANK = 51
|
||||
|
||||
# Errors that can surface from ProwlarrClient.get_indexer_seed_settings(). The
|
||||
# Errors that can surface from a ProwlarrClient call that talks to Prowlarr. The
|
||||
# client raises requests exceptions (subclasses of OSError via IOError lineage
|
||||
# is not guaranteed), so include RequestException explicitly.
|
||||
_PROWLARR_SEED_SETTINGS_ERRORS = (*_PROWLARR_SOURCE_ERRORS, requests.exceptions.RequestException)
|
||||
_PROWLARR_REQUEST_ERRORS = (*_PROWLARR_SOURCE_ERRORS, requests.exceptions.RequestException)
|
||||
|
||||
|
||||
def _raise_timeout_error(message: str) -> NoReturn:
|
||||
@@ -232,6 +238,35 @@ ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS
|
||||
# Backend safeguard: cap total Prowlarr search time per request.
|
||||
PROWLARR_SEARCH_TIMEOUT_SECONDS = 120.0
|
||||
|
||||
# The overall budget has to leave room for at least a couple of indexers to spend
|
||||
# their full per-indexer timeout, otherwise raising PROWLARR_INDEXER_TIMEOUT for a
|
||||
# Cloudflare-fronted tracker just moves the cutoff here. Capped short of the
|
||||
# gunicorn worker timeout (300s) so the worker is never the thing that gives up.
|
||||
_MAX_SEARCH_BUDGET_SECONDS = 240.0
|
||||
|
||||
|
||||
def _search_budget_seconds(indexer_timeout: int) -> float:
|
||||
"""Total time one Prowlarr search may spend, scaled to the per-indexer timeout."""
|
||||
return min(
|
||||
_MAX_SEARCH_BUDGET_SECONDS,
|
||||
max(PROWLARR_SEARCH_TIMEOUT_SECONDS, indexer_timeout * 2.0),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _IndexerSearchOutcome:
|
||||
"""What one pass over the target indexers produced.
|
||||
|
||||
Separates "every indexer answered, none had this book" from "the indexers
|
||||
never answered", which the caller has to tell apart before it decides to
|
||||
auto-expand or to report the search as failed.
|
||||
"""
|
||||
|
||||
results: list[dict]
|
||||
attempted: int = 0
|
||||
failed: int = 0
|
||||
last_error: str | None = None
|
||||
|
||||
|
||||
def _extract_format(title: str) -> str | None:
|
||||
"""Extract ebook/audiobook format from release title (extension, bracketed, or standalone)."""
|
||||
@@ -539,7 +574,7 @@ def _fetch_indexer_seed_settings(
|
||||
"""Fetch per-indexer share limits, falling back to last-known-good on failure."""
|
||||
try:
|
||||
fetched = client.get_indexer_seed_settings(restrict_to=indexer_ids)
|
||||
except _PROWLARR_SEED_SETTINGS_ERRORS:
|
||||
except _PROWLARR_REQUEST_ERRORS:
|
||||
with _seed_settings_lock:
|
||||
fallback = dict(_last_known_seed_settings)
|
||||
logger.warning(
|
||||
@@ -895,8 +930,16 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
try:
|
||||
auto_expand_enabled = config.get("PROWLARR_AUTO_EXPAND", False)
|
||||
deadline = time.monotonic() + PROWLARR_SEARCH_TIMEOUT_SECONDS
|
||||
enabled_indexers = client.get_enabled_indexers_detailed()
|
||||
search_budget = _search_budget_seconds(client.indexer_timeout)
|
||||
deadline = time.monotonic() + search_budget
|
||||
try:
|
||||
enabled_indexers = client.get_enabled_indexers_detailed(raise_on_error=True)
|
||||
except _PROWLARR_REQUEST_ERRORS as e:
|
||||
# Prowlarr itself is unreachable. Swallowing this leaves the search
|
||||
# with no indexers to query, which the UI renders as "No releases
|
||||
# found for this book" - the same lie as a swallowed timeout (#1249).
|
||||
msg = f"could not reach Prowlarr: {e}"
|
||||
raise SourceUnavailableError(msg) from e
|
||||
indexer_priority = _build_indexer_priority(enabled_indexers)
|
||||
# Some indexers benefit from title+author queries and extra format detection.
|
||||
enriched_indexer_ids = client.get_enriched_indexer_ids(
|
||||
@@ -911,18 +954,16 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
def _check_timeout() -> None:
|
||||
if time.monotonic() > deadline:
|
||||
_raise_timeout_error(
|
||||
f"Prowlarr search timed out after {int(PROWLARR_SEARCH_TIMEOUT_SECONDS)}s"
|
||||
)
|
||||
_raise_timeout_error(f"Prowlarr search timed out after {int(search_budget)}s")
|
||||
|
||||
def search_indexers(
|
||||
query: str, cats: list[int] | None, *, enriched_query: str | None = None
|
||||
) -> list[dict]:
|
||||
) -> _IndexerSearchOutcome:
|
||||
"""Search indexers with given categories via Torznab/Newznab."""
|
||||
results: list[dict] = []
|
||||
outcome = _IndexerSearchOutcome(results=[])
|
||||
target_indexer_ids = self._get_search_indexer_ids(client, indexer_ids, cats)
|
||||
if not target_indexer_ids:
|
||||
return results
|
||||
return outcome
|
||||
|
||||
for indexer_id in target_indexer_ids:
|
||||
_check_timeout()
|
||||
@@ -931,19 +972,31 @@ class ProwlarrSource(ReleaseSource):
|
||||
if indexer_id in enriched_indexer_ids_set and enriched_query
|
||||
else query
|
||||
)
|
||||
raw = client.torznab_search(
|
||||
indexer_id=indexer_id,
|
||||
query=indexer_query,
|
||||
categories=cats,
|
||||
search_type="book",
|
||||
)
|
||||
outcome.attempted += 1
|
||||
try:
|
||||
raw = client.torznab_search(
|
||||
indexer_id=indexer_id,
|
||||
query=indexer_query,
|
||||
categories=cats,
|
||||
search_type="book",
|
||||
)
|
||||
except ProwlarrSearchError as e:
|
||||
# One unreachable indexer must not sink the others, but it
|
||||
# is not "no results" either - record it so the caller can
|
||||
# report a failed search instead of an empty one.
|
||||
outcome.failed += 1
|
||||
outcome.last_error = str(e)
|
||||
continue
|
||||
if raw:
|
||||
results.extend(raw)
|
||||
outcome.results.extend(raw)
|
||||
|
||||
return results
|
||||
return outcome
|
||||
|
||||
seen_keys: set[tuple[int | None, str]] = set()
|
||||
all_results: list[dict] = []
|
||||
attempted_searches = 0
|
||||
failed_searches = 0
|
||||
last_search_error: str | None = None
|
||||
|
||||
for idx, variant in enumerate(variants, start=1):
|
||||
_check_timeout()
|
||||
@@ -953,23 +1006,39 @@ class ProwlarrSource(ReleaseSource):
|
||||
if len(variants) > 1:
|
||||
logger.debug("Prowlarr query %s/%s: '%s'", idx, len(variants), query)
|
||||
|
||||
raw_results = search_indexers(
|
||||
outcome = search_indexers(
|
||||
query=query, cats=categories, enriched_query=enriched_query
|
||||
)
|
||||
|
||||
# Auto-expand: if no results with categories and auto-expand enabled, retry without
|
||||
if not raw_results and categories and auto_expand_enabled:
|
||||
# Auto-expand: if no results with categories and auto-expand enabled, retry without.
|
||||
# Only when every indexer actually answered: a failed search says nothing about
|
||||
# whether the category filter is what hid the book, and retrying it stacks a second
|
||||
# request on an indexer that is still busy solving a Cloudflare challenge (#1249).
|
||||
if (
|
||||
not outcome.results
|
||||
and not outcome.failed
|
||||
and categories
|
||||
and auto_expand_enabled
|
||||
):
|
||||
_check_timeout()
|
||||
logger.info(
|
||||
"Prowlarr: no results for query '%s' with category filter, auto-expanding search",
|
||||
query,
|
||||
)
|
||||
raw_results = search_indexers(
|
||||
expanded = search_indexers(
|
||||
query=query, cats=None, enriched_query=enriched_query
|
||||
)
|
||||
outcome.results = expanded.results
|
||||
outcome.attempted += expanded.attempted
|
||||
outcome.failed += expanded.failed
|
||||
outcome.last_error = expanded.last_error or outcome.last_error
|
||||
self.last_search_type = "expanded"
|
||||
|
||||
for r in raw_results:
|
||||
attempted_searches += outcome.attempted
|
||||
failed_searches += outcome.failed
|
||||
last_search_error = outcome.last_error or last_search_error
|
||||
|
||||
for r in outcome.results:
|
||||
key = _result_dedup_key(r)
|
||||
if key is not None:
|
||||
if key in seen_keys:
|
||||
@@ -977,6 +1046,14 @@ class ProwlarrSource(ReleaseSource):
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
|
||||
if failed_searches:
|
||||
logger.warning(
|
||||
"Prowlarr: %s of %s indexer searches failed (%s)",
|
||||
failed_searches,
|
||||
attempted_searches,
|
||||
last_search_error,
|
||||
)
|
||||
|
||||
if config.get("PROWLARR_COLLAPSE_DUPLICATES", True):
|
||||
before_collapse = len(all_results)
|
||||
all_results = _collapse_duplicate_indexer_results(all_results, indexer_priority)
|
||||
@@ -1033,6 +1110,10 @@ class ProwlarrSource(ReleaseSource):
|
||||
else:
|
||||
logger.debug("Prowlarr: no results found")
|
||||
|
||||
except SourceUnavailableError:
|
||||
# Already carries its own message for the caller to surface; the blanket
|
||||
# handler below would turn it back into a silent empty result.
|
||||
raise
|
||||
except TimeoutError as e:
|
||||
logger.warning("Prowlarr search timed out: %s", e)
|
||||
raise
|
||||
@@ -1040,6 +1121,15 @@ class ProwlarrSource(ReleaseSource):
|
||||
logger.exception("Prowlarr search failed")
|
||||
return []
|
||||
else:
|
||||
# An empty list is the UI's "No releases found for this book", so it has
|
||||
# to mean the indexers answered and had nothing. When they failed instead,
|
||||
# say so rather than blaming the book (#1249).
|
||||
if not results and failed_searches:
|
||||
msg = (
|
||||
f"{failed_searches} of {attempted_searches} indexer searches failed "
|
||||
f"({last_search_error})"
|
||||
)
|
||||
raise SourceUnavailableError(msg)
|
||||
return results
|
||||
|
||||
def is_available(self) -> bool:
|
||||
|
||||
Generated
+89
-142
@@ -19,7 +19,7 @@
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.4",
|
||||
"@vitejs/plugin-react": "^6.0.5",
|
||||
"knip": "^6.32.1",
|
||||
"knip": "^6.32.2",
|
||||
"oxfmt": "^0.63.0",
|
||||
"oxlint": "^1.78.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
@@ -130,9 +130,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-android-arm-eabi": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.142.0.tgz",
|
||||
"integrity": "sha512-ZiRGDutGsv1G6bL/ozy/koC0Sv39T1DqyoC4KD1DOy9ZoACm1O5UWhEK2c02Qdk+4lfLVkvFa/mQ0fm/4h1BtQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.143.0.tgz",
|
||||
"integrity": "sha512-n9uozULWflPqBtdmI8lAabLqGKNgLVNN0ZH8HfgCwpKGNtzRzauB76jTiW/3YLkcA7N1zskpi9GdVnZuu1SAvg==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -147,9 +147,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-android-arm64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.142.0.tgz",
|
||||
"integrity": "sha512-WZkvGRLNQTz8lR9zP5nLjUdlroRCopBu3g9zF1p/laE6DzT1UbQo8Rdz5MWhaJUPYg/6gp+jo7HUgsyKaN1FtQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.143.0.tgz",
|
||||
"integrity": "sha512-9BbdjHETk6O3zH/DDid9IgBtF0GlpLabNKN231uraXpRDSfY+iiZxTP5bk1Z63GBownVdhdINFIeddmMz4MzpQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -164,9 +164,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-darwin-arm64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.142.0.tgz",
|
||||
"integrity": "sha512-l4khS8LQOOVYsGRVARo1gSaCT/aBSceUVXgtovWc2+drnxVuDr082WA3OCHVdVzIz5JIrP/y9CWsSKxBDNmYGg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.143.0.tgz",
|
||||
"integrity": "sha512-gh+6ecoHUy4/sUcolBl/1qPXKBbYNxFY0Pk0ujgQvINTMSftJY7o4yb8gOkDJPeZeB8+a+u7xTe6umoP8N5HFA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -181,9 +181,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-darwin-x64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.142.0.tgz",
|
||||
"integrity": "sha512-QBsNF3nqlXmcH2B1YOPqQYmCJoy4HuIjUxGbBO/k5JAJUl68ghU2psRY2zPk+RyBaWqKP/qfL4oaFgEMCdwskA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.143.0.tgz",
|
||||
"integrity": "sha512-qd1hl2d+lXgHv/VQ/M9qm8TrMC5T4RqDBwtOnl+1D0QMjwcz+8AaB4JSg8STgeag0GP6a6L74XEGAsrTSJWNzQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -198,9 +198,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-freebsd-x64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.142.0.tgz",
|
||||
"integrity": "sha512-b7Q7m4Cqc6XqNhri3R+QhU+GVy646Pn+bkdhrDdWym/Fdi0ZUa+d73H9dm5H91JtbtAQ/z1d8XKMW3oOV8a4tQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.143.0.tgz",
|
||||
"integrity": "sha512-M5XXcNa7aOqLPKTR41msfghKu2yQ4xWvCm11/gwU0JzOzHNk5sgW//rVEjJ+LO48+VDAMzXTSzurUVxIDKwozw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -215,9 +215,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm-gnueabihf": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.142.0.tgz",
|
||||
"integrity": "sha512-3riVS5IhdH3uCZj1Y9ftDQlR0dvLsIlw/edrRqk8JhgNd5K0XSs+UBtgh50N13CAlW9/TXj6sVGXaKNBocd0Yg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.143.0.tgz",
|
||||
"integrity": "sha512-T/GXusuOkPNQhCQCSBbcU/N8j0rAypuDBl1IyFK+lyYT594XsVz80clPC/OtbSSpBGyJxj8uYEfctxVuxVYoww==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -232,9 +232,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm-musleabihf": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.142.0.tgz",
|
||||
"integrity": "sha512-NmXUOpgpTSkhl795TiXmWppTwmSJ92RC1qvD6e4XOF+slgmo3e6Ah+kEu+6AN8s7NAOEwqGmir58MgSQSWmBSA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.143.0.tgz",
|
||||
"integrity": "sha512-oKu4RcBlXSqo3OC62dp6YTnQaZIurNDpCX3BnAM3+bJxt7s8J2TJKMnC0UYer1qhlRaDCg6wkTaTw+2IlsZ12w==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -249,9 +249,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-gc0EXsKtXgerujmU2Bql3u1L1HsSQ2774R83idq/FoNMPVV/RY/1ErFsvnit7KoiP/sLvzQixeUo4Ut0ic0wmw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-WJBbD186AZmMGaSIhlktC+rPl8L3peCTXAh88Ih9uEvK0en2mPojGyCGYiL6mHtV1RPV3JyfJW5t6n5hh0lXhA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -269,9 +269,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm64-musl": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.142.0.tgz",
|
||||
"integrity": "sha512-F2XvmWSE0uWpie+jHKKIFgdVOe9ypGhkEZxKx5DuW215K6cbAC274yYaPkcM7EqY4Df3Weyhpcz3lsURyH2LVg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.143.0.tgz",
|
||||
"integrity": "sha512-t1AcYOwEzgceadT4v5e+vaCCb0AncCA3v5AyzfBAz/tMq11qzVccXKzNHtkWdjBsgvTKwRkaUF3QvT4kot8vcQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -289,9 +289,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-ppc64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-wLMbT21U/QxknQsk+VvNF0b9D2/aGWhcaQQQ+VYlE8FwD5+GoWZIPPXNzyHmkYyhm0KB3itL+TBavjMatqNnYA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-RsnO/NoD8376LMJq8JS8TwI0ieNaFRTuNe2GVJntQg6gwZNMENZsEbknHdVwjpOmxdGLGodcwaGSbAeRr5Bgjw==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
@@ -309,9 +309,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-riscv64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-+G8F/4ckwT7FCJV4H2bt09xEzJbjNCfuL4Sp1AYNaFtFMVtgIGMuJlteT82U+K0UIZ/DzAR/LDlMFnEuajG7Kw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-48fSVfR9TZi5CASZFyv0VC6z6BCoeihFsX031mAD/oSH7d9PYsPgIqza7d9mjP7Z2KTEpTFyH6SIu0Ui6R1vdg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -329,9 +329,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-riscv64-musl": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.142.0.tgz",
|
||||
"integrity": "sha512-hTsHtTLxMAfCo+rpF5K3qZJKW2NpPN/CHd4mYB3y7XlSdspHkd2gehDIofP64AacA9nWQw2tY3O7wR6UY8IVOA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.143.0.tgz",
|
||||
"integrity": "sha512-T8CpdD+SfE01DnIOD4HpVxu0ZJOfMJ/VhCvikKfaXAxkZ+9veyLM/D2hpi7Y2hFUyPmVQO3FNZHmYzV/WlVR4g==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -349,9 +349,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-s390x-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-6y7qYY3TCUDYjqswImdTGl92y+KA/80twALegQPN27kfY+bG7Ib1+L3jbmrCZQx6wrVnai9IPsEZp07I0hx7JQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-QLdeMsCcacenPEFsfxnBUDF1y6opyz5+fmOz9bfD5Y7fiGCMupUCuB3KTPQhNwshIG1P9fPqar9MHxuBDd4bwQ==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
@@ -369,9 +369,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-x64-gnu": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.142.0.tgz",
|
||||
"integrity": "sha512-i69kAWU+2LgoH5bR+zWiiu+UzAw7Oxkwv7COeJTeY19pn4e70nKQcr9Pm6cL2Z0Z54d+gl9qADlK/0yyuCPiBA==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.143.0.tgz",
|
||||
"integrity": "sha512-659ujfqLy6k7cuH3sbzhd8b+ztSq+i6E2E9pG78Q0BmHjAExfGIdgc8cGgMdwAozDXeZFHkJ+LXYJdWsaGdgyw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -389,9 +389,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-x64-musl": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.142.0.tgz",
|
||||
"integrity": "sha512-4SQs678MmjYVrmhAgCWD4o0vpaFszXw9xLX5p2Z9MMFcltxiLkA88wQjh80YHjPrXtpyZ2CWI5m+1yNKM0m2Pw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.143.0.tgz",
|
||||
"integrity": "sha512-/Mw/9j4TfZcnKphPrzOE6t4MMknXadcAAuVUlDRTF/ETWB5xOgQvOJV2Mh9We/bWxZdoxaGAdc+hy4GuYwQ2yQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -409,9 +409,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-openharmony-arm64": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.142.0.tgz",
|
||||
"integrity": "sha512-YHpx9N7Ln3a++Tc8rv+H7mrK1zyJQOAwCFg8LZ3lTs1T5afGWeZrLPhPT9HLnIwSjCyJqPWVMIrMxbjcmBr2oQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.143.0.tgz",
|
||||
"integrity": "sha512-8rIKWR2BFuifbIK/1XB9wTaSdtuJ25dlE7ZQYDnEwj/2xH2vHsxnvIjHT3ZjSVuLLwGGlSslIG/fbOJ8TV8rTw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -425,52 +425,10 @@
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-wasm32-wasi": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.142.0.tgz",
|
||||
"integrity": "sha512-3pLDyY3+oogW73RM5uehNgAiR/Xfb7fvO2Q1Z1gIqZ2+50XDVQmBVlRkHXZTU4gKnQHpwETNsYQVsJ3joVB2iA==",
|
||||
"cpu": [
|
||||
"wasm32"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/core": "1.11.2",
|
||||
"@emnapi/runtime": "1.11.2",
|
||||
"@napi-rs/wasm-runtime": "^1.1.6"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@emnapi/core": {
|
||||
"version": "1.11.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz",
|
||||
"integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.2.2",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@emnapi/runtime": {
|
||||
"version": "1.11.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz",
|
||||
"integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-arm64-msvc": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.142.0.tgz",
|
||||
"integrity": "sha512-Had/VeVY28Oyb0K+Q4FV8KCzoBycIh93oDK6pCbya9lkzdq+ikMHMgBubsdqqlybjJmQRawCQRrnBRHyQwYvcQ==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.143.0.tgz",
|
||||
"integrity": "sha512-5U9kQYMfRRI6Zq7KDxgbIP0RMnKrfn3gLepRMgJuRkPSUALTiRCk9d/uyhb4lGDjUdzwK7mBkKqhLgzBPCmLpQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -485,9 +443,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-ia32-msvc": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.142.0.tgz",
|
||||
"integrity": "sha512-GGi3+YphVHavvgs6gum2UXoNCqzHAmPt/nXkn8ZQZstV2Q1qZD1Mn8fz/nWrDkefHQtrG/+1/XrbMxsBTo6Svw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.143.0.tgz",
|
||||
"integrity": "sha512-25P7AaHk4R88Yv2XH4gToDVmh0cOu+bEURQU10CRrmvgabfRArSGAP5osmwUKeSUHj0VS50upbpbRWWW/m7mHA==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
@@ -502,9 +460,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-x64-msvc": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.142.0.tgz",
|
||||
"integrity": "sha512-Ny/Wv4Us1LGC/ljwNTp+Hx3r/pH15EFfeDF0p+n898gt+TtRd6C9SccHcuUhDiNTb8s5tt7jdeAMDRQZ4Vq6hg==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.143.0.tgz",
|
||||
"integrity": "sha512-ORMh3JE1s6V7ySicdRK7vgaDQnn5o+UHg9ct989PlWHbel8O9ARrmWXM6kZjrBMtNucxNayQ8g69G0VfWzhANw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2979,9 +2937,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/knip": {
|
||||
"version": "6.32.1",
|
||||
"resolved": "https://registry.npmjs.org/knip/-/knip-6.32.1.tgz",
|
||||
"integrity": "sha512-mIiIHMTJVUgSlz0mxEgPt7wg8DmfbCp1Txqab3WpbMCJF7YHvHtC9jeAHHXfISMl72N8WzhyG71SQlaqCOGZtg==",
|
||||
"version": "6.32.2",
|
||||
"resolved": "https://registry.npmjs.org/knip/-/knip-6.32.2.tgz",
|
||||
"integrity": "sha512-WXTXbmocrw7gqm1A1TQvFN0OgJ7hUSU6E1g6SPRIzzHFogUBhXByc7cYeOFVtJ2uODg7DP4VbESYBYnfbtBYsg==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -2999,13 +2957,13 @@
|
||||
"formatly": "^0.3.0",
|
||||
"get-tsconfig": "4.14.1",
|
||||
"jiti": "^2.7.0",
|
||||
"oxc-parser": "^0.142.0",
|
||||
"oxc-parser": "^0.143.0",
|
||||
"oxc-resolver": "11.24.2",
|
||||
"picomatch": "^4.0.5",
|
||||
"smol-toml": "^1.7.1",
|
||||
"strip-json-comments": "5.0.3",
|
||||
"tinyglobby": "^0.2.17",
|
||||
"unbash": "^4.0.4",
|
||||
"unbash": "^4.0.9",
|
||||
"yaml": "^2.9.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
@@ -3311,13 +3269,13 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/oxc-parser": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.142.0.tgz",
|
||||
"integrity": "sha512-kKR+jPiRJYJDexVoziIg/FVGvr1fT1FZSSJOk6tVoMKKSlsf1Cso+cgGCJkOEDWOP174vRntCPFKg+AS7InWvw==",
|
||||
"version": "0.143.0",
|
||||
"resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.143.0.tgz",
|
||||
"integrity": "sha512-ov0NzaDCOInknS7mP1cwKdJERt3utPW8ldjtdUXQ8Ty0GEFD08wk422vCUN0d7pST6kqtV7dxoI9w1Zi0l/9TA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@oxc-project/types": "^0.142.0"
|
||||
"@oxc-project/types": "^0.143.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
@@ -3326,36 +3284,25 @@
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oxc-parser/binding-android-arm-eabi": "0.142.0",
|
||||
"@oxc-parser/binding-android-arm64": "0.142.0",
|
||||
"@oxc-parser/binding-darwin-arm64": "0.142.0",
|
||||
"@oxc-parser/binding-darwin-x64": "0.142.0",
|
||||
"@oxc-parser/binding-freebsd-x64": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm-gnueabihf": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm-musleabihf": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-arm64-musl": "0.142.0",
|
||||
"@oxc-parser/binding-linux-ppc64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-riscv64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-riscv64-musl": "0.142.0",
|
||||
"@oxc-parser/binding-linux-s390x-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-x64-gnu": "0.142.0",
|
||||
"@oxc-parser/binding-linux-x64-musl": "0.142.0",
|
||||
"@oxc-parser/binding-openharmony-arm64": "0.142.0",
|
||||
"@oxc-parser/binding-wasm32-wasi": "0.142.0",
|
||||
"@oxc-parser/binding-win32-arm64-msvc": "0.142.0",
|
||||
"@oxc-parser/binding-win32-ia32-msvc": "0.142.0",
|
||||
"@oxc-parser/binding-win32-x64-msvc": "0.142.0"
|
||||
}
|
||||
},
|
||||
"node_modules/oxc-parser/node_modules/@oxc-project/types": {
|
||||
"version": "0.142.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz",
|
||||
"integrity": "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
"@oxc-parser/binding-android-arm-eabi": "0.143.0",
|
||||
"@oxc-parser/binding-android-arm64": "0.143.0",
|
||||
"@oxc-parser/binding-darwin-arm64": "0.143.0",
|
||||
"@oxc-parser/binding-darwin-x64": "0.143.0",
|
||||
"@oxc-parser/binding-freebsd-x64": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm-gnueabihf": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm-musleabihf": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-arm64-musl": "0.143.0",
|
||||
"@oxc-parser/binding-linux-ppc64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-riscv64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-riscv64-musl": "0.143.0",
|
||||
"@oxc-parser/binding-linux-s390x-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-x64-gnu": "0.143.0",
|
||||
"@oxc-parser/binding-linux-x64-musl": "0.143.0",
|
||||
"@oxc-parser/binding-openharmony-arm64": "0.143.0",
|
||||
"@oxc-parser/binding-win32-arm64-msvc": "0.143.0",
|
||||
"@oxc-parser/binding-win32-ia32-msvc": "0.143.0",
|
||||
"@oxc-parser/binding-win32-x64-msvc": "0.143.0"
|
||||
}
|
||||
},
|
||||
"node_modules/oxc-resolver": {
|
||||
@@ -3874,9 +3821,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/unbash": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/unbash/-/unbash-4.0.4.tgz",
|
||||
"integrity": "sha512-60m9IVGbavD6jholbxt0jVBXZkEB/HsMZq7Tyaghseve2/Sf0zQRAIfWsD34sde+DKP2tBxJS2wP88ZM0D1FhA==",
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/unbash/-/unbash-4.0.10.tgz",
|
||||
"integrity": "sha512-b7zoBQvpWp0vuN5q2vK2RRBR2SvuruQAs50DApdDveBSn3eSYd84IaHodFqQIMlvY9K2VnyBUEXgwOBuGU9GBg==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
"@types/react": "^19.2.18",
|
||||
"@types/react-dom": "^19.2.4",
|
||||
"@vitejs/plugin-react": "^6.0.5",
|
||||
"knip": "^6.32.1",
|
||||
"knip": "^6.32.2",
|
||||
"oxfmt": "^0.63.0",
|
||||
"oxlint": "^1.78.0",
|
||||
"oxlint-tsgolint": "^7.0.2001",
|
||||
|
||||
@@ -223,6 +223,14 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
);
|
||||
const showActiveTargetLabel = queryTargets.length > 0 && activeTarget.source !== 'general';
|
||||
|
||||
// Manual search browses release sources directly, one media type at a time — the
|
||||
// combined ("both") flow doesn't apply. Present a plain, switchable Books/Audiobooks
|
||||
// choice for it, even when combined search is forced on for metadata targets.
|
||||
const isManualTarget = activeTarget?.source === 'manual';
|
||||
const combinedSelectionActive = combinedMode && !isManualTarget;
|
||||
const combinedSelectorLocked = combinedModeLocked && !isManualTarget;
|
||||
const combinedToggleAvailable = !!onCombinedModeChange && !isManualTarget;
|
||||
|
||||
useDismiss(isSelectorOpen, [selectorRef], () => setIsSelectorOpen(false));
|
||||
useDismiss(isSelectOpen, [selectPanelRef, selectTriggerRef], () => setIsSelectOpen(false));
|
||||
useDismiss(isAutocompleteOpen, [autocompletePanelRef, inputRef], () =>
|
||||
@@ -356,7 +364,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
contentType,
|
||||
activeTarget,
|
||||
placeholder,
|
||||
combinedMode,
|
||||
combinedSelectionActive,
|
||||
);
|
||||
const effectiveInputAriaLabel = activeTarget
|
||||
? `${inputAriaLabel}: ${activeTarget.label}`
|
||||
@@ -537,7 +545,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
|
||||
let selectorContentTypeLabel = 'audiobooks';
|
||||
let selectorIcon = <AudiobookIcon />;
|
||||
if (combinedMode) {
|
||||
if (combinedSelectionActive) {
|
||||
selectorContentTypeLabel = 'books and audiobooks';
|
||||
selectorIcon = <BothIcon />;
|
||||
} else if (contentType === 'ebook') {
|
||||
@@ -665,7 +673,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
<div className="max-h-[min(24rem,calc(100vh-8rem))] overflow-y-auto p-3">
|
||||
{showContentTypeSelector && (
|
||||
<div
|
||||
className={`border-b ${onCombinedModeChange ? 'pb-0' : 'pb-3'}`}
|
||||
className={`border-b ${combinedToggleAvailable ? 'pb-0' : 'pb-3'}`}
|
||||
style={{ borderColor: 'var(--border-muted)' }}
|
||||
>
|
||||
<div className="flex items-center justify-between px-1 pb-2">
|
||||
@@ -712,34 +720,38 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('ebook')}
|
||||
className={`flex items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'ebook' || combinedMode
|
||||
contentType === 'ebook' || combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'ebook' || combinedMode
|
||||
contentType === 'ebook' || combinedSelectionActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{contentType === 'ebook' || combinedMode ? <CheckIcon /> : <BookIcon />}
|
||||
{contentType === 'ebook' || combinedSelectionActive ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<BookIcon />
|
||||
)}
|
||||
<span>Books</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleContentTypeSelect('audiobook')}
|
||||
className={`flex items-center gap-2 rounded-xl border px-3 py-2 text-sm font-medium transition-colors ${
|
||||
contentType === 'audiobook' || combinedMode
|
||||
contentType === 'audiobook' || combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'hover-surface'
|
||||
}`}
|
||||
style={
|
||||
contentType === 'audiobook' || combinedMode
|
||||
contentType === 'audiobook' || combinedSelectionActive
|
||||
? { borderColor: 'rgb(16 185 129 / 0.7)' }
|
||||
: { color: 'var(--text)', borderColor: 'var(--border-muted)' }
|
||||
}
|
||||
>
|
||||
{contentType === 'audiobook' || combinedMode ? (
|
||||
{contentType === 'audiobook' || combinedSelectionActive ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<AudiobookIcon />
|
||||
@@ -747,9 +759,9 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
<span>Audiobooks</span>
|
||||
</button>
|
||||
</div>
|
||||
{onCombinedModeChange &&
|
||||
{combinedToggleAvailable &&
|
||||
(() => {
|
||||
const lineColor = combinedMode
|
||||
const lineColor = combinedSelectionActive
|
||||
? 'bg-emerald-500'
|
||||
: 'bg-(--border-muted) group-hover:bg-zinc-400 dark:group-hover:bg-zinc-500';
|
||||
return (
|
||||
@@ -787,7 +799,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
{/* Chain icon centered at bottom */}
|
||||
<div
|
||||
className={`relative z-10 mx-auto rounded-full p-1 transition-colors ${
|
||||
combinedMode
|
||||
combinedSelectionActive
|
||||
? 'bg-emerald-600 text-white'
|
||||
: 'bg-(--bg) text-zinc-400 group-hover:bg-zinc-200 group-hover:text-zinc-600 dark:text-zinc-500 dark:group-hover:bg-zinc-700 dark:group-hover:text-zinc-300'
|
||||
}`}
|
||||
@@ -800,7 +812,7 @@ export const SearchBar = forwardRef<SearchBarHandle, SearchBarProps>(
|
||||
stroke="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
{combinedModeLocked ? (
|
||||
{combinedSelectorLocked ? (
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
|
||||
@@ -317,6 +317,44 @@ class TestSearchAudiobookbay:
|
||||
assert "s=%22test+query%22" in requested_url
|
||||
assert "cat=undefined%2Cundefined" in requested_url
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_normalizes_curly_apostrophe(self, mock_config_get, mock_html_get):
|
||||
"""Test curly apostrophes are searched as the ASCII form ABB stores."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
|
||||
|
||||
scraper.search_audiobookbay(
|
||||
"the stranger’s wife",
|
||||
max_pages=1,
|
||||
hostname="audiobookbay.lu",
|
||||
)
|
||||
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert "s=the+stranger%27s+wife" in requested_url
|
||||
assert "%E2%80%99" not in requested_url
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_percent_encodes_reserved_characters(
|
||||
self, mock_config_get, mock_html_get
|
||||
):
|
||||
"""Test reserved characters cannot break out of the search parameter."""
|
||||
mock_config_get.return_value = 0.0
|
||||
mock_html_get.return_value = (SAMPLE_SEARCH_HTML, "https://audiobookbay.lu/?s=x")
|
||||
|
||||
scraper.search_audiobookbay(
|
||||
"sense & sensibility 100% c++",
|
||||
max_pages=1,
|
||||
hostname="audiobookbay.lu",
|
||||
)
|
||||
|
||||
requested_url = mock_html_get.call_args.args[0]
|
||||
assert "s=sense+%26+sensibility+100%25+c%2B%2B" in requested_url
|
||||
# The only surviving '&' introduces the legacy category parameter.
|
||||
assert requested_url.count("&") == 1
|
||||
assert requested_url.endswith("&cat=undefined%2Cundefined")
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.downloader.html_get_page")
|
||||
@patch("shelfmark.release_sources.audiobookbay.scraper.config.get")
|
||||
def test_search_audiobookbay_always_uses_legacy_category_query(
|
||||
|
||||
@@ -306,6 +306,42 @@ class TestAudiobookBaySource:
|
||||
assert len(results) == 1
|
||||
assert results[0].title == "Test Book by Test Author"
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
|
||||
def test_search_relevance_filtering_spans_typographic_punctuation(self, mock_search):
|
||||
"""Test an ASCII query still matches the typographic title ABB renders."""
|
||||
mock_search.return_value = [
|
||||
{
|
||||
"title": "The Stranger’s Wife — Anna‑Lou Weatherley",
|
||||
"link": "https://audiobookbay.lu/abss/the-strangers-wife/",
|
||||
"format": "M4B",
|
||||
"size": "259 MB",
|
||||
"language": "English",
|
||||
},
|
||||
]
|
||||
|
||||
source = AudiobookBaySource()
|
||||
book = BookMetadata(
|
||||
provider="test",
|
||||
provider_id="123",
|
||||
title="Stranger's",
|
||||
authors=["Anna-Lou Weatherley"],
|
||||
)
|
||||
# Every query word carries punctuation, so the result survives only when
|
||||
# both sides of the comparison are normalized.
|
||||
plan = ReleaseSearchPlan(
|
||||
languages=["en"],
|
||||
isbn_candidates=[],
|
||||
author="",
|
||||
title_variants=[ReleaseSearchVariant(title="Stranger's", author="")],
|
||||
grouped_title_variants=[],
|
||||
)
|
||||
|
||||
results = source.search(book, plan, content_type="audiobook")
|
||||
|
||||
assert len(results) == 1
|
||||
# The release keeps the title as ABB rendered it; only matching normalizes.
|
||||
assert results[0].title == "The Stranger’s Wife — Anna‑Lou Weatherley"
|
||||
|
||||
@patch("shelfmark.release_sources.audiobookbay.source.scraper.search_audiobookbay")
|
||||
def test_search_result_mapping(self, mock_search):
|
||||
"""Test conversion of scraper results to Release objects."""
|
||||
|
||||
@@ -2,7 +2,52 @@
|
||||
Tests for AudiobookBay utility functions.
|
||||
"""
|
||||
|
||||
from shelfmark.release_sources.audiobookbay.utils import parse_size
|
||||
from shelfmark.release_sources.audiobookbay.utils import normalize_search_punctuation, parse_size
|
||||
|
||||
|
||||
class TestNormalizeSearchPunctuation:
|
||||
"""Tests for the normalize_search_punctuation function."""
|
||||
|
||||
def test_curly_apostrophe_becomes_ascii(self):
|
||||
"""ABB matches the stored ASCII apostrophe, not the rendered curly one."""
|
||||
assert normalize_search_punctuation("The Stranger’s Wife") == "The Stranger's Wife"
|
||||
|
||||
def test_all_single_quote_variants(self):
|
||||
"""Every single-quote lookalike collapses to the ASCII apostrophe."""
|
||||
for variant in ("‘", "’", "‚", "‛", "′", "´", "`"):
|
||||
assert normalize_search_punctuation(f"don{variant}t") == "don't"
|
||||
|
||||
def test_all_double_quote_variants(self):
|
||||
"""Every double-quote lookalike collapses to the ASCII double quote."""
|
||||
for variant in ("“", "”", "„", "‟", "″"):
|
||||
assert normalize_search_punctuation(f"{variant}quoted{variant}") == '"quoted"'
|
||||
|
||||
def test_all_dash_variants(self):
|
||||
"""Every dash lookalike collapses to the ASCII hyphen."""
|
||||
for variant in ("‐", "‑", "‒", "–", "—", "―", "−", "﹘", "﹣", "－"):
|
||||
assert normalize_search_punctuation(f"anna{variant}lou") == "anna-lou"
|
||||
|
||||
def test_ellipsis_expands_to_three_dots(self):
|
||||
"""WordPress renders '...' as a single ellipsis character."""
|
||||
assert normalize_search_punctuation("And Then…") == "And Then..."
|
||||
|
||||
def test_ascii_query_is_unchanged(self):
|
||||
"""A query that is already ASCII passes through untouched."""
|
||||
assert normalize_search_punctuation("The Stranger's Wife") == "The Stranger's Wife"
|
||||
|
||||
def test_ascii_dash_runs_are_not_collapsed(self):
|
||||
"""Only characters ABB cannot have stored are rewritten."""
|
||||
assert normalize_search_punctuation("Book -- Subtitle") == "Book -- Subtitle"
|
||||
|
||||
def test_other_punctuation_is_preserved(self):
|
||||
"""Colons and commas carry search signal and are left alone."""
|
||||
assert normalize_search_punctuation("Weatherley: Book 3, Part 1") == (
|
||||
"Weatherley: Book 3, Part 1"
|
||||
)
|
||||
|
||||
def test_empty_query(self):
|
||||
"""An empty query is returned as-is."""
|
||||
assert normalize_search_punctuation("") == ""
|
||||
|
||||
|
||||
class TestParseSize:
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""DDoS-Guard cookie reuse between requests.
|
||||
|
||||
Anna's Archive issues nine cookies after a solve, and they are not equivalent:
|
||||
|
||||
__ddg1_/__ddg2_/__ddgid_ ~1 year clearance
|
||||
__ddgmark_ ~1 day
|
||||
__ddg5_ session
|
||||
__ddg8_/__ddg9_/__ddg10_ ~40 min one check: token, CLIENT IP, TIMESTAMP
|
||||
|
||||
Replaying the last three is what produces the ?check=1 redirect loop. They describe a
|
||||
single check, so once the timestamp ages out - or the egress IP changes, routine
|
||||
behind a VPN - DDoS-Guard stops recognising the caller and re-arms the challenge on
|
||||
every request. Storing an expired cookie and sending it forever has the same effect.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import shelfmark.bypass.cookie_store as cs
|
||||
import shelfmark.bypass.internal_bypasser as ib
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_cookie_store(monkeypatch):
|
||||
monkeypatch.setattr(cs, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cs, "_cf_user_agents", {})
|
||||
|
||||
|
||||
class _Cookie:
|
||||
"""Stand-in for the CDP cookie objects the bypasser extracts."""
|
||||
|
||||
def __init__(self, name, value="v", expires=None, domain="annas-archive.gl"):
|
||||
self.name = name
|
||||
self.value = value
|
||||
self.expires = expires
|
||||
self.domain = domain
|
||||
self.path = "/"
|
||||
self.secure = True
|
||||
|
||||
|
||||
def _store(cookies, url="https://annas-archive.gl/search"):
|
||||
cs.store_extracted_cookies(url=url, cookies=cookies, user_agent="UA/1.0")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Per-check cookies must not be persisted for replay
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_per_check_cookies_are_not_stored():
|
||||
"""The IP/timestamp trio describes one check and must not outlive it."""
|
||||
_store(
|
||||
[
|
||||
_Cookie("__ddg1_", "clearance"),
|
||||
_Cookie("__ddg2_", "clearance2"),
|
||||
_Cookie("__ddg8_", "opaque"),
|
||||
_Cookie("__ddg9_", "203.0.113.7"),
|
||||
_Cookie("__ddg10_", "1786826304"),
|
||||
_Cookie("ddg_last_challenge", "1786826304"),
|
||||
]
|
||||
)
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(stored) == {"__ddg1_", "__ddg2_"}
|
||||
for ephemeral in ("__ddg8_", "__ddg9_", "__ddg10_", "ddg_last_challenge"):
|
||||
assert ephemeral not in stored
|
||||
|
||||
|
||||
def test_clearance_cookies_survive():
|
||||
_store([_Cookie("__ddg1_", "a"), _Cookie("__ddg2_", "b"), _Cookie("__ddgid_", "c")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"__ddg1_": "a", "__ddg2_": "b", "__ddgid_": "c"}
|
||||
|
||||
|
||||
def test_cloudflare_cookies_are_unaffected():
|
||||
_store([_Cookie("cf_clearance", "token"), _Cookie("__cf_bm", "bm")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"cf_clearance": "token", "__cf_bm": "bm"}
|
||||
|
||||
|
||||
def test_per_check_cookies_are_excluded_even_for_full_session_domains(monkeypatch):
|
||||
"""extract_all exists for Z-Library sessions; it must not resurrect the trio."""
|
||||
monkeypatch.setattr(cs, "_get_full_cookie_domains", lambda: {"annas-archive.gl"})
|
||||
_store([_Cookie("sessionid", "s"), _Cookie("__ddg9_", "203.0.113.7")])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert "sessionid" in stored
|
||||
assert "__ddg9_" not in stored
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Expiry must be honoured for every cookie, not only cf_clearance
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_expired_ddg_cookies_are_dropped():
|
||||
"""The old code only expiry-checked cf_clearance, so DDoS-Guard domains - which
|
||||
have none - replayed dead cookies forever."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "live"), _Cookie("__ddgmark_", "dead", expires=past)])
|
||||
|
||||
stored = ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert stored == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_all_cookies_expired_returns_empty_so_caller_re_solves():
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "dead", expires=past)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
assert cs.has_valid_cf_cookies("annas-archive.gl") is False
|
||||
|
||||
|
||||
def test_unexpired_cookies_are_kept():
|
||||
future = int(time.time()) + 3600
|
||||
_store([_Cookie("__ddg1_", "live", expires=future)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_session_cookies_never_expire():
|
||||
"""expires<=0 means a session cookie, not an already-expired one."""
|
||||
_store([_Cookie("__ddg5_", "s", expires=0), _Cookie("__ddg1_", "a", expires=None)])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg5_": "s", "__ddg1_": "a"}
|
||||
|
||||
|
||||
def test_expired_cf_clearance_still_drops_the_whole_domain():
|
||||
"""Pre-existing Cloudflare behaviour must not regress."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("cf_clearance", "dead", expires=past), _Cookie("__cf_bm", "bm")])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_expired_cookies_are_pruned_from_the_store():
|
||||
"""A dropped cookie must not linger and be re-evaluated on every request."""
|
||||
past = int(time.time()) - 60
|
||||
_store([_Cookie("__ddg1_", "live"), _Cookie("__ddgmark_", "dead", expires=past)])
|
||||
|
||||
ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
assert set(cs._cf_cookies["annas-archive.gl"]) == {"__ddg1_"}
|
||||
|
||||
|
||||
def test_solve_that_yields_only_per_check_cookies_stores_nothing():
|
||||
"""No clearance means no reuse - the caller must go back to the bypasser rather
|
||||
than believe it holds a valid session."""
|
||||
_store([_Cookie("__ddg9_", "203.0.113.7"), _Cookie("__ddg10_", "1786826304")])
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Rejected cookies are discarded, never retried forever
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _Resp:
|
||||
def __init__(self, status_code, text="page"):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
|
||||
def _seed(monkeypatch):
|
||||
_store([_Cookie("__ddg1_", "clearance"), _Cookie("__ddg2_", "c2")])
|
||||
monkeypatch.setattr(ib, "get_proxies", lambda _url: None)
|
||||
monkeypatch.setattr(ib, "get_ssl_verify", lambda _url: True)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl")
|
||||
|
||||
|
||||
def test_rejected_cached_cookies_are_discarded(monkeypatch):
|
||||
"""A 403 while presenting cookies proves they are dead - keep them and every
|
||||
later request re-presents a known-rejected cookie."""
|
||||
_seed(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
assert (
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl") is None
|
||||
)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_redirect_loop_on_cached_cookies_discards_them(monkeypatch):
|
||||
"""DDoS-Guard answers dead clearance with an endless ?check=1 bounce, which
|
||||
surfaces as an exception rather than a status code."""
|
||||
_seed(monkeypatch)
|
||||
|
||||
def boom(*_a, **_k):
|
||||
raise ib.requests.exceptions.TooManyRedirects("Exceeded 30 redirects")
|
||||
|
||||
monkeypatch.setattr(ib.requests, "get", boom)
|
||||
|
||||
assert (
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl") is None
|
||||
)
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_working_cookies_are_kept(monkeypatch):
|
||||
_seed(monkeypatch)
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(200, "the page"))
|
||||
|
||||
result = ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl")
|
||||
|
||||
assert result == "the page"
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") != {}
|
||||
|
||||
|
||||
def test_failure_only_clears_the_failing_host(monkeypatch):
|
||||
"""clear_cf_cookies('') means every host - a blank hostname must not wipe
|
||||
clearance for sites that are working fine."""
|
||||
_seed(monkeypatch)
|
||||
_store([_Cookie("__ddg1_", "other")], url="https://other-site.test/x")
|
||||
monkeypatch.setattr(ib.requests, "get", lambda *a, **k: _Resp(403))
|
||||
|
||||
ib._try_with_cached_cookies("https://annas-archive.gl/search", "annas-archive.gl")
|
||||
|
||||
assert ib.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
assert ib.get_cf_cookies_for_domain("other-site.test") == {"__ddg1_": "other"}
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Tests for the external bypasser flow."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
@@ -55,6 +57,165 @@ def test_fetch_via_bypasser_posts_expected_payload_and_uses_ssl_verify(monkeypat
|
||||
]
|
||||
|
||||
|
||||
def _stub_solution(monkeypatch, external_bypasser, solution: dict) -> None:
|
||||
"""Answer one bypass with `solution`, with config and SSL stubbed out."""
|
||||
|
||||
def fake_get(key, default=""):
|
||||
values = {
|
||||
"EXT_BYPASSER_URL": "https://bypass.example",
|
||||
"EXT_BYPASSER_PATH": "/v1",
|
||||
"EXT_BYPASSER_TIMEOUT": 60000,
|
||||
}
|
||||
return values.get(key, default)
|
||||
|
||||
monkeypatch.setattr(external_bypasser.config, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
external_bypasser.requests,
|
||||
"post",
|
||||
lambda *_a, **_k: _FakeResponse({"status": "ok", "solution": solution}),
|
||||
)
|
||||
monkeypatch.setattr(external_bypasser, "get_ssl_verify", lambda _url: False)
|
||||
|
||||
|
||||
def test_solved_clearance_is_stored_for_reuse(monkeypatch):
|
||||
"""A solve costs tens of seconds of real browser; its clearance must be kept.
|
||||
|
||||
Without this every request paid a 403 plus a full solve, and the file download -
|
||||
which the solver cannot proxy - presented no clearance at all.
|
||||
"""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"userAgent": "Mozilla/5.0 (solver)",
|
||||
"cookies": [
|
||||
{"name": "__ddg1_", "value": "clearance", "domain": ".annas-archive.gl"},
|
||||
{"name": "__ddg2_", "value": "c2", "domain": ".annas-archive.gl"},
|
||||
# Per-check cookies: kept out of the store, same as the internal path.
|
||||
{"name": "__ddg9_", "value": "203.0.113.7", "domain": ".annas-archive.gl"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {
|
||||
"__ddg1_": "clearance",
|
||||
"__ddg2_": "c2",
|
||||
}
|
||||
# Cloudflare ties clearance to the solving UA, so replaying one without the other fails.
|
||||
assert cookie_store.get_cf_user_agent_for_domain("annas-archive.gl") == "Mozilla/5.0 (solver)"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "shape"),
|
||||
[
|
||||
# Byparr drives Playwright/camoufox, whose cookies spell it "expires".
|
||||
("expires", "playwright"),
|
||||
# FlareSolverr assigns driver.get_cookies() - the WebDriver cookie object,
|
||||
# which spells it "expiry". Reading only "expires" made every FlareSolverr
|
||||
# cookie immortal, so dead clearance was replayed forever.
|
||||
("expiry", "webdriver"),
|
||||
],
|
||||
)
|
||||
def test_expired_solution_cookie_is_not_replayed(monkeypatch, field, shape):
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"cookies": [{"name": "__ddg1_", "value": "dead", field: int(time.time()) - 60}],
|
||||
},
|
||||
)
|
||||
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}, (
|
||||
f"a dead {shape} cookie was kept for replay"
|
||||
)
|
||||
|
||||
|
||||
def test_solution_cookie_expiry_is_coerced_not_trusted(monkeypatch):
|
||||
"""The solver is not ours; a stringified expiry must be read, not raised on."""
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{
|
||||
"response": "<html>ok</html>",
|
||||
"cookies": [
|
||||
{"name": "__ddg1_", "value": "live", "expires": str(int(time.time()) + 3600)},
|
||||
{"name": "__ddg2_", "value": "dead", "expires": str(int(time.time()) - 60)},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {"__ddg1_": "live"}
|
||||
|
||||
|
||||
def test_storing_clearance_can_never_discard_the_solved_page(monkeypatch):
|
||||
"""A solve costs ~30s; a surprise in the cookie shape must not throw it away.
|
||||
|
||||
The store call sits inside the request try/except, whose handler returns None -
|
||||
so without its own guard a raising store turned a good page into a failed fetch
|
||||
and sent the caller round for up to MAX_RETRY more solves.
|
||||
"""
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
_stub_solution(
|
||||
monkeypatch,
|
||||
external_bypasser,
|
||||
{"response": "<html>ok</html>", "cookies": [{"name": "__ddg1_", "value": "v"}]},
|
||||
)
|
||||
|
||||
def boom(*_args, **_kwargs):
|
||||
raise TypeError("unexpected cookie shape")
|
||||
|
||||
monkeypatch.setattr(external_bypasser, "store_extracted_cookies", boom)
|
||||
|
||||
assert (
|
||||
external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
== "<html>ok</html>"
|
||||
)
|
||||
|
||||
|
||||
def test_solution_without_cookies_is_still_returned(monkeypatch):
|
||||
"""A solver that returns no cookie list must not break the page fetch."""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
_stub_solution(monkeypatch, external_bypasser, {"response": "<html>ok</html>"})
|
||||
|
||||
result = external_bypasser._fetch_via_bypasser("https://annas-archive.gl/search?q=dune")
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert cookie_store.get_cf_cookies_for_domain("annas-archive.gl") == {}
|
||||
|
||||
|
||||
def test_get_bypassed_page_retries_and_rotates_selector_between_attempts(monkeypatch):
|
||||
import shelfmark.bypass.external_bypasser as external_bypasser
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -113,7 +115,9 @@ def test_extract_cookies_from_cdp_keeps_full_session_cookies_for_configured_zlib
|
||||
async def evaluate(self, _expr):
|
||||
return "TestUA/1.0"
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "_get_full_cookie_domains", lambda: {"z-lib.fm"})
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_get_full_cookie_domains", lambda: {"z-lib.fm"})
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
asyncio.run(
|
||||
@@ -165,12 +169,14 @@ def test_extract_cookies_from_cdp_normalizes_session_expiry():
|
||||
)
|
||||
)
|
||||
|
||||
stored = internal_bypasser._cf_cookies.get("example.com", {})
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
stored = cookie_store._cf_cookies.get("example.com", {})
|
||||
assert stored["cf_clearance"]["expiry"] is None
|
||||
assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {"cf_clearance": "abc"}
|
||||
|
||||
# Verify fallback to "expires" key for expiry checks
|
||||
internal_bypasser._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10
|
||||
cookie_store._cf_cookies["example.com"]["cf_clearance"]["expires"] = int(time.time()) - 10
|
||||
assert internal_bypasser.get_cf_cookies_for_domain("example.com") == {}
|
||||
|
||||
|
||||
@@ -374,7 +380,9 @@ def test_try_with_cached_cookies_returns_none_on_request_exception(monkeypatch):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
internal_bypasser._cf_cookies["example.com"] = {
|
||||
from shelfmark.bypass import cookie_store
|
||||
|
||||
cookie_store._cf_cookies["example.com"] = {
|
||||
"cf_clearance": {
|
||||
"value": "abc",
|
||||
"domain": "example.com",
|
||||
@@ -482,3 +490,239 @@ def test_run_bypass_in_current_process_bounds_its_wait(monkeypatch):
|
||||
|
||||
assert result == "html"
|
||||
assert observed["timeout"] == internal_bypasser._IN_PROCESS_BYPASS_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def _write_fake_proc_entry(proc_root, pid: int, pgid: int, argv: list[str]) -> None:
|
||||
"""Create a /proc-shaped entry for a fake process."""
|
||||
entry = proc_root / str(pid)
|
||||
entry.mkdir()
|
||||
(entry / "cmdline").write_bytes(b"\0".join(arg.encode() for arg in argv) + b"\0")
|
||||
# pid (comm) state ppid pgrp ... - comm is parenthesised and may contain spaces.
|
||||
(entry / "stat").write_text(f"{pid} (some (odd) name) S 1 {pgid} {pgid} 0 -1 4194304 0 0")
|
||||
|
||||
|
||||
def test_cleanup_only_kills_own_and_abandoned_browser_sessions(monkeypatch, tmp_path):
|
||||
"""Regression test for issue #1231: the sweep used a container-wide `pkill -f chrome`,
|
||||
so every worker that started a bypass killed the browsers the other workers were
|
||||
still driving. Only our own process group and groups whose leader is gone are ours."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
proc_root = tmp_path / "proc"
|
||||
proc_root.mkdir()
|
||||
_write_fake_proc_entry(proc_root, 1000, 1000, ["python", "-m", "shelfmark.bypass"])
|
||||
_write_fake_proc_entry(proc_root, 1001, 1000, ["/usr/bin/chromium", "--headless"])
|
||||
_write_fake_proc_entry(proc_root, 1002, 1000, ["Xvfb", ":99"])
|
||||
# Live sibling session: another worker is solving a challenge with these right now.
|
||||
_write_fake_proc_entry(proc_root, 2000, 2000, ["python", "-m", "shelfmark.bypass"])
|
||||
_write_fake_proc_entry(proc_root, 2001, 2000, ["/usr/bin/chromium", "--headless"])
|
||||
# Abandoned session: its leader (pid 3000) is gone, so its browser really is an orphan.
|
||||
_write_fake_proc_entry(proc_root, 3001, 3000, ["/usr/bin/chromium", "--headless"])
|
||||
|
||||
killed: list[int] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.env, "DOCKERMODE", True)
|
||||
monkeypatch.setattr(internal_bypasser, "_PROC_ROOT", proc_root)
|
||||
monkeypatch.setattr(internal_bypasser.os, "getpid", lambda: 1000)
|
||||
monkeypatch.setattr(internal_bypasser.os, "getpgrp", lambda: 1000)
|
||||
monkeypatch.setattr(internal_bypasser.os, "kill", lambda pid, _sig: killed.append(pid))
|
||||
monkeypatch.setattr(internal_bypasser.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert internal_bypasser._cleanup_orphan_processes() == 3
|
||||
assert sorted(killed) == [1001, 1002, 3001]
|
||||
|
||||
|
||||
def test_cleanup_is_skipped_without_proc(monkeypatch, tmp_path):
|
||||
"""Without /proc there is no way to tell sessions apart, so kill nothing."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.env, "DOCKERMODE", True)
|
||||
monkeypatch.setattr(internal_bypasser, "_PROC_ROOT", tmp_path / "missing")
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.os, "kill", lambda *_args: pytest.fail("must not kill anything")
|
||||
)
|
||||
|
||||
assert internal_bypasser._cleanup_orphan_processes() == 0
|
||||
|
||||
|
||||
class _FakeHelperStdin:
|
||||
"""The request pipe: a write is how the helper receives one request."""
|
||||
|
||||
def __init__(self, process):
|
||||
self._process = process
|
||||
self.closed = False
|
||||
|
||||
def write(self, data):
|
||||
self._process.serve(data)
|
||||
|
||||
def flush(self):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeHelperProcess:
|
||||
"""Stand-in for the bypass helper subprocess.
|
||||
|
||||
The helper serves one request per line of stdin and answers by writing the result file
|
||||
the request named, so that is what this fakes: a write produces an answer.
|
||||
"""
|
||||
|
||||
def __init__(self, *_args, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.pid = 4242
|
||||
self.returncode = None
|
||||
self.answers = True
|
||||
self.killed = False
|
||||
self.waited = False
|
||||
self.stdin = _FakeHelperStdin(self)
|
||||
self.requests: list[dict] = []
|
||||
|
||||
def serve(self, payload):
|
||||
request = json.loads(payload)
|
||||
self.requests.append(request)
|
||||
if not self.answers:
|
||||
return
|
||||
result = {"ok": True, "html": "<html>solved</html>", "cookies": {}, "user_agents": {}}
|
||||
Path(request["result_path"]).write_text(json.dumps(result), encoding="utf-8")
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def kill(self):
|
||||
self.killed = True
|
||||
self.returncode = -9
|
||||
|
||||
def wait(self, timeout=None):
|
||||
self.waited = True
|
||||
if self.returncode is None:
|
||||
self.returncode = 0
|
||||
return self.returncode
|
||||
|
||||
|
||||
def _patch_helper_subprocess(monkeypatch, internal_bypasser, process, killed_groups):
|
||||
monkeypatch.setattr(internal_bypasser.subprocess, "Popen", lambda *a, **kw: process(*a, **kw))
|
||||
monkeypatch.setattr(internal_bypasser.network, "get_dns_config", dict)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.os, "killpg", lambda pgid, _sig: killed_groups.append(pgid)
|
||||
)
|
||||
# A fresh helper per test: the module-level one is shared, and a process parked by one
|
||||
# test would be handed to the next.
|
||||
helper = internal_bypasser._BypassHelper()
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 0.0)
|
||||
monkeypatch.setattr(internal_bypasser, "_BYPASS_HELPER", helper)
|
||||
return helper
|
||||
|
||||
|
||||
def test_helper_runs_in_its_own_session_and_is_torn_down(monkeypatch):
|
||||
"""Regression test for issue #1231: the helper's Chrome and Xvfb must belong to the
|
||||
helper's own process group, and the whole group must die with it - otherwise the
|
||||
leftovers break the next worker's browser and can only be cleared by a sweep broad
|
||||
enough to kill a concurrent worker's browser too.
|
||||
|
||||
The helper outlives a single request, so the teardown happens when it is dropped rather
|
||||
than after every solve. Each bypass still closes its own browser, so what survives in
|
||||
between is the process, not a Chrome.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
helper = _patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
|
||||
assert internal_bypasser._get_via_subprocess("https://example.com", 1) == "<html>solved</html>"
|
||||
assert processes[0].kwargs["start_new_session"] is True
|
||||
assert killed_groups == [], "the helper was torn down after a single request"
|
||||
|
||||
helper._discard()
|
||||
|
||||
assert killed_groups == [processes[0].pid]
|
||||
|
||||
|
||||
def test_helper_serves_a_second_request_without_respawning(monkeypatch):
|
||||
"""The interpreter start and imports are paid once, not per protected request."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
_patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
|
||||
internal_bypasser._get_via_subprocess("https://example.com/one", 1)
|
||||
internal_bypasser._get_via_subprocess("https://example.com/two", 1)
|
||||
|
||||
assert len(processes) == 1
|
||||
assert [request["url"] for request in processes[0].requests] == [
|
||||
"https://example.com/one",
|
||||
"https://example.com/two",
|
||||
]
|
||||
|
||||
|
||||
def test_helper_timeout_kills_the_whole_session(monkeypatch):
|
||||
"""A timed-out solve must not leave a live browser behind for the next worker."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
processes: list[_FakeHelperProcess] = []
|
||||
killed_groups: list[int] = []
|
||||
|
||||
def _make_process(*args, **kwargs):
|
||||
process = _FakeHelperProcess(*args, **kwargs)
|
||||
process.answers = False # accepts the request, never writes a result
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
_patch_helper_subprocess(monkeypatch, internal_bypasser, _make_process, killed_groups)
|
||||
monkeypatch.setattr(internal_bypasser, "_BYPASS_SUBPROCESS_TIMEOUT_SECONDS", 0.1)
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
internal_bypasser._get_via_subprocess("https://example.com", 1)
|
||||
|
||||
assert killed_groups == [processes[0].pid]
|
||||
assert processes[0].killed is True
|
||||
|
||||
|
||||
def test_helper_takes_the_browser_down_when_its_parent_dies(monkeypatch):
|
||||
"""Cleanup only reclaims process groups whose leader is gone (#1231), so an orphaned
|
||||
helper must not sit there holding a browser no later bypass is allowed to touch."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
terminated: list[str] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.os, "getppid", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser, "_terminate_own_session", lambda: terminated.append("terminated")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser.time, "sleep", lambda _seconds: pytest.fail("should not wait")
|
||||
)
|
||||
|
||||
internal_bypasser._watch_parent_process(999, interval=0.0)
|
||||
|
||||
assert terminated == ["terminated"]
|
||||
|
||||
|
||||
def test_helper_watchdog_waits_while_its_parent_is_alive(monkeypatch):
|
||||
"""The watchdog must only fire on a changed ppid, not on every poll."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
ppids = iter([999, 999, 1])
|
||||
sleeps: list[float] = []
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.os, "getppid", lambda: next(ppids))
|
||||
monkeypatch.setattr(internal_bypasser, "_terminate_own_session", lambda: None)
|
||||
monkeypatch.setattr(internal_bypasser.time, "sleep", sleeps.append)
|
||||
|
||||
internal_bypasser._watch_parent_process(999, interval=0.5)
|
||||
|
||||
assert sleeps == [0.5, 0.5]
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
"""Tests for keeping the bypass helper process alive between requests.
|
||||
|
||||
The browser is deliberately not kept: every bypass starts and closes its own Chrome. What
|
||||
survives is the helper process, whose interpreter start and imports are pure overhead.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _FakeStdin:
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
self.written: list[str] = []
|
||||
|
||||
def write(self, data: str) -> None:
|
||||
if self.closed:
|
||||
raise BrokenPipeError("stdin is closed")
|
||||
self.written.append(data)
|
||||
|
||||
def flush(self) -> None:
|
||||
return None
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeProc:
|
||||
"""Enough of subprocess.Popen for the helper's process bookkeeping."""
|
||||
|
||||
_next_pid = 90001
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.stdin = _FakeStdin()
|
||||
self.returncode: int | None = None
|
||||
self.waited = False
|
||||
# A pid nothing may actually be signalled by: _terminate_helper_session is patched
|
||||
# out in these tests, and a stray killpg on a live pid would take out the test run.
|
||||
type(self)._next_pid += 1
|
||||
self.pid = type(self)._next_pid
|
||||
|
||||
def poll(self) -> int | None:
|
||||
return self.returncode
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
self.waited = True
|
||||
if self.returncode is None:
|
||||
self.returncode = 0
|
||||
return self.returncode
|
||||
|
||||
def kill(self) -> None:
|
||||
self.returncode = -9
|
||||
|
||||
|
||||
def _helper_with_fake_spawn(monkeypatch, procs: list[_FakeProc], terminated=None):
|
||||
"""Build a helper that hands out fake processes and never arms a real timer."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
def _spawn(_self) -> _FakeProc:
|
||||
proc = _FakeProc()
|
||||
procs.append(proc)
|
||||
return proc
|
||||
|
||||
def _terminate(proc) -> None:
|
||||
if terminated is not None:
|
||||
terminated.append(proc)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_spawn", _spawn)
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 0.0)
|
||||
monkeypatch.setattr(internal_bypasser, "_terminate_helper_session", _terminate)
|
||||
return internal_bypasser._BypassHelper()
|
||||
|
||||
|
||||
def _answered_payload(tmp_path, name: str = "result.json") -> dict:
|
||||
"""A request whose result file already exists, so the helper resolves immediately."""
|
||||
result_path = tmp_path / name
|
||||
result_path.write_text(json.dumps({"ok": True, "html": "<html/>"}), encoding="utf-8")
|
||||
return {"url": "https://example.com", "retry": 1, "result_path": str(result_path)}
|
||||
|
||||
|
||||
def test_helper_serves_consecutive_requests_from_one_process(monkeypatch, tmp_path):
|
||||
"""The point of the whole thing: request two and three must not re-pay the spawn."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
for i in range(3):
|
||||
result = helper.run(_answered_payload(tmp_path, f"r{i}.json"), timeout=5, cancel_flag=None)
|
||||
assert result["ok"] is True
|
||||
|
||||
assert len(procs) == 1, "each request spawned its own helper"
|
||||
assert len(procs[0].stdin.written) == 3
|
||||
assert all(line.endswith("\n") for line in procs[0].stdin.written), (
|
||||
"requests must be newline-delimited or the helper's loop cannot split them"
|
||||
)
|
||||
|
||||
|
||||
def test_helper_respawns_after_the_previous_one_died(monkeypatch, tmp_path):
|
||||
"""A helper can be reaped while idle; the next request must not fail on it."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path, "a.json"), timeout=5, cancel_flag=None)
|
||||
procs[0].returncode = 1 # died between requests
|
||||
|
||||
result = helper.run(_answered_payload(tmp_path, "b.json"), timeout=5, cancel_flag=None)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert len(procs) == 2
|
||||
|
||||
|
||||
def test_helper_retries_once_when_the_pipe_breaks_on_write(monkeypatch, tmp_path):
|
||||
"""poll() can still say alive when the far end is already gone."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path, "a.json"), timeout=5, cancel_flag=None)
|
||||
procs[0].stdin.closed = True # pipe gone, but poll() still reports running
|
||||
|
||||
result = helper.run(_answered_payload(tmp_path, "b.json"), timeout=5, cancel_flag=None)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert len(procs) == 2
|
||||
|
||||
|
||||
def test_helper_reports_a_helper_that_exits_without_answering(monkeypatch, tmp_path):
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
payload = {
|
||||
"url": "https://example.com",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / "never-written.json"),
|
||||
}
|
||||
|
||||
def _die_on_write(_self, proc, _line) -> None:
|
||||
proc.returncode = 3
|
||||
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_write", _die_on_write)
|
||||
|
||||
with pytest.raises(RuntimeError, match="exited without a result"):
|
||||
helper.run(payload, timeout=5, cancel_flag=None)
|
||||
|
||||
|
||||
def test_helper_times_out_and_discards_the_wedged_process(monkeypatch, tmp_path):
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
payload = {
|
||||
"url": "https://example.com",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / "never-written.json"),
|
||||
}
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
helper.run(payload, timeout=0.05, cancel_flag=None)
|
||||
|
||||
assert helper._proc is None, "a wedged helper must not be handed to the next request"
|
||||
|
||||
|
||||
def test_idle_reaper_rearms_when_work_arrived_while_it_waited(monkeypatch, tmp_path):
|
||||
"""The timer fires on its own thread and can lose the race against a new request."""
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
|
||||
rearmed: list[bool] = []
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 3600.0)
|
||||
monkeypatch.setattr(
|
||||
internal_bypasser._BypassHelper, "_arm_idle_timer", lambda _self: rearmed.append(True)
|
||||
)
|
||||
helper._last_used = time.monotonic()
|
||||
|
||||
helper._reap_if_idle()
|
||||
|
||||
assert rearmed == [True]
|
||||
assert helper._proc is not None, "helper was killed despite recent work"
|
||||
|
||||
|
||||
def test_idle_reaper_closes_a_genuinely_idle_helper(monkeypatch, tmp_path):
|
||||
import time
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser._BypassHelper, "_idle_timeout", lambda _self: 60.0)
|
||||
helper._last_used = time.monotonic() - 120
|
||||
|
||||
helper._reap_if_idle()
|
||||
|
||||
assert helper._proc is None
|
||||
assert procs[0].stdin.closed
|
||||
|
||||
|
||||
def test_discard_tears_down_the_whole_session(monkeypatch, tmp_path):
|
||||
"""Dropping the helper must reach its browser tree, not just the helper itself.
|
||||
|
||||
The helper is a session leader (start_new_session), so a Chrome left behind by one
|
||||
killed mid-bypass would keep a process group alive that the cleanup sweep is then not
|
||||
allowed to reclaim - the leak #1231 was about.
|
||||
"""
|
||||
procs: list[_FakeProc] = []
|
||||
terminated: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs, terminated)
|
||||
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
helper._discard()
|
||||
|
||||
assert terminated == [procs[0]]
|
||||
|
||||
|
||||
def test_helper_asks_before_it_kills(monkeypatch, tmp_path):
|
||||
"""An idle helper should get to exit on its own; the kill is the fallback."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
helper._discard()
|
||||
|
||||
assert procs[0].stdin.closed, "stdin must be closed to end the helper's request loop"
|
||||
assert procs[0].returncode == 0, "an idle helper should have exited on its own"
|
||||
|
||||
|
||||
def _bypass_with_recorded_driver(monkeypatch, get_impl):
|
||||
"""Wire up a bypass whose browser creation and closing are observable."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
driver = object()
|
||||
closed: list[object] = []
|
||||
|
||||
async def _create(_url):
|
||||
return driver
|
||||
|
||||
async def _close(drv):
|
||||
closed.append(drv)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "_create_cdp_browser", _create)
|
||||
monkeypatch.setattr(internal_bypasser, "_get", get_impl)
|
||||
monkeypatch.setattr(internal_bypasser, "_close_cdp_driver", _close)
|
||||
return driver, closed
|
||||
|
||||
|
||||
def test_successful_bypass_closes_its_browser(monkeypatch):
|
||||
"""A living helper must not accumulate browsers: each bypass ends with Chrome gone."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
return "<html>ok</html>"
|
||||
|
||||
driver, closed = _bypass_with_recorded_driver(monkeypatch, _get)
|
||||
|
||||
result = internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert result == "<html>ok</html>"
|
||||
assert closed == [driver]
|
||||
|
||||
|
||||
def test_failed_bypass_closes_its_browser(monkeypatch):
|
||||
"""The same has to hold when the bypass raises on its way out."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
raise internal_bypasser.BypassCancelledError("cancelled")
|
||||
|
||||
driver, closed = _bypass_with_recorded_driver(monkeypatch, _get)
|
||||
|
||||
with pytest.raises(internal_bypasser.BypassCancelledError):
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert closed == [driver]
|
||||
|
||||
|
||||
def test_child_process_serves_every_line_it_is_given(monkeypatch, tmp_path):
|
||||
"""One helper, several requests: the loop is what saves the repeated process start."""
|
||||
import io
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
urls: list[str] = []
|
||||
|
||||
def _fake_get(url, retry=None, cancel_flag=None):
|
||||
urls.append(url)
|
||||
return f"<html>{url}</html>"
|
||||
|
||||
requests = [
|
||||
{"url": "https://example.com/one", "retry": 1, "result_path": str(tmp_path / "1.json")},
|
||||
{"url": "https://example.com/two", "retry": 1, "result_path": str(tmp_path / "2.json")},
|
||||
]
|
||||
stdin = io.StringIO("\n".join(json.dumps(request) for request in requests) + "\n")
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "get", _fake_get)
|
||||
monkeypatch.setattr(internal_bypasser.sys, "stdin", stdin)
|
||||
|
||||
assert internal_bypasser._run_child_process() == 0
|
||||
assert urls == ["https://example.com/one", "https://example.com/two"]
|
||||
|
||||
for index, request in enumerate(requests, start=1):
|
||||
result = json.loads((tmp_path / f"{index}.json").read_text(encoding="utf-8"))
|
||||
assert result["ok"] is True
|
||||
assert result["html"] == f"<html>{request['url']}</html>"
|
||||
|
||||
|
||||
def test_child_process_keeps_serving_after_a_failed_request(monkeypatch, tmp_path):
|
||||
"""One failing URL must not take the helper - and everything queued - down."""
|
||||
import io
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
def _fake_get(url, retry=None, cancel_flag=None):
|
||||
if url.endswith("boom"):
|
||||
raise RuntimeError("bypass exploded")
|
||||
return "<html>ok</html>"
|
||||
|
||||
requests = [
|
||||
{"url": "https://example.com/boom", "retry": 1, "result_path": str(tmp_path / "1.json")},
|
||||
{"url": "https://example.com/fine", "retry": 1, "result_path": str(tmp_path / "2.json")},
|
||||
]
|
||||
stdin = io.StringIO("\n".join(json.dumps(request) for request in requests) + "\n")
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "get", _fake_get)
|
||||
monkeypatch.setattr(internal_bypasser.sys, "stdin", stdin)
|
||||
|
||||
assert internal_bypasser._run_child_process() == 0
|
||||
|
||||
failed = json.loads((tmp_path / "1.json").read_text(encoding="utf-8"))
|
||||
assert failed["ok"] is False
|
||||
assert failed["error"] == "bypass exploded"
|
||||
|
||||
served = json.loads((tmp_path / "2.json").read_text(encoding="utf-8"))
|
||||
assert served["ok"] is True
|
||||
|
||||
|
||||
def test_result_file_becomes_visible_only_when_complete(tmp_path):
|
||||
"""The parent treats the file's existence as the answer, so no partial writes."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
result_path = tmp_path / "result.json"
|
||||
internal_bypasser._publish_result(result_path, {"ok": True, "html": "<html/>"})
|
||||
|
||||
assert json.loads(result_path.read_text(encoding="utf-8"))["ok"] is True
|
||||
assert list(tmp_path.iterdir()) == [result_path], "temporary file was left behind"
|
||||
|
||||
|
||||
def test_child_bypass_runs_on_the_long_lived_worker_loop(monkeypatch):
|
||||
"""A helper serving many requests must not build and close a loop per bypass.
|
||||
|
||||
asyncio.run() owns the loop for one call and closes it on the way out, which is why the
|
||||
child goes through the worker unconditionally: one loop for the process's lifetime.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setenv("SHELFMARK_INTERNAL_BYPASSER_CHILD", "1")
|
||||
|
||||
loops: list[asyncio.AbstractEventLoop] = []
|
||||
|
||||
async def _record_loop(_url, _driver, _cancel=None):
|
||||
loops.append(asyncio.get_running_loop())
|
||||
return "<html>ok</html>"
|
||||
|
||||
_bypass_with_recorded_driver(monkeypatch, _record_loop)
|
||||
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert len(loops) == 2
|
||||
assert loops[0] is loops[1], "second bypass ran on a different loop than the first"
|
||||
assert not loops[0].is_closed()
|
||||
|
||||
|
||||
def test_child_bypass_carries_its_own_deadline(monkeypatch):
|
||||
"""The child bounds itself, rather than relying only on the parent's deadline."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setenv("SHELFMARK_INTERNAL_BYPASSER_CHILD", "1")
|
||||
|
||||
timeouts: list[float | None] = []
|
||||
real_run = internal_bypasser._CDP_WORKER.run
|
||||
|
||||
def _record_timeout(coro, timeout=None):
|
||||
timeouts.append(timeout)
|
||||
return real_run(coro, timeout=timeout)
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
return "<html>ok</html>"
|
||||
|
||||
_bypass_with_recorded_driver(monkeypatch, _get)
|
||||
monkeypatch.setattr(internal_bypasser._CDP_WORKER, "run", _record_timeout)
|
||||
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert timeouts == [internal_bypasser._CHILD_BYPASS_TIMEOUT_SECONDS]
|
||||
|
||||
|
||||
def test_child_deadline_leaves_the_parent_room_to_hear_the_answer(monkeypatch):
|
||||
"""If the parent gave up first it could only kill the helper, losing a warm process."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
# The child's worst case is its deadline plus the grace it is given to close the
|
||||
# browser after that deadline cancels the bypass, and all of it has to fit inside the
|
||||
# parent's wait - otherwise the parent gives up first and kills a helper that was
|
||||
# about to answer.
|
||||
assert (
|
||||
internal_bypasser._CHILD_BYPASS_TIMEOUT_SECONDS
|
||||
+ internal_bypasser._CDP_UNWIND_GRACE_SECONDS
|
||||
< internal_bypasser._BYPASS_SUBPROCESS_TIMEOUT_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def test_in_process_bypass_keeps_the_parents_budget(monkeypatch):
|
||||
"""Non-Docker installs run in-process, where there is no helper to outlive anything."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.delenv("SHELFMARK_INTERNAL_BYPASSER_CHILD", raising=False)
|
||||
|
||||
timeouts: list[float | None] = []
|
||||
real_run = internal_bypasser._CDP_WORKER.run
|
||||
|
||||
def _record_timeout(coro, timeout=None):
|
||||
timeouts.append(timeout)
|
||||
return real_run(coro, timeout=timeout)
|
||||
|
||||
async def _get(_url, _driver, _cancel=None):
|
||||
return "<html>ok</html>"
|
||||
|
||||
_bypass_with_recorded_driver(monkeypatch, _get)
|
||||
monkeypatch.setattr(internal_bypasser._CDP_WORKER, "run", _record_timeout)
|
||||
|
||||
internal_bypasser._run_bypass_in_current_process("https://example.com", 1)
|
||||
|
||||
assert timeouts == [internal_bypasser._IN_PROCESS_BYPASS_TIMEOUT_SECONDS]
|
||||
|
||||
|
||||
def test_timed_out_bypass_finishes_unwinding_before_the_call_returns():
|
||||
"""A helper serving the next request must not race the browser teardown of the last.
|
||||
|
||||
The deadline cancels the bypass, but cancelling from the calling thread only schedules
|
||||
that - it returns while `finally: await _close_cdp_driver(driver)` is still running.
|
||||
In a helper that now outlives the request, the next bypass would open its Chrome on the
|
||||
same loop while the abandoned one was still closing its own, sharing the DISPLAY
|
||||
globals and one process group.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
events: list[str] = []
|
||||
|
||||
async def _wedged():
|
||||
try:
|
||||
await asyncio.sleep(30)
|
||||
finally:
|
||||
# Teardown that yields, the way closing websockets and Chrome does.
|
||||
await asyncio.sleep(0.05)
|
||||
events.append("browser closed")
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
internal_bypasser._CDP_WORKER.run(_wedged(), timeout=0.1)
|
||||
|
||||
assert events == ["browser closed"], "run() returned before the bypass had unwound"
|
||||
|
||||
|
||||
def test_unwind_that_wedges_does_not_hold_the_caller_forever(monkeypatch):
|
||||
"""The grace is a bound, not a promise: cleanup can hang on a dead browser too."""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "_CDP_UNWIND_GRACE_SECONDS", 0.1)
|
||||
|
||||
async def _wedged_on_both_ends():
|
||||
try:
|
||||
await asyncio.sleep(30)
|
||||
finally:
|
||||
await asyncio.sleep(30)
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
internal_bypasser._CDP_WORKER.run(_wedged_on_both_ends(), timeout=0.1)
|
||||
|
||||
|
||||
def test_cancelling_does_not_wait_out_the_shutdown_grace(monkeypatch, tmp_path):
|
||||
"""The grace only helps a helper that can still read its stdin.
|
||||
|
||||
One dropped mid-bypass is blocked inside the solve and will never reach its read loop,
|
||||
so waiting it out cannot end in anything but the kill - while the user who asked to
|
||||
cancel, and every bypass queued behind them on LOCKED, waits for it.
|
||||
"""
|
||||
import threading
|
||||
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
cancel_flag = threading.Event()
|
||||
cancel_flag.set()
|
||||
payload = {
|
||||
"url": "https://example.com",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / "never-written.json"),
|
||||
}
|
||||
|
||||
with pytest.raises(internal_bypasser.BypassCancelledError):
|
||||
helper.run(payload, timeout=5, cancel_flag=cancel_flag)
|
||||
|
||||
assert not procs[0].waited, "a helper wedged mid-bypass was given the full exit grace"
|
||||
assert procs[0].stdin.closed
|
||||
|
||||
|
||||
def test_idle_helper_still_gets_its_grace(monkeypatch, tmp_path):
|
||||
"""The reaper drops a helper that *is* in its read loop, and that one gets to exit."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
helper.run(_answered_payload(tmp_path), timeout=5, cancel_flag=None)
|
||||
helper._discard()
|
||||
|
||||
assert procs[0].waited, "an idle helper should be asked to exit before being killed"
|
||||
|
||||
|
||||
def test_failed_request_leaves_no_result_files_behind(monkeypatch, tmp_path):
|
||||
"""Result paths are unique per request, so anything left is left for good."""
|
||||
procs: list[_FakeProc] = []
|
||||
helper = _helper_with_fake_spawn(monkeypatch, procs)
|
||||
|
||||
result_path = tmp_path / "result.json"
|
||||
# A helper killed part-way through _publish_result leaves the staging file.
|
||||
(tmp_path / "result.json.part").write_text('{"ok": tr', encoding="utf-8")
|
||||
payload = {"url": "https://example.com", "retry": 1, "result_path": str(result_path)}
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
helper.run(payload, timeout=0.05, cancel_flag=None)
|
||||
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_child_does_not_export_cookies_left_by_an_earlier_request(monkeypatch, tmp_path):
|
||||
"""The parent owns the store; a warm helper must not push its own history back over it.
|
||||
|
||||
http.py purges a host's clearance the moment that host challenges a request carrying
|
||||
it. A helper that kept its store across requests would still be holding the purged
|
||||
cookies, and the next solve - for some entirely different host - would export them and
|
||||
the parent would merge them straight back in.
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
def _solve_host(url, retry=None, cancel_flag=None):
|
||||
# A solve fills the store for the host it solved, which is all it should report.
|
||||
host = url.rsplit("/", 1)[-1]
|
||||
internal_bypasser.import_store({host: {"cf_clearance": "fresh"}}, {host: "UA"})
|
||||
return "<html>ok</html>"
|
||||
|
||||
monkeypatch.setattr(internal_bypasser, "get", _solve_host)
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
|
||||
for index, host in enumerate(("first.example", "second.example")):
|
||||
internal_bypasser._handle_child_request(
|
||||
json.dumps(
|
||||
{
|
||||
"url": f"https://example.com/{host}",
|
||||
"retry": 1,
|
||||
"result_path": str(tmp_path / f"{index}.json"),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
second = json.loads((tmp_path / "1.json").read_text(encoding="utf-8"))
|
||||
assert list(second["cookies"]) == ["second.example"], (
|
||||
"the helper exported clearance won by an earlier request"
|
||||
)
|
||||
assert list(second["user_agents"]) == ["second.example"]
|
||||
|
||||
internal_bypasser.clear_cf_cookies()
|
||||
|
||||
|
||||
def _record_dns_calls(monkeypatch):
|
||||
"""Stand in for the network module: report a resolver state, record changes to it.
|
||||
|
||||
A helper starts on system DNS, which is what the parent reports as "auto".
|
||||
"""
|
||||
import shelfmark.bypass.internal_bypasser as internal_bypasser
|
||||
|
||||
calls: list[tuple] = []
|
||||
state = {"provider": "auto", "servers": [], "doh_enabled": False}
|
||||
|
||||
def _set(provider, servers=None, use_doh=None):
|
||||
calls.append((provider, servers, use_doh))
|
||||
state.update({"provider": provider, "servers": servers or [], "doh_enabled": bool(use_doh)})
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.network, "set_dns_provider", _set)
|
||||
monkeypatch.setattr(internal_bypasser.network, "get_dns_config", lambda: dict(state))
|
||||
return internal_bypasser, calls
|
||||
|
||||
|
||||
def test_helper_follows_the_parent_back_to_auto_dns(monkeypatch):
|
||||
"""A user flipping CUSTOM_DNS back to auto applies live - the helper has to hear it.
|
||||
|
||||
The old early-return on "auto" was correct only because a fresh helper had never been
|
||||
told anything else. One that outlives the request has, and would go on resolving AA
|
||||
through a resolver the parent has already abandoned.
|
||||
"""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "cloudflare", "servers": [], "doh_enabled": True}
|
||||
)
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "auto", "servers": [], "doh_enabled": False}
|
||||
)
|
||||
|
||||
assert calls == [("cloudflare", None, True), ("auto", None, False)]
|
||||
|
||||
|
||||
def test_helper_does_not_reinitialize_dns_for_an_unchanged_config(monkeypatch):
|
||||
"""set_dns_provider() rebuilds resolvers; every request would pay for it otherwise."""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
for _ in range(3):
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "quad9", "servers": [], "doh_enabled": True}
|
||||
)
|
||||
|
||||
assert calls == [("quad9", None, True)]
|
||||
|
||||
|
||||
def test_fresh_helper_leaves_auto_dns_alone(monkeypatch):
|
||||
"""A helper starts on system DNS, which is what the parent reports as auto."""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
internal_bypasser._apply_parent_dns_config(
|
||||
{"provider": "auto", "servers": [], "doh_enabled": False}
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_failed_dns_apply_is_retried_on_the_next_request(monkeypatch):
|
||||
"""A provider that did not land leaves the resolver where it was, so the next request
|
||||
sees the same mismatch and tries again."""
|
||||
internal_bypasser, calls = _record_dns_calls(monkeypatch)
|
||||
|
||||
def _explode(provider, servers=None, use_doh=None):
|
||||
calls.append((provider, servers, use_doh))
|
||||
msg = "resolver unreachable"
|
||||
raise OSError(msg)
|
||||
|
||||
monkeypatch.setattr(internal_bypasser.network, "set_dns_provider", _explode)
|
||||
|
||||
config = {"provider": "google", "servers": [], "doh_enabled": True}
|
||||
internal_bypasser._apply_parent_dns_config(config)
|
||||
internal_bypasser._apply_parent_dns_config(config)
|
||||
|
||||
assert calls == [("google", None, True), ("google", None, True)]
|
||||
@@ -66,6 +66,32 @@ def test_download_settings_booklore_destination_field_defaults_to_library():
|
||||
assert option_values == {"library", "bookdrop"}
|
||||
|
||||
|
||||
def test_download_settings_audiobook_grouping_is_opt_in():
|
||||
from shelfmark.config.settings import download_settings
|
||||
|
||||
fields = download_settings()
|
||||
organization_field = next(
|
||||
field for field in fields if getattr(field, "key", None) == "FILE_ORGANIZATION_AUDIOBOOK"
|
||||
)
|
||||
rename_template = next(
|
||||
field
|
||||
for field in fields
|
||||
if getattr(field, "key", None) == "template_audiobook_rename_editor"
|
||||
)
|
||||
|
||||
assert organization_field.default == "rename"
|
||||
assert [option["value"] for option in organization_field.options] == [
|
||||
"none",
|
||||
"rename",
|
||||
"organize",
|
||||
"rename_and_group",
|
||||
]
|
||||
assert rename_template.show_when == {
|
||||
"field": "FILE_ORGANIZATION_AUDIOBOOK",
|
||||
"value": ["rename", "rename_and_group"],
|
||||
}
|
||||
|
||||
|
||||
def test_download_settings_grimmory_copy_is_exposed_in_ui_metadata():
|
||||
from shelfmark.config.settings import download_settings
|
||||
|
||||
|
||||
@@ -385,14 +385,15 @@ class TestSettingsValidation:
|
||||
assert "Naming Template" in result["message"]
|
||||
assert "Organize" in result["message"]
|
||||
|
||||
def test_downloads_audiobooks_rename_template_rejects_path_separators(self):
|
||||
@pytest.mark.parametrize("organization_mode", ["rename", "rename_and_group"])
|
||||
def test_downloads_audiobooks_rename_template_rejects_path_separators(self, organization_mode):
|
||||
import shelfmark.config.settings # noqa: F401
|
||||
from shelfmark.core.settings_registry import update_settings
|
||||
|
||||
result = update_settings(
|
||||
"downloads",
|
||||
{
|
||||
"FILE_ORGANIZATION_AUDIOBOOK": "rename",
|
||||
"FILE_ORGANIZATION_AUDIOBOOK": organization_mode,
|
||||
"TEMPLATE_AUDIOBOOK_RENAME": "{Author}/{Title}",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -66,6 +66,20 @@ def test_get_file_organization_uses_current_keys(monkeypatch):
|
||||
assert policy.get_file_organization(is_audiobook=True) == "none"
|
||||
|
||||
|
||||
def test_get_file_organization_accepts_audiobook_grouping(monkeypatch):
|
||||
import shelfmark.download.postprocess.policy as policy
|
||||
|
||||
monkeypatch.setattr(
|
||||
policy.core_config.config,
|
||||
"get",
|
||||
lambda key, default=None: {
|
||||
"FILE_ORGANIZATION_AUDIOBOOK": "rename_and_group",
|
||||
}.get(key, default),
|
||||
)
|
||||
|
||||
assert policy.get_file_organization(is_audiobook=True) == "rename_and_group"
|
||||
|
||||
|
||||
def test_get_file_organization_ignores_pre_release_processing_mode_keys(monkeypatch):
|
||||
import shelfmark.download.postprocess.policy as policy
|
||||
|
||||
|
||||
+168
-3
@@ -14,6 +14,7 @@ Two approaches to preserve torrent files for seeding:
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -1003,8 +1004,13 @@ class TestTorrentSourceCleanupProtection:
|
||||
Each test simulates a specific real-world content type and file structure.
|
||||
"""
|
||||
|
||||
def _make_config_mock(self, library_path: str, hardlink: bool = True):
|
||||
"""Create config mock for library/organize mode with hardlinking."""
|
||||
def _make_config_mock(
|
||||
self,
|
||||
library_path: str,
|
||||
hardlink: bool = True,
|
||||
organization_mode: str = "organize",
|
||||
):
|
||||
"""Create a folder-output config mock for torrent transfers."""
|
||||
return MagicMock(
|
||||
side_effect=lambda key, default=None, **_kwargs: {
|
||||
# Destination paths (what _get_final_destination uses)
|
||||
@@ -1013,9 +1019,10 @@ class TestTorrentSourceCleanupProtection:
|
||||
# Templates (what _get_template uses)
|
||||
"TEMPLATE_ORGANIZE": "{Author}/{Title}",
|
||||
"TEMPLATE_AUDIOBOOK_ORGANIZE": "{Author}/{Title}{ - PartNumber}",
|
||||
"TEMPLATE_AUDIOBOOK_RENAME": "{Author} - {Title}",
|
||||
# File organization mode
|
||||
"FILE_ORGANIZATION": "organize",
|
||||
"FILE_ORGANIZATION_AUDIOBOOK": "organize",
|
||||
"FILE_ORGANIZATION_AUDIOBOOK": organization_mode,
|
||||
# Hardlink toggle
|
||||
"HARDLINK_TORRENTS": hardlink,
|
||||
"HARDLINK_TORRENTS_AUDIOBOOK": hardlink,
|
||||
@@ -1122,6 +1129,163 @@ class TestTorrentSourceCleanupProtection:
|
||||
|
||||
# ==================== AUDIOBOOK TESTS ====================
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("organization_mode", "grouped"),
|
||||
[("rename", False), ("rename_and_group", True)],
|
||||
)
|
||||
@pytest.mark.parametrize("hardlink", [True, False])
|
||||
def test_torrent_audiobook_multifile_grouping_is_opt_in(
|
||||
self, tmp_path, organization_mode, grouped, hardlink
|
||||
):
|
||||
"""Multi-file audiobooks retain their torrent folder only when opted in."""
|
||||
from shelfmark.core.models import DownloadTask, SearchMode
|
||||
from shelfmark.download.postprocess.router import (
|
||||
post_process_download as _post_process_download,
|
||||
)
|
||||
|
||||
torrent_dir = tmp_path / "downloads" / "Project: Hail Mary Audiobook"
|
||||
torrent_dir.mkdir(parents=True)
|
||||
source_files = [torrent_dir / "Part 01.mp3", torrent_dir / "Part 02.mp3"]
|
||||
for index, source_file in enumerate(source_files, start=1):
|
||||
source_file.write_bytes(f"audio {index}".encode())
|
||||
|
||||
library = tmp_path / "library"
|
||||
library.mkdir()
|
||||
task = DownloadTask(
|
||||
task_id=f"audiobook_{organization_mode}_{hardlink}",
|
||||
source="prowlarr",
|
||||
title="Project Hail Mary",
|
||||
author="Andy Weir",
|
||||
format="mp3",
|
||||
content_type="audiobook",
|
||||
search_mode=SearchMode.UNIVERSAL,
|
||||
original_download_path=str(torrent_dir),
|
||||
)
|
||||
|
||||
with patch("shelfmark.core.config.config") as mock_orch:
|
||||
mock_orch.get = self._make_config_mock(
|
||||
str(library),
|
||||
hardlink=hardlink,
|
||||
organization_mode=organization_mode,
|
||||
)
|
||||
mock_orch.CUSTOM_SCRIPT = None
|
||||
result = _post_process_download(torrent_dir, task, Event(), MagicMock())
|
||||
|
||||
transfer_dir = library / "Project_ Hail Mary Audiobook" if grouped else library
|
||||
assert result is not None
|
||||
assert Path(result).parent == transfer_dir
|
||||
assert sorted(path.name for path in transfer_dir.glob("*.mp3")) == [
|
||||
"Part 01.mp3",
|
||||
"Part 02.mp3",
|
||||
]
|
||||
assert bool(list(library.glob("*.mp3"))) is not grouped
|
||||
|
||||
for source_file in source_files:
|
||||
destination_file = transfer_dir / source_file.name
|
||||
assert source_file.exists()
|
||||
assert destination_file.exists()
|
||||
if hardlink:
|
||||
assert source_file.stat().st_ino == destination_file.stat().st_ino
|
||||
else:
|
||||
assert source_file.stat().st_ino != destination_file.stat().st_ino
|
||||
|
||||
@pytest.mark.parametrize("organization_mode", ["rename", "none", "rename_and_group"])
|
||||
def test_torrent_audiobook_single_file_stays_in_destination_root(
|
||||
self, tmp_path, organization_mode
|
||||
):
|
||||
"""Single-file audiobooks do not gain a source-folder wrapper."""
|
||||
from shelfmark.core.models import DownloadTask, SearchMode
|
||||
from shelfmark.download.postprocess.router import (
|
||||
post_process_download as _post_process_download,
|
||||
)
|
||||
|
||||
torrent_file = tmp_path / "downloads" / "Project Hail Mary.mp3"
|
||||
torrent_file.parent.mkdir()
|
||||
torrent_file.write_bytes(b"audio")
|
||||
library = tmp_path / "library"
|
||||
library.mkdir()
|
||||
task = DownloadTask(
|
||||
task_id=f"single_audiobook_{organization_mode}",
|
||||
source="prowlarr",
|
||||
title="Project Hail Mary",
|
||||
author="Andy Weir",
|
||||
format="mp3",
|
||||
content_type="audiobook",
|
||||
search_mode=SearchMode.UNIVERSAL,
|
||||
original_download_path=str(torrent_file),
|
||||
)
|
||||
|
||||
with patch("shelfmark.core.config.config") as mock_orch:
|
||||
mock_orch.get = self._make_config_mock(
|
||||
str(library), organization_mode=organization_mode
|
||||
)
|
||||
mock_orch.CUSTOM_SCRIPT = None
|
||||
result = _post_process_download(torrent_file, task, Event(), MagicMock())
|
||||
|
||||
expected_name = (
|
||||
"Andy Weir - Project Hail Mary.mp3"
|
||||
if organization_mode in {"rename", "rename_and_group"}
|
||||
else torrent_file.name
|
||||
)
|
||||
assert result is not None
|
||||
assert Path(result) == library / expected_name
|
||||
assert torrent_file.exists()
|
||||
|
||||
def test_torrent_audiobook_archive_groups_under_the_release_name(self, tmp_path):
|
||||
"""Torrent: a single archive of chapters groups under the release name.
|
||||
|
||||
Simulates: a torrent whose only file is "Project Hail Mary.zip" holding
|
||||
two chapters. Extraction only runs with hardlinking off, and the archive
|
||||
itself must stay put for seeding.
|
||||
"""
|
||||
from shelfmark.core.models import DownloadTask, SearchMode
|
||||
from shelfmark.download.postprocess.router import (
|
||||
post_process_download as _post_process_download,
|
||||
)
|
||||
|
||||
torrent_file = tmp_path / "downloads" / "Project Hail Mary.zip"
|
||||
torrent_file.parent.mkdir(parents=True)
|
||||
with zipfile.ZipFile(torrent_file, "w") as zf:
|
||||
zf.writestr("Part 01.mp3", "audio 1")
|
||||
zf.writestr("Part 02.mp3", "audio 2")
|
||||
|
||||
library = tmp_path / "library"
|
||||
library.mkdir()
|
||||
task = DownloadTask(
|
||||
task_id="torrent_archive_audiobook",
|
||||
source="prowlarr",
|
||||
title="Project Hail Mary",
|
||||
author="Andy Weir",
|
||||
format="mp3",
|
||||
content_type="audiobook",
|
||||
search_mode=SearchMode.UNIVERSAL,
|
||||
original_download_path=str(torrent_file),
|
||||
)
|
||||
|
||||
with (
|
||||
patch("shelfmark.core.config.config") as mock_orch,
|
||||
patch("shelfmark.config.env.TMP_DIR", tmp_path / "staging"),
|
||||
):
|
||||
mock_orch.get = self._make_config_mock(
|
||||
str(library),
|
||||
hardlink=False,
|
||||
organization_mode="rename_and_group",
|
||||
)
|
||||
mock_orch.CUSTOM_SCRIPT = None
|
||||
result = _post_process_download(torrent_file, task, Event(), MagicMock())
|
||||
|
||||
grouped_dir = library / "Project Hail Mary"
|
||||
assert result is not None
|
||||
assert Path(result).parent == grouped_dir
|
||||
assert sorted(path.name for path in grouped_dir.glob("*.mp3")) == [
|
||||
"Part 01.mp3",
|
||||
"Part 02.mp3",
|
||||
]
|
||||
assert not list(library.glob("*.mp3"))
|
||||
# The archive stays behind for seeding, and never names the folder.
|
||||
assert torrent_file.exists()
|
||||
assert not (library / "Project Hail Mary.zip").exists()
|
||||
|
||||
def test_torrent_audiobook_multifile_hardlink(self, tmp_path):
|
||||
"""Torrent: Multi-file audiobook - all source files preserved for seeding.
|
||||
|
||||
@@ -1184,6 +1348,7 @@ class TestTorrentSourceCleanupProtection:
|
||||
# Verify library has all 12 files
|
||||
library_files = list((library / "Andy Weir").glob("*.mp3"))
|
||||
assert len(library_files) == 12
|
||||
assert not (library / torrent_dir.name).exists()
|
||||
|
||||
# ==================== COMIC/CBZ TESTS ====================
|
||||
|
||||
|
||||
@@ -748,6 +748,94 @@ def test_archive_extraction_organize_multifile_assigns_part_numbers(tmp_path):
|
||||
assert files[1].name == "Archive Audio - 02.mp3"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("organization_mode", "grouped"),
|
||||
[("rename_and_group", True), ("rename", False)],
|
||||
)
|
||||
def test_archive_extraction_groups_chapters_under_the_archive_stem(
|
||||
tmp_path, organization_mode, grouped
|
||||
):
|
||||
"""An extracted multi-chapter archive groups into `Book/`, never `Book.zip/`."""
|
||||
from shelfmark.download.postprocess.router import (
|
||||
post_process_download as _post_process_download,
|
||||
)
|
||||
|
||||
staging = tmp_path / "staging"
|
||||
ingest = tmp_path / "ingest"
|
||||
staging.mkdir()
|
||||
ingest.mkdir()
|
||||
|
||||
archive_path = staging / "Book.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as zf:
|
||||
zf.writestr("Part 1.mp3", "audio1")
|
||||
zf.writestr("Part 2.mp3", "audio2")
|
||||
|
||||
task = DownloadTask(
|
||||
task_id=f"direct-archive-audio-{organization_mode}",
|
||||
source="direct_download",
|
||||
title="Archive Audio",
|
||||
author="Tester",
|
||||
format="mp3",
|
||||
content_type="audiobook",
|
||||
search_mode=SearchMode.DIRECT,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("shelfmark.core.config.config") as mock_config,
|
||||
patch("shelfmark.config.env.TMP_DIR", staging),
|
||||
):
|
||||
mock_config.get = _build_config(ingest, organization=organization_mode)
|
||||
mock_config.CUSTOM_SCRIPT = None
|
||||
_sync_config(mock_config, mock_config)
|
||||
|
||||
result = _post_process_download(archive_path, task, Event(), lambda *_args: None)
|
||||
|
||||
transfer_dir = ingest / "Book" if grouped else ingest
|
||||
assert result is not None
|
||||
assert Path(result).parent == transfer_dir
|
||||
assert sorted(path.name for path in transfer_dir.glob("*.mp3")) == ["Part 1.mp3", "Part 2.mp3"]
|
||||
assert bool(list(ingest.glob("*.mp3"))) is not grouped
|
||||
# The suffix is packaging, not part of the release name.
|
||||
assert not (ingest / "Book.zip").exists()
|
||||
|
||||
|
||||
def test_usenet_audiobook_multifile_preserves_source_folder(tmp_path):
|
||||
from shelfmark.download.postprocess.router import (
|
||||
post_process_download as _post_process_download,
|
||||
)
|
||||
|
||||
source_dir = tmp_path / "downloads" / "Usenet Audiobook"
|
||||
source_dir.mkdir(parents=True)
|
||||
for part in (1, 2):
|
||||
(source_dir / f"Part {part}.mp3").write_text(f"audio{part}")
|
||||
|
||||
ingest = tmp_path / "ingest"
|
||||
ingest.mkdir()
|
||||
task = DownloadTask(
|
||||
task_id="usenet-audio-grouped",
|
||||
source="prowlarr",
|
||||
title="Usenet Audio",
|
||||
author="Tester",
|
||||
format="mp3",
|
||||
content_type="audiobook",
|
||||
search_mode=SearchMode.UNIVERSAL,
|
||||
)
|
||||
|
||||
with patch("shelfmark.core.config.config") as mock_config:
|
||||
mock_config.get = _build_config(ingest, organization="rename_and_group")
|
||||
mock_config.CUSTOM_SCRIPT = None
|
||||
|
||||
result = _post_process_download(source_dir, task, Event(), lambda *_args: None)
|
||||
|
||||
grouped_dir = ingest / "Usenet Audiobook"
|
||||
assert result is not None
|
||||
assert Path(result).parent == grouped_dir
|
||||
assert sorted(path.name for path in grouped_dir.glob("*.mp3")) == [
|
||||
"Part 1.mp3",
|
||||
"Part 2.mp3",
|
||||
]
|
||||
|
||||
|
||||
def test_archive_extraction_organize_multifile_can_use_original_name(tmp_path):
|
||||
from shelfmark.download.postprocess.router import (
|
||||
post_process_download as _post_process_download,
|
||||
|
||||
@@ -5,6 +5,7 @@ parking page is indistinguishable from a broken search, so the mirror stays in
|
||||
rotation and every later search pays for it again.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from bs4 import Tag
|
||||
|
||||
PARKED_PAGE = """<!doctype html><html><head><title>annas-archive.li</title></head>
|
||||
@@ -85,15 +86,19 @@ def test_genuinely_empty_aa_result_does_not_quarantine(monkeypatch):
|
||||
assert "No files found." in html
|
||||
|
||||
|
||||
def test_challenge_page_does_not_quarantine(monkeypatch):
|
||||
"""A DDoS-Guard interstitial means the mirror is alive and holds our clearance."""
|
||||
def test_challenge_page_is_reported_not_passed_off_as_an_empty_result(monkeypatch):
|
||||
"""An unsolved interstitial means the search never ran.
|
||||
|
||||
The mirror is alive and holds our clearance, so it must not be quarantined - but
|
||||
returning it as "no table" made the caller tell the user their query found nothing.
|
||||
"""
|
||||
dd, _calls = _patch_pages(monkeypatch, [DDOS_GUARD_PAGE])
|
||||
selector = _Selector(["https://real.test", "https://other.test"])
|
||||
|
||||
_html, table = dd._fetch_search_table("https://real.test/search?q=dune", selector)
|
||||
with pytest.raises(dd.SearchUnavailableError, match="protection challenge"):
|
||||
dd._fetch_search_table("https://real.test/search?q=dune", selector)
|
||||
|
||||
assert selector.quarantined == []
|
||||
assert table is None
|
||||
|
||||
|
||||
def test_unreachable_mirror_raises_search_unavailable(monkeypatch):
|
||||
@@ -105,3 +110,22 @@ def test_unreachable_mirror_raises_search_unavailable(monkeypatch):
|
||||
except dd.SearchUnavailableError:
|
||||
return
|
||||
raise AssertionError("expected SearchUnavailableError")
|
||||
|
||||
|
||||
def test_recorded_failure_reason_is_surfaced_to_the_caller(monkeypatch):
|
||||
"""The concrete give-up reason html_get_page stashed replaces the generic line."""
|
||||
import shelfmark.release_sources.direct_download as dd
|
||||
|
||||
reason = "Anna's Archive returned 403 (blocked) and no bypasser is enabled."
|
||||
|
||||
def fake_get(url, *, selector=None, **_kwargs):
|
||||
selector.last_failure = reason
|
||||
return ""
|
||||
|
||||
monkeypatch.setattr(dd.downloader, "html_get_page", fake_get)
|
||||
monkeypatch.setattr(dd.network, "get_available_aa_urls", lambda: ["a"])
|
||||
selector = _Selector(["https://real.test"])
|
||||
selector.last_failure = None
|
||||
|
||||
with pytest.raises(dd.SearchUnavailableError, match="403 .blocked."):
|
||||
dd._fetch_search_table("https://real.test/search?q=dune", selector)
|
||||
|
||||
@@ -11,6 +11,134 @@ class _FakeResponse:
|
||||
self.url = url
|
||||
|
||||
|
||||
def test_external_bypasser_clearance_is_presented_on_the_next_request(monkeypatch):
|
||||
"""Clearance is read from the shared store whichever bypasser filled it.
|
||||
|
||||
Guards the regression where the external path returned {} unconditionally: every
|
||||
request re-paid a 403 plus a full solve, and a download - which the solver cannot
|
||||
proxy - presented no clearance at all.
|
||||
"""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: True)
|
||||
|
||||
cookie_store.store_extracted_cookies(
|
||||
url="https://annas-archive.gl/search",
|
||||
cookies=[{"name": "__ddg1_", "value": "clearance"}],
|
||||
user_agent="Mozilla/5.0 (solver)",
|
||||
)
|
||||
|
||||
headers: dict[str, str] = {}
|
||||
cookies = http._apply_cf_bypass("https://annas-archive.gl/md5/abc", headers)
|
||||
|
||||
assert cookies == {"__ddg1_": "clearance"}
|
||||
assert headers["User-Agent"] == "Mozilla/5.0 (solver)"
|
||||
|
||||
|
||||
def test_external_bypasser_solve_is_reused_instead_of_re_solved(monkeypatch):
|
||||
"""One solve should clear the following requests, not just the one that paid for it.
|
||||
|
||||
A solve is tens of seconds of real browser, so re-running it per request is what
|
||||
made direct download unusable behind an external bypasser.
|
||||
"""
|
||||
import shelfmark.bypass.cookie_store as cookie_store
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(cookie_store, "_cf_cookies", {})
|
||||
monkeypatch.setattr(cookie_store, "_cf_user_agents", {})
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: True)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
class _Cleared:
|
||||
is_redirect = False
|
||||
status_code = 200
|
||||
cookies: dict[str, str] = {}
|
||||
text = "<table>results</table>"
|
||||
url = "https://annas-archive.gl/search?q=dune"
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def gated_get(url: str, **kwargs):
|
||||
if kwargs.get("cookies", {}).get("cf_clearance") != "token":
|
||||
error = requests.exceptions.HTTPError("forbidden")
|
||||
error.response = _FakeResponse(403, url=url)
|
||||
raise error
|
||||
return _Cleared()
|
||||
|
||||
solves: list[str] = []
|
||||
|
||||
def fake_solve(url: str, *_args, **_kwargs):
|
||||
solves.append(url)
|
||||
cookie_store.store_extracted_cookies(
|
||||
url=url,
|
||||
cookies=[{"name": "cf_clearance", "value": "token"}],
|
||||
user_agent="Mozilla/5.0 (solver)",
|
||||
)
|
||||
return "<table>results</table>"
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", gated_get)
|
||||
monkeypatch.setattr(http, "get_bypassed_page", fake_solve)
|
||||
|
||||
url = "https://annas-archive.gl/search?q=dune"
|
||||
first = http.html_get_page(url, retry=2, allow_bypasser_fallback=True, success_delay=0)
|
||||
second = http.html_get_page(url, retry=2, allow_bypasser_fallback=True, success_delay=0)
|
||||
|
||||
assert first == "<table>results</table>"
|
||||
assert second == "<table>results</table>"
|
||||
# The second request rode the stored clearance instead of paying for another solve.
|
||||
assert solves == [url]
|
||||
|
||||
|
||||
def test_403_with_a_concurrently_won_clearance_still_reaches_the_bypasser(monkeypatch):
|
||||
"""The last attempt must hand off, not `continue` into the end of the loop.
|
||||
|
||||
Another worker's solve can land between our request and its 403, which used to
|
||||
send this branch back round the retry loop - but on the final attempt (and
|
||||
MAX_RETRY=1 is the supported setting) `continue` just ends it, abandoning the
|
||||
request without ever offering the URL to the bypasser.
|
||||
"""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
|
||||
# A concurrent solve has filled the store, but this request went out before it did.
|
||||
monkeypatch.setattr(http, "get_cf_cookies_for_domain", lambda _hostname: {"__ddg1_": "fresh"})
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
def gated(url: str, **_kwargs):
|
||||
error = requests.exceptions.HTTPError("forbidden")
|
||||
error.response = _FakeResponse(403, url=url)
|
||||
raise error
|
||||
|
||||
bypassed: list[str] = []
|
||||
monkeypatch.setattr(http.requests, "get", gated)
|
||||
monkeypatch.setattr(
|
||||
http,
|
||||
"get_bypassed_page",
|
||||
lambda url, *_a, **_k: bypassed.append(url) or "<table>results</table>",
|
||||
)
|
||||
|
||||
url = "https://annas-archive.gl/search?q=dune"
|
||||
html = http.html_get_page(url, retry=1, allow_bypasser_fallback=True, success_delay=0)
|
||||
|
||||
assert html == "<table>results</table>"
|
||||
assert bypassed == [url]
|
||||
|
||||
|
||||
def test_html_get_page_ignores_status_callback_failure(monkeypatch):
|
||||
"""A raising status_callback must not break the bypass it was reporting on."""
|
||||
import shelfmark.download.http as http
|
||||
@@ -190,15 +318,13 @@ def test_redirect_loop_purges_stale_cookies_and_switches_to_bypasser(monkeypatch
|
||||
stale = {"__ddg8_": "stale"}
|
||||
cleared: list[str] = []
|
||||
|
||||
class _FakeInternalBypasser:
|
||||
@staticmethod
|
||||
def clear_cf_cookies(domain: str) -> None:
|
||||
cleared.append(domain)
|
||||
stale.clear()
|
||||
def fake_clear(domain: str) -> None:
|
||||
cleared.append(domain)
|
||||
stale.clear()
|
||||
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
monkeypatch.setattr(http, "_is_using_external_bypasser", lambda: False)
|
||||
monkeypatch.setattr(http, "_get_internal_bypasser", lambda: _FakeInternalBypasser)
|
||||
monkeypatch.setattr(http.cookie_store, "clear_cf_cookies", fake_clear)
|
||||
monkeypatch.setattr(http, "_bypass_grace_seconds", lambda: 100.0)
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: dict(stale))
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
@@ -349,17 +475,11 @@ def test_html_get_page_redirect_loop_purges_cookies_and_bypasses(monkeypatch):
|
||||
|
||||
cleared: list[str] = []
|
||||
|
||||
class FakeInternalBypasser:
|
||||
def clear_cf_cookies(self, domain: str) -> None:
|
||||
cleared.append(domain)
|
||||
|
||||
def get_cf_cookies_for_domain(self, _domain: str) -> dict[str, str]:
|
||||
return {"__ddg2_": "stale"}
|
||||
|
||||
def get_cf_user_agent_for_domain(self, _domain: str) -> str | None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(http, "_get_internal_bypasser", lambda: FakeInternalBypasser())
|
||||
monkeypatch.setattr(http.cookie_store, "clear_cf_cookies", cleared.append)
|
||||
monkeypatch.setattr(
|
||||
http.cookie_store, "get_cf_cookies_for_domain", lambda _domain: {"__ddg2_": "stale"}
|
||||
)
|
||||
monkeypatch.setattr(http.cookie_store, "get_cf_user_agent_for_domain", lambda _domain: None)
|
||||
monkeypatch.setattr(http, "get_bypassed_page", lambda *_args, **_kwargs: "SOLVED")
|
||||
|
||||
class _FakeRedirect:
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Tests for handing a 503 that carries a browser challenge to the bypasser."""
|
||||
|
||||
import requests
|
||||
|
||||
_CHALLENGE_HTML = (
|
||||
"<html><head><title>Checking your browser before accessing z-lib.gd</title>"
|
||||
"<script src='/.well-known/ddos-guard/check.js'></script></head>"
|
||||
"<body>Please wait...</body></html>"
|
||||
)
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
"""Minimal stand-in for requests.Response covering what html_get_page touches."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
*,
|
||||
url: str = "https://z-lib.gd/md5/abc",
|
||||
text: str = "",
|
||||
cookies: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self.url = url
|
||||
self.text = text
|
||||
self.cookies = cookies or {}
|
||||
self.headers = {"Content-Type": "text/html;charset=utf-8"}
|
||||
self.is_redirect = False
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status_code >= 400:
|
||||
error = requests.exceptions.HTTPError(f"{self.status_code} Error")
|
||||
error.response = self
|
||||
raise error
|
||||
|
||||
|
||||
def _neutralize_network(monkeypatch, http):
|
||||
monkeypatch.setattr(http, "_apply_cf_bypass", lambda _url, _headers: {})
|
||||
monkeypatch.setattr(http, "get_proxies", lambda _url: {})
|
||||
monkeypatch.setattr(http, "get_ssl_verify", lambda _url: True)
|
||||
monkeypatch.setattr(http.network, "should_rotate_dns_for_url", lambda _url: False)
|
||||
monkeypatch.setattr(http.time, "sleep", lambda _seconds: None)
|
||||
|
||||
|
||||
def test_503_challenge_is_handed_to_the_bypasser(monkeypatch):
|
||||
"""The reissued-cookie 503 from #1233 reaches the bypasser instead of retrying."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
_neutralize_network(monkeypatch, http)
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
|
||||
attempts: list[dict[str, str]] = []
|
||||
bypassed: list[str] = []
|
||||
|
||||
def fake_get(_url: str, **kwargs):
|
||||
attempts.append(dict(kwargs["cookies"]))
|
||||
# Hit 1 issues the cookie; every later hit re-serves the challenge unchanged,
|
||||
# which is what leaves the handshake with nothing to echo back.
|
||||
if len(attempts) == 1:
|
||||
return _FakeResponse(503, cookies={"bsrv": "1"})
|
||||
return _FakeResponse(503, text=_CHALLENGE_HTML, cookies={"bsrv": "1"})
|
||||
|
||||
def fake_bypass(url: str, _selector=None, _cancel_flag=None):
|
||||
bypassed.append(url)
|
||||
return "<html>real page</html>"
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", fake_get)
|
||||
monkeypatch.setattr(http, "get_bypassed_page", fake_bypass)
|
||||
|
||||
html = http.html_get_page("https://z-lib.gd/md5/abc", retry=10, success_delay=0)
|
||||
|
||||
assert html == "<html>real page</html>"
|
||||
assert bypassed == ["https://z-lib.gd/md5/abc"]
|
||||
# The handshake still gets its echo; the challenge ends the loop on the second hit
|
||||
# rather than burning all ten attempts.
|
||||
assert attempts == [{}, {"bsrv": "1"}]
|
||||
|
||||
|
||||
def test_plain_503_still_retries_without_bypassing(monkeypatch):
|
||||
"""An overloaded origin has no challenge marker, so its retry path is untouched."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
_neutralize_network(monkeypatch, http)
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
|
||||
attempts: list[dict[str, str]] = []
|
||||
bypassed: list[str] = []
|
||||
|
||||
def fake_get(_url: str, **kwargs):
|
||||
attempts.append(dict(kwargs["cookies"]))
|
||||
return _FakeResponse(503, text="<html><body>Service Unavailable</body></html>")
|
||||
|
||||
monkeypatch.setattr(http.requests, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or ""
|
||||
)
|
||||
|
||||
html = http.html_get_page("https://z-lib.gd/md5/abc", retry=3, success_delay=0)
|
||||
|
||||
assert html == ""
|
||||
assert bypassed == []
|
||||
assert attempts == [{}, {}, {}]
|
||||
|
||||
|
||||
def test_503_challenge_respects_disabled_bypasser_fallback(monkeypatch):
|
||||
"""Best-effort fetches must not stall on a minutes-long solve."""
|
||||
import shelfmark.download.http as http
|
||||
|
||||
_neutralize_network(monkeypatch, http)
|
||||
monkeypatch.setattr(http, "_is_cf_bypass_enabled", lambda: True)
|
||||
|
||||
bypassed: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
http.requests,
|
||||
"get",
|
||||
lambda _url, **_kwargs: _FakeResponse(503, text=_CHALLENGE_HTML),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
http, "get_bypassed_page", lambda url, *_a, **_k: bypassed.append(url) or ""
|
||||
)
|
||||
|
||||
html = http.html_get_page(
|
||||
"https://z-lib.gd/md5/abc", retry=1, success_delay=0, allow_bypasser_fallback=False
|
||||
)
|
||||
|
||||
assert html == ""
|
||||
assert bypassed == []
|
||||
@@ -144,3 +144,27 @@ def test_start_does_not_run_the_search_inline(monkeypatch, warmup):
|
||||
|
||||
if warmup._warmup_thread:
|
||||
warmup._warmup_thread.cancel()
|
||||
|
||||
|
||||
def test_env_var_can_disable_the_warmup(monkeypatch, warmup):
|
||||
"""SEARCH_WARMUP_ENABLED is not in the settings registry, so config.get never
|
||||
sees it - the documented off-switch only works if os.environ is consulted."""
|
||||
_patch_config(monkeypatch, warmup, {}) # config knows nothing about the key
|
||||
monkeypatch.setenv("SEARCH_WARMUP_ENABLED", "false")
|
||||
|
||||
assert warmup.is_enabled() is False
|
||||
assert warmup.start() is False
|
||||
|
||||
|
||||
def test_env_var_can_set_the_query(monkeypatch, warmup):
|
||||
_patch_config(monkeypatch, warmup, {})
|
||||
monkeypatch.setenv("SEARCH_WARMUP_QUERY", "Moby Dick")
|
||||
|
||||
assert warmup.warmup_query() == "Moby Dick"
|
||||
|
||||
|
||||
def test_env_var_absent_falls_back_to_config(monkeypatch, warmup):
|
||||
monkeypatch.delenv("SEARCH_WARMUP_QUERY", raising=False)
|
||||
_patch_config(monkeypatch, warmup, {"SEARCH_WARMUP_QUERY": "From Config"})
|
||||
|
||||
assert warmup.warmup_query() == "From Config"
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import pytest
|
||||
|
||||
from shelfmark.metadata_providers import hardcover
|
||||
from shelfmark.metadata_providers.hardcover import (
|
||||
HardcoverProvider,
|
||||
_test_hardcover_connection,
|
||||
)
|
||||
|
||||
# Hardcover replaced its ~500 char JWTs with short opaque tokens.
|
||||
PAT = "hc_pat_" + "a" * 32
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_config_writes(monkeypatch):
|
||||
"""Keep the connection test from touching the on-disk provider config."""
|
||||
monkeypatch.setattr(hardcover, "_save_connected_user", lambda user_id, username: None)
|
||||
|
||||
|
||||
class TestHardcoverApiKey:
|
||||
def test_personal_access_token_is_accepted(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
HardcoverProvider,
|
||||
"_execute_query",
|
||||
lambda self, query, variables: {"me": [{"id": 1, "username": "alex"}]},
|
||||
)
|
||||
|
||||
result = _test_hardcover_connection({"HARDCOVER_API_KEY": PAT})
|
||||
|
||||
assert result == {"success": True, "message": "Connected as: alex"}
|
||||
|
||||
def test_short_key_without_the_prefix_is_rejected(self):
|
||||
result = _test_hardcover_connection({"HARDCOVER_API_KEY": "eyJhbGciOiJIUzI1NiJ9.short"})
|
||||
|
||||
assert result["success"] is False
|
||||
assert "too short" in result["message"]
|
||||
|
||||
def test_short_prefixed_key_still_reaches_the_api(self, monkeypatch):
|
||||
"""A key wearing the hc_pat_ prefix is Hardcover's to accept or reject."""
|
||||
monkeypatch.setattr(
|
||||
HardcoverProvider,
|
||||
"_execute_query",
|
||||
lambda self, query, variables: None,
|
||||
)
|
||||
|
||||
result = _test_hardcover_connection({"HARDCOVER_API_KEY": "hc_pat_ab"})
|
||||
|
||||
assert result == {"success": False, "message": "API request failed - check your API key"}
|
||||
|
||||
@pytest.mark.parametrize("pasted", [f"Bearer {PAT}", f"bearer {PAT}", f" {PAT} "])
|
||||
def test_pasted_auth_header_noise_is_stripped(self, pasted):
|
||||
provider = HardcoverProvider(api_key=pasted)
|
||||
|
||||
assert provider.session.headers["Authorization"] == f"Bearer {PAT}"
|
||||
@@ -1,4 +1,8 @@
|
||||
from shelfmark.metadata_providers.hardcover import HardcoverProvider
|
||||
from shelfmark.metadata_providers.hardcover import (
|
||||
TITLE_SUGGESTION_FIELDS,
|
||||
TITLE_SUGGESTION_WEIGHTS,
|
||||
HardcoverProvider,
|
||||
)
|
||||
|
||||
|
||||
class TestHardcoverFieldOptions:
|
||||
@@ -130,8 +134,10 @@ class TestHardcoverFieldOptions:
|
||||
"limit": 7,
|
||||
"page": 1,
|
||||
"sort": "_text_match:desc,users_count:desc",
|
||||
"fields": "title,alternative_titles",
|
||||
"weights": "5,2",
|
||||
# Hardcover rejects a Book search that narrows to fewer fields than its
|
||||
# preset expects, so the typeahead sends the full list and leans on weights.
|
||||
"fields": TITLE_SUGGESTION_FIELDS,
|
||||
"weights": TITLE_SUGGESTION_WEIGHTS,
|
||||
}
|
||||
|
||||
def test_get_search_field_options_skips_short_text_queries(self):
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Guards on the shape of Hardcover's `fields`/`weights` search parameters.
|
||||
|
||||
Hardcover turns `fields` into Typesense's `query_by` but keeps `num_typos` and
|
||||
`query_by_weights` as fixed-length presets per query_type. A field list of the
|
||||
wrong length is not searched loosely -- the whole search is rejected with a null
|
||||
results body, which used to surface as "0 results". These tests pin the counts
|
||||
so a narrower field list cannot silently ship again.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.metadata_providers.hardcover import (
|
||||
AUTHOR_SUGGESTION_FIELDS,
|
||||
AUTHOR_SUGGESTION_WEIGHTS,
|
||||
BOOK_SEARCH_FIELD_COUNT,
|
||||
BOOK_SEARCH_FIELDS,
|
||||
BOOK_TITLE_AUTHOR_WEIGHTS,
|
||||
BOOK_TITLE_WEIGHTS,
|
||||
SERIES_SEARCH_FIELDS,
|
||||
SERIES_SEARCH_WEIGHTS,
|
||||
TITLE_SUGGESTION_FIELDS,
|
||||
TITLE_SUGGESTION_WEIGHTS,
|
||||
HardcoverProvider,
|
||||
)
|
||||
|
||||
|
||||
def _count(value: str) -> int:
|
||||
return len([part for part in value.split(",") if part.strip()])
|
||||
|
||||
|
||||
class TestBookSearchFieldCounts:
|
||||
def test_book_field_list_matches_hardcovers_preset_length(self):
|
||||
assert _count(BOOK_SEARCH_FIELDS) == BOOK_SEARCH_FIELD_COUNT
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("label", "weights"),
|
||||
[
|
||||
("title", BOOK_TITLE_WEIGHTS),
|
||||
("title+author", BOOK_TITLE_AUTHOR_WEIGHTS),
|
||||
("title typeahead", TITLE_SUGGESTION_WEIGHTS),
|
||||
],
|
||||
)
|
||||
def test_book_weights_line_up_with_the_field_list(self, label, weights):
|
||||
assert _count(weights) == BOOK_SEARCH_FIELD_COUNT, label
|
||||
|
||||
def test_title_typeahead_uses_the_full_book_field_list(self):
|
||||
assert TITLE_SUGGESTION_FIELDS == BOOK_SEARCH_FIELDS
|
||||
|
||||
|
||||
class TestNonBookSearchFieldCounts:
|
||||
@pytest.mark.parametrize(
|
||||
("fields", "weights"),
|
||||
[
|
||||
(AUTHOR_SUGGESTION_FIELDS, AUTHOR_SUGGESTION_WEIGHTS),
|
||||
(SERIES_SEARCH_FIELDS, SERIES_SEARCH_WEIGHTS),
|
||||
],
|
||||
)
|
||||
def test_weights_line_up_with_their_field_list(self, fields, weights):
|
||||
assert _count(fields) == _count(weights)
|
||||
|
||||
|
||||
class TestBuildSearchParams:
|
||||
@pytest.mark.parametrize(
|
||||
("author", "title", "series"),
|
||||
[
|
||||
("", "Dune", ""),
|
||||
("Herbert", "Dune", ""),
|
||||
("Herbert", "", ""),
|
||||
("", "", ""),
|
||||
],
|
||||
)
|
||||
def test_every_branch_sends_a_usable_field_weight_pair(self, author, title, series):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
|
||||
_query, fields, weights = provider._build_search_params("dune", author, title, series)
|
||||
|
||||
if fields is None:
|
||||
# No override: Hardcover applies its own preset, so weights must be absent too.
|
||||
assert weights is None
|
||||
return
|
||||
assert _count(fields) == BOOK_SEARCH_FIELD_COUNT
|
||||
assert _count(weights) == BOOK_SEARCH_FIELD_COUNT
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -6,9 +7,10 @@ from shelfmark.metadata_providers import MetadataSearchOptions
|
||||
from shelfmark.metadata_providers.hardcover import HardcoverProvider
|
||||
|
||||
# Hardcover answers a rejected search with HTTP 200, no GraphQL errors, and a
|
||||
# null results body. A search that genuinely matched nothing still returns a
|
||||
# results object with found: 0.
|
||||
REJECTED = {"search": {"results": None}}
|
||||
# null results body; the reason shows up in the sibling error field. A search
|
||||
# that genuinely matched nothing still returns a results object with found: 0.
|
||||
REJECTED = {"search": {"error": "Parameter `sort_by` is malformed.", "results": None}}
|
||||
REJECTED_SILENTLY = {"search": {"results": None}}
|
||||
EMPTY = {"search": {"results": {"hits": [], "found": 0}}}
|
||||
ONE_HIT = {"search": {"results": {"hits": [{"document": {"id": 7, "title": "Dune"}}], "found": 1}}}
|
||||
|
||||
@@ -19,6 +21,29 @@ def _reset_sort_fallback(monkeypatch):
|
||||
monkeypatch.setattr("shelfmark.metadata_providers.hardcover._sort_fallback_until", 0.0)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hardcover_logs():
|
||||
"""Collect Hardcover log messages.
|
||||
|
||||
The provider's logger is built outside the standard hierarchy, so its
|
||||
records never reach the root handler that caplog installs.
|
||||
"""
|
||||
from shelfmark.metadata_providers import hardcover
|
||||
|
||||
messages: list[str] = []
|
||||
|
||||
class _Capture(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
messages.append(record.getMessage())
|
||||
|
||||
handler = _Capture()
|
||||
hardcover.logger.addHandler(handler)
|
||||
try:
|
||||
yield messages
|
||||
finally:
|
||||
hardcover.logger.removeHandler(handler)
|
||||
|
||||
|
||||
def _reject_sorted(calls: list[dict[str, Any]], *, success=ONE_HIT):
|
||||
"""Build an _execute_query stand-in that rejects any request carrying a sort."""
|
||||
|
||||
@@ -38,7 +63,9 @@ class TestHardcoverSortFallback:
|
||||
result = provider._execute_search_query("query", {"query": "dune", "sort": "relevance"})
|
||||
|
||||
assert result == ONE_HIT
|
||||
assert [call["sort"] for call in calls] == ["relevance", ""]
|
||||
# The retry drops sort entirely -- an empty sort is a value Hardcover can reject too.
|
||||
assert [call.get("sort") for call in calls] == ["relevance", None]
|
||||
assert "sort" not in calls[1]
|
||||
|
||||
def test_treats_an_empty_result_set_as_success(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
@@ -64,6 +91,24 @@ class TestHardcoverSortFallback:
|
||||
assert result is None
|
||||
assert len(calls) == 2
|
||||
|
||||
def test_keeps_sorting_when_the_sort_was_not_the_culprit(self, monkeypatch):
|
||||
"""A rejection that survives dropping the sort must not disable sorting globally."""
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
calls: list[dict[str, Any]] = []
|
||||
monkeypatch.setattr(
|
||||
provider, "_execute_query", lambda query, variables: calls.append(variables) or REJECTED
|
||||
)
|
||||
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": "rating:desc"})
|
||||
provider._execute_search_query("query", {"query": "hyperion", "sort": "rating:desc"})
|
||||
|
||||
assert [call.get("sort") for call in calls] == [
|
||||
"rating:desc",
|
||||
None,
|
||||
"rating:desc",
|
||||
None,
|
||||
]
|
||||
|
||||
def test_reports_failure_for_an_unsorted_rejection(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
calls: list[dict[str, Any]] = []
|
||||
@@ -76,6 +121,24 @@ class TestHardcoverSortFallback:
|
||||
assert result is None
|
||||
assert len(calls) == 1
|
||||
|
||||
def test_logs_the_reason_hardcover_gave(self, monkeypatch, hardcover_logs):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
monkeypatch.setattr(provider, "_execute_query", lambda query, variables: REJECTED)
|
||||
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": ""})
|
||||
|
||||
assert any("Parameter `sort_by` is malformed." in message for message in hardcover_logs)
|
||||
|
||||
def test_falls_back_to_a_placeholder_when_hardcover_says_nothing(
|
||||
self, monkeypatch, hardcover_logs
|
||||
):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
monkeypatch.setattr(provider, "_execute_query", lambda query, variables: REJECTED_SILENTLY)
|
||||
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": ""})
|
||||
|
||||
assert any("no error message" in message for message in hardcover_logs)
|
||||
|
||||
def test_skips_the_doomed_request_on_later_searches(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
calls: list[dict[str, Any]] = []
|
||||
@@ -84,7 +147,7 @@ class TestHardcoverSortFallback:
|
||||
provider._execute_search_query("query", {"query": "dune", "sort": "relevance"})
|
||||
provider._execute_search_query("query", {"query": "hyperion", "sort": "relevance"})
|
||||
|
||||
assert [call["sort"] for call in calls] == ["relevance", "", ""]
|
||||
assert [call.get("sort") for call in calls] == ["relevance", None, None]
|
||||
|
||||
def test_search_returns_results_despite_a_rejected_sort(self, monkeypatch):
|
||||
provider = HardcoverProvider(api_key="test-token")
|
||||
@@ -97,7 +160,7 @@ class TestHardcoverSortFallback:
|
||||
|
||||
assert result.total_found == 1
|
||||
assert [book.title for book in result.books] == ["Dune"]
|
||||
assert [call["sort"] for call in calls] == ["_text_match:desc,users_count:desc", ""]
|
||||
assert [call.get("sort") for call in calls] == ["_text_match:desc,users_count:desc", None]
|
||||
|
||||
|
||||
class TestSearchPayloadRejection:
|
||||
@@ -105,9 +168,19 @@ class TestSearchPayloadRejection:
|
||||
from shelfmark.metadata_providers.hardcover import _search_payload_rejected
|
||||
|
||||
assert _search_payload_rejected(REJECTED) is True
|
||||
assert _search_payload_rejected(REJECTED_SILENTLY) is True
|
||||
assert _search_payload_rejected(EMPTY) is False
|
||||
assert _search_payload_rejected(ONE_HIT) is False
|
||||
assert _search_payload_rejected(None) is False
|
||||
assert _search_payload_rejected({}) is False
|
||||
# Non-search payloads (list lookups, book fetches) must pass through.
|
||||
assert _search_payload_rejected({"series": [{"id": 1}]}) is False
|
||||
|
||||
def test_reads_the_error_hardcover_attached(self):
|
||||
from shelfmark.metadata_providers.hardcover import _search_rejection_reason
|
||||
|
||||
assert _search_rejection_reason(REJECTED) == "Parameter `sort_by` is malformed."
|
||||
assert _search_rejection_reason(REJECTED_SILENTLY) == ""
|
||||
assert _search_rejection_reason(EMPTY) == ""
|
||||
assert _search_rejection_reason(None) == ""
|
||||
assert _search_rejection_reason({"search": {"error": None, "results": None}}) == ""
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Prowlarr Torznab timeout handling (#1249).
|
||||
|
||||
A Torznab search is Prowlarr proxying a live request to the tracker, so for a
|
||||
Cloudflare-fronted indexer it waits on FlareSolverr. Those searches need their
|
||||
own budget, and when one runs out the caller has to hear about it instead of
|
||||
receiving an empty list that reads as "this book has no releases".
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
import shelfmark.release_sources.prowlarr.api as prowlarr_api
|
||||
from shelfmark.release_sources.prowlarr.api import (
|
||||
DEFAULT_INDEXER_TIMEOUT_SECONDS,
|
||||
MAX_INDEXER_TIMEOUT_SECONDS,
|
||||
MIN_INDEXER_TIMEOUT_SECONDS,
|
||||
ProwlarrClient,
|
||||
ProwlarrSearchError,
|
||||
resolve_indexer_timeout,
|
||||
)
|
||||
|
||||
_TORZNAB_EMPTY = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0"><channel></channel></rss>"""
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, text="", status_code=200, reason="OK"):
|
||||
self.text = text
|
||||
self.status_code = status_code
|
||||
self.reason = reason
|
||||
self.ok = status_code < 400
|
||||
|
||||
def raise_for_status(self):
|
||||
if not self.ok:
|
||||
raise requests.exceptions.HTTPError(f"{self.status_code} {self.reason}", response=self)
|
||||
|
||||
def json(self):
|
||||
return {}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_config(monkeypatch):
|
||||
"""Keep the client off any persisted PROWLARR_INDEXER_TIMEOUT."""
|
||||
monkeypatch.setattr(prowlarr_api.config, "get", lambda key, default=None: default)
|
||||
|
||||
|
||||
class TestResolveIndexerTimeout:
|
||||
def test_defaults_when_unset(self, no_config):
|
||||
assert resolve_indexer_timeout() == DEFAULT_INDEXER_TIMEOUT_SECONDS
|
||||
|
||||
def test_reads_config(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
prowlarr_api.config,
|
||||
"get",
|
||||
lambda key, default=None: 150 if key == "PROWLARR_INDEXER_TIMEOUT" else default,
|
||||
)
|
||||
assert resolve_indexer_timeout() == 150
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
(1, MIN_INDEXER_TIMEOUT_SECONDS),
|
||||
(9000, MAX_INDEXER_TIMEOUT_SECONDS),
|
||||
("120", 120),
|
||||
],
|
||||
)
|
||||
def test_clamps_and_coerces(self, value, expected):
|
||||
assert resolve_indexer_timeout(value) == expected
|
||||
|
||||
def test_unparsable_value_falls_back_rather_than_raising(self):
|
||||
assert resolve_indexer_timeout("soon") == DEFAULT_INDEXER_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
class TestTorznabSearchFailures:
|
||||
def _client(self, monkeypatch, response_or_error):
|
||||
client = ProwlarrClient("http://prowlarr:9696", "apikey", indexer_timeout=90)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_get(**kwargs):
|
||||
captured.update(kwargs)
|
||||
if isinstance(response_or_error, Exception):
|
||||
raise response_or_error
|
||||
return response_or_error
|
||||
|
||||
monkeypatch.setattr(client._session, "get", fake_get)
|
||||
return client, captured
|
||||
|
||||
def test_read_timeout_raises_instead_of_returning_empty(self, monkeypatch):
|
||||
client, _ = self._client(monkeypatch, requests.exceptions.ReadTimeout("read timeout=90"))
|
||||
|
||||
with pytest.raises(ProwlarrSearchError, match="did not respond within 90s"):
|
||||
client.torznab_search(indexer_id=1, query="Dune")
|
||||
|
||||
def test_http_error_raises_instead_of_returning_empty(self, monkeypatch):
|
||||
"""Prowlarr answers 429 once it has disabled an indexer for recent failures."""
|
||||
client, _ = self._client(monkeypatch, _Response("", status_code=429, reason="Too Many"))
|
||||
|
||||
with pytest.raises(ProwlarrSearchError, match="indexer 1 search failed"):
|
||||
client.torznab_search(indexer_id=1, query="Dune")
|
||||
|
||||
def test_an_indexer_with_nothing_still_returns_empty(self, monkeypatch):
|
||||
client, _ = self._client(monkeypatch, _Response(_TORZNAB_EMPTY))
|
||||
|
||||
assert client.torznab_search(indexer_id=1, query="Dune") == []
|
||||
|
||||
def test_search_uses_the_indexer_timeout_with_a_short_connect_timeout(self, monkeypatch):
|
||||
client, captured = self._client(monkeypatch, _Response(_TORZNAB_EMPTY))
|
||||
client.torznab_search(indexer_id=1, query="Dune")
|
||||
|
||||
connect_timeout, read_timeout = captured["timeout"]
|
||||
assert read_timeout == 90
|
||||
assert connect_timeout < read_timeout
|
||||
|
||||
def test_json_endpoints_keep_the_short_timeout(self, monkeypatch, no_config):
|
||||
client = ProwlarrClient("http://prowlarr:9696", "apikey")
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_request(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _Response("{}")
|
||||
|
||||
monkeypatch.setattr(client._session, "request", fake_request)
|
||||
client.test_connection()
|
||||
|
||||
assert captured["timeout"] == client.timeout
|
||||
assert client.timeout < client.indexer_timeout
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Debrid clients must resolve a real torrent before uploading (#1250).
|
||||
|
||||
Prowlarr hands out a proxy download URL - no magnetUrl, no infoHash - for any
|
||||
indexer that only publishes torrent files. Posting that URL as if it were a
|
||||
magnet is what made Real-Debrid answer /torrents/addMagnet with a bare 404.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from shelfmark.download.clients.alldebrid import AllDebridClient
|
||||
from shelfmark.download.clients.realdebrid import RealDebridClient
|
||||
from shelfmark.download.clients.torrent_utils import (
|
||||
DebridMagnet,
|
||||
DebridTorrentFile,
|
||||
bencode_encode,
|
||||
resolve_debrid_upload,
|
||||
)
|
||||
|
||||
_PROWLARR_PROXY_URL = "https://prowlarr.example/api/v1/indexer/1/download?apikey=k&link=abc"
|
||||
_MAGNET = "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=Dune"
|
||||
|
||||
|
||||
def _valid_torrent() -> tuple[bytes, str]:
|
||||
info_dict = {
|
||||
b"name": b"book.epub",
|
||||
b"length": 100,
|
||||
b"piece length": 16384,
|
||||
b"pieces": b"\x00" * 20,
|
||||
}
|
||||
return (
|
||||
bencode_encode({b"info": info_dict}),
|
||||
hashlib.sha1(bencode_encode(info_dict)).hexdigest().lower(),
|
||||
)
|
||||
|
||||
|
||||
def _mock_fetch(monkeypatch, *, content=b"", status_code=200, error=None):
|
||||
"""Stand in for the .torrent prefetch inside extract_torrent_info."""
|
||||
if error is not None:
|
||||
mock_get = MagicMock(side_effect=error)
|
||||
else:
|
||||
response = MagicMock(status_code=status_code, content=content)
|
||||
response.raise_for_status = MagicMock()
|
||||
mock_get = MagicMock(return_value=response)
|
||||
monkeypatch.setattr("shelfmark.download.clients.torrent_utils.requests.get", mock_get)
|
||||
return mock_get
|
||||
|
||||
|
||||
class TestResolveDebridUpload:
|
||||
def test_magnet_url_passes_through_untouched(self):
|
||||
assert resolve_debrid_upload(_MAGNET) == DebridMagnet(magnet_url=_MAGNET)
|
||||
|
||||
def test_proxy_url_becomes_the_torrent_file(self, monkeypatch):
|
||||
torrent_data, _ = _valid_torrent()
|
||||
_mock_fetch(monkeypatch, content=torrent_data)
|
||||
|
||||
assert resolve_debrid_upload(_PROWLARR_PROXY_URL) == DebridTorrentFile(
|
||||
torrent_data=torrent_data
|
||||
)
|
||||
|
||||
def test_proxy_url_returning_a_magnet_body_becomes_that_magnet(self, monkeypatch):
|
||||
_mock_fetch(monkeypatch, content=_MAGNET.encode())
|
||||
|
||||
assert resolve_debrid_upload(_PROWLARR_PROXY_URL) == DebridMagnet(magnet_url=_MAGNET)
|
||||
|
||||
def test_falls_back_to_a_magnet_built_from_the_prowlarr_info_hash(self, monkeypatch):
|
||||
_mock_fetch(monkeypatch, error=OSError("tracker unreachable"))
|
||||
info_hash = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
upload = resolve_debrid_upload(_PROWLARR_PROXY_URL, expected_hash=info_hash)
|
||||
|
||||
assert upload == DebridMagnet(magnet_url=f"magnet:?xt=urn:btih:{info_hash}")
|
||||
|
||||
def test_prefers_the_torrent_file_over_the_info_hash(self, monkeypatch):
|
||||
"""The file carries the tracker list; a bare btih magnet does not."""
|
||||
torrent_data, info_hash = _valid_torrent()
|
||||
_mock_fetch(monkeypatch, content=torrent_data)
|
||||
|
||||
upload = resolve_debrid_upload(_PROWLARR_PROXY_URL, expected_hash=info_hash)
|
||||
|
||||
assert upload == DebridTorrentFile(torrent_data=torrent_data)
|
||||
|
||||
def test_unresolvable_url_raises_rather_than_handing_back_the_url(self, monkeypatch):
|
||||
_mock_fetch(monkeypatch, error=OSError("tracker unreachable"))
|
||||
|
||||
with pytest.raises(ValueError, match="Could not resolve a torrent to send") as excinfo:
|
||||
resolve_debrid_upload(_PROWLARR_PROXY_URL)
|
||||
|
||||
assert "tracker unreachable" in str(excinfo.value)
|
||||
|
||||
|
||||
class TestRealDebridAdd:
|
||||
@staticmethod
|
||||
def _client(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.realdebrid.config.get",
|
||||
lambda key, default="": {"REALDEBRID_API_KEY": "rd-key"}.get(key, default),
|
||||
)
|
||||
return RealDebridClient()
|
||||
|
||||
@staticmethod
|
||||
def _mock_api(monkeypatch):
|
||||
ok = MagicMock(status_code=201)
|
||||
ok.raise_for_status = MagicMock()
|
||||
ok.json = MagicMock(return_value={"id": "RD1"})
|
||||
post = MagicMock(return_value=ok)
|
||||
put = MagicMock(return_value=ok)
|
||||
monkeypatch.setattr("shelfmark.download.clients.realdebrid.requests.post", post)
|
||||
monkeypatch.setattr("shelfmark.download.clients.realdebrid.requests.put", put)
|
||||
return post, put
|
||||
|
||||
def test_proxy_url_is_uploaded_as_a_torrent_file_not_posted_as_a_magnet(self, monkeypatch):
|
||||
torrent_data, _ = _valid_torrent()
|
||||
_mock_fetch(monkeypatch, content=torrent_data)
|
||||
post, put = self._mock_api(monkeypatch)
|
||||
client = self._client(monkeypatch)
|
||||
|
||||
assert client.add_download(_PROWLARR_PROXY_URL, "Dune") == "RD1"
|
||||
|
||||
put.assert_called_once()
|
||||
assert put.call_args.args[0].endswith("/torrents/addTorrent")
|
||||
assert put.call_args.kwargs["data"] == torrent_data
|
||||
# The only POST is selectFiles; addMagnet must never see the proxy URL.
|
||||
assert [c.args[0] for c in post.call_args_list] == [
|
||||
"https://api.real-debrid.com/rest/1.0/torrents/selectFiles/RD1"
|
||||
]
|
||||
|
||||
def test_magnet_still_goes_to_add_magnet(self, monkeypatch):
|
||||
post, put = self._mock_api(monkeypatch)
|
||||
client = self._client(monkeypatch)
|
||||
|
||||
assert client.add_download(_MAGNET, "Dune") == "RD1"
|
||||
|
||||
put.assert_not_called()
|
||||
assert post.call_args_list[0].args[0].endswith("/torrents/addMagnet")
|
||||
assert post.call_args_list[0].kwargs["data"] == {"magnet": _MAGNET}
|
||||
|
||||
def test_unresolvable_url_reports_the_reason_instead_of_a_service_error(self, monkeypatch):
|
||||
_mock_fetch(monkeypatch, error=OSError("tracker unreachable"))
|
||||
post, put = self._mock_api(monkeypatch)
|
||||
client = self._client(monkeypatch)
|
||||
|
||||
with pytest.raises(ValueError, match="Could not resolve a torrent to send"):
|
||||
client.add_download(_PROWLARR_PROXY_URL, "Dune")
|
||||
|
||||
post.assert_not_called()
|
||||
put.assert_not_called()
|
||||
|
||||
|
||||
class TestAllDebridAdd:
|
||||
@staticmethod
|
||||
def _client(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"shelfmark.download.clients.alldebrid.config.get",
|
||||
lambda key, default="": {"ALLDEBRID_API_KEY": "ad-key"}.get(key, default),
|
||||
)
|
||||
return AllDebridClient()
|
||||
|
||||
@staticmethod
|
||||
def _mock_api(monkeypatch, entries_key="files"):
|
||||
ok = MagicMock(status_code=200)
|
||||
ok.raise_for_status = MagicMock()
|
||||
ok.json = MagicMock(
|
||||
return_value={"status": "success", "data": {entries_key: [{"id": 4242}]}}
|
||||
)
|
||||
post = MagicMock(return_value=ok)
|
||||
monkeypatch.setattr("shelfmark.download.clients.alldebrid.requests.post", post)
|
||||
return post
|
||||
|
||||
def test_proxy_url_is_uploaded_to_the_file_endpoint(self, monkeypatch):
|
||||
torrent_data, _ = _valid_torrent()
|
||||
_mock_fetch(monkeypatch, content=torrent_data)
|
||||
post = self._mock_api(monkeypatch)
|
||||
client = self._client(monkeypatch)
|
||||
|
||||
assert client.add_download(_PROWLARR_PROXY_URL, "Dune") == "4242"
|
||||
|
||||
assert post.call_args.args[0].endswith("/magnet/upload/file")
|
||||
assert post.call_args.kwargs["files"]["files[]"][1] == torrent_data
|
||||
|
||||
def test_magnet_still_goes_to_the_magnet_endpoint(self, monkeypatch):
|
||||
post = self._mock_api(monkeypatch, entries_key="magnets")
|
||||
client = self._client(monkeypatch)
|
||||
|
||||
assert client.add_download(_MAGNET, "Dune") == "4242"
|
||||
|
||||
assert post.call_args.args[0].endswith("/magnet/upload")
|
||||
assert post.call_args.kwargs["data"] == {"magnets[]": _MAGNET}
|
||||
@@ -727,6 +727,62 @@ class TestProwlarrHandlerSeedCriteria:
|
||||
assert call_kwargs["ratio_limit"] == 1.25
|
||||
|
||||
|
||||
class TestProwlarrHandlerContentType:
|
||||
"""Regression tests for issue #1235 — content type must reach the client."""
|
||||
|
||||
def test_download_passes_content_type_to_client(self):
|
||||
"""rTorrent picks its audiobook label from content_type, not category."""
|
||||
mock_client = MagicMock()
|
||||
mock_client.name = "rtorrent"
|
||||
mock_client.find_existing.return_value = None
|
||||
mock_client.add_download.return_value = "download_id"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"shelfmark.release_sources.prowlarr.handler.get_release",
|
||||
return_value={
|
||||
"protocol": "torrent",
|
||||
"title": "Test Release",
|
||||
"magnetUrl": "magnet:?xt=urn:btih:abc123",
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"shelfmark.release_sources.prowlarr.handler.get_client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"shelfmark.release_sources.prowlarr.handler.remove_release",
|
||||
),
|
||||
patch("shelfmark.release_sources.prowlarr.handler.config.get", return_value=True),
|
||||
patch.object(
|
||||
ProwlarrHandler,
|
||||
"_poll_and_complete",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
handler = ProwlarrHandler()
|
||||
task = DownloadTask(
|
||||
task_id="content-type-pass-through",
|
||||
source="prowlarr",
|
||||
title="Test Audiobook",
|
||||
content_type="audiobook",
|
||||
)
|
||||
cancel_flag = Event()
|
||||
recorder = ProgressRecorder()
|
||||
|
||||
handler.download(
|
||||
task=task,
|
||||
cancel_flag=cancel_flag,
|
||||
progress_callback=recorder.progress_callback,
|
||||
status_callback=recorder.status_callback,
|
||||
)
|
||||
|
||||
call_kwargs = mock_client.add_download.call_args.kwargs
|
||||
assert call_kwargs["content_type"] == "audiobook"
|
||||
# rTorrent gets no category, so content_type is its only audiobook signal.
|
||||
assert call_kwargs["category"] is None
|
||||
|
||||
|
||||
class TestProwlarrHandlerExistingDownload:
|
||||
"""Tests for handling existing downloads."""
|
||||
|
||||
|
||||
@@ -563,6 +563,36 @@ class TestRTorrentClientAudiobookLabel:
|
||||
assert "d.custom1.set=books" in args[2]
|
||||
assert "d.custom1.set=audiobooks" not in args[2]
|
||||
|
||||
def test_uses_audiobook_label_for_compound_content_type(self, monkeypatch):
|
||||
"""Issue #1235 — an exact "audiobook" match missed forms like "book (audiobook)"."""
|
||||
config_values = {
|
||||
"RTORRENT_URL": "http://localhost:8080/RPC2",
|
||||
"RTORRENT_LABEL": "books",
|
||||
"RTORRENT_AUDIOBOOK_LABEL": "audiobooks",
|
||||
"RTORRENT_DOWNLOAD_DIR": "/downloads",
|
||||
}
|
||||
mock_rpc, mock_xmlrpc, mock_torrent_info = self._make_client(monkeypatch, config_values)
|
||||
|
||||
with patch.dict("sys.modules", {"xmlrpc.client": mock_xmlrpc}):
|
||||
with patch(
|
||||
"shelfmark.download.clients.torrent_utils.extract_torrent_info",
|
||||
return_value=mock_torrent_info,
|
||||
):
|
||||
if "shelfmark.download.clients.rtorrent" in sys.modules:
|
||||
del sys.modules["shelfmark.download.clients.rtorrent"]
|
||||
from shelfmark.download.clients.rtorrent import RTorrentClient
|
||||
|
||||
client = RTorrentClient()
|
||||
client.add_download(
|
||||
"magnet:?xt=urn:btih:abc123",
|
||||
"Test Audiobook",
|
||||
content_type="Book (Audiobook)",
|
||||
)
|
||||
|
||||
args = mock_rpc.load.start.call_args[0]
|
||||
assert "d.custom1.set=audiobooks" in args[2]
|
||||
assert "d.custom1.set=books" not in args[2]
|
||||
|
||||
|
||||
class TestRTorrentClientGetStatus:
|
||||
"""Tests for RTorrentClient.get_status()."""
|
||||
|
||||
@@ -6,10 +6,13 @@ Tests the utility functions for parsing release metadata.
|
||||
|
||||
# Import the functions to test
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from shelfmark.metadata_providers import BookMetadata
|
||||
from shelfmark.release_sources.prowlarr.api import ProwlarrClient
|
||||
from shelfmark.release_sources import SourceUnavailableError
|
||||
from shelfmark.release_sources.prowlarr.api import ProwlarrClient, ProwlarrSearchError
|
||||
from shelfmark.release_sources.prowlarr.source import (
|
||||
PROWLARR_SEARCH_TIMEOUT_SECONDS,
|
||||
ProwlarrSource,
|
||||
_build_indexer_priority,
|
||||
_collapse_duplicate_indexer_results,
|
||||
@@ -21,6 +24,7 @@ from shelfmark.release_sources.prowlarr.source import (
|
||||
_parse_size,
|
||||
_release_identity,
|
||||
_result_dedup_key,
|
||||
_search_budget_seconds,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.utils import (
|
||||
build_source_id,
|
||||
@@ -244,6 +248,7 @@ class FakeTorznabClient:
|
||||
self.seed_settings_calls: list[object] = []
|
||||
self.search_results = search_results or []
|
||||
self.seed_settings = seed_settings or {}
|
||||
self.indexer_timeout = 90
|
||||
|
||||
def get_enabled_indexers_detailed(self, *, raise_on_error=False):
|
||||
return [
|
||||
@@ -788,6 +793,7 @@ class _MultiIndexerClient:
|
||||
def __init__(self, results_by_indexer: dict[int, list[dict]], priorities=None):
|
||||
self.results_by_indexer = results_by_indexer
|
||||
self.priorities = priorities or {}
|
||||
self.indexer_timeout = 90
|
||||
|
||||
def get_enabled_indexers_detailed(self, *, raise_on_error=False):
|
||||
del raise_on_error
|
||||
@@ -1222,3 +1228,173 @@ class TestIndexerPrioritySortOption:
|
||||
assert options["Indexer priority"]["sort_key"] == "extra.indexer_priority"
|
||||
assert options["Indexer priority"]["default_direction"] == "asc"
|
||||
assert options["Peers"]["default_direction"] == "desc"
|
||||
|
||||
|
||||
class _FailingIndexerClient:
|
||||
"""Torznab client where chosen indexers raise instead of answering.
|
||||
|
||||
Mirrors #1249: Prowlarr proxies the search to a Cloudflare-fronted tracker,
|
||||
FlareSolverr is still solving the challenge when the HTTP read times out.
|
||||
"""
|
||||
|
||||
def __init__(self, failing_indexers: set[int], results_by_indexer=None):
|
||||
self.failing_indexers = failing_indexers
|
||||
self.results_by_indexer = results_by_indexer or {}
|
||||
self.indexer_timeout = 90
|
||||
self.calls: list[tuple[int, object]] = []
|
||||
|
||||
def get_enabled_indexers_detailed(self, *, raise_on_error=False):
|
||||
del raise_on_error
|
||||
indexer_ids = sorted(self.failing_indexers | set(self.results_by_indexer))
|
||||
return [
|
||||
{
|
||||
"id": indexer_id,
|
||||
"enable": True,
|
||||
"capabilities": {"categories": [{"id": 7000, "subCategories": []}]},
|
||||
}
|
||||
for indexer_id in indexer_ids
|
||||
]
|
||||
|
||||
def torznab_search(
|
||||
self,
|
||||
*,
|
||||
indexer_id: int,
|
||||
query: str,
|
||||
categories=None,
|
||||
search_type="book",
|
||||
limit=100,
|
||||
offset=0,
|
||||
):
|
||||
del query, search_type, limit, offset
|
||||
self.calls.append((indexer_id, categories))
|
||||
if indexer_id in self.failing_indexers:
|
||||
msg = f"indexer {indexer_id} did not respond within 90s"
|
||||
raise ProwlarrSearchError(msg)
|
||||
return self.results_by_indexer.get(indexer_id, [])
|
||||
|
||||
def get_enriched_indexer_ids(self, restrict_to=None, indexers=None):
|
||||
del restrict_to, indexers
|
||||
return []
|
||||
|
||||
def get_indexer_seed_settings(self, restrict_to=None):
|
||||
del restrict_to
|
||||
return {}
|
||||
|
||||
|
||||
class TestFailedIndexerSearchIsNotNoResults:
|
||||
"""A Torznab timeout must not read as "this book has no releases" (#1249)."""
|
||||
|
||||
def _search(self, monkeypatch, client, config_values=None):
|
||||
import shelfmark.release_sources.prowlarr.source as prowlarr_source
|
||||
from shelfmark.core.search_plan import build_release_search_plan
|
||||
|
||||
values = {"PROWLARR_INDEXERS": "", "PROWLARR_AUTO_EXPAND": False}
|
||||
values.update(config_values or {})
|
||||
monkeypatch.setattr(
|
||||
prowlarr_source.config, "get", lambda key, default=None: values.get(key, default)
|
||||
)
|
||||
|
||||
source = ProwlarrSource()
|
||||
monkeypatch.setattr(source, "_get_client", lambda: client)
|
||||
|
||||
book = BookMetadata(
|
||||
provider="hardcover", provider_id="123", title="Dune", authors=["Frank Herbert"]
|
||||
)
|
||||
plan = build_release_search_plan(book, languages=["en"])
|
||||
return source.search(book, plan)
|
||||
|
||||
def test_sole_indexer_timing_out_raises_instead_of_returning_empty(self, monkeypatch):
|
||||
client = _FailingIndexerClient({1})
|
||||
|
||||
with pytest.raises(SourceUnavailableError) as excinfo:
|
||||
self._search(monkeypatch, client)
|
||||
|
||||
assert "1 of 1 indexer searches failed" in str(excinfo.value)
|
||||
assert "did not respond within 90s" in str(excinfo.value)
|
||||
|
||||
def test_one_failure_does_not_sink_an_indexer_that_answered(self, monkeypatch):
|
||||
client = _FailingIndexerClient(
|
||||
{1},
|
||||
{
|
||||
2: [
|
||||
{
|
||||
"guid": "g2",
|
||||
"title": "Dune",
|
||||
"indexerId": 2,
|
||||
"indexer": "working",
|
||||
"protocol": "torrent",
|
||||
"size": 1048576,
|
||||
"seeders": 5,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
releases = self._search(monkeypatch, client)
|
||||
|
||||
assert [r.indexer for r in releases] == ["working"]
|
||||
|
||||
def test_partial_failure_with_no_results_still_reports_the_failure(self, monkeypatch):
|
||||
client = _FailingIndexerClient({1}, {2: []})
|
||||
|
||||
with pytest.raises(SourceUnavailableError) as excinfo:
|
||||
self._search(monkeypatch, client)
|
||||
|
||||
assert "1 of 2 indexer searches failed" in str(excinfo.value)
|
||||
|
||||
def test_auto_expand_does_not_retry_on_top_of_a_failed_search(self, monkeypatch):
|
||||
"""The second search is what crashes FlareSolverr's Chrome on a small host."""
|
||||
client = _FailingIndexerClient({1})
|
||||
|
||||
with pytest.raises(SourceUnavailableError):
|
||||
self._search(monkeypatch, client, {"PROWLARR_AUTO_EXPAND": True})
|
||||
|
||||
assert client.calls == [(1, [7000])]
|
||||
|
||||
def test_auto_expand_still_retries_when_the_indexer_answered_empty(self, monkeypatch):
|
||||
client = _FailingIndexerClient(set(), {1: []})
|
||||
|
||||
assert self._search(monkeypatch, client, {"PROWLARR_AUTO_EXPAND": True}) == []
|
||||
assert client.calls == [(1, [7000]), (1, None)]
|
||||
|
||||
|
||||
class TestUnreachableProwlarrIsNotNoResults:
|
||||
"""Prowlarr itself being down must not read as "this book has no releases" (#1249)."""
|
||||
|
||||
class _UnreachableClient:
|
||||
indexer_timeout = 90
|
||||
|
||||
def get_enabled_indexers_detailed(self, *, raise_on_error=False):
|
||||
del raise_on_error
|
||||
raise requests.exceptions.ConnectionError("connection refused")
|
||||
|
||||
def test_search_reports_the_connection_failure(self, monkeypatch):
|
||||
import shelfmark.release_sources.prowlarr.source as prowlarr_source
|
||||
from shelfmark.core.search_plan import build_release_search_plan
|
||||
|
||||
monkeypatch.setattr(
|
||||
prowlarr_source.config,
|
||||
"get",
|
||||
lambda key, default=None: {"PROWLARR_INDEXERS": ""}.get(key, default),
|
||||
)
|
||||
source = ProwlarrSource()
|
||||
monkeypatch.setattr(source, "_get_client", lambda: self._UnreachableClient())
|
||||
|
||||
book = BookMetadata(
|
||||
provider="hardcover", provider_id="123", title="Dune", authors=["Frank Herbert"]
|
||||
)
|
||||
plan = build_release_search_plan(book, languages=["en"])
|
||||
|
||||
with pytest.raises(SourceUnavailableError, match="could not reach Prowlarr"):
|
||||
source.search(book, plan)
|
||||
|
||||
|
||||
class TestSearchBudgetScalesWithIndexerTimeout:
|
||||
def test_default_budget_is_unchanged(self):
|
||||
assert _search_budget_seconds(30) == PROWLARR_SEARCH_TIMEOUT_SECONDS
|
||||
|
||||
def test_a_long_indexer_timeout_widens_the_budget(self):
|
||||
assert _search_budget_seconds(120) == 240.0
|
||||
|
||||
def test_budget_stays_under_the_gunicorn_worker_timeout(self):
|
||||
assert _search_budget_seconds(300) == 240.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
version = 1
|
||||
revision = 2
|
||||
revision = 3
|
||||
requires-python = ">=3.14"
|
||||
|
||||
[[package]]
|
||||
@@ -16,7 +16,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "apprise"
|
||||
version = "1.12.0"
|
||||
version = "1.13.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
@@ -27,9 +27,9 @@ dependencies = [
|
||||
{ name = "requests-oauthlib" },
|
||||
{ name = "tzdata", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ff/f7/95a52cda9355f32c5db9ef77bac18e85977472d29c555fafd825cf60c309/apprise-1.12.0.tar.gz", hash = "sha256:ffe9860d99cbde05fd156c610d4bcb49d953200c4ed3cb2db001ed50722fc142", size = 2477874, upload-time = "2026-07-04T16:15:36.67Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0c/36/80a0e7f99f59902ca5a30051c2034938dc9387eeca7d78f750fb1585c890/apprise-1.13.0.tar.gz", hash = "sha256:9a56964bf3ca004b3e0db98ab8a8d87fad051d8edb377bc166dc4be826666c81", size = 2512349, upload-time = "2026-08-20T14:50:33.404Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/e9/acfea54abdd66a059ff82995d3703eca3a27df3dc1bdc6afaf8a30a854fe/apprise-1.12.0-py3-none-any.whl", hash = "sha256:28edabfec5a9d5dbcd9aa28bdfd8928ecb68764e40214b6e648eab6d974b5a93", size = 1797824, upload-time = "2026-07-04T16:15:34.228Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/ab/199a88aa1cee354a0e9bc0d9fce9d14969214e3c95b466d75d63756112c5/apprise-1.13.0-py3-none-any.whl", hash = "sha256:cc8f4e2b138a46ff0397f649d73622ab268ddb94a0954399ff318a29a30146c8", size = 1817510, upload-time = "2026-08-20T14:50:31.346Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -56,14 +56,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "basedpyright"
|
||||
version = "1.39.9"
|
||||
version = "1.39.10"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nodejs-wheel-binaries" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7a/4b/c1f4e211e50389304d6af32b9280e026a7133e3ad59bbdf8f7a3250f8bee/basedpyright-1.39.9.tar.gz", hash = "sha256:32cbea5fc8273e89df3db20daea56cb7286e419ccdfdc479c64759d2dc071901", size = 24412216, upload-time = "2026-06-27T02:19:49.834Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/68/43/ad2999f3b09eb2b1e59931d88fac0f7bcc9c17fc18c903268779bd10cc97/basedpyright-1.39.10.tar.gz", hash = "sha256:c8eaf5302f3265e275c7df4fba194d7afa7c1cb53fbfd448e90098360aca2c2e", size = 24740347, upload-time = "2026-08-13T17:09:02.51Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/d4/e1fa108710d0498a18c77b1e13897f31eab47c69aa8cfe2d2a4df746541e/basedpyright-1.39.9-py3-none-any.whl", hash = "sha256:6b0837b9eba972c71895167ab9b127e6afdbc17abc92312e3f8d15ca82a5611c", size = 13374276, upload-time = "2026-06-27T02:19:54.431Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/2a/a224054d75a58786c482f63b8ff2a09fc3362268bd1ebd0a61fb3f982153/basedpyright-1.39.10-py3-none-any.whl", hash = "sha256:cbd75d83c0be841329bcfef2d2f1182f152a6d975b8eb199e75cf5b8e9a3de78", size = 13482322, upload-time = "2026-08-13T17:08:59.074Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -181,37 +181,96 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "charset-normalizer"
|
||||
version = "3.4.9"
|
||||
version = "3.5.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -392,11 +451,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "filelock"
|
||||
version = "3.32.2"
|
||||
version = "3.32.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6d/30/03b03951873a1a0ffc7e8ca0e10c15597b59e8d0e39260704cd2ea087bc4/filelock-3.32.4.tar.gz", hash = "sha256:2bde2e4cf732e0153406d8a7bc80620ecf5e621fe0d25e41143c4e3b4733ff30", size = 222126, upload-time = "2026-08-23T17:37:55.363Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/a4/9b63d595d748e3aff8812b65eacc1a2c4bd90b7c2012e08e72373b4835eb/filelock-3.32.4-py3-none-any.whl", hash = "sha256:22e58ca3b1ae3b98993b762d7338367ae64fe50252bf78d59da3bfebcdf1cedd", size = 99864, upload-time = "2026-08-23T17:37:53.913Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -539,14 +598,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "gunicorn"
|
||||
version = "26.0.0"
|
||||
version = "26.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6d/b7/a4a3f632f823e432ce6bc65f62961b7980c898c77f075a2f7118cb3846fe/gunicorn-26.0.0.tar.gz", hash = "sha256:ca9346f85e3a4aeeb64d491045c16b9a35647abd37ea15efe53080eb8b090baf", size = 727286, upload-time = "2026-05-05T06:38:25.529Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/38/b8/ec4ba3f6cace4091c34e27478b576bb80f2f06fab80fd42c0ecc785b308f/gunicorn-26.1.0.tar.gz", hash = "sha256:1413d777bf99d31ebeb08acd354b01f1ecc44db0aa7b811ae7b86c669232e4f7", size = 755923, upload-time = "2026-08-18T11:49:39.438Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/40/9c2384fc2be4ad25dd4a49decd5ad9ea5a3639814c11bd40ab77cb9f0a14/gunicorn-26.0.0-py3-none-any.whl", hash = "sha256:40233d26a5f0d1872916188c276e21641155111c2853f0c2cd55260aec0d24fc", size = 212009, upload-time = "2026-05-05T06:38:23.007Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/dc/7a55fc605543fd5cb11c003fbbb21a1911d5e88a582cce6c5e063bf5c176/gunicorn-26.1.0-py3-none-any.whl", hash = "sha256:9f45bcddec5e9dc7a25a3bdccb0c6832f11fd5d4739b1ee36c8d2fec25f1dc86", size = 216237, upload-time = "2026-08-18T11:49:38.001Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -624,11 +680,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.18"
|
||||
version = "3.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -873,26 +929,26 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "prek"
|
||||
version = "0.4.13"
|
||||
version = "0.4.14"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d4/79/19f47eeb4d6092d36f94f47a056e7ae7a421d60220c772e8513483521b63/prek-0.4.13.tar.gz", hash = "sha256:9bf3dce400ef38a281836e4fe6429aa5f1690848be77cd97cb53c93769db4681", size = 533200, upload-time = "2026-08-10T08:54:15.477Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cf/51/135dc6ba2c021ce32b40700c8c337db72d802893e15291f4b3056076582f/prek-0.4.14.tar.gz", hash = "sha256:f6d0952e31ffd6e508660749dd51b8d8de96e955ed12c40e411f3224f502fed2", size = 537668, upload-time = "2026-08-17T04:27:55.031Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/12/661dc1c63c322000580dffa8503d28d633cb5d4c3181e662cbb86eded12e/prek-0.4.13-py3-none-linux_armv6l.whl", hash = "sha256:6a313f5f041b2fcbd33bceb6b6e11ee9b8621c8c6ad8ef13787a6d90541b624d", size = 5801508, upload-time = "2026-08-10T08:53:52.18Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/4a/c50597a45d08b22e5704a5d4d5c03269a581b46cf1f060861f9ba96cdade/prek-0.4.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:40436cd7247d2a2fc036ef07d7efcd828acf1aab9d648a8a3f21475e3ad3f789", size = 6141612, upload-time = "2026-08-10T08:53:53.86Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/82/93/abc084bbd76bb6c34efbf7db441392f17b7bfc50a54c4d53d3e37c7ad6e0/prek-0.4.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:019a33b477b7b949fb6dcd6cb33ed494c4a28117e53c370c637ec0aba60211aa", size = 5625418, upload-time = "2026-08-10T08:53:55.39Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/4d/a310ff9adcb4b822d935eb55f0f5cdcfe4c18a16610bc5e220a11215dc38/prek-0.4.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:5a2851b6e60912e73be1bf2cf61fab5b314add15ba7bcf230f860282bdbaf16b", size = 5942132, upload-time = "2026-08-10T08:53:56.789Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/d8/811230ff285000abdc0fb98b56b9ef2d23b2dc530cf2da86fe1665f844ba/prek-0.4.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9ba9b94bd3f47b5f94a4ae504c44f9529cb3258ebaf577a5a6070ea0c6a853d6", size = 5710181, upload-time = "2026-08-10T08:53:58.278Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/84/01f7163cc4267daba6b01cfac726327c6d848d2bb7349bd2fd877d88f744/prek-0.4.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33fc9e0d435cb9e650ec108f840febe7a94fd266c77149860091cead212dc82d", size = 6157705, upload-time = "2026-08-10T08:53:59.586Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/e3/061a0e9ebd7064edf70bb783afa575d045ac0cc9035c2143b76401ebcaf5/prek-0.4.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91066d8978eab83c111e7c34ee3046a003819d2df15ef7dc2bddeb83dce62bc0", size = 6898016, upload-time = "2026-08-10T08:54:00.927Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/68/5e8e2b9ff6a30b46bf35ca3b50e4d164c26406d9e7457d5b4633c6e1fbc0/prek-0.4.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85646fcc940f30bd946d63b6bbd1000d946994c88e154c5bdde7773a2c358dcf", size = 6365056, upload-time = "2026-08-10T08:54:02.704Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/23/afe543b69fb7f35016645eb4b766191e814cbcfa5545d695a2a7dd9b712e/prek-0.4.13-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:961859c3ddddb8e10367afcf93742da36fa213eda080f168fc1dcbe33e0b004d", size = 5952174, upload-time = "2026-08-10T08:54:04.079Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/6e/9e651ce51aa0b03244277f5e0660cf1b946a270cb62635a8a100389a67a2/prek-0.4.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ccf4fcbfc686ad6b589f2908ed6012a315091c45163e1f4420648b9669651a9e", size = 5761875, upload-time = "2026-08-10T08:54:05.378Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/64/e70e18734d93df272476d2f1b30d92c71d41ee7141070f1dd13cfbb2eff8/prek-0.4.13-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:eb4b7edfe00ca73e58d7e26fb871abddc5854e8d21067a202e43418fb7f98ef2", size = 5685365, upload-time = "2026-08-10T08:54:06.729Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/78/686fd5d3f12249368a0fdd8f7705e3ba540402a6157ef0b888e48734ff83/prek-0.4.13-py3-none-musllinux_1_1_i686.whl", hash = "sha256:c29449443f89da1647331742984b7046a55084759ad642373a9cc0a973494339", size = 5999013, upload-time = "2026-08-10T08:54:08.183Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/aa/93ffac2460b44f6182dbbb3194db976d18332edcb8208c4a796f8f9955f8/prek-0.4.13-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:7a04d5ac901819b115ecd0bf79cee091f0034323767abcc4a53626eeb96c3be0", size = 6486759, upload-time = "2026-08-10T08:54:09.775Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/3c/9f12c0d469ea345c249d5b0a027a1f2bf1ca05041d7785c34c455a9b605c/prek-0.4.13-py3-none-win32.whl", hash = "sha256:6d1bdcc1699ae18270f9bac9c4b4d29c6f1512b7a067e17ce5e220c30636f88c", size = 5515046, upload-time = "2026-08-10T08:54:11.456Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/3c/ef9ec67c560e60525d6a49b48a2a60434906f25ec2efa5e010fe2d42bbfa/prek-0.4.13-py3-none-win_amd64.whl", hash = "sha256:2d8fd796ed7944154fbee6d5a6d2490b9d4f14ce3626b1a0c9ca698455b25d9b", size = 5894683, upload-time = "2026-08-10T08:54:12.88Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/ce/8fe8fdf8154108a552d5578400da44144697ed11e5bd71d5a4980d7e202e/prek-0.4.13-py3-none-win_arm64.whl", hash = "sha256:65d6811b0220444bcdf539e157d7d5cb8edab01a5ed89534c70d73d675266ecb", size = 5650616, upload-time = "2026-08-10T08:54:14.21Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/75/727724174d419cab6e11e0c22c3fbcdd083312b0a0202dfa6befdb7f1bcb/prek-0.4.14-py3-none-linux_armv6l.whl", hash = "sha256:cf7fe2e07c99948ca3326fbd4254054cf4caa71a69cf9032be09c3938544e1e3", size = 5878406, upload-time = "2026-08-17T04:27:30.135Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/b6/82dfb41347342b4c53c1ddd27cfa81d3e1cf4ad0d5d76493c51e8da58dc9/prek-0.4.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6c8043aa5555c2ada561f4c69429e5117f370ad92f7d708340613b0965e725bd", size = 6216669, upload-time = "2026-08-17T04:27:31.795Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/68/ade0ba8b8c0044a3f7ca1a5cd7c97bc10656398c67466b04235f0f0b7418/prek-0.4.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:355570f0d8366a56817e55f44edf736ffc9ff2323894e21cd739ab519d343541", size = 5728085, upload-time = "2026-08-17T04:27:33.322Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/cc/8ddb67fb000d1ad4c95288e6c7e27989d3c172cd88893e3a6f9cdd7199fb/prek-0.4.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:334f48a2b19e63c5ab741e601cb6088caaa11d1b75146900dc298b055452d551", size = 6043650, upload-time = "2026-08-17T04:27:34.699Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/49/2780489798147fa1e81fa017af384a40229802cf36680cf72c74478d143c/prek-0.4.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5fbb17270df2dcb3c1aa6edfaa68f850d7968862413e58b1572e41c981f644f9", size = 5786094, upload-time = "2026-08-17T04:27:36.282Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/2a/27a4cdfa767b46663eb32a2b5f78ebaaeefc6e5caa87a21587216a504e31/prek-0.4.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ec46be0b1b45943a0746fdf69feeede274b0230bad32b11e2489bb600846081", size = 6239874, upload-time = "2026-08-17T04:27:37.984Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/ca/5ae58d95bc9dbad75ea13aea83fe0acfaca37d027c6bd7f9f1d1d97f3666/prek-0.4.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a21eea6ee996dbba351c1af8c6bd1959a61c37949757c0b97aef3edde82c126a", size = 6968009, upload-time = "2026-08-17T04:27:39.48Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/1c/c6a7800406f987559fc8e7c2cfd257a5a68096e806e953f143b488e00e4f/prek-0.4.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34169cb1c8dfbe4b9cb7af164f5c1c3dff68929c61ca1e1de28d378b32720836", size = 6443370, upload-time = "2026-08-17T04:27:41.181Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/38/4bf84223216ce2d31b500691d1644a9f35d6e468cb8ecb9a7dfbc66bc82c/prek-0.4.14-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:e39772ebf579f957fdbbc66ef6a7f7433bf603128cd60d5c804c70caac5f69c0", size = 6056449, upload-time = "2026-08-17T04:27:42.722Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/0a/f1980039ab7bf8342c4aa4da2a4cafe40e6a840f456d5bbf86ba024fcdca/prek-0.4.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a79e5940354a789e1311172a960c5daa75e3f643a47ee2be85b7d2ca1d7d980a", size = 5839941, upload-time = "2026-08-17T04:27:44.306Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/e2/67f067dcb912e157bc7f4cd2aeb422a0de65e7744c675d7f439676f6c832/prek-0.4.14-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:60b6539c1b28804807849173a6b49d467f6f36ca98bbb4536f5e34114d79942a", size = 5762723, upload-time = "2026-08-17T04:27:45.737Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/39/6669fff5c3336a2b79cf853c86b95547cf9be720a2ee7e4ae5eaa6cb46a1/prek-0.4.14-py3-none-musllinux_1_1_i686.whl", hash = "sha256:24cfabd9e5b8c5546ecaef562841a41a255a9dec1b7ae7761ce5a7a024ab9dd9", size = 6090040, upload-time = "2026-08-17T04:27:47.185Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/23/c23210be4c89795a854e76146c2465aaffc3d85ea83df61c1d4d5bb41bca/prek-0.4.14-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1f08d368da45439f949885bb0b637bc7dfe6910140d5285fa8bae058697ddd06", size = 6570273, upload-time = "2026-08-17T04:27:48.673Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/9c/4f10728ed36295347e84f01bb3e4c9078c9e337edc66215f5e2066b30551/prek-0.4.14-py3-none-win32.whl", hash = "sha256:5bfb30808ce2099c67d2a2d4cd68dc031e3fec1eeb61d73ef51a8f7c04d021cb", size = 5586715, upload-time = "2026-08-17T04:27:50.471Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/a7/d080a157d4e92927f7618d62d6a23979ef64a10c76a040cb8a8a194c50d7/prek-0.4.14-py3-none-win_amd64.whl", hash = "sha256:29364012d5704475d1092eb8a96ea30b163279096ad5e0c80a620fffa79bc639", size = 5969946, upload-time = "2026-08-17T04:27:51.971Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/e1/6fc64bb82e7270f61707e00b6d3154a4592ae0b2d3bd308173aa7aabe0a1/prek-0.4.14-py3-none-win_arm64.whl", hash = "sha256:ff588c02e10c8d05150763607671a22d5585c0ff7036c884d3489c2726eb215c", size = 5729906, upload-time = "2026-08-17T04:27:53.52Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -953,11 +1009,11 @@ sdist = { url = "https://files.pythonhosted.org/packages/e1/70/c7a4f46dbf06048c6
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.20.0"
|
||||
version = "2.21.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1133,15 +1189,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pytest-rerunfailures"
|
||||
version = "16.4"
|
||||
version = "16.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
{ name = "pytest" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/47/60/a90ca1cc6cffcb97b4260ed0ad2b7934b999d7c48abe4ea0840344862a3b/pytest_rerunfailures-16.4.tar.gz", hash = "sha256:8222d17c37eb7b9e4d6fc96a3c724ff4e1a5c97a5cc7cbb2c19e9282cfd21a11", size = 36635, upload-time = "2026-07-01T06:30:56.813Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ed/63/0114e45d4b2fcd5f6297dac655c067b47de28be4d33e088f200f0f2c4c28/pytest_rerunfailures-16.6.tar.gz", hash = "sha256:29dbfee46f542073c888e0ed4e81c51e15b9096f49a299eb1a759629c601684a", size = 42806, upload-time = "2026-08-17T07:11:00.447Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/93/3cdcc4033444e822e01b573414b03fd37fd5533070c750477b8f5fa5224b/pytest_rerunfailures-16.4-py3-none-any.whl", hash = "sha256:f69b5beb39622c90d1e44bd945d826eff6db545dcf0b68f52b7e4ad15eaf6d6c", size = 16955, upload-time = "2026-07-01T06:30:55.333Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/5e/1e994889673d7a0da11651f17ef789b6c83bfe349f29f871873dd3802445/pytest_rerunfailures-16.6-py3-none-any.whl", hash = "sha256:6af2d1ebd6e5cb79666ac408942cd6a0672a49fefd8523664770718560795e13", size = 19137, upload-time = "2026-08-17T07:10:59.121Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1243,16 +1299,16 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "qbittorrent-api"
|
||||
version = "2026.8.0"
|
||||
version = "2026.8.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
{ name = "requests" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/eb/d2/4a6d6ae6aaca1ba83039d27f28e5bf30bacef27ff4eb635c729d2d582363/qbittorrent_api-2026.8.0.tar.gz", hash = "sha256:4127a9c1d5c7ad9cd4e8de16f83daeea2e8b2ed962f99f284d238f927479391a", size = 1430678, upload-time = "2026-08-01T21:05:55.88Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/04/00/b7f41dfe8af6c3991f7182f9c14c46851379b8e66ac74c819f0952e9923d/qbittorrent_api-2026.8.1.tar.gz", hash = "sha256:9642c4528baee67216eab87022ce297048ac96df2f8c9f1cf920de078eb50b62", size = 1436760, upload-time = "2026-08-16T20:30:28.35Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/81/b456688c20e75cae5964700b58712630930b9dfaf32ce7c6a8881658d0d0/qbittorrent_api-2026.8.0-py3-none-any.whl", hash = "sha256:03b3db598d351194fd3c65f1d3e53843f65f85bef1d5077791efaa2e26869d2b", size = 68196, upload-time = "2026-08-01T21:05:54.376Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/60/77c7b63675f12ba492dee9f8aabab0fc61ced5d2131a83bd7e92dcc335c1/qbittorrent_api-2026.8.1-py3-none-any.whl", hash = "sha256:c17260e416ec5832c3ed7aff46fb0e2b47e70b90de182e89c34133b9d4ce5a6e", size = 71970, upload-time = "2026-08-16T20:30:26.67Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1321,27 +1377,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.16.2"
|
||||
version = "0.16.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/00/8f/d8074b1f25e003164087a8bfe79a0f1a3945135764dbb6aaab04103dcaf9/ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc", size = 4899731, upload-time = "2026-08-20T17:43:59.196Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ff/80/779895ef584e089d22f2c6df0d0e99a65ec2df0805f1fffd439415b8c1f0/ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7", size = 10006909, upload-time = "2026-08-20T17:43:16.888Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/e6/f553199b5e8927a05cb5c422d921fd0656b29ab976e91c44802107c6b0da/ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604", size = 10240201, upload-time = "2026-08-20T17:43:19.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/70/4a6dc4bb34da4dee35e30f09bbd1bfbdd26f33b62fb9b8df31f08a199cd2/ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255", size = 9835122, upload-time = "2026-08-20T17:43:21.708Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/12/c6e22d686372c15bcb7af99831f1a1be96df696491babf4f24e4f942c527/ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade", size = 9977162, upload-time = "2026-08-20T17:43:24.236Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/49/72b10ec912f5ab5854992eaf7aa7cd36729b6937d9dc4e0fb41b3bf428ec/ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff", size = 9829789, upload-time = "2026-08-20T17:43:26.966Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/80/0f30e32e7f6ee26edc39075502db9d368d788a44a79b55f763eb4ab03796/ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80", size = 10527949, upload-time = "2026-08-20T17:43:29.384Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/3d/86e8ad3542169e56cac3859a343afdb9df2ad54d35a59ce1e67baee83421/ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07", size = 11333695, upload-time = "2026-08-20T17:43:31.872Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/16/481c29b380c20a0054a8261066665e1b3488e23636c49d0a43e75975b9bb/ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7", size = 10727741, upload-time = "2026-08-20T17:43:34.596Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/b6/56bc0b8cf45b54b28b3a5e6381c8945d51b5b18adf659454c32295209a31/ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3", size = 10286522, upload-time = "2026-08-20T17:43:37.288Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/8b/b345b4fb110f2fbe2bd31eabd271e5e8b3b7e4ee6c0e02f2dc6be78db000/ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454", size = 10584182, upload-time = "2026-08-20T17:43:39.984Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/e5/827b34041c35f58774a9681a4213994c164fc987800f4dddabcf451da0bf/ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b", size = 10134195, upload-time = "2026-08-20T17:43:42.351Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/10/d0bffcdd6729b87afc82ba0ef377173356a7dc8e972f5179968cf2fdf98c/ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d", size = 9825821, upload-time = "2026-08-20T17:43:44.532Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/32/0db2a863b796ca62d83e92a07a3ccf00921b14db02059347576a2fda3d4b/ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0", size = 10267658, upload-time = "2026-08-20T17:43:46.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/a0/fbdeb59e48c6261f523e56c8f12e9c08fbe693786595cc7e3959207a9232/ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d", size = 10697071, upload-time = "2026-08-20T17:43:49.891Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/28/0c6dd865859c6d17bc8ccc34cb72b0e02d6c7eb25e8a1e22b5bea681e2c0/ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e", size = 10021687, upload-time = "2026-08-20T17:43:52.281Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/03/e724450f621698117f9aa6dd241c94d0274ae96781378dc86745ae29f0e7/ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c", size = 10567657, upload-time = "2026-08-20T17:43:54.78Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1355,7 +1411,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "selenium"
|
||||
version = "4.46.0"
|
||||
version = "4.47.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
@@ -1365,14 +1421,14 @@ dependencies = [
|
||||
{ name = "urllib3", extra = ["socks"] },
|
||||
{ name = "websocket-client" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a0/53/053df98ef0c38535a74ea732217eb2dbfddc7ff5442a47a4b315ba577f85/selenium-4.46.0.tar.gz", hash = "sha256:54f7e1a4df5f7508ac8c38ce2ea584db1b27083dd79962b22524219219df5cbe", size = 1006000, upload-time = "2026-07-11T00:38:41.11Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c3/a2/213190a606bc036b4db1b8129f399964988872a555b50dfbfddf612d333c/selenium-4.47.0.tar.gz", hash = "sha256:4f6667c23080646e045fb91d2039687e88f549d667961f6ce85832b17384b68e", size = 1014095, upload-time = "2026-08-10T17:54:11.99Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/30/f4/365c723a37c7e9ba6a6a19bf0c9e407137703f56a72c6201d0e4cb1432cb/selenium-4.46.0-py3-none-any.whl", hash = "sha256:f03e864770a21ab32009961c9d015cb2c6275eba45c926c272e02d90af423aad", size = 9428587, upload-time = "2026-07-11T00:38:38.84Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/0b/652575986d2ed03d29103d8574580a03aefe50d243d225b441e2375bd0f6/selenium-4.47.0-py3-none-any.whl", hash = "sha256:2eac6b8e7c017f57ecc40820383da8881a6fd7a90ea555c1b0af322f2344b347", size = 9511195, upload-time = "2026-08-10T17:54:09.369Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "seleniumbase"
|
||||
version = "4.51.12"
|
||||
version = "4.52.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
@@ -1436,9 +1492,9 @@ dependencies = [
|
||||
{ name = "wheel" },
|
||||
{ name = "wsproto" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8a/6d/b852196360200892bc0efb8daeb5ad2278e09eec5b6aaee9f35fdf7b6860/seleniumbase-4.51.12.tar.gz", hash = "sha256:a914610bbe5c9084bce045243e3134987064bbe6e1f2725fe369ce4dbce313d1", size = 672929, upload-time = "2026-08-10T19:28:37.957Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fb/9e/a702bc91ac93ba4e4bc29ae3da16a5ac6b436f3d8885646f82555ebf0282/seleniumbase-4.52.1.tar.gz", hash = "sha256:b6c0d67c32465e888231a06839197e73165518dbb06165f7af6b42dfa925a362", size = 672812, upload-time = "2026-08-19T16:31:52.135Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/93/15/199f8ed68290e553a6bc3c10920817d41a006ab3d9aaac06b980256df966/seleniumbase-4.51.12-py3-none-any.whl", hash = "sha256:64ea9fa52b8c1ac8d1b8ccbb3dcf7daa02c1cb1ffdd7e1f358677e36cbbc071c", size = 677511, upload-time = "2026-08-10T19:28:34.24Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/99/a42b49b356aff2682e1808ef7c6ec28479b96e07d40b5d79ae4858ee9235/seleniumbase-4.52.1-py3-none-any.whl", hash = "sha256:a21a9eb44cabf896a1d21edf7b4569f282e521f913efd69c1becd9202ee35e30", size = 677476, upload-time = "2026-08-19T16:31:48.9Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1498,7 +1554,7 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "apprise", specifier = ">=1.12.0" },
|
||||
{ name = "apprise", specifier = ">=1.13.0" },
|
||||
{ name = "authlib", specifier = ">=1.7.2,<1.8" },
|
||||
{ name = "beautifulsoup4" },
|
||||
{ name = "defusedxml" },
|
||||
@@ -1510,16 +1566,16 @@ requires-dist = [
|
||||
{ name = "gevent" },
|
||||
{ name = "gevent-websocket" },
|
||||
{ name = "gunicorn" },
|
||||
{ name = "httpx", extras = ["http2"], specifier = ">=0.27" },
|
||||
{ name = "httpx", extras = ["http2"], specifier = ">=0.28.1" },
|
||||
{ name = "psutil" },
|
||||
{ name = "pyautogui", marker = "extra == 'browser'" },
|
||||
{ name = "python-socketio" },
|
||||
{ name = "python-xlib", marker = "extra == 'browser'" },
|
||||
{ name = "pyvirtualdisplay", marker = "extra == 'browser'" },
|
||||
{ name = "qbittorrent-api", specifier = ">=2026.8.0" },
|
||||
{ name = "qbittorrent-api", specifier = ">=2026.8.1" },
|
||||
{ name = "rarfile" },
|
||||
{ name = "requests", extras = ["socks"] },
|
||||
{ name = "seleniumbase", marker = "extra == 'browser'", specifier = "==4.51.12" },
|
||||
{ name = "seleniumbase", marker = "extra == 'browser'", specifier = "==4.52.1" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "transmission-rpc" },
|
||||
]
|
||||
@@ -1527,12 +1583,12 @@ provides-extras = ["browser"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
{ name = "basedpyright", specifier = ">=1.39.9" },
|
||||
{ name = "basedpyright", specifier = ">=1.39.10" },
|
||||
{ name = "prek" },
|
||||
{ name = "pytest" },
|
||||
{ name = "pytest-cov" },
|
||||
{ name = "pytest-xdist", specifier = ">=3.8.0" },
|
||||
{ name = "ruff", specifier = "==0.16.2" },
|
||||
{ name = "ruff", specifier = "==0.16.4" },
|
||||
{ name = "vulture", specifier = ">=2.14" },
|
||||
]
|
||||
|
||||
@@ -1623,7 +1679,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "trio"
|
||||
version = "0.33.0"
|
||||
version = "0.34.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
@@ -1633,9 +1689,9 @@ dependencies = [
|
||||
{ name = "sniffio" },
|
||||
{ name = "sortedcontainers" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/92/dc/a2d25ed73ad49cfd79bf18d262577c3731c98e382284e28d522f49a0df35/trio-0.34.0.tar.gz", hash = "sha256:63b9485408bdfdde544fced107045a8c0086cdc4bd0ef2f797b9e0dd111b964b", size = 607457, upload-time = "2026-08-11T00:33:42.198Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/1f/555f1364bed52a92a864181962b77f1b15adadeacf23b86105324363e461/trio-0.34.0-py3-none-any.whl", hash = "sha256:6c7c9f49917694dcdcd5f67abd168df5599eca480d61f29854d17a61a75c2f05", size = 511840, upload-time = "2026-08-11T00:33:40.552Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1765,14 +1821,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "wheel"
|
||||
version = "0.47.0"
|
||||
version = "0.48.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/39/62/75f18a0f03b4219c456652c7780e4d749b929eb605c098ce3a5b6b6bc081/wheel-0.47.0.tar.gz", hash = "sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3", size = 63854, upload-time = "2026-04-22T15:51:27.727Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d0/20/50ed6bdf27dec98b568a8ae25dc599f35baa3d9709f9e83fd1edb56b9a90/wheel-0.48.0.tar.gz", hash = "sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322", size = 66471, upload-time = "2026-08-11T22:02:27.327Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/87/1b/9e33c09813d65e248f7f773119148a612516a4bea93e9c6f545f78455b7c/wheel-0.47.0-py3-none-any.whl", hash = "sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced", size = 32218, upload-time = "2026-04-22T15:51:26.296Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl", hash = "sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab", size = 33320, upload-time = "2026-08-11T22:02:26.1Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user