From 2007384058d303e61d37a020e58b15f7858dcafa Mon Sep 17 00:00:00 2001 From: misko903 <41118056+misko903@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:30:01 +0200 Subject: [PATCH] feat(sources): add Webshare.cz as a release source (#1543) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds [Webshare.cz](https://webshare.cz), the Czech file host, as a release source for ebooks and audiobooks. It follows the Libgen source: disabled by default, with a settings tab for credentials and a **Test Login** button that reports whether the account is VIP. ## Why Webshare is one of the main places Czech and Slovak ebooks and audiobooks are shared, and none of the existing sources reach it. Together with the Databáze knih metadata provider (#1479), this lets Universal mode find Czech and Slovak titles end to end. ## How - **API client** (`webshare/api.py`): Webshare's XML API (`POST /api//`). Login is the salted exchange the API documents: fetch the salt, send `SHA1(MD5_CRYPT(password, salt))` plus an MD5 digest. Python dropped the `crypt` module in 3.13, so md5-crypt is a small pure-Python port, checked against `openssl passwd -1`. The session token is cached per account and refreshed once when an authenticated call fails. File-level errors (`FILE_LINK_FATAL_*`) are not retried. XML is parsed with `defusedxml`. - **Search** (`webshare/source.py`): Webshare matches file names only, so the file name becomes the release title and its extension the format. Queries are title + author, then title only, or the manual query. Results are filtered to the configured `SUPPORTED_FORMATS` / `SUPPORTED_AUDIOBOOK_FORMATS`, plus `zip`/`rar` for audiobooks, since they are usually uploaded as archives. Password-protected files are skipped. The language is guessed from tags in the name (`CZ`, `SK`, `ENG`, ...), and the vote balance is a sortable column. API failures raise `SourceUnavailableError`. - **Download** (`webshare/handler.py`): resolves a time-limited link per attempt with `file_link`, then streams it into `TMP_DIR` with progress and cancel support. Large audiobook archives are not buffered in memory. - **Settings**: `WEBSHARE_ENABLED`, `WEBSHARE_USERNAME`, `WEBSHARE_PASSWORD` (a `PasswordField`, already covered by the `_PASSWORD` redaction in `genDebug.sh`), and `WEBSHARE_MAX_RESULTS`. The `docs/environment-variables.md` section was added. ## Testing - `tests/webshare/`: 31 offline tests. They cover md5-crypt vectors, password hashing, XML parsing, token reuse and re-login, non-retried file errors, availability gating, format filtering, query fallback, release mapping, language guessing, and the handler's streaming, small-file rejection, link errors and cancel. - `ruff check` and `ruff format --check` are clean. The rest of the unit suite fails the same way with and without this change: Windows-only tests on my machine. - Verified live against webshare.cz: search parsing and the salt endpoint's error path. - Still a draft: I haven't yet verified login and download with a real account. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: CaliBrain --- docs/environment-variables.md | 54 +++ shelfmark/release_sources/__init__.py | 1 + .../release_sources/webshare/__init__.py | 6 + shelfmark/release_sources/webshare/api.py | 333 ++++++++++++++++++ shelfmark/release_sources/webshare/handler.py | 124 +++++++ .../release_sources/webshare/settings.py | 102 ++++++ shelfmark/release_sources/webshare/source.py | 268 ++++++++++++++ tests/webshare/__init__.py | 0 tests/webshare/test_api.py | 175 +++++++++ tests/webshare/test_handler.py | 87 +++++ tests/webshare/test_source.py | 164 +++++++++ 11 files changed, 1314 insertions(+) create mode 100644 shelfmark/release_sources/webshare/__init__.py create mode 100644 shelfmark/release_sources/webshare/api.py create mode 100644 shelfmark/release_sources/webshare/handler.py create mode 100644 shelfmark/release_sources/webshare/settings.py create mode 100644 shelfmark/release_sources/webshare/source.py create mode 100644 tests/webshare/__init__.py create mode 100644 tests/webshare/test_api.py create mode 100644 tests/webshare/test_handler.py create mode 100644 tests/webshare/test_source.py diff --git a/docs/environment-variables.md b/docs/environment-variables.md index d403e289..5ae2540e 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -19,6 +19,7 @@ This document lists all configuration options that can be set via environment va - [Soulseek (slskd)](#soulseek-slskd) - [AudiobookBay](#audiobookbay) - [Libgen Search](#libgen-search) +- [Webshare.cz](#webshare.cz) - [IRC](#irc) - [Download Clients](#download-clients) - [Metadata Providers](#metadata-providers) @@ -1779,6 +1780,59 @@ Maximum number of results to request per search (1-100). +## Webshare.cz + +| Variable | Description | Type | Default | +|----------|-------------|------|---------| +| `WEBSHARE_ENABLED` | Search Webshare.cz by file name and download matching ebooks and audiobooks. Requires a Webshare account; a VIP account is recommended for full download speed. | boolean | `false` | +| `WEBSHARE_USERNAME` | Your Webshare.cz login. | string | _none_ | +| `WEBSHARE_PASSWORD` | Your Webshare.cz password. | string (secret) | _none_ | +| `WEBSHARE_MAX_RESULTS` | Maximum number of files to request per search (1-200). Results are then filtered to your supported ebook or audiobook formats. | number | `50` | + +
+Detailed descriptions + +#### `WEBSHARE_ENABLED` + +**Enable Webshare.cz** + +Search Webshare.cz by file name and download matching ebooks and audiobooks. Requires a Webshare account; a VIP account is recommended for full download speed. + +- **Type:** boolean +- **Default:** `false` + +#### `WEBSHARE_USERNAME` + +**Username or Email** + +Your Webshare.cz login. + +- **Type:** string +- **Default:** _none_ +- **Required:** Yes + +#### `WEBSHARE_PASSWORD` + +**Password** + +Your Webshare.cz password. + +- **Type:** string (secret) +- **Default:** _none_ +- **Required:** Yes + +#### `WEBSHARE_MAX_RESULTS` + +**Max Results** + +Maximum number of files to request per search (1-200). Results are then filtered to your supported ebook or audiobook formats. + +- **Type:** number +- **Default:** `50` +- **Constraints:** min: 1, max: 200 + +
+ ## IRC | Variable | Description | Type | Default | diff --git a/shelfmark/release_sources/__init__.py b/shelfmark/release_sources/__init__.py index 8c1951e8..7fc10c14 100644 --- a/shelfmark/release_sources/__init__.py +++ b/shelfmark/release_sources/__init__.py @@ -494,6 +494,7 @@ _BUILTIN_SOURCE_MODULES = ( "shelfmark.release_sources.newznab", "shelfmark.release_sources.prowlarr", "shelfmark.release_sources.slskd", + "shelfmark.release_sources.webshare", ) _builtin_source_state = {"loaded": False} diff --git a/shelfmark/release_sources/webshare/__init__.py b/shelfmark/release_sources/webshare/__init__.py new file mode 100644 index 00000000..3892b158 --- /dev/null +++ b/shelfmark/release_sources/webshare/__init__.py @@ -0,0 +1,6 @@ +"""Webshare.cz release source - file-name search and downloads via the Webshare API.""" + +# Import to trigger registration +from shelfmark.release_sources.webshare import handler as handler +from shelfmark.release_sources.webshare import settings as settings +from shelfmark.release_sources.webshare import source as source diff --git a/shelfmark/release_sources/webshare/api.py b/shelfmark/release_sources/webshare/api.py new file mode 100644 index 00000000..96d555d6 --- /dev/null +++ b/shelfmark/release_sources/webshare/api.py @@ -0,0 +1,333 @@ +"""Webshare.cz XML API client: login, search and download-link resolution. + +Every endpoint is ``POST https://webshare.cz/api//`` with a form-encoded body and +an XML ```` whose ```` is ``OK`` or ``FATAL``. Authenticated calls carry +the session token as the ``wst`` form field. Login is a two-step salted exchange: fetch +the user's salt, then send ``SHA1(MD5_CRYPT(password, salt))`` plus an MD5 digest. + +All shelfmark-stateful behaviour (config, task naming) lives in source.py/handler.py. +""" + +import hashlib +import hmac +import threading +from dataclasses import dataclass +from typing import Any + +import requests +from defusedxml import ElementTree as DefusedElementTree +from defusedxml.common import DefusedXmlException + +from shelfmark.core.logger import setup_logger +from shelfmark.download import http as downloader +from shelfmark.download import network + +logger = setup_logger(__name__) + +API_BASE = "https://webshare.cz/api" +FILE_PAGE_BASE = "https://webshare.cz/#/file" +DEFAULT_TIMEOUT = (5, 15) + +# Webshare's API answers with an empty/HTML body unless the request looks like the +# site's own XHR calls. +_API_HEADERS = { + **downloader.DOWNLOAD_HEADERS, + "Accept": "text/xml; charset=UTF-8", + "X-Requested-With": "XMLHttpRequest", +} + + +class WebshareError(Exception): + """A Webshare API call failed (network error, bad XML or a FATAL status).""" + + def __init__(self, message: str, code: str | None = None) -> None: + """Keep the Webshare error code (e.g. ``LOGIN_FATAL_1``) next to the message.""" + super().__init__(message) + self.code = code + + +@dataclass(frozen=True) +class WebshareFile: + """One file from a Webshare search result.""" + + ident: str + name: str + type: str | None + size_bytes: int | None + positive_votes: int + negative_votes: int + password_protected: bool + + @property + def info_url(self) -> str: + """Public Webshare page for this file.""" + return f"{FILE_PAGE_BASE}/{self.ident}" + + +@dataclass(frozen=True) +class WebshareAccount: + """The logged-in account's identity and VIP state.""" + + username: str + vip: bool + vip_days: int | None + + +# --- md5-crypt --- + +_ITOA64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + +def _to64(value: int, length: int) -> str: + chars = [] + for _ in range(length): + chars.append(_ITOA64[value & 0x3F]) + value >>= 6 + return "".join(chars) + + +def md5_crypt(password: str, salt: str, magic: str = "$1$") -> str: + """FreeBSD md5-crypt (``$1$salt$hash``), the scheme Webshare salts passwords with. + + Python dropped the ``crypt`` module in 3.13, so this is a straight port of the + reference algorithm (the same one ``openssl passwd -1`` implements). MD5 here is + dictated by Webshare's login protocol, not chosen to protect anything at rest, hence + ``usedforsecurity=False`` throughout this module. + """ + salt = salt.removeprefix(magic) + salt = salt.split("$", 1)[0][:8] + + pw = password.encode() + salt_b = salt.encode() + magic_b = magic.encode() + + alternate = hashlib.md5(pw + salt_b + pw, usedforsecurity=False).digest() + ctx = pw + magic_b + salt_b + for remaining in range(len(pw), 0, -16): + ctx += alternate[: min(16, remaining)] + length = len(pw) + while length: + ctx += b"\x00" if length & 1 else pw[:1] + length >>= 1 + final = hashlib.md5(ctx, usedforsecurity=False).digest() + + for i in range(1000): + round_ctx = pw if i & 1 else final + if i % 3: + round_ctx += salt_b + if i % 7: + round_ctx += pw + round_ctx += final if i & 1 else pw + final = hashlib.md5(round_ctx, usedforsecurity=False).digest() + + encoded = "".join( + _to64((final[a] << 16) | (final[b] << 8) | final[c], 4) + for a, b, c in ((0, 6, 12), (1, 7, 13), (2, 8, 14), (3, 9, 15), (4, 10, 5)) + ) + encoded += _to64(final[11], 2) + return f"{magic}{salt}${encoded}" + + +def hash_password(username: str, password: str, salt: str) -> tuple[str, str]: + """Return the ``(password, digest)`` pair Webshare's login endpoint expects.""" + crypted = md5_crypt(password, salt).encode() + password_hash = hashlib.sha1(crypted, usedforsecurity=False).hexdigest() + digest_input = f"{username}:Webshare:{password_hash}".encode() + digest = hashlib.md5(digest_input, usedforsecurity=False).hexdigest() + return password_hash, digest + + +# --- transport --- + + +def _text(node: Any, tag: str) -> str | None: + child = node.find(tag) + if child is None or child.text is None: + return None + value = child.text.strip() + return value or None + + +def _int(node: Any, tag: str) -> int | None: + value = _text(node, tag) + if value is None: + return None + try: + return int(value) + except ValueError: + return None + + +def _post( + endpoint: str, + data: dict[str, Any], + timeout: tuple[int, int] = DEFAULT_TIMEOUT, +) -> Any: + """POST to an API endpoint and return the parsed ```` root on OK.""" + url = f"{API_BASE}/{endpoint}/" + try: + response = requests.post( + url, + data=data, + headers=_API_HEADERS, + timeout=timeout, + proxies=network.get_proxies(url), + verify=network.get_ssl_verify(url), + ) + response.raise_for_status() + except requests.exceptions.RequestException as exc: + msg = f"Webshare request failed: {exc}" + raise WebshareError(msg) from exc + + try: + root = DefusedElementTree.fromstring(response.content) + except DefusedElementTree.ParseError, DefusedXmlException: + msg = f"Webshare returned an unreadable response for {endpoint}" + raise WebshareError(msg) from None + + if _text(root, "status") != "OK": + code = _text(root, "code") + message = _text(root, "message") or f"Webshare {endpoint} failed" + raise WebshareError(message, code) + return root + + +# --- client --- + + +class WebshareClient: + """Session-aware Webshare client. + + The login token is cached per username at module level so searches and + downloads share one session instead of logging in for every request. A call that + fails while authenticated drops the token and retries once with a fresh login, which + covers tokens Webshare expired server-side. A cached token is only reused while the + password it was obtained with is unchanged, so editing the password forces a login. + """ + + _token_lock = threading.Lock() + _tokens: dict[str, tuple[str, str]] = {} # noqa: RUF012 - username -> (password, token) + + def __init__(self, username: str, password: str) -> None: + """Bind the client to one Webshare account.""" + self.username = username + self.password = password + + def login(self) -> str: + """Log in and return a fresh session token (WST).""" + if not self.username or not self.password: + msg = "Webshare username and password are required" + raise WebshareError(msg) + + salt_root = _post("salt", {"username_or_email": self.username}) + salt = _text(salt_root, "salt") + if not salt: + msg = "Webshare did not return a password salt" + raise WebshareError(msg) + + password_hash, digest = hash_password(self.username, self.password, salt) + login_root = _post( + "login", + { + "username_or_email": self.username, + "password": password_hash, + "digest": digest, + "keep_logged_in": 1, + }, + ) + token = _text(login_root, "token") + if not token: + msg = "Webshare login did not return a session token" + raise WebshareError(msg) + + with self._token_lock: + self._tokens[self.username] = (self.password, token) + return token + + def _token(self) -> str: + with self._token_lock: + cached = self._tokens.get(self.username) + if cached and hmac.compare_digest(cached[0].encode(), self.password.encode()): + return cached[1] + return self.login() + + def _invalidate_token(self) -> None: + with self._token_lock: + self._tokens.pop(self.username, None) + + def _authed_post( + self, + endpoint: str, + data: dict[str, Any], + timeout: tuple[int, int] = DEFAULT_TIMEOUT, + ) -> Any: + try: + return _post(endpoint, {**data, "wst": self._token()}, timeout) + except WebshareError as exc: + # Errors that are about the file itself won't change after re-login. + if exc.code and exc.code.startswith(("FILE_LINK_FATAL", "FILE_INFO_FATAL")): + raise + logger.debug("Webshare %s failed (%s), retrying with a fresh login", endpoint, exc) + self._invalidate_token() + return _post(endpoint, {**data, "wst": self.login()}, timeout) + + def account(self) -> WebshareAccount: + """Return the logged-in account's username and VIP state.""" + root = self._authed_post("user_data", {}) + return WebshareAccount( + username=_text(root, "username") or self.username, + vip=_text(root, "vip") == "1", + vip_days=_int(root, "vip_days"), + ) + + def search( + self, + query: str, + *, + category: str = "", + limit: int = 50, + sort: str = "", + timeout: tuple[int, int] = DEFAULT_TIMEOUT, + ) -> list[WebshareFile]: + """Search Webshare by file name.""" + root = self._authed_post( + "search", + { + "what": query, + "category": category, + "sort": sort, + "limit": limit, + "offset": 0, + }, + timeout, + ) + files: list[WebshareFile] = [] + for node in root.findall("file"): + ident = _text(node, "ident") + name = _text(node, "name") + if not ident or not name: + continue + files.append( + WebshareFile( + ident=ident, + name=name, + type=(_text(node, "type") or "").lower() or None, + size_bytes=_int(node, "size"), + positive_votes=_int(node, "positive_votes") or 0, + negative_votes=_int(node, "negative_votes") or 0, + password_protected=_text(node, "password") == "1", + ) + ) + return files + + def file_link(self, ident: str) -> str: + """Resolve a file ident to a time-limited direct download URL.""" + root = self._authed_post( + "file_link", + {"ident": ident, "download_type": "file_download", "force_https": 1}, + ) + link = _text(root, "link") + if not link: + msg = "Webshare did not return a download link" + raise WebshareError(msg) + return link diff --git a/shelfmark/release_sources/webshare/handler.py b/shelfmark/release_sources/webshare/handler.py new file mode 100644 index 00000000..e2595243 --- /dev/null +++ b/shelfmark/release_sources/webshare/handler.py @@ -0,0 +1,124 @@ +"""Webshare.cz download handler - resolves a file ident to a link and streams it. + +Selected by ``get_handler(task.source)`` for ``source == "webshare"``. Download links +are time-limited and tied to the session, so one is resolved per attempt rather than at +search time. Audiobook archives can run to gigabytes, so the body is streamed straight +into TMP_DIR instead of buffered in memory like the Libgen handler does. +""" + +from pathlib import PurePosixPath +from typing import TYPE_CHECKING +from urllib.parse import unquote, urlsplit + +import requests + +from shelfmark.config.env import TMP_DIR +from shelfmark.core.config import config +from shelfmark.core.logger import setup_logger +from shelfmark.core.models import build_filename +from shelfmark.download import http as downloader +from shelfmark.download import network +from shelfmark.release_sources import DownloadHandler, register_handler +from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError +from shelfmark.release_sources.webshare.source import get_credentials + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from threading import Event + + from shelfmark.core.models import DownloadTask + +logger = setup_logger(__name__) + +# Files under this size are almost certainly an error page, not a book. +_MIN_VALID_FILE_SIZE = 10 * 1024 +_CHUNK_SIZE = 256 * 1024 +_DOWNLOAD_TIMEOUT = (10, 60) + + +def _target_path(task: DownloadTask, ident: str, link: str) -> Path: + """Staging path: the naming template, or Webshare's own file name for "none".""" + if config.get("FILE_ORGANIZATION", "rename") != "none": + return TMP_DIR / build_filename(task.title, task.author, task.year, task.format) + link_name = PurePosixPath(unquote(urlsplit(link).path)).name + return TMP_DIR / (link_name or f"{ident}.{task.format or 'bin'}") + + +@register_handler("webshare") +class WebshareHandler(DownloadHandler): + """Download handler for Webshare.cz releases.""" + + def download( + self, + task: DownloadTask, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, str | None], None], + ) -> str | None: + """Resolve a download link for the file and stream it into TMP_DIR.""" + ident = task.task_id.split(":", 1)[-1] + book_path: Path | None = None + try: + if cancel_flag.is_set(): + status_callback("cancelled", "Cancelled") + return None + + status_callback("resolving", "Resolving Webshare link") + link = WebshareClient(*get_credentials()).file_link(ident) + + book_path = _target_path(task, ident, link) + status_callback("downloading", None) + if not self._stream(link, book_path, cancel_flag, progress_callback): + book_path.unlink(missing_ok=True) + status_callback("cancelled", "Cancelled") + return None + + if book_path.stat().st_size < _MIN_VALID_FILE_SIZE: + book_path.unlink(missing_ok=True) + status_callback("error", "Webshare returned an invalid file") + return None + return str(book_path) + except (WebshareError, requests.exceptions.RequestException, OSError) as exc: + if book_path is not None: + book_path.unlink(missing_ok=True) + if cancel_flag.is_set(): + status_callback("cancelled", "Cancelled") + else: + logger.warning("Webshare download failed for %s: %s", ident, exc) + status_callback("error", str(exc)) + return None + + @staticmethod + def _stream( + link: str, + book_path: Path, + cancel_flag: Event, + progress_callback: Callable[[float], None], + ) -> bool: + """Stream ``link`` into ``book_path``; return False if cancelled midway.""" + with requests.get( + link, + headers=downloader.DOWNLOAD_HEADERS, + stream=True, + timeout=_DOWNLOAD_TIMEOUT, + proxies=network.get_proxies(link), + verify=network.get_ssl_verify(link), + ) as response: + response.raise_for_status() + total = int(response.headers.get("Content-Length") or 0) + written = 0 + with book_path.open("wb") as file: + for chunk in response.iter_content(chunk_size=_CHUNK_SIZE): + if cancel_flag.is_set(): + return False + file.write(chunk) + written += len(chunk) + if total: + progress_callback(min(100.0, written * 100 / total)) + progress_callback(100.0) + return True + + def cancel(self, task_id: str) -> bool: + """Cancellation is handled by the orchestrator via the cancel_flag.""" + return False diff --git a/shelfmark/release_sources/webshare/settings.py b/shelfmark/release_sources/webshare/settings.py new file mode 100644 index 00000000..53b164ad --- /dev/null +++ b/shelfmark/release_sources/webshare/settings.py @@ -0,0 +1,102 @@ +"""Webshare.cz settings registration.""" + +from typing import Any + +from shelfmark.core.request_helpers import normalize_optional_text +from shelfmark.core.settings_registry import ( + ActionButton, + CheckboxField, + NumberField, + PasswordField, + SettingsField, + TextField, + register_settings, +) + + +def _resolve_setting_text(current_values: dict[str, Any], key: str) -> str: + """Prefer current form values, then fall back to persisted config text.""" + from shelfmark.core.config import config + + current_value = normalize_optional_text(current_values.get(key)) + if current_value is not None: + return current_value + return normalize_optional_text(config.get(key, "")) or "" + + +def _test_webshare_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]: + """Log in with the current form credentials and report the account's VIP state.""" + from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError + + current_values = current_values or {} + username = _resolve_setting_text(current_values, "WEBSHARE_USERNAME") + password = _resolve_setting_text(current_values, "WEBSHARE_PASSWORD") + if not username or not password: + return {"success": False, "message": "Username and password are required"} + + try: + client = WebshareClient(username, password) + client.login() + account = client.account() + except WebshareError as exc: + return {"success": False, "message": f"Login failed: {exc}"} + + if account.vip: + days = f" ({account.vip_days} days left)" if account.vip_days is not None else "" + return {"success": True, "message": f"Logged in as {account.username}, VIP{days}"} + return { + "success": True, + "message": f"Logged in as {account.username} (free account - downloads are slow)", + } + + +@register_settings("webshare_config", "Webshare.cz", icon="download", order=47) +def webshare_config_settings() -> list[SettingsField]: + """Webshare.cz configuration settings.""" + enabled = {"field": "WEBSHARE_ENABLED", "value": True} + return [ + CheckboxField( + key="WEBSHARE_ENABLED", + label="Enable Webshare.cz", + description=( + "Search Webshare.cz by file name and download matching ebooks and " + "audiobooks. Requires a Webshare account; a VIP account is recommended " + "for full download speed." + ), + default=False, + ), + TextField( + key="WEBSHARE_USERNAME", + label="Username or Email", + description="Your Webshare.cz login.", + required=True, + show_when=enabled, + ), + PasswordField( + key="WEBSHARE_PASSWORD", + label="Password", + description="Your Webshare.cz password.", + required=True, + show_when=enabled, + ), + ActionButton( + key="test_webshare", + label="Test Login", + description="Log in to Webshare.cz and show the account's VIP status.", + style="primary", + callback=_test_webshare_connection, + show_when=enabled, + ), + NumberField( + key="WEBSHARE_MAX_RESULTS", + label="Max Results", + description=( + "Maximum number of files to request per search (1-200). Results are then " + "filtered to your supported ebook or audiobook formats." + ), + default=50, + min_value=1, + max_value=200, + show_when=enabled, + ), + ] diff --git a/shelfmark/release_sources/webshare/source.py b/shelfmark/release_sources/webshare/source.py new file mode 100644 index 00000000..9aede7bb --- /dev/null +++ b/shelfmark/release_sources/webshare/source.py @@ -0,0 +1,268 @@ +"""Webshare.cz release source - searches Webshare's file index by name. + +Webshare is a Czech file host whose search matches file names only, so there is no +book metadata to read: the file name is the release title, its extension the format, +and results are filtered down to the user's supported ebook/audiobook formats. +Downloads resolve a time-limited link per file through the logged-in account (see +handler.py). +""" + +import re +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, ClassVar + +from shelfmark.core.config import config +from shelfmark.core.logger import setup_logger +from shelfmark.download.postprocess.policy import ( + get_supported_audiobook_formats, + get_supported_formats, +) +from shelfmark.release_sources import ( + ColumnAlign, + ColumnColorHint, + ColumnRenderType, + ColumnSchema, + Release, + ReleaseColumnConfig, + ReleaseProtocol, + ReleaseSource, + SourceUnavailableError, + register_source, +) +from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError, WebshareFile + +if TYPE_CHECKING: + from shelfmark.core.search_plan import ReleaseSearchPlan + from shelfmark.metadata_providers import BookMetadata + +logger = setup_logger(__name__) + +_DEFAULT_MAX_RESULTS = 50 + +# Audiobooks on Webshare are usually uploaded as a single archive of mp3s; the +# orchestrator extracts zip/rar before import, so accept them for audiobooks. +_AUDIOBOOK_ARCHIVE_FORMATS = ("zip", "rar") + +# Language tags commonly embedded in Webshare file names ("Kniha (CZ).epub"). +_LANGUAGE_TAGS = { + "cz": "cs", + "cze": "cs", + "ces": "cs", + "cesky": "cs", + "czech": "cs", + "sk": "sk", + "slo": "sk", + "slk": "sk", + "slovensky": "sk", + "slovak": "sk", + "en": "en", + "eng": "en", + "english": "en", + "de": "de", + "ger": "de", + "deu": "de", +} +_TOKEN_RE = re.compile(r"[a-z]+") + + +def get_credentials() -> tuple[str, str]: + """Return the configured ``(username, password)``, stripped, possibly empty.""" + username = config.get("WEBSHARE_USERNAME", "") + password = config.get("WEBSHARE_PASSWORD", "") + return ( + username.strip() if isinstance(username, str) else "", + password if isinstance(password, str) else "", + ) + + +def _coerce_positive_int(value: object, default: int) -> int: + """Return a positive integer config value or the provided default.""" + if isinstance(value, bool): + return default + if isinstance(value, int | float) and value > 0: + return int(value) + return default + + +def _build_query_candidates(plan: ReleaseSearchPlan, book: BookMetadata) -> list[str]: + """Build ordered, de-duplicated search queries from the plan (mirrors Libgen).""" + candidates: list[str] = [] + if plan.manual_query: + candidates.append(plan.manual_query.strip()) + elif plan.title_variants: + variant = plan.title_variants[0] + combined = f"{variant.title} {variant.author}".strip() + title_only = (variant.title or "").strip() + if combined: + candidates.append(combined) + if title_only and title_only.lower() != combined.lower(): + candidates.append(title_only) + elif book.title: + candidates.append(book.title.strip()) + + deduped: list[str] = [] + seen: set[str] = set() + for candidate in candidates: + normalized = candidate.strip() + if not normalized or normalized.lower() in seen: + continue + seen.add(normalized.lower()) + deduped.append(normalized) + return deduped + + +def _file_format(file: WebshareFile) -> str | None: + """File extension, preferring the name's suffix over Webshare's ``type`` field.""" + suffix = PurePosixPath(file.name).suffix.lstrip(".").lower() + return suffix or file.type + + +def _guess_language(name: str) -> str | None: + """Pick a language code from a tag in the file name, if there is exactly one.""" + stem = PurePosixPath(name).stem.lower() + found = {_LANGUAGE_TAGS[token] for token in _TOKEN_RE.findall(stem) if token in _LANGUAGE_TAGS} + return found.pop() if len(found) == 1 else None + + +def _format_size(size_bytes: int | None) -> str | None: + if size_bytes is None: + return None + size = float(size_bytes) + for unit in ("B", "KB", "MB", "GB"): + if size < 1024 or unit == "GB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return None + + +def _allowed_formats(content_type: str) -> set[str]: + if content_type == "audiobook": + return {*get_supported_audiobook_formats(), *_AUDIOBOOK_ARCHIVE_FORMATS} + return set(get_supported_formats()) + + +@register_source("webshare") +class WebshareSource(ReleaseSource): + """Release source that searches Webshare.cz for ebook and audiobook files.""" + + name = "webshare" + display_name = "Webshare.cz" + supported_content_types: ClassVar[list[str]] = ["ebook", "audiobook"] + + def is_available(self) -> bool: + """Available only when explicitly enabled and credentials are configured.""" + username, password = get_credentials() + return config.get("WEBSHARE_ENABLED", False) is True and bool(username and password) + + def search( + self, + book: BookMetadata, + plan: ReleaseSearchPlan, + *, + expand_search: bool = False, + content_type: str = "ebook", + ) -> list[Release]: + """Search Webshare for files of a book in the user's supported formats.""" + if content_type not in self.supported_content_types or not self.is_available(): + return [] + + queries = _build_query_candidates(plan, book) + if not queries: + return [] + + max_results = _coerce_positive_int( + config.get("WEBSHARE_MAX_RESULTS", _DEFAULT_MAX_RESULTS), _DEFAULT_MAX_RESULTS + ) + allowed = _allowed_formats(content_type) + client = WebshareClient(*get_credentials()) + + for query in queries: + logger.info("Searching Webshare for: %s", query) + try: + files = client.search(query, limit=max_results) + except WebshareError as exc: + msg = f"Webshare search failed: {exc}" + raise SourceUnavailableError(msg) from exc + + releases = [ + self._file_to_release(file, content_type) + for file in files + if not file.password_protected and _file_format(file) in allowed + ] + if releases: + return releases + return [] + + def _file_to_release(self, file: WebshareFile, content_type: str) -> Release: + """Normalize a Webshare search hit into a Release. + + ``source_id`` is namespaced ``webshare:`` so it can never collide with + another source's queue key; the handler strips the prefix back to the ident. + """ + fmt = _file_format(file) + language = _guess_language(file.name) + return Release( + source="webshare", + source_id=f"webshare:{file.ident}", + title=PurePosixPath(file.name).stem or file.name, + format=fmt, + language=language, + size=_format_size(file.size_bytes), + size_bytes=file.size_bytes, + download_url=None, # links are time-limited; the handler resolves one per download + info_url=file.info_url, + protocol=ReleaseProtocol.HTTP, + indexer="Webshare.cz", + content_type=content_type, + extra={ + "ident": file.ident, + "file_name": file.name, + "language": language, + "votes": file.positive_votes - file.negative_votes, + }, + ) + + def get_column_config(self) -> ReleaseColumnConfig: + """Language, format, size and the file's Webshare vote balance.""" + return ReleaseColumnConfig( + columns=[ + ColumnSchema( + key="extra.language", + label="Language", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="60px", + color_hint=ColumnColorHint(type="map", value="language"), + uppercase=True, + ), + ColumnSchema( + key="format", + label="Format", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="80px", + color_hint=ColumnColorHint(type="map", value="format"), + uppercase=True, + ), + ColumnSchema( + key="size", + label="Size", + render_type=ColumnRenderType.SIZE, + align=ColumnAlign.CENTER, + width="80px", + sortable=True, + sort_key="size_bytes", + ), + ColumnSchema( + key="extra.votes", + label="Votes", + render_type=ColumnRenderType.NUMBER, + align=ColumnAlign.CENTER, + width="60px", + hide_mobile=True, + sortable=True, + ), + ], + grid_template="minmax(0,2fr) 60px 80px 80px 60px", + supported_filters=["format", "language"], + ) diff --git a/tests/webshare/__init__.py b/tests/webshare/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/webshare/test_api.py b/tests/webshare/test_api.py new file mode 100644 index 00000000..1b52cfb7 --- /dev/null +++ b/tests/webshare/test_api.py @@ -0,0 +1,175 @@ +"""Tests for the Webshare API client: password hashing, XML parsing, token retry.""" + +import hashlib +from unittest.mock import MagicMock, patch + +import pytest + +from shelfmark.release_sources.webshare import api +from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError + + +def _xml(body: str, status: str = "OK") -> bytes: + return f"{status}{body}".encode() + + +def _response(content: bytes) -> MagicMock: + response = MagicMock() + response.content = content + response.raise_for_status.return_value = None + return response + + +@pytest.fixture(autouse=True) +def _clear_tokens(): + WebshareClient._tokens.clear() + yield + WebshareClient._tokens.clear() + + +@pytest.mark.parametrize( + ("password", "salt", "expected"), + [ + # Vectors from `openssl passwd -1 -salt `. + ("password", "saltsalt", "$1$saltsalt$qjXMvbEw8oaL.CzflDtaK/"), + ("heslo123ľ", "UtvP0QwB", "$1$UtvP0QwB$BNPMnAafFtaDfoqW.SVSy/"), + ("x" * 40, "$1$abc$", "$1$abc$b/oSMeDNO7QqHD3W/TXaY0"), + ], +) +def test_md5_crypt_matches_openssl(password, salt, expected): + assert api.md5_crypt(password, salt) == expected + + +def test_hash_password_is_sha1_of_md5crypt_and_md5_digest(): + password_hash, digest = api.hash_password("user", "password", "saltsalt") + expected_hash = hashlib.sha1(b"$1$saltsalt$qjXMvbEw8oaL.CzflDtaK/").hexdigest() + assert password_hash == expected_hash + assert digest == hashlib.md5(f"user:Webshare:{expected_hash}".encode()).hexdigest() + + +def test_search_parses_files_and_sends_token(): + calls = [] + + def post(url, data, **kwargs): + calls.append((url, dict(data))) + if url.endswith("/salt/"): + return _response(_xml("saltsalt")) + if url.endswith("/login/"): + return _response(_xml("TOKEN1")) + return _response( + _xml( + "2" + "abc123Kniha (CZ).epubepub" + "1234563" + "10" + "broken.epub" + ) + ) + + with patch.object(api.requests, "post", side_effect=post): + files = WebshareClient("user", "pw").search("kniha", limit=10) + + assert [f.ident for f in files] == ["abc123"] + assert files[0].size_bytes == 123456 + assert files[0].positive_votes == 3 + assert files[0].info_url == "https://webshare.cz/#/file/abc123" + search_url, search_data = calls[-1] + assert search_url == "https://webshare.cz/api/search/" + assert search_data["wst"] == "TOKEN1" + assert search_data["what"] == "kniha" + + +def test_token_is_reused_between_calls(): + logins = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + logins.append(1) + return _response(_xml("T")) + return _response(_xml("https://dl.example/file.epub")) + + with patch.object(api.requests, "post", side_effect=post): + WebshareClient("user", "pw").file_link("a") + WebshareClient("user", "pw").file_link("b") + + assert len(logins) == 1 + + +def test_expired_token_triggers_one_relogin(): + tokens = iter(["OLD", "NEW"]) + seen_tokens = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + return _response(_xml(f"{next(tokens)}")) + seen_tokens.append(data["wst"]) + if data["wst"] == "OLD": + return _response( + _xml("SEARCH_FATAL_1Bad token", "FATAL") + ) + return _response(_xml("0")) + + with patch.object(api.requests, "post", side_effect=post): + assert WebshareClient("user", "pw").search("x") == [] + + assert seen_tokens == ["OLD", "NEW"] + + +def test_file_errors_are_not_retried(): + link_calls = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + return _response(_xml("T")) + link_calls.append(1) + return _response( + _xml("FILE_LINK_FATAL_1File not found.", "FATAL") + ) + + with ( + patch.object(api.requests, "post", side_effect=post), + pytest.raises(WebshareError) as exc_info, + ): + WebshareClient("user", "pw").file_link("gone") + + assert exc_info.value.code == "FILE_LINK_FATAL_1" + assert str(exc_info.value) == "File not found." + assert len(link_calls) == 1 + + +def test_login_requires_credentials(): + with pytest.raises(WebshareError): + WebshareClient("", "").login() + + +def test_unreadable_response_raises(): + with ( + patch.object(api.requests, "post", return_value=_response(b"nope")), + pytest.raises(WebshareError), + ): + WebshareClient("user", "pw").login() + + +def test_changed_password_forces_new_login(): + logins = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + logins.append(data["password"]) + return _response(_xml(f"T{len(logins)}")) + return _response(_xml("https://dl.example/file.epub")) + + with patch.object(api.requests, "post", side_effect=post): + WebshareClient("user", "old").file_link("a") + WebshareClient("user", "new").file_link("a") + + assert len(logins) == 2 + assert logins[0] != logins[1] diff --git a/tests/webshare/test_handler.py b/tests/webshare/test_handler.py new file mode 100644 index 00000000..e8e55739 --- /dev/null +++ b/tests/webshare/test_handler.py @@ -0,0 +1,87 @@ +"""Tests for WebshareHandler: link resolution, streaming, cleanup on failure.""" + +import threading +from unittest.mock import MagicMock, patch + +from shelfmark.core.models import DownloadTask +from shelfmark.release_sources.webshare import handler as webshare_handler +from shelfmark.release_sources.webshare.api import WebshareError +from shelfmark.release_sources.webshare.handler import WebshareHandler + + +def _task(task_id="webshare:abc123", fmt="epub"): + return DownloadTask(task_id=task_id, source="webshare", title="Krysar", format=fmt) + + +def _http_response(body: bytes): + response = MagicMock() + response.__enter__.return_value = response + response.__exit__.return_value = False + response.headers = {"Content-Length": str(len(body))} + response.raise_for_status.return_value = None + response.iter_content.return_value = [body[i : i + 4096] for i in range(0, len(body), 4096)] + return response + + +def _run(tmp_path, *, file_link, body=b"x" * 20000, cancel=None, task=None): + status = MagicMock() + progress = MagicMock() + cancel = cancel or threading.Event() + with ( + patch.object(webshare_handler, "TMP_DIR", tmp_path), + patch.object( + webshare_handler.config, + "get", + side_effect=lambda k, d=None: "none" if k == "FILE_ORGANIZATION" else d, + ), + patch.object(webshare_handler, "get_credentials", return_value=("user", "pw")), + patch.object(webshare_handler.WebshareClient, "file_link", side_effect=file_link), + patch.object(webshare_handler.requests, "get", return_value=_http_response(body)), + ): + result = WebshareHandler().download(task or _task(), cancel, progress, status) + return result, status, progress + + +def test_download_streams_file_named_after_link(tmp_path): + idents = [] + + def file_link(ident): + idents.append(ident) + return "https://dl.webshare.cz/abc/Krysar%20%28CZ%29.epub" + + result, _, progress = _run(tmp_path, file_link=file_link) + + expected = tmp_path / "Krysar (CZ).epub" + assert result == str(expected) + assert expected.read_bytes() == b"x" * 20000 + assert idents == ["abc123"] + progress.assert_called_with(100.0) + + +def test_too_small_file_is_rejected_and_removed(tmp_path): + result, status, _ = _run( + tmp_path, file_link=lambda ident: "https://dl.webshare.cz/f.epub", body=b"tiny" + ) + assert result is None + assert not (tmp_path / "f.epub").exists() + status.assert_called_with("error", "Webshare returned an invalid file") + + +def test_link_error_reports_message(tmp_path): + def file_link(ident): + msg = "File not found." + raise WebshareError(msg, "FILE_LINK_FATAL_1") + + result, status, _ = _run(tmp_path, file_link=file_link) + assert result is None + status.assert_called_with("error", "File not found.") + + +def test_cancel_before_start(tmp_path): + cancel = threading.Event() + cancel.set() + file_link = MagicMock() + result, status, _ = _run(tmp_path, file_link=file_link, cancel=cancel) + assert result is None + file_link.assert_not_called() + status.assert_called_with("cancelled", "Cancelled") diff --git a/tests/webshare/test_source.py b/tests/webshare/test_source.py new file mode 100644 index 00000000..5a37729d --- /dev/null +++ b/tests/webshare/test_source.py @@ -0,0 +1,164 @@ +"""Tests for WebshareSource: availability, format filtering, release mapping.""" + +from unittest.mock import patch + +import pytest + +from shelfmark.core.search_plan import ReleaseSearchPlan, ReleaseSearchVariant +from shelfmark.metadata_providers import BookMetadata +from shelfmark.release_sources import SourceUnavailableError, get_source +from shelfmark.release_sources.webshare import source as webshare_source +from shelfmark.release_sources.webshare.api import WebshareError, WebshareFile + +_CONFIG = { + "WEBSHARE_ENABLED": True, + "WEBSHARE_USERNAME": " user ", + "WEBSHARE_PASSWORD": "pw", + "WEBSHARE_MAX_RESULTS": 20, + "SUPPORTED_FORMATS": ["epub", "mobi"], + "SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"], +} + + +def _file(ident, name, *, size=200_000, password=False, up=0, down=0): + return WebshareFile( + ident=ident, + name=name, + type=None, + size_bytes=size, + positive_votes=up, + negative_votes=down, + password_protected=password, + ) + + +def _book(): + return BookMetadata(provider="test", provider_id="1", title="Krysař", authors=["Viktor Dyk"]) + + +def _plan(**kwargs): + return ReleaseSearchPlan( + languages=None, + isbn_candidates=[], + author="Viktor Dyk", + title_variants=[ReleaseSearchVariant(title="Krysař", author="Viktor Dyk")], + grouped_title_variants=[], + **kwargs, + ) + + +@pytest.fixture +def config_values(): + values = dict(_CONFIG) + with ( + patch.object(webshare_source.config, "get", side_effect=lambda k, d=None: values.get(k, d)), + patch( + "shelfmark.download.postprocess.policy.core_config.config.get", + side_effect=lambda k, d=None: values.get(k, d), + ), + ): + yield values + + +def test_registered_under_webshare(): + assert get_source("webshare").display_name == "Webshare.cz" + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + ({}, True), + ({"WEBSHARE_ENABLED": False}, False), + ({"WEBSHARE_ENABLED": "true"}, False), + ({"WEBSHARE_USERNAME": ""}, False), + ({"WEBSHARE_PASSWORD": ""}, False), + ], +) +def test_is_available(config_values, overrides, expected): + config_values.update(overrides) + assert webshare_source.WebshareSource().is_available() is expected + + +def test_ebook_search_filters_formats_and_maps_release(config_values): + files = [ + _file("a1", "Dyk Viktor - Krysar (CZ).epub", up=5, down=1), + _file("a2", "Krysar.mobi"), + _file("a3", "Krysar.mp3"), + _file("a4", "Krysar.zip"), + _file("a5", "Krysar locked.epub", password=True), + ] + with patch.object(webshare_source.WebshareClient, "search", return_value=files) as search: + releases = webshare_source.WebshareSource().search(_book(), _plan()) + + assert [r.source_id for r in releases] == ["webshare:a1", "webshare:a2"] + search.assert_called_once_with("Krysař Viktor Dyk", limit=20) + first = releases[0] + assert first.title == "Dyk Viktor - Krysar (CZ)" + assert first.format == "epub" + assert first.language == "cs" + assert first.size == "195.3 KB" + assert first.size_bytes == 200_000 + assert first.info_url == "https://webshare.cz/#/file/a1" + assert first.extra["votes"] == 4 + assert first.content_type == "ebook" + + +def test_audiobook_search_accepts_archives(config_values): + files = [_file("b1", "Krysar audiokniha.zip"), _file("b2", "Krysar.epub")] + with patch.object(webshare_source.WebshareClient, "search", return_value=files): + releases = webshare_source.WebshareSource().search( + _book(), _plan(), content_type="audiobook" + ) + assert [r.source_id for r in releases] == ["webshare:b1"] + assert releases[0].content_type == "audiobook" + + +def test_falls_back_to_title_only_query(config_values): + calls = [] + + def search(query, limit): + calls.append(query) + return [] if len(calls) == 1 else [_file("c1", "Krysar.epub")] + + with patch.object(webshare_source.WebshareClient, "search", side_effect=search): + releases = webshare_source.WebshareSource().search(_book(), _plan()) + + assert calls == ["Krysař Viktor Dyk", "Krysař"] + assert len(releases) == 1 + + +def test_manual_query_is_used_verbatim(config_values): + with patch.object(webshare_source.WebshareClient, "search", return_value=[]) as search: + webshare_source.WebshareSource().search(_book(), _plan(manual_query=" dyk krysar ")) + search.assert_called_once_with("dyk krysar", limit=20) + + +def test_search_error_marks_source_unavailable(config_values): + with ( + patch.object( + webshare_source.WebshareClient, "search", side_effect=WebshareError("Login failed") + ), + pytest.raises(SourceUnavailableError), + ): + webshare_source.WebshareSource().search(_book(), _plan()) + + +def test_disabled_source_does_not_search(config_values): + config_values["WEBSHARE_ENABLED"] = False + with patch.object(webshare_source.WebshareClient, "search") as search: + assert webshare_source.WebshareSource().search(_book(), _plan()) == [] + search.assert_not_called() + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("Kniha (CZ).epub", "cs"), + ("kniha.sk.epub", "sk"), + ("Book [ENG].epub", "en"), + ("Kniha CZ + EN.epub", None), + ("Kniha.epub", None), + ], +) +def test_guess_language(name, expected): + assert webshare_source._guess_language(name) == expected