diff --git a/docs/environment-variables.md b/docs/environment-variables.md index d403e289..5ae2540e 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -19,6 +19,7 @@ This document lists all configuration options that can be set via environment va - [Soulseek (slskd)](#soulseek-slskd) - [AudiobookBay](#audiobookbay) - [Libgen Search](#libgen-search) +- [Webshare.cz](#webshare.cz) - [IRC](#irc) - [Download Clients](#download-clients) - [Metadata Providers](#metadata-providers) @@ -1779,6 +1780,59 @@ Maximum number of results to request per search (1-100). +## Webshare.cz + +| Variable | Description | Type | Default | +|----------|-------------|------|---------| +| `WEBSHARE_ENABLED` | Search Webshare.cz by file name and download matching ebooks and audiobooks. Requires a Webshare account; a VIP account is recommended for full download speed. | boolean | `false` | +| `WEBSHARE_USERNAME` | Your Webshare.cz login. | string | _none_ | +| `WEBSHARE_PASSWORD` | Your Webshare.cz password. | string (secret) | _none_ | +| `WEBSHARE_MAX_RESULTS` | Maximum number of files to request per search (1-200). Results are then filtered to your supported ebook or audiobook formats. | number | `50` | + +
+Detailed descriptions + +#### `WEBSHARE_ENABLED` + +**Enable Webshare.cz** + +Search Webshare.cz by file name and download matching ebooks and audiobooks. Requires a Webshare account; a VIP account is recommended for full download speed. + +- **Type:** boolean +- **Default:** `false` + +#### `WEBSHARE_USERNAME` + +**Username or Email** + +Your Webshare.cz login. + +- **Type:** string +- **Default:** _none_ +- **Required:** Yes + +#### `WEBSHARE_PASSWORD` + +**Password** + +Your Webshare.cz password. + +- **Type:** string (secret) +- **Default:** _none_ +- **Required:** Yes + +#### `WEBSHARE_MAX_RESULTS` + +**Max Results** + +Maximum number of files to request per search (1-200). Results are then filtered to your supported ebook or audiobook formats. + +- **Type:** number +- **Default:** `50` +- **Constraints:** min: 1, max: 200 + +
+ ## IRC | Variable | Description | Type | Default | diff --git a/shelfmark/release_sources/__init__.py b/shelfmark/release_sources/__init__.py index 8c1951e8..7fc10c14 100644 --- a/shelfmark/release_sources/__init__.py +++ b/shelfmark/release_sources/__init__.py @@ -494,6 +494,7 @@ _BUILTIN_SOURCE_MODULES = ( "shelfmark.release_sources.newznab", "shelfmark.release_sources.prowlarr", "shelfmark.release_sources.slskd", + "shelfmark.release_sources.webshare", ) _builtin_source_state = {"loaded": False} diff --git a/shelfmark/release_sources/webshare/__init__.py b/shelfmark/release_sources/webshare/__init__.py new file mode 100644 index 00000000..3892b158 --- /dev/null +++ b/shelfmark/release_sources/webshare/__init__.py @@ -0,0 +1,6 @@ +"""Webshare.cz release source - file-name search and downloads via the Webshare API.""" + +# Import to trigger registration +from shelfmark.release_sources.webshare import handler as handler +from shelfmark.release_sources.webshare import settings as settings +from shelfmark.release_sources.webshare import source as source diff --git a/shelfmark/release_sources/webshare/api.py b/shelfmark/release_sources/webshare/api.py new file mode 100644 index 00000000..96d555d6 --- /dev/null +++ b/shelfmark/release_sources/webshare/api.py @@ -0,0 +1,333 @@ +"""Webshare.cz XML API client: login, search and download-link resolution. + +Every endpoint is ``POST https://webshare.cz/api//`` with a form-encoded body and +an XML ```` whose ```` is ``OK`` or ``FATAL``. Authenticated calls carry +the session token as the ``wst`` form field. Login is a two-step salted exchange: fetch +the user's salt, then send ``SHA1(MD5_CRYPT(password, salt))`` plus an MD5 digest. + +All shelfmark-stateful behaviour (config, task naming) lives in source.py/handler.py. +""" + +import hashlib +import hmac +import threading +from dataclasses import dataclass +from typing import Any + +import requests +from defusedxml import ElementTree as DefusedElementTree +from defusedxml.common import DefusedXmlException + +from shelfmark.core.logger import setup_logger +from shelfmark.download import http as downloader +from shelfmark.download import network + +logger = setup_logger(__name__) + +API_BASE = "https://webshare.cz/api" +FILE_PAGE_BASE = "https://webshare.cz/#/file" +DEFAULT_TIMEOUT = (5, 15) + +# Webshare's API answers with an empty/HTML body unless the request looks like the +# site's own XHR calls. +_API_HEADERS = { + **downloader.DOWNLOAD_HEADERS, + "Accept": "text/xml; charset=UTF-8", + "X-Requested-With": "XMLHttpRequest", +} + + +class WebshareError(Exception): + """A Webshare API call failed (network error, bad XML or a FATAL status).""" + + def __init__(self, message: str, code: str | None = None) -> None: + """Keep the Webshare error code (e.g. ``LOGIN_FATAL_1``) next to the message.""" + super().__init__(message) + self.code = code + + +@dataclass(frozen=True) +class WebshareFile: + """One file from a Webshare search result.""" + + ident: str + name: str + type: str | None + size_bytes: int | None + positive_votes: int + negative_votes: int + password_protected: bool + + @property + def info_url(self) -> str: + """Public Webshare page for this file.""" + return f"{FILE_PAGE_BASE}/{self.ident}" + + +@dataclass(frozen=True) +class WebshareAccount: + """The logged-in account's identity and VIP state.""" + + username: str + vip: bool + vip_days: int | None + + +# --- md5-crypt --- + +_ITOA64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + +def _to64(value: int, length: int) -> str: + chars = [] + for _ in range(length): + chars.append(_ITOA64[value & 0x3F]) + value >>= 6 + return "".join(chars) + + +def md5_crypt(password: str, salt: str, magic: str = "$1$") -> str: + """FreeBSD md5-crypt (``$1$salt$hash``), the scheme Webshare salts passwords with. + + Python dropped the ``crypt`` module in 3.13, so this is a straight port of the + reference algorithm (the same one ``openssl passwd -1`` implements). MD5 here is + dictated by Webshare's login protocol, not chosen to protect anything at rest, hence + ``usedforsecurity=False`` throughout this module. + """ + salt = salt.removeprefix(magic) + salt = salt.split("$", 1)[0][:8] + + pw = password.encode() + salt_b = salt.encode() + magic_b = magic.encode() + + alternate = hashlib.md5(pw + salt_b + pw, usedforsecurity=False).digest() + ctx = pw + magic_b + salt_b + for remaining in range(len(pw), 0, -16): + ctx += alternate[: min(16, remaining)] + length = len(pw) + while length: + ctx += b"\x00" if length & 1 else pw[:1] + length >>= 1 + final = hashlib.md5(ctx, usedforsecurity=False).digest() + + for i in range(1000): + round_ctx = pw if i & 1 else final + if i % 3: + round_ctx += salt_b + if i % 7: + round_ctx += pw + round_ctx += final if i & 1 else pw + final = hashlib.md5(round_ctx, usedforsecurity=False).digest() + + encoded = "".join( + _to64((final[a] << 16) | (final[b] << 8) | final[c], 4) + for a, b, c in ((0, 6, 12), (1, 7, 13), (2, 8, 14), (3, 9, 15), (4, 10, 5)) + ) + encoded += _to64(final[11], 2) + return f"{magic}{salt}${encoded}" + + +def hash_password(username: str, password: str, salt: str) -> tuple[str, str]: + """Return the ``(password, digest)`` pair Webshare's login endpoint expects.""" + crypted = md5_crypt(password, salt).encode() + password_hash = hashlib.sha1(crypted, usedforsecurity=False).hexdigest() + digest_input = f"{username}:Webshare:{password_hash}".encode() + digest = hashlib.md5(digest_input, usedforsecurity=False).hexdigest() + return password_hash, digest + + +# --- transport --- + + +def _text(node: Any, tag: str) -> str | None: + child = node.find(tag) + if child is None or child.text is None: + return None + value = child.text.strip() + return value or None + + +def _int(node: Any, tag: str) -> int | None: + value = _text(node, tag) + if value is None: + return None + try: + return int(value) + except ValueError: + return None + + +def _post( + endpoint: str, + data: dict[str, Any], + timeout: tuple[int, int] = DEFAULT_TIMEOUT, +) -> Any: + """POST to an API endpoint and return the parsed ```` root on OK.""" + url = f"{API_BASE}/{endpoint}/" + try: + response = requests.post( + url, + data=data, + headers=_API_HEADERS, + timeout=timeout, + proxies=network.get_proxies(url), + verify=network.get_ssl_verify(url), + ) + response.raise_for_status() + except requests.exceptions.RequestException as exc: + msg = f"Webshare request failed: {exc}" + raise WebshareError(msg) from exc + + try: + root = DefusedElementTree.fromstring(response.content) + except DefusedElementTree.ParseError, DefusedXmlException: + msg = f"Webshare returned an unreadable response for {endpoint}" + raise WebshareError(msg) from None + + if _text(root, "status") != "OK": + code = _text(root, "code") + message = _text(root, "message") or f"Webshare {endpoint} failed" + raise WebshareError(message, code) + return root + + +# --- client --- + + +class WebshareClient: + """Session-aware Webshare client. + + The login token is cached per username at module level so searches and + downloads share one session instead of logging in for every request. A call that + fails while authenticated drops the token and retries once with a fresh login, which + covers tokens Webshare expired server-side. A cached token is only reused while the + password it was obtained with is unchanged, so editing the password forces a login. + """ + + _token_lock = threading.Lock() + _tokens: dict[str, tuple[str, str]] = {} # noqa: RUF012 - username -> (password, token) + + def __init__(self, username: str, password: str) -> None: + """Bind the client to one Webshare account.""" + self.username = username + self.password = password + + def login(self) -> str: + """Log in and return a fresh session token (WST).""" + if not self.username or not self.password: + msg = "Webshare username and password are required" + raise WebshareError(msg) + + salt_root = _post("salt", {"username_or_email": self.username}) + salt = _text(salt_root, "salt") + if not salt: + msg = "Webshare did not return a password salt" + raise WebshareError(msg) + + password_hash, digest = hash_password(self.username, self.password, salt) + login_root = _post( + "login", + { + "username_or_email": self.username, + "password": password_hash, + "digest": digest, + "keep_logged_in": 1, + }, + ) + token = _text(login_root, "token") + if not token: + msg = "Webshare login did not return a session token" + raise WebshareError(msg) + + with self._token_lock: + self._tokens[self.username] = (self.password, token) + return token + + def _token(self) -> str: + with self._token_lock: + cached = self._tokens.get(self.username) + if cached and hmac.compare_digest(cached[0].encode(), self.password.encode()): + return cached[1] + return self.login() + + def _invalidate_token(self) -> None: + with self._token_lock: + self._tokens.pop(self.username, None) + + def _authed_post( + self, + endpoint: str, + data: dict[str, Any], + timeout: tuple[int, int] = DEFAULT_TIMEOUT, + ) -> Any: + try: + return _post(endpoint, {**data, "wst": self._token()}, timeout) + except WebshareError as exc: + # Errors that are about the file itself won't change after re-login. + if exc.code and exc.code.startswith(("FILE_LINK_FATAL", "FILE_INFO_FATAL")): + raise + logger.debug("Webshare %s failed (%s), retrying with a fresh login", endpoint, exc) + self._invalidate_token() + return _post(endpoint, {**data, "wst": self.login()}, timeout) + + def account(self) -> WebshareAccount: + """Return the logged-in account's username and VIP state.""" + root = self._authed_post("user_data", {}) + return WebshareAccount( + username=_text(root, "username") or self.username, + vip=_text(root, "vip") == "1", + vip_days=_int(root, "vip_days"), + ) + + def search( + self, + query: str, + *, + category: str = "", + limit: int = 50, + sort: str = "", + timeout: tuple[int, int] = DEFAULT_TIMEOUT, + ) -> list[WebshareFile]: + """Search Webshare by file name.""" + root = self._authed_post( + "search", + { + "what": query, + "category": category, + "sort": sort, + "limit": limit, + "offset": 0, + }, + timeout, + ) + files: list[WebshareFile] = [] + for node in root.findall("file"): + ident = _text(node, "ident") + name = _text(node, "name") + if not ident or not name: + continue + files.append( + WebshareFile( + ident=ident, + name=name, + type=(_text(node, "type") or "").lower() or None, + size_bytes=_int(node, "size"), + positive_votes=_int(node, "positive_votes") or 0, + negative_votes=_int(node, "negative_votes") or 0, + password_protected=_text(node, "password") == "1", + ) + ) + return files + + def file_link(self, ident: str) -> str: + """Resolve a file ident to a time-limited direct download URL.""" + root = self._authed_post( + "file_link", + {"ident": ident, "download_type": "file_download", "force_https": 1}, + ) + link = _text(root, "link") + if not link: + msg = "Webshare did not return a download link" + raise WebshareError(msg) + return link diff --git a/shelfmark/release_sources/webshare/handler.py b/shelfmark/release_sources/webshare/handler.py new file mode 100644 index 00000000..e2595243 --- /dev/null +++ b/shelfmark/release_sources/webshare/handler.py @@ -0,0 +1,124 @@ +"""Webshare.cz download handler - resolves a file ident to a link and streams it. + +Selected by ``get_handler(task.source)`` for ``source == "webshare"``. Download links +are time-limited and tied to the session, so one is resolved per attempt rather than at +search time. Audiobook archives can run to gigabytes, so the body is streamed straight +into TMP_DIR instead of buffered in memory like the Libgen handler does. +""" + +from pathlib import PurePosixPath +from typing import TYPE_CHECKING +from urllib.parse import unquote, urlsplit + +import requests + +from shelfmark.config.env import TMP_DIR +from shelfmark.core.config import config +from shelfmark.core.logger import setup_logger +from shelfmark.core.models import build_filename +from shelfmark.download import http as downloader +from shelfmark.download import network +from shelfmark.release_sources import DownloadHandler, register_handler +from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError +from shelfmark.release_sources.webshare.source import get_credentials + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from threading import Event + + from shelfmark.core.models import DownloadTask + +logger = setup_logger(__name__) + +# Files under this size are almost certainly an error page, not a book. +_MIN_VALID_FILE_SIZE = 10 * 1024 +_CHUNK_SIZE = 256 * 1024 +_DOWNLOAD_TIMEOUT = (10, 60) + + +def _target_path(task: DownloadTask, ident: str, link: str) -> Path: + """Staging path: the naming template, or Webshare's own file name for "none".""" + if config.get("FILE_ORGANIZATION", "rename") != "none": + return TMP_DIR / build_filename(task.title, task.author, task.year, task.format) + link_name = PurePosixPath(unquote(urlsplit(link).path)).name + return TMP_DIR / (link_name or f"{ident}.{task.format or 'bin'}") + + +@register_handler("webshare") +class WebshareHandler(DownloadHandler): + """Download handler for Webshare.cz releases.""" + + def download( + self, + task: DownloadTask, + cancel_flag: Event, + progress_callback: Callable[[float], None], + status_callback: Callable[[str, str | None], None], + ) -> str | None: + """Resolve a download link for the file and stream it into TMP_DIR.""" + ident = task.task_id.split(":", 1)[-1] + book_path: Path | None = None + try: + if cancel_flag.is_set(): + status_callback("cancelled", "Cancelled") + return None + + status_callback("resolving", "Resolving Webshare link") + link = WebshareClient(*get_credentials()).file_link(ident) + + book_path = _target_path(task, ident, link) + status_callback("downloading", None) + if not self._stream(link, book_path, cancel_flag, progress_callback): + book_path.unlink(missing_ok=True) + status_callback("cancelled", "Cancelled") + return None + + if book_path.stat().st_size < _MIN_VALID_FILE_SIZE: + book_path.unlink(missing_ok=True) + status_callback("error", "Webshare returned an invalid file") + return None + return str(book_path) + except (WebshareError, requests.exceptions.RequestException, OSError) as exc: + if book_path is not None: + book_path.unlink(missing_ok=True) + if cancel_flag.is_set(): + status_callback("cancelled", "Cancelled") + else: + logger.warning("Webshare download failed for %s: %s", ident, exc) + status_callback("error", str(exc)) + return None + + @staticmethod + def _stream( + link: str, + book_path: Path, + cancel_flag: Event, + progress_callback: Callable[[float], None], + ) -> bool: + """Stream ``link`` into ``book_path``; return False if cancelled midway.""" + with requests.get( + link, + headers=downloader.DOWNLOAD_HEADERS, + stream=True, + timeout=_DOWNLOAD_TIMEOUT, + proxies=network.get_proxies(link), + verify=network.get_ssl_verify(link), + ) as response: + response.raise_for_status() + total = int(response.headers.get("Content-Length") or 0) + written = 0 + with book_path.open("wb") as file: + for chunk in response.iter_content(chunk_size=_CHUNK_SIZE): + if cancel_flag.is_set(): + return False + file.write(chunk) + written += len(chunk) + if total: + progress_callback(min(100.0, written * 100 / total)) + progress_callback(100.0) + return True + + def cancel(self, task_id: str) -> bool: + """Cancellation is handled by the orchestrator via the cancel_flag.""" + return False diff --git a/shelfmark/release_sources/webshare/settings.py b/shelfmark/release_sources/webshare/settings.py new file mode 100644 index 00000000..53b164ad --- /dev/null +++ b/shelfmark/release_sources/webshare/settings.py @@ -0,0 +1,102 @@ +"""Webshare.cz settings registration.""" + +from typing import Any + +from shelfmark.core.request_helpers import normalize_optional_text +from shelfmark.core.settings_registry import ( + ActionButton, + CheckboxField, + NumberField, + PasswordField, + SettingsField, + TextField, + register_settings, +) + + +def _resolve_setting_text(current_values: dict[str, Any], key: str) -> str: + """Prefer current form values, then fall back to persisted config text.""" + from shelfmark.core.config import config + + current_value = normalize_optional_text(current_values.get(key)) + if current_value is not None: + return current_value + return normalize_optional_text(config.get(key, "")) or "" + + +def _test_webshare_connection(current_values: dict[str, Any] | None = None) -> dict[str, Any]: + """Log in with the current form credentials and report the account's VIP state.""" + from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError + + current_values = current_values or {} + username = _resolve_setting_text(current_values, "WEBSHARE_USERNAME") + password = _resolve_setting_text(current_values, "WEBSHARE_PASSWORD") + if not username or not password: + return {"success": False, "message": "Username and password are required"} + + try: + client = WebshareClient(username, password) + client.login() + account = client.account() + except WebshareError as exc: + return {"success": False, "message": f"Login failed: {exc}"} + + if account.vip: + days = f" ({account.vip_days} days left)" if account.vip_days is not None else "" + return {"success": True, "message": f"Logged in as {account.username}, VIP{days}"} + return { + "success": True, + "message": f"Logged in as {account.username} (free account - downloads are slow)", + } + + +@register_settings("webshare_config", "Webshare.cz", icon="download", order=47) +def webshare_config_settings() -> list[SettingsField]: + """Webshare.cz configuration settings.""" + enabled = {"field": "WEBSHARE_ENABLED", "value": True} + return [ + CheckboxField( + key="WEBSHARE_ENABLED", + label="Enable Webshare.cz", + description=( + "Search Webshare.cz by file name and download matching ebooks and " + "audiobooks. Requires a Webshare account; a VIP account is recommended " + "for full download speed." + ), + default=False, + ), + TextField( + key="WEBSHARE_USERNAME", + label="Username or Email", + description="Your Webshare.cz login.", + required=True, + show_when=enabled, + ), + PasswordField( + key="WEBSHARE_PASSWORD", + label="Password", + description="Your Webshare.cz password.", + required=True, + show_when=enabled, + ), + ActionButton( + key="test_webshare", + label="Test Login", + description="Log in to Webshare.cz and show the account's VIP status.", + style="primary", + callback=_test_webshare_connection, + show_when=enabled, + ), + NumberField( + key="WEBSHARE_MAX_RESULTS", + label="Max Results", + description=( + "Maximum number of files to request per search (1-200). Results are then " + "filtered to your supported ebook or audiobook formats." + ), + default=50, + min_value=1, + max_value=200, + show_when=enabled, + ), + ] diff --git a/shelfmark/release_sources/webshare/source.py b/shelfmark/release_sources/webshare/source.py new file mode 100644 index 00000000..9aede7bb --- /dev/null +++ b/shelfmark/release_sources/webshare/source.py @@ -0,0 +1,268 @@ +"""Webshare.cz release source - searches Webshare's file index by name. + +Webshare is a Czech file host whose search matches file names only, so there is no +book metadata to read: the file name is the release title, its extension the format, +and results are filtered down to the user's supported ebook/audiobook formats. +Downloads resolve a time-limited link per file through the logged-in account (see +handler.py). +""" + +import re +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, ClassVar + +from shelfmark.core.config import config +from shelfmark.core.logger import setup_logger +from shelfmark.download.postprocess.policy import ( + get_supported_audiobook_formats, + get_supported_formats, +) +from shelfmark.release_sources import ( + ColumnAlign, + ColumnColorHint, + ColumnRenderType, + ColumnSchema, + Release, + ReleaseColumnConfig, + ReleaseProtocol, + ReleaseSource, + SourceUnavailableError, + register_source, +) +from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError, WebshareFile + +if TYPE_CHECKING: + from shelfmark.core.search_plan import ReleaseSearchPlan + from shelfmark.metadata_providers import BookMetadata + +logger = setup_logger(__name__) + +_DEFAULT_MAX_RESULTS = 50 + +# Audiobooks on Webshare are usually uploaded as a single archive of mp3s; the +# orchestrator extracts zip/rar before import, so accept them for audiobooks. +_AUDIOBOOK_ARCHIVE_FORMATS = ("zip", "rar") + +# Language tags commonly embedded in Webshare file names ("Kniha (CZ).epub"). +_LANGUAGE_TAGS = { + "cz": "cs", + "cze": "cs", + "ces": "cs", + "cesky": "cs", + "czech": "cs", + "sk": "sk", + "slo": "sk", + "slk": "sk", + "slovensky": "sk", + "slovak": "sk", + "en": "en", + "eng": "en", + "english": "en", + "de": "de", + "ger": "de", + "deu": "de", +} +_TOKEN_RE = re.compile(r"[a-z]+") + + +def get_credentials() -> tuple[str, str]: + """Return the configured ``(username, password)``, stripped, possibly empty.""" + username = config.get("WEBSHARE_USERNAME", "") + password = config.get("WEBSHARE_PASSWORD", "") + return ( + username.strip() if isinstance(username, str) else "", + password if isinstance(password, str) else "", + ) + + +def _coerce_positive_int(value: object, default: int) -> int: + """Return a positive integer config value or the provided default.""" + if isinstance(value, bool): + return default + if isinstance(value, int | float) and value > 0: + return int(value) + return default + + +def _build_query_candidates(plan: ReleaseSearchPlan, book: BookMetadata) -> list[str]: + """Build ordered, de-duplicated search queries from the plan (mirrors Libgen).""" + candidates: list[str] = [] + if plan.manual_query: + candidates.append(plan.manual_query.strip()) + elif plan.title_variants: + variant = plan.title_variants[0] + combined = f"{variant.title} {variant.author}".strip() + title_only = (variant.title or "").strip() + if combined: + candidates.append(combined) + if title_only and title_only.lower() != combined.lower(): + candidates.append(title_only) + elif book.title: + candidates.append(book.title.strip()) + + deduped: list[str] = [] + seen: set[str] = set() + for candidate in candidates: + normalized = candidate.strip() + if not normalized or normalized.lower() in seen: + continue + seen.add(normalized.lower()) + deduped.append(normalized) + return deduped + + +def _file_format(file: WebshareFile) -> str | None: + """File extension, preferring the name's suffix over Webshare's ``type`` field.""" + suffix = PurePosixPath(file.name).suffix.lstrip(".").lower() + return suffix or file.type + + +def _guess_language(name: str) -> str | None: + """Pick a language code from a tag in the file name, if there is exactly one.""" + stem = PurePosixPath(name).stem.lower() + found = {_LANGUAGE_TAGS[token] for token in _TOKEN_RE.findall(stem) if token in _LANGUAGE_TAGS} + return found.pop() if len(found) == 1 else None + + +def _format_size(size_bytes: int | None) -> str | None: + if size_bytes is None: + return None + size = float(size_bytes) + for unit in ("B", "KB", "MB", "GB"): + if size < 1024 or unit == "GB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return None + + +def _allowed_formats(content_type: str) -> set[str]: + if content_type == "audiobook": + return {*get_supported_audiobook_formats(), *_AUDIOBOOK_ARCHIVE_FORMATS} + return set(get_supported_formats()) + + +@register_source("webshare") +class WebshareSource(ReleaseSource): + """Release source that searches Webshare.cz for ebook and audiobook files.""" + + name = "webshare" + display_name = "Webshare.cz" + supported_content_types: ClassVar[list[str]] = ["ebook", "audiobook"] + + def is_available(self) -> bool: + """Available only when explicitly enabled and credentials are configured.""" + username, password = get_credentials() + return config.get("WEBSHARE_ENABLED", False) is True and bool(username and password) + + def search( + self, + book: BookMetadata, + plan: ReleaseSearchPlan, + *, + expand_search: bool = False, + content_type: str = "ebook", + ) -> list[Release]: + """Search Webshare for files of a book in the user's supported formats.""" + if content_type not in self.supported_content_types or not self.is_available(): + return [] + + queries = _build_query_candidates(plan, book) + if not queries: + return [] + + max_results = _coerce_positive_int( + config.get("WEBSHARE_MAX_RESULTS", _DEFAULT_MAX_RESULTS), _DEFAULT_MAX_RESULTS + ) + allowed = _allowed_formats(content_type) + client = WebshareClient(*get_credentials()) + + for query in queries: + logger.info("Searching Webshare for: %s", query) + try: + files = client.search(query, limit=max_results) + except WebshareError as exc: + msg = f"Webshare search failed: {exc}" + raise SourceUnavailableError(msg) from exc + + releases = [ + self._file_to_release(file, content_type) + for file in files + if not file.password_protected and _file_format(file) in allowed + ] + if releases: + return releases + return [] + + def _file_to_release(self, file: WebshareFile, content_type: str) -> Release: + """Normalize a Webshare search hit into a Release. + + ``source_id`` is namespaced ``webshare:`` so it can never collide with + another source's queue key; the handler strips the prefix back to the ident. + """ + fmt = _file_format(file) + language = _guess_language(file.name) + return Release( + source="webshare", + source_id=f"webshare:{file.ident}", + title=PurePosixPath(file.name).stem or file.name, + format=fmt, + language=language, + size=_format_size(file.size_bytes), + size_bytes=file.size_bytes, + download_url=None, # links are time-limited; the handler resolves one per download + info_url=file.info_url, + protocol=ReleaseProtocol.HTTP, + indexer="Webshare.cz", + content_type=content_type, + extra={ + "ident": file.ident, + "file_name": file.name, + "language": language, + "votes": file.positive_votes - file.negative_votes, + }, + ) + + def get_column_config(self) -> ReleaseColumnConfig: + """Language, format, size and the file's Webshare vote balance.""" + return ReleaseColumnConfig( + columns=[ + ColumnSchema( + key="extra.language", + label="Language", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="60px", + color_hint=ColumnColorHint(type="map", value="language"), + uppercase=True, + ), + ColumnSchema( + key="format", + label="Format", + render_type=ColumnRenderType.BADGE, + align=ColumnAlign.CENTER, + width="80px", + color_hint=ColumnColorHint(type="map", value="format"), + uppercase=True, + ), + ColumnSchema( + key="size", + label="Size", + render_type=ColumnRenderType.SIZE, + align=ColumnAlign.CENTER, + width="80px", + sortable=True, + sort_key="size_bytes", + ), + ColumnSchema( + key="extra.votes", + label="Votes", + render_type=ColumnRenderType.NUMBER, + align=ColumnAlign.CENTER, + width="60px", + hide_mobile=True, + sortable=True, + ), + ], + grid_template="minmax(0,2fr) 60px 80px 80px 60px", + supported_filters=["format", "language"], + ) diff --git a/tests/webshare/__init__.py b/tests/webshare/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/webshare/test_api.py b/tests/webshare/test_api.py new file mode 100644 index 00000000..1b52cfb7 --- /dev/null +++ b/tests/webshare/test_api.py @@ -0,0 +1,175 @@ +"""Tests for the Webshare API client: password hashing, XML parsing, token retry.""" + +import hashlib +from unittest.mock import MagicMock, patch + +import pytest + +from shelfmark.release_sources.webshare import api +from shelfmark.release_sources.webshare.api import WebshareClient, WebshareError + + +def _xml(body: str, status: str = "OK") -> bytes: + return f"{status}{body}".encode() + + +def _response(content: bytes) -> MagicMock: + response = MagicMock() + response.content = content + response.raise_for_status.return_value = None + return response + + +@pytest.fixture(autouse=True) +def _clear_tokens(): + WebshareClient._tokens.clear() + yield + WebshareClient._tokens.clear() + + +@pytest.mark.parametrize( + ("password", "salt", "expected"), + [ + # Vectors from `openssl passwd -1 -salt `. + ("password", "saltsalt", "$1$saltsalt$qjXMvbEw8oaL.CzflDtaK/"), + ("heslo123ľ", "UtvP0QwB", "$1$UtvP0QwB$BNPMnAafFtaDfoqW.SVSy/"), + ("x" * 40, "$1$abc$", "$1$abc$b/oSMeDNO7QqHD3W/TXaY0"), + ], +) +def test_md5_crypt_matches_openssl(password, salt, expected): + assert api.md5_crypt(password, salt) == expected + + +def test_hash_password_is_sha1_of_md5crypt_and_md5_digest(): + password_hash, digest = api.hash_password("user", "password", "saltsalt") + expected_hash = hashlib.sha1(b"$1$saltsalt$qjXMvbEw8oaL.CzflDtaK/").hexdigest() + assert password_hash == expected_hash + assert digest == hashlib.md5(f"user:Webshare:{expected_hash}".encode()).hexdigest() + + +def test_search_parses_files_and_sends_token(): + calls = [] + + def post(url, data, **kwargs): + calls.append((url, dict(data))) + if url.endswith("/salt/"): + return _response(_xml("saltsalt")) + if url.endswith("/login/"): + return _response(_xml("TOKEN1")) + return _response( + _xml( + "2" + "abc123Kniha (CZ).epubepub" + "1234563" + "10" + "broken.epub" + ) + ) + + with patch.object(api.requests, "post", side_effect=post): + files = WebshareClient("user", "pw").search("kniha", limit=10) + + assert [f.ident for f in files] == ["abc123"] + assert files[0].size_bytes == 123456 + assert files[0].positive_votes == 3 + assert files[0].info_url == "https://webshare.cz/#/file/abc123" + search_url, search_data = calls[-1] + assert search_url == "https://webshare.cz/api/search/" + assert search_data["wst"] == "TOKEN1" + assert search_data["what"] == "kniha" + + +def test_token_is_reused_between_calls(): + logins = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + logins.append(1) + return _response(_xml("T")) + return _response(_xml("https://dl.example/file.epub")) + + with patch.object(api.requests, "post", side_effect=post): + WebshareClient("user", "pw").file_link("a") + WebshareClient("user", "pw").file_link("b") + + assert len(logins) == 1 + + +def test_expired_token_triggers_one_relogin(): + tokens = iter(["OLD", "NEW"]) + seen_tokens = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + return _response(_xml(f"{next(tokens)}")) + seen_tokens.append(data["wst"]) + if data["wst"] == "OLD": + return _response( + _xml("SEARCH_FATAL_1Bad token", "FATAL") + ) + return _response(_xml("0")) + + with patch.object(api.requests, "post", side_effect=post): + assert WebshareClient("user", "pw").search("x") == [] + + assert seen_tokens == ["OLD", "NEW"] + + +def test_file_errors_are_not_retried(): + link_calls = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + return _response(_xml("T")) + link_calls.append(1) + return _response( + _xml("FILE_LINK_FATAL_1File not found.", "FATAL") + ) + + with ( + patch.object(api.requests, "post", side_effect=post), + pytest.raises(WebshareError) as exc_info, + ): + WebshareClient("user", "pw").file_link("gone") + + assert exc_info.value.code == "FILE_LINK_FATAL_1" + assert str(exc_info.value) == "File not found." + assert len(link_calls) == 1 + + +def test_login_requires_credentials(): + with pytest.raises(WebshareError): + WebshareClient("", "").login() + + +def test_unreadable_response_raises(): + with ( + patch.object(api.requests, "post", return_value=_response(b"nope")), + pytest.raises(WebshareError), + ): + WebshareClient("user", "pw").login() + + +def test_changed_password_forces_new_login(): + logins = [] + + def post(url, data, **kwargs): + if url.endswith("/salt/"): + return _response(_xml("s")) + if url.endswith("/login/"): + logins.append(data["password"]) + return _response(_xml(f"T{len(logins)}")) + return _response(_xml("https://dl.example/file.epub")) + + with patch.object(api.requests, "post", side_effect=post): + WebshareClient("user", "old").file_link("a") + WebshareClient("user", "new").file_link("a") + + assert len(logins) == 2 + assert logins[0] != logins[1] diff --git a/tests/webshare/test_handler.py b/tests/webshare/test_handler.py new file mode 100644 index 00000000..e8e55739 --- /dev/null +++ b/tests/webshare/test_handler.py @@ -0,0 +1,87 @@ +"""Tests for WebshareHandler: link resolution, streaming, cleanup on failure.""" + +import threading +from unittest.mock import MagicMock, patch + +from shelfmark.core.models import DownloadTask +from shelfmark.release_sources.webshare import handler as webshare_handler +from shelfmark.release_sources.webshare.api import WebshareError +from shelfmark.release_sources.webshare.handler import WebshareHandler + + +def _task(task_id="webshare:abc123", fmt="epub"): + return DownloadTask(task_id=task_id, source="webshare", title="Krysar", format=fmt) + + +def _http_response(body: bytes): + response = MagicMock() + response.__enter__.return_value = response + response.__exit__.return_value = False + response.headers = {"Content-Length": str(len(body))} + response.raise_for_status.return_value = None + response.iter_content.return_value = [body[i : i + 4096] for i in range(0, len(body), 4096)] + return response + + +def _run(tmp_path, *, file_link, body=b"x" * 20000, cancel=None, task=None): + status = MagicMock() + progress = MagicMock() + cancel = cancel or threading.Event() + with ( + patch.object(webshare_handler, "TMP_DIR", tmp_path), + patch.object( + webshare_handler.config, + "get", + side_effect=lambda k, d=None: "none" if k == "FILE_ORGANIZATION" else d, + ), + patch.object(webshare_handler, "get_credentials", return_value=("user", "pw")), + patch.object(webshare_handler.WebshareClient, "file_link", side_effect=file_link), + patch.object(webshare_handler.requests, "get", return_value=_http_response(body)), + ): + result = WebshareHandler().download(task or _task(), cancel, progress, status) + return result, status, progress + + +def test_download_streams_file_named_after_link(tmp_path): + idents = [] + + def file_link(ident): + idents.append(ident) + return "https://dl.webshare.cz/abc/Krysar%20%28CZ%29.epub" + + result, _, progress = _run(tmp_path, file_link=file_link) + + expected = tmp_path / "Krysar (CZ).epub" + assert result == str(expected) + assert expected.read_bytes() == b"x" * 20000 + assert idents == ["abc123"] + progress.assert_called_with(100.0) + + +def test_too_small_file_is_rejected_and_removed(tmp_path): + result, status, _ = _run( + tmp_path, file_link=lambda ident: "https://dl.webshare.cz/f.epub", body=b"tiny" + ) + assert result is None + assert not (tmp_path / "f.epub").exists() + status.assert_called_with("error", "Webshare returned an invalid file") + + +def test_link_error_reports_message(tmp_path): + def file_link(ident): + msg = "File not found." + raise WebshareError(msg, "FILE_LINK_FATAL_1") + + result, status, _ = _run(tmp_path, file_link=file_link) + assert result is None + status.assert_called_with("error", "File not found.") + + +def test_cancel_before_start(tmp_path): + cancel = threading.Event() + cancel.set() + file_link = MagicMock() + result, status, _ = _run(tmp_path, file_link=file_link, cancel=cancel) + assert result is None + file_link.assert_not_called() + status.assert_called_with("cancelled", "Cancelled") diff --git a/tests/webshare/test_source.py b/tests/webshare/test_source.py new file mode 100644 index 00000000..5a37729d --- /dev/null +++ b/tests/webshare/test_source.py @@ -0,0 +1,164 @@ +"""Tests for WebshareSource: availability, format filtering, release mapping.""" + +from unittest.mock import patch + +import pytest + +from shelfmark.core.search_plan import ReleaseSearchPlan, ReleaseSearchVariant +from shelfmark.metadata_providers import BookMetadata +from shelfmark.release_sources import SourceUnavailableError, get_source +from shelfmark.release_sources.webshare import source as webshare_source +from shelfmark.release_sources.webshare.api import WebshareError, WebshareFile + +_CONFIG = { + "WEBSHARE_ENABLED": True, + "WEBSHARE_USERNAME": " user ", + "WEBSHARE_PASSWORD": "pw", + "WEBSHARE_MAX_RESULTS": 20, + "SUPPORTED_FORMATS": ["epub", "mobi"], + "SUPPORTED_AUDIOBOOK_FORMATS": ["m4b", "mp3"], +} + + +def _file(ident, name, *, size=200_000, password=False, up=0, down=0): + return WebshareFile( + ident=ident, + name=name, + type=None, + size_bytes=size, + positive_votes=up, + negative_votes=down, + password_protected=password, + ) + + +def _book(): + return BookMetadata(provider="test", provider_id="1", title="Krysař", authors=["Viktor Dyk"]) + + +def _plan(**kwargs): + return ReleaseSearchPlan( + languages=None, + isbn_candidates=[], + author="Viktor Dyk", + title_variants=[ReleaseSearchVariant(title="Krysař", author="Viktor Dyk")], + grouped_title_variants=[], + **kwargs, + ) + + +@pytest.fixture +def config_values(): + values = dict(_CONFIG) + with ( + patch.object(webshare_source.config, "get", side_effect=lambda k, d=None: values.get(k, d)), + patch( + "shelfmark.download.postprocess.policy.core_config.config.get", + side_effect=lambda k, d=None: values.get(k, d), + ), + ): + yield values + + +def test_registered_under_webshare(): + assert get_source("webshare").display_name == "Webshare.cz" + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + ({}, True), + ({"WEBSHARE_ENABLED": False}, False), + ({"WEBSHARE_ENABLED": "true"}, False), + ({"WEBSHARE_USERNAME": ""}, False), + ({"WEBSHARE_PASSWORD": ""}, False), + ], +) +def test_is_available(config_values, overrides, expected): + config_values.update(overrides) + assert webshare_source.WebshareSource().is_available() is expected + + +def test_ebook_search_filters_formats_and_maps_release(config_values): + files = [ + _file("a1", "Dyk Viktor - Krysar (CZ).epub", up=5, down=1), + _file("a2", "Krysar.mobi"), + _file("a3", "Krysar.mp3"), + _file("a4", "Krysar.zip"), + _file("a5", "Krysar locked.epub", password=True), + ] + with patch.object(webshare_source.WebshareClient, "search", return_value=files) as search: + releases = webshare_source.WebshareSource().search(_book(), _plan()) + + assert [r.source_id for r in releases] == ["webshare:a1", "webshare:a2"] + search.assert_called_once_with("Krysař Viktor Dyk", limit=20) + first = releases[0] + assert first.title == "Dyk Viktor - Krysar (CZ)" + assert first.format == "epub" + assert first.language == "cs" + assert first.size == "195.3 KB" + assert first.size_bytes == 200_000 + assert first.info_url == "https://webshare.cz/#/file/a1" + assert first.extra["votes"] == 4 + assert first.content_type == "ebook" + + +def test_audiobook_search_accepts_archives(config_values): + files = [_file("b1", "Krysar audiokniha.zip"), _file("b2", "Krysar.epub")] + with patch.object(webshare_source.WebshareClient, "search", return_value=files): + releases = webshare_source.WebshareSource().search( + _book(), _plan(), content_type="audiobook" + ) + assert [r.source_id for r in releases] == ["webshare:b1"] + assert releases[0].content_type == "audiobook" + + +def test_falls_back_to_title_only_query(config_values): + calls = [] + + def search(query, limit): + calls.append(query) + return [] if len(calls) == 1 else [_file("c1", "Krysar.epub")] + + with patch.object(webshare_source.WebshareClient, "search", side_effect=search): + releases = webshare_source.WebshareSource().search(_book(), _plan()) + + assert calls == ["Krysař Viktor Dyk", "Krysař"] + assert len(releases) == 1 + + +def test_manual_query_is_used_verbatim(config_values): + with patch.object(webshare_source.WebshareClient, "search", return_value=[]) as search: + webshare_source.WebshareSource().search(_book(), _plan(manual_query=" dyk krysar ")) + search.assert_called_once_with("dyk krysar", limit=20) + + +def test_search_error_marks_source_unavailable(config_values): + with ( + patch.object( + webshare_source.WebshareClient, "search", side_effect=WebshareError("Login failed") + ), + pytest.raises(SourceUnavailableError), + ): + webshare_source.WebshareSource().search(_book(), _plan()) + + +def test_disabled_source_does_not_search(config_values): + config_values["WEBSHARE_ENABLED"] = False + with patch.object(webshare_source.WebshareClient, "search") as search: + assert webshare_source.WebshareSource().search(_book(), _plan()) == [] + search.assert_not_called() + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("Kniha (CZ).epub", "cs"), + ("kniha.sk.epub", "sk"), + ("Book [ENG].epub", "en"), + ("Kniha CZ + EN.epub", None), + ("Kniha.epub", None), + ], +) +def test_guess_language(name, expected): + assert webshare_source._guess_language(name) == expected