From cfba44b75494c07becd7c4ef8f08cd6d423759d1 Mon Sep 17 00:00:00 2001 From: splitsec2 <35583321+splitsec2@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:48:36 -0600 Subject: [PATCH] refactor: one module for taking credentials and links out of text (#1578) Draft until #1571 merges: this is stacked on it, so the first two commits here are that PR's and only the last one is new. I will rebase once #1571 is in. Shelfmark strips credentials and links out of text in five places, written in different PRs with different patterns: the log formatter, `_safe_url` and `_redact_urls_in_text` in `torrent_utils.py`, `redact_status_message` in `viewer_redaction.py`, the `sed` rules in `genDebug.sh`, and the password handling in the settings registry. Between them there were three regexes that find a URL in text, two separate userinfo patterns and no shared list of what counts as a secret. #1571 made the gap visible, because every pattern added to the logger had to be remembered somewhere else too. This moves the text and URL part into `shelfmark/core/redaction.py`, which imports nothing from the app so the logger can depend on it: - `redact_secrets` (mask in place, for logs) moved from `core/logger.py`, patterns unchanged. The logger imports it. - `safe_url` and `redact_urls_in_text` (keep scheme, host and path, drop the query) moved from `torrent_utils.py` and are public now, so `alldebrid.py` no longer imports a private name from another module. - `strip_links` (remove links entirely) is what `redact_status_message` now calls. It and `redact_urls_in_text` share one link pattern, the one the viewer redaction already used. Two small behaviour changes come from sharing that pattern. A closing bracket or quote right after a URL is no longer swallowed into it when scrubbing an exception message. And the scheme-less path that requests prints after `url: ` now keeps only what is before its query in `redact_urls_in_text`, where before it passed through as it was (the log formatter masked the key, but the path and query stayed in the message). Also here: the two `sed` rules in `genDebug.sh` that mask userinfo now stop at `?` and `#` like the logger pattern does, with a test that fails on the old script. Not in this PR: the debug bundle zips the live log directory as it is, so log lines written before #1571 are not redacted in it. Fixing that means copying the logs to a staging folder and running them through this module, which I would do as a separate change. Tests: a new file covers `safe_url`, `redact_urls_in_text` and `strip_links` directly, and two checks that the viewer and the logger really use these. Full suite passes. --- genDebug.sh | 4 +- shelfmark/core/logger.py | 49 +-------- shelfmark/core/redaction.py | 112 ++++++++++++++++++++ shelfmark/core/viewer_redaction.py | 18 +--- shelfmark/download/clients/alldebrid.py | 8 +- shelfmark/download/clients/torrent_utils.py | 37 ++----- tests/core/test_gen_debug_env_filter.py | 20 ++++ tests/core/test_redaction.py | 74 +++++++++++++ tests/prowlarr/test_debrid_clients.py | 5 +- 9 files changed, 228 insertions(+), 99 deletions(-) create mode 100644 shelfmark/core/redaction.py create mode 100644 tests/core/test_redaction.py diff --git a/genDebug.sh b/genDebug.sh index cf2c47dd..144aa208 100755 --- a/genDebug.sh +++ b/genDebug.sh @@ -141,14 +141,14 @@ fi # Add environment variables (redacting sensitive info and the credentials in URLs # such as a proxy's http://user:pass@host:port) -env | grep -v -E "^[^=]*(_KEY|_SECRET|_PASSWORD|_TOKEN|_MAM_ID)[^=]*=" | sed -E 's#(://)[^/@[:space:]]+@#\1[REDACTED]@#g' | sort > "$LOG_DIR/environment.txt" +env | grep -v -E "^[^=]*(_KEY|_SECRET|_PASSWORD|_TOKEN|_MAM_ID)[^=]*=" | sed -E 's#(://)[^/?#@[:space:]]+@#\1[REDACTED]@#g' | sort > "$LOG_DIR/environment.txt" # Redact sensitive fields (API keys, passwords, tokens, the MAM session id), the # credentials in URLs, and notification URLs, which carry their service's token redact_config_json() { sed -E \ -e 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN|[^"]*_MAM_ID)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \ - -e 's#(://)[^/@[:space:]"]+@#\1[REDACTED]@#g' \ + -e 's#(://)[^/?#@[:space:]"]+@#\1[REDACTED]@#g' \ -e 's#("url"[[:space:]]*:[[:space:]]*"([[:alnum:]+.-]+://)?)[^"]+"#\1[REDACTED]"#g' \ "$1" } diff --git a/shelfmark/core/logger.py b/shelfmark/core/logger.py index 5ab56076..97277077 100644 --- a/shelfmark/core/logger.py +++ b/shelfmark/core/logger.py @@ -1,7 +1,6 @@ """Logging configuration and custom logger with error tracing.""" import logging -import re import sys from collections.abc import Mapping from logging.handlers import RotatingFileHandler @@ -9,6 +8,7 @@ from threading import Lock from typing import TYPE_CHECKING from shelfmark.config.env import ENABLE_LOGGING, LOG_FILE, LOG_LEVEL +from shelfmark.core.redaction import redact_secrets if TYPE_CHECKING: from pathlib import Path @@ -17,53 +17,6 @@ if TYPE_CHECKING: _file_handlers: dict[Path, RotatingFileHandler] = {} _file_handlers_lock = Lock() -# Query parameters that carry tracker/indexer credentials. Prowlarr guids are often -# the download URL itself (apikey included), and they end up in task ids that are -# logged all over the download pipeline, so mask them at the formatter instead. -_SECRET_QUERY_PARAM_PATTERN = re.compile( - r"(?<=[?&;])" - r"([a-z0-9_-]*api[_-]?key|api|apitoken|access_token|token|passkey|authkey|rsskey" - r"|torrent_pass|auth|key|password|passwd|pwd|pass|secret|sid|uid)" - r"=[^&\s#'\"<>;:,)\]]+", - re.IGNORECASE, -) - - -# Credentials in a URL's userinfo, as in a proxy's socks5://user:pass@host:port. The authority -# ends at the first "/", "?" or "#" (RFC 3986), so an "@" after one of those is not userinfo. -_URL_USERINFO_PATTERN = re.compile(r"(://)[^/?#@\s'\"<>]+@") - -# A credential carried as a path segment: a tracker passkey (/rss/), or the token in the -# URL of a chat service's bot or webhook, which notification URLs end up logging. Only segments -# that name a secret are matched, so ids in our own routes (/api/download/) stay readable. -_SECRET_PATH_SEGMENT_PATTERN = re.compile( - r"(?<=/)((?:passkey|apikey|api[_-]key|rsskey|authkey|token|secret|rss|torznab)/)" - r"[A-Za-z0-9_.~-]{16,}", - re.IGNORECASE, -) -_CHAT_SERVICE_TOKEN_PATTERNS = ( - # https://api.telegram.org/bot:/sendMessage - re.compile(r"(/bot)\d+:[A-Za-z0-9_-]{20,}"), - # https://discord.com/api/webhooks// - re.compile(r"(/webhooks/\d+/)[A-Za-z0-9_-]{20,}"), - # https://hooks.slack.com/services/T000/B000/ - re.compile(r"(/services/[A-Z0-9]+/[A-Z0-9]+/)[A-Za-z0-9]{16,}"), -) - - -def redact_secrets(text: str) -> str: - """Mask credentials in text, keeping the rest of a URL readable. - - Covers credential query parameters, userinfo (``user:pass@host``), path segments that - name a secret and the tokens of Telegram, Discord and Slack URLs. - """ - text = _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text) - text = _URL_USERINFO_PATTERN.sub(r"\1(removed)@", text) - text = _SECRET_PATH_SEGMENT_PATTERN.sub(r"\1(removed)", text) - for pattern in _CHAT_SERVICE_TOKEN_PATTERNS: - text = pattern.sub(r"\1(removed)", text) - return text - class RedactingFormatter(logging.Formatter): """Formatter that masks credentials in the message, traceback and stack info.""" diff --git a/shelfmark/core/redaction.py b/shelfmark/core/redaction.py new file mode 100644 index 00000000..894f7bd3 --- /dev/null +++ b/shelfmark/core/redaction.py @@ -0,0 +1,112 @@ +"""Taking credentials and links out of text. + +Three jobs, which used to live in three places with three sets of patterns: + +* ``redact_secrets`` masks credentials where they stand and leaves the rest of a URL readable. + The log formatter uses it, so nothing logged carries a key. +* ``safe_url`` and ``redact_urls_in_text`` keep a URL's scheme, host and path and drop its query, + for an error message or a log line that names a link. +* ``strip_links`` removes links altogether, for text shown to someone who has no use for them. + +This module imports nothing from the rest of the app, so the logger can depend on it. +""" + +from __future__ import annotations + +import re +from urllib.parse import urlparse, urlunparse + +# Query parameters that carry tracker/indexer credentials. Prowlarr guids are often +# the download URL itself (apikey included), and they end up in task ids that are +# logged all over the download pipeline, so mask them at the formatter instead. +_SECRET_QUERY_PARAM_PATTERN = re.compile( + r"(?<=[?&;])" + r"([a-z0-9_-]*api[_-]?key|api|apitoken|access_token|token|passkey|authkey|rsskey" + r"|torrent_pass|auth|key|password|passwd|pwd|pass|secret|sid|uid)" + r"=[^&\s#'\"<>;:,)\]]+", + re.IGNORECASE, +) + +# Credentials in a URL's userinfo, as in a proxy's socks5://user:pass@host:port. The authority +# ends at the first "/", "?" or "#" (RFC 3986), so an "@" after one of those is not userinfo. +_URL_USERINFO_PATTERN = re.compile(r"(://)[^/?#@\s'\"<>]+@") + +# A credential carried as a path segment: a tracker passkey (/rss/), or the token in the +# URL of a chat service's bot or webhook, which notification URLs end up logging. Only segments +# that name a secret are matched, so ids in our own routes (/api/download/) stay readable. +_SECRET_PATH_SEGMENT_PATTERN = re.compile( + r"(?<=/)((?:passkey|apikey|api[_-]key|rsskey|authkey|token|secret|rss|torznab)/)" + r"[A-Za-z0-9_.~-]{16,}", + re.IGNORECASE, +) +_CHAT_SERVICE_TOKEN_PATTERNS = ( + # https://api.telegram.org/bot:/sendMessage + re.compile(r"(/bot)\d+:[A-Za-z0-9_-]{20,}"), + # https://discord.com/api/webhooks// + re.compile(r"(/webhooks/\d+/)[A-Za-z0-9_-]{20,}"), + # https://hooks.slack.com/services/T000/B000/ + re.compile(r"(/services/[A-Z0-9]+/[A-Z0-9]+/)[A-Za-z0-9]{16,}"), +) + +# A link inside a message. A client error quotes the request: "401 Client Error: Unauthorized +# for url: ", or for a refused connection "Max retries exceeded with url: " with no scheme. Schemes are case-insensitive, and a closing quote or bracket is not +# part of the link. +_LINK_IN_TEXT_PATTERN = re.compile( + r"https?://[^\s'\"<>)\]]+|(?<=url: )/[^\s'\"<>)\]]*", + re.IGNORECASE, +) +_REMOVED_LINK = "(link removed)" + + +def redact_secrets(text: str) -> str: + """Mask credentials in text, keeping the rest of a URL readable. + + Covers credential query parameters, userinfo (``user:pass@host``), path segments that + name a secret and the tokens of Telegram, Discord and Slack URLs. + """ + text = _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text) + text = _URL_USERINFO_PATTERN.sub(r"\1(removed)@", text) + text = _SECRET_PATH_SEGMENT_PATTERN.sub(r"\1(removed)", text) + for pattern in _CHAT_SERVICE_TOKEN_PATTERNS: + text = pattern.sub(r"\1(removed)", text) + return text + + +def safe_url(url: str, *, limit: int = 120) -> str: + """Return a log-safe URL: scheme/host/path kept, query and fragment dropped. + + Torrent download URLs commonly carry credentials in their query string + (Prowlarr's ``apikey=...`` proxy links among them), so raw URLs must never + reach logs or exception messages. + """ + parsed = urlparse(url) + if not parsed.scheme or not parsed.netloc: + return f"" + safe = urlunparse(parsed._replace(query="", fragment="")) + return safe[:limit] + + +def _link_without_query(link: str) -> str: + if link.startswith("/"): + return link.split("?", 1)[0].split("#", 1)[0] + return safe_url(link) + + +def redact_urls_in_text(text: str) -> str: + """Scrub credential-bearing links out of free-form text such as exception messages. + + A full URL keeps its scheme, host and path, and the path a failed request prints after + ``url: `` keeps everything before its query. + """ + return _LINK_IN_TEXT_PATTERN.sub(lambda match: _link_without_query(match.group(0)), text) + + +def strip_links(text: str) -> str: + """Replace every link in text with a marker, keeping the rest of the text. + + A link is a full http(s) URL, or the scheme-less path after ``url: `` that requests + prints for a failed connection. The link can carry an indexer key or a tracker passkey in + its query string or path, and a non-admin has no use for it. + """ + return _LINK_IN_TEXT_PATTERN.sub(_REMOVED_LINK, text) diff --git a/shelfmark/core/viewer_redaction.py b/shelfmark/core/viewer_redaction.py index 16fe6e8e..62fe98fc 100644 --- a/shelfmark/core/viewer_redaction.py +++ b/shelfmark/core/viewer_redaction.py @@ -19,10 +19,10 @@ from __future__ import annotations import hashlib import hmac import os -import re from pathlib import PurePath from typing import TYPE_CHECKING, Any +from shelfmark.core.redaction import strip_links from shelfmark.core.request_helpers import normalize_positive_int if TYPE_CHECKING: @@ -30,16 +30,6 @@ if TYPE_CHECKING: PUBLIC_DOWNLOAD_ID_PREFIX = "dl_" -# A client error is stored as the download's status message, and it quotes the request: -# "401 Client Error: Unauthorized for url: ", or for a refused connection -# "Max retries exceeded with url: " with no scheme. The link can carry an -# indexer key or a tracker passkey in its query string or path, and a non-admin has no use -# for it. Schemes are case-insensitive. -_LINK_IN_MESSAGE_PATTERN = re.compile( - r"https?://[^\s'\"<>)\]]+|(?<=url: )/[^\s'\"<>)\]]*", - re.IGNORECASE, -) -_REMOVED_LINK = "(link removed)" # Release fields a non-admin's activity cards display. Everything else (source_id, # indexer, info and download URLs, torrent attributes) stays server-side. @@ -124,10 +114,10 @@ def redact_request_row(row: dict[str, Any]) -> dict[str, Any]: def redact_status_message(message: str) -> str: """Replace every link in a status message, keeping the rest of the text. - A link is a full http(s) URL, or the scheme-less path after ``url: `` that requests - prints for a failed connection. + A client error is stored as the download's status message and quotes the request, so it + can carry an indexer key or a tracker passkey that a non-admin has no use for. """ - return _LINK_IN_MESSAGE_PATTERN.sub(_REMOVED_LINK, message) + return strip_links(message) def redact_download_payload(payload: dict[str, Any], task_id: str | None = None) -> dict[str, Any]: diff --git a/shelfmark/download/clients/alldebrid.py b/shelfmark/download/clients/alldebrid.py index 000b4760..0555bca1 100644 --- a/shelfmark/download/clients/alldebrid.py +++ b/shelfmark/download/clients/alldebrid.py @@ -19,6 +19,7 @@ import requests from shelfmark.config.env import TMP_DIR from shelfmark.core.config import config from shelfmark.core.logger import setup_logger +from shelfmark.core.redaction import safe_url from shelfmark.download.clients import ( DownloadClient, DownloadState, @@ -29,7 +30,6 @@ from shelfmark.download.clients._coercion import config_text from shelfmark.download.clients.torrent_utils import ( DebridMagnet, DebridUpload, - _safe_url, resolve_debrid_upload, ) from shelfmark.download.http import download_url @@ -572,7 +572,7 @@ class AllDebridClient(DownloadClient): if "alldebrid.com/f/" in link: logger.info( "Using apikey fallback for AllDebrid file link: %s", - _safe_url(link), + safe_url(link), ) if "apikey=" not in link: sep = "&" if "?" in link else "?" @@ -581,7 +581,7 @@ class AllDebridClient(DownloadClient): logger.error( "AllDebrid unlock failed for '%s': %s", - _safe_url(link), + safe_url(link), err_msg, ) msg = f"AllDebrid unlock failed: {err_msg}" @@ -664,7 +664,7 @@ class AllDebridClient(DownloadClient): referer="https://alldebrid.com/", ) if not buf: - msg = f"Failed to download from {_safe_url(direct_link)}" + msg = f"Failed to download from {safe_url(direct_link)}" _raise_runtime_error(msg) with dest.open("wb") as fh: diff --git a/shelfmark/download/clients/torrent_utils.py b/shelfmark/download/clients/torrent_utils.py index dc7dc7fb..a1b270d4 100644 --- a/shelfmark/download/clients/torrent_utils.py +++ b/shelfmark/download/clients/torrent_utils.py @@ -12,12 +12,13 @@ from binascii import Error as BinasciiError from dataclasses import dataclass from pathlib import Path, PurePosixPath, PureWindowsPath from threading import Lock -from urllib.parse import ParseResult, parse_qs, urljoin, urlparse, urlunparse +from urllib.parse import ParseResult, parse_qs, urljoin, urlparse import requests from shelfmark.core.config import config from shelfmark.core.logger import setup_logger +from shelfmark.core.redaction import redact_urls_in_text, safe_url from shelfmark.core.utils import normalize_http_url from shelfmark.download.network import get_ssl_verify from shelfmark.download.postprocess.packs import PackFile @@ -55,28 +56,6 @@ _torrent_fetch_cache: dict[str, tuple[float, TorrentInfo]] = {} type BencodeValue = dict[str | bytes, BencodeValue] | list[BencodeValue] | int | bytes | str -def _safe_url(url: str, *, limit: int = 120) -> str: - """Return a log-safe URL: scheme/host/path kept, query and fragment dropped. - - Torrent download URLs commonly carry credentials in their query string - (Prowlarr's ``apikey=...`` proxy links among them), so raw URLs must never - reach logs or exception messages. - """ - parsed = urlparse(url) - if not parsed.scheme or not parsed.netloc: - return f"" - safe = urlunparse(parsed._replace(query="", fragment="")) - return safe[:limit] - - -_URL_IN_TEXT_PATTERN = re.compile(r"https?://\S+") - - -def _redact_urls_in_text(text: str) -> str: - """Scrub credential-bearing URLs out of free-form text such as exception messages.""" - return _URL_IN_TEXT_PATTERN.sub(lambda match: _safe_url(match.group(0)), text) - - @dataclass class TorrentInfo: """Parsed information from a torrent URL.""" @@ -159,7 +138,7 @@ def resolve_debrid_upload(url: str, *, expected_hash: str | None = None) -> Debr return DebridMagnet(magnet_url=f"magnet:?xt=urn:btih:{info.info_hash}") reason = info.fetch_error or "no magnet link, info hash, or torrent file was available" - msg = f"Could not resolve a torrent to send from {_safe_url(url)} ({reason})" + msg = f"Could not resolve a torrent to send from {safe_url(url)} ({reason})" raise ValueError(msg) @@ -211,7 +190,7 @@ def _get_cached_torrent_fetch(url: str) -> TorrentInfo | None: if time.monotonic() - fetched_at > _TORRENT_FETCH_CACHE_TTL_SECONDS: del _torrent_fetch_cache[url] return None - logger.debug("Reusing recently fetched torrent data for: %s...", _safe_url(url)) + logger.debug("Reusing recently fetched torrent data for: %s...", safe_url(url)) return info @@ -257,7 +236,7 @@ def _fetch_torrent_info(url: str) -> TorrentInfo: return urljoin(current, location) try: - logger.debug("Fetching torrent file from: %s...", _safe_url(url)) + logger.debug("Fetching torrent file from: %s...", safe_url(url)) # Redirects are followed manually: some indexers redirect download URLs # to magnet links, and each hop must decide anew whether it may see the @@ -295,7 +274,7 @@ def _fetch_torrent_info(url: str) -> TorrentInfo: magnet_url=redirect_url, ) if redirects_remaining <= 0: - logger.warning("Too many redirects fetching torrent file: %s...", _safe_url(url)) + logger.warning("Too many redirects fetching torrent file: %s...", safe_url(url)) return TorrentInfo( info_hash=None, torrent_data=None, @@ -303,7 +282,7 @@ def _fetch_torrent_info(url: str) -> TorrentInfo: fetch_error="too many redirects", ) redirects_remaining -= 1 - logger.debug("Following redirect to: %s...", _safe_url(redirect_url)) + logger.debug("Following redirect to: %s...", safe_url(redirect_url)) current_url = redirect_url resp.raise_for_status() @@ -331,7 +310,7 @@ def _fetch_torrent_info(url: str) -> TorrentInfo: except _TORRENT_FETCH_ERRORS as e: # Exception messages can repeat the source or redirect URL, including its # credentials; scrub them before logging or storing the reason. - message = _redact_urls_in_text(str(e)) + message = redact_urls_in_text(str(e)) logger.warning("Could not fetch torrent file: %s: %s", type(e).__name__, message) return TorrentInfo(info_hash=None, torrent_data=None, is_magnet=False, fetch_error=message) diff --git a/tests/core/test_gen_debug_env_filter.py b/tests/core/test_gen_debug_env_filter.py index 0ef31bd6..662b0b0f 100644 --- a/tests/core/test_gen_debug_env_filter.py +++ b/tests/core/test_gen_debug_env_filter.py @@ -98,6 +98,26 @@ def test_proxy_credentials_are_redacted() -> None: assert "NO_PROXY=localhost" in lines +def test_an_address_after_a_query_or_fragment_is_not_taken_for_credentials( + tmp_path: Path, +) -> None: + """The authority of a URL ends at "/", "?" or "#", as in the log redaction.""" + lines = _filtered_env_lines( + { + "PATH": "/usr/bin:/bin", + "TRACKER_URL": "https://tracker.example?contact=ops@tracker.example", + "DOCS_URL": "https://docs.example#reach-us@docs.example", + } + ) + redacted = _redacted_config( + tmp_path, {"TRACKER_URL": "https://tracker.example?contact=ops@tracker.example"} + ) + + assert "TRACKER_URL=https://tracker.example?contact=ops@tracker.example" in lines + assert "DOCS_URL=https://docs.example#reach-us@docs.example" in lines + assert redacted["TRACKER_URL"] == "https://tracker.example?contact=ops@tracker.example" + + def test_config_secrets_are_redacted(tmp_path: Path) -> None: redacted = _redacted_config( tmp_path, diff --git a/tests/core/test_redaction.py b/tests/core/test_redaction.py new file mode 100644 index 00000000..dc81f1b8 --- /dev/null +++ b/tests/core/test_redaction.py @@ -0,0 +1,74 @@ +"""The shared helpers that take credentials and links out of text.""" + +import pytest + +from shelfmark.core import logger as logger_module +from shelfmark.core import redaction +from shelfmark.core.viewer_redaction import redact_status_message + + +class TestSafeUrl: + def test_keeps_scheme_host_and_path_and_drops_the_query_and_fragment(self): + url = "https://prowlarr.example:9696/1/download?apikey=s3cr3t&link=abc#frag" + + assert redaction.safe_url(url) == "https://prowlarr.example:9696/1/download" + + def test_is_cut_to_the_limit(self): + assert len(redaction.safe_url("https://t.example/" + "a" * 500, limit=40)) == 40 + + @pytest.mark.parametrize("value", ["magnet:?xt=urn:btih:abc", "not a url", ""]) + def test_a_value_that_is_not_a_url_is_not_echoed(self, value): + assert redaction.safe_url(value) == "" + + +class TestRedactUrlsInText: + def test_every_url_loses_its_query(self): + text = "GET https://a.example/x?apikey=k1 then https://b.example/y?token=k2 failed" + + assert redaction.redact_urls_in_text(text) == ( + "GET https://a.example/x then https://b.example/y failed" + ) + + def test_a_closing_bracket_or_quote_is_not_part_of_the_link(self): + text = "failed (https://a.example/x?apikey=k1) and 'https://b.example/y?key=k2'" + + assert redaction.redact_urls_in_text(text) == ( + "failed (https://a.example/x) and 'https://b.example/y'" + ) + + def test_the_path_a_failed_request_prints_keeps_everything_before_its_query(self): + text = ( + "Max retries exceeded with url: /api/v1/indexer/1/download?apikey=s3cr3t&link=abc (x)" + ) + + assert redaction.redact_urls_in_text(text) == ( + "Max retries exceeded with url: /api/v1/indexer/1/download (x)" + ) + + def test_text_without_a_link_is_untouched(self): + assert redaction.redact_urls_in_text("Connection refused") == "Connection refused" + + +class TestStripLinks: + def test_replaces_a_full_url(self): + assert redaction.strip_links("401 for url: https://i.example/dl?apikey=k") == ( + "401 for url: (link removed)" + ) + + def test_replaces_the_scheme_less_path_after_url(self): + assert redaction.strip_links("Max retries exceeded with url: /a/b?apikey=k (Caused)") == ( + "Max retries exceeded with url: (link removed) (Caused)" + ) + + def test_the_scheme_is_case_insensitive(self): + assert redaction.strip_links("HTTPS://I.example/dl?k=1 failed") == "(link removed) failed" + + +def test_the_viewer_message_redaction_is_the_shared_one(): + message = "Unauthorized for url: https://i.example/dl?apikey=k" + + assert redact_status_message(message) == redaction.strip_links(message) + + +def test_the_log_formatter_masks_with_the_shared_function(): + assert logger_module.redact_secrets is redaction.redact_secrets diff --git a/tests/prowlarr/test_debrid_clients.py b/tests/prowlarr/test_debrid_clients.py index 45ad8cd2..ea3299fd 100644 --- a/tests/prowlarr/test_debrid_clients.py +++ b/tests/prowlarr/test_debrid_clients.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock, patch import pytest +from shelfmark.core.redaction import safe_url from shelfmark.download.clients import DownloadState from shelfmark.download.clients.alldebrid import AllDebridClient from shelfmark.download.clients.realdebrid import RealDebridClient, _DownloadState @@ -103,7 +104,7 @@ class TestResolveDebridUpload: logged = str(logger.mock_calls) assert _PROWLARR_PROXY_URL not in logged - assert torrent_utils._safe_url(_PROWLARR_PROXY_URL) in logged + assert safe_url(_PROWLARR_PROXY_URL) in logged def test_resolver_does_not_log_a_credential_bearing_redirect(self, monkeypatch): import shelfmark.download.clients.torrent_utils as torrent_utils @@ -124,7 +125,7 @@ class TestResolveDebridUpload: logged = str(logger.mock_calls) assert redirect_url not in logged - assert torrent_utils._safe_url(redirect_url) in logged + assert safe_url(redirect_url) in logged def test_resolver_failure_message_keeps_out_the_source_url(self, monkeypatch): _mock_fetch(monkeypatch, error=OSError("tracker unreachable"))