From 47f0acdd31670a10fb0250106fcdaca332871883 Mon Sep 17 00:00:00 2001 From: splitsec2 <35583321+splitsec2@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:19:53 -0600 Subject: [PATCH] fix(logging): mask userinfo and path-segment credentials, route stray loggers through the formatter (#1571) Logs get pasted into issues, and the formatter from #1488 only masks `?key=value` query parameters. I went through what else commonly ends up in a log line and found these unmasked: - credentials in a URL's userinfo, like a proxy's `socks5://user:pass@host:1080` or `http://admin:secret@qbittorrent:8080` - a passkey as a path segment, like `/rss/` or `/passkey//get.torrent`, which private trackers use - the token in a Telegram bot, Discord webhook or Slack webhook URL, which is what a notification URL looks like when it gets logged All three are masked now, in `redact_secrets`, so they apply to every handler. The path rule only fires after a segment that names a secret (`passkey`, `apikey`, `rsskey`, `authkey`, `token`, `secret`, `rss`, `torznab`) followed by 16 or more token characters. I left out generic words like `download` on purpose, because `/api/download//cancel` appears in our own request log and the ids there are useful when debugging. That is the tradeoff: a passkey in a `/download//` path still gets through. Separately, four modules created their logger with plain `logging.getLogger`, so nothing they logged went through the formatter at all: `api/websocket.py`, `download/clients/__init__.py`, and the Jackett, Newznab and Prowlarr package inits. They use `setup_logger` now like everything else. Tests: ten cases, seven for the new shapes and three that must stay readable (an md5 route id, a short `/rss/latest`, an email address), plus two that check the stray modules now carry the redacting formatter. Nine of them fail without the change. Full suite passes. Not covered here: `/api/debug` still zips the existing log files as they are on disk, so anything logged before this change rides along in a bundle. That is a separate change in `genDebug.sh`. --- shelfmark/api/websocket.py | 4 +- shelfmark/core/logger.py | 35 ++++++++- shelfmark/download/clients/__init__.py | 5 +- shelfmark/release_sources/jackett/__init__.py | 4 +- shelfmark/release_sources/newznab/__init__.py | 4 +- .../release_sources/prowlarr/__init__.py | 4 +- tests/core/test_logger.py | 71 +++++++++++++++++++ 7 files changed, 115 insertions(+), 12 deletions(-) diff --git a/shelfmark/api/websocket.py b/shelfmark/api/websocket.py index 6c1be11e..60fd32ea 100644 --- a/shelfmark/api/websocket.py +++ b/shelfmark/api/websocket.py @@ -2,12 +2,12 @@ from __future__ import annotations -import logging import threading from typing import TYPE_CHECKING, Any from flask_socketio import SocketIO, join_room, leave_room +from shelfmark.core.logger import setup_logger from shelfmark.core.viewer_redaction import viewer_download_id if TYPE_CHECKING: @@ -15,7 +15,7 @@ if TYPE_CHECKING: from flask import Flask -logger = logging.getLogger(__name__) +logger = setup_logger(__name__) class WebSocketManager: diff --git a/shelfmark/core/logger.py b/shelfmark/core/logger.py index 6a171344..5ab56076 100644 --- a/shelfmark/core/logger.py +++ b/shelfmark/core/logger.py @@ -29,9 +29,40 @@ _SECRET_QUERY_PARAM_PATTERN = re.compile( ) +# Credentials in a URL's userinfo, as in a proxy's socks5://user:pass@host:port. The authority +# ends at the first "/", "?" or "#" (RFC 3986), so an "@" after one of those is not userinfo. +_URL_USERINFO_PATTERN = re.compile(r"(://)[^/?#@\s'\"<>]+@") + +# A credential carried as a path segment: a tracker passkey (/rss/), or the token in the +# URL of a chat service's bot or webhook, which notification URLs end up logging. Only segments +# that name a secret are matched, so ids in our own routes (/api/download/) stay readable. +_SECRET_PATH_SEGMENT_PATTERN = re.compile( + r"(?<=/)((?:passkey|apikey|api[_-]key|rsskey|authkey|token|secret|rss|torznab)/)" + r"[A-Za-z0-9_.~-]{16,}", + re.IGNORECASE, +) +_CHAT_SERVICE_TOKEN_PATTERNS = ( + # https://api.telegram.org/bot:/sendMessage + re.compile(r"(/bot)\d+:[A-Za-z0-9_-]{20,}"), + # https://discord.com/api/webhooks// + re.compile(r"(/webhooks/\d+/)[A-Za-z0-9_-]{20,}"), + # https://hooks.slack.com/services/T000/B000/ + re.compile(r"(/services/[A-Z0-9]+/[A-Z0-9]+/)[A-Za-z0-9]{16,}"), +) + + def redact_secrets(text: str) -> str: - """Mask credential query parameters in text, keeping the rest of the URL readable.""" - return _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text) + """Mask credentials in text, keeping the rest of a URL readable. + + Covers credential query parameters, userinfo (``user:pass@host``), path segments that + name a secret and the tokens of Telegram, Discord and Slack URLs. + """ + text = _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text) + text = _URL_USERINFO_PATTERN.sub(r"\1(removed)@", text) + text = _SECRET_PATH_SEGMENT_PATTERN.sub(r"\1(removed)", text) + for pattern in _CHAT_SERVICE_TOKEN_PATTERNS: + text = pattern.sub(r"\1(removed)", text) + return text class RedactingFormatter(logging.Formatter): diff --git a/shelfmark/download/clients/__init__.py b/shelfmark/download/clients/__init__.py index 3b9e4151..1214c458 100644 --- a/shelfmark/download/clients/__init__.py +++ b/shelfmark/download/clients/__init__.py @@ -9,7 +9,6 @@ This module provides: Clients register themselves via the @register_client decorator. """ -import logging import os import random import time @@ -23,10 +22,12 @@ from typing import TYPE_CHECKING, TypeVar, cast import requests +from shelfmark.core.logger import setup_logger + if TYPE_CHECKING: from collections.abc import Callable -_logger = logging.getLogger(__name__) +_logger = setup_logger(__name__) # Type variable for generic return type T = TypeVar("T") diff --git a/shelfmark/release_sources/jackett/__init__.py b/shelfmark/release_sources/jackett/__init__.py index 670c374b..9b20be0c 100644 --- a/shelfmark/release_sources/jackett/__init__.py +++ b/shelfmark/release_sources/jackett/__init__.py @@ -26,6 +26,6 @@ try: import_module("shelfmark.download.clients") import_module("shelfmark.download.clients.settings") except ImportError as e: - import logging + from shelfmark.core.logger import setup_logger - logging.getLogger(__name__).debug("Download clients not loaded: %s", e) + setup_logger(__name__).debug("Download clients not loaded: %s", e) diff --git a/shelfmark/release_sources/newznab/__init__.py b/shelfmark/release_sources/newznab/__init__.py index 4723904d..91b45a58 100644 --- a/shelfmark/release_sources/newznab/__init__.py +++ b/shelfmark/release_sources/newznab/__init__.py @@ -27,6 +27,6 @@ try: import_module("shelfmark.download.clients") import_module("shelfmark.download.clients.settings") except ImportError as e: - import logging + from shelfmark.core.logger import setup_logger - logging.getLogger(__name__).debug("Download clients not loaded: %s", e) + setup_logger(__name__).debug("Download clients not loaded: %s", e) diff --git a/shelfmark/release_sources/prowlarr/__init__.py b/shelfmark/release_sources/prowlarr/__init__.py index 19a7ac78..6d1e7e3b 100644 --- a/shelfmark/release_sources/prowlarr/__init__.py +++ b/shelfmark/release_sources/prowlarr/__init__.py @@ -27,6 +27,6 @@ try: import_module("shelfmark.download.clients") import_module("shelfmark.download.clients.settings") except ImportError as e: - import logging + from shelfmark.core.logger import setup_logger - logging.getLogger(__name__).debug("Download clients not loaded: %s", e) + setup_logger(__name__).debug("Download clients not loaded: %s", e) diff --git a/tests/core/test_logger.py b/tests/core/test_logger.py index 45f33932..1eb33215 100644 --- a/tests/core/test_logger.py +++ b/tests/core/test_logger.py @@ -91,6 +91,77 @@ def test_redact_secrets_masks_credential_query_params(text, expected): assert logger_module.redact_secrets(text) == expected +@pytest.mark.parametrize( + ("text", "expected"), + [ + ( + "proxy socks5://alice:p4ss@proxy.example:1080 unreachable", + "proxy socks5://(removed)@proxy.example:1080 unreachable", + ), + ( + "GET http://admin:secret@qbittorrent:8080/api/v2/torrents/info failed", + "GET http://(removed)@qbittorrent:8080/api/v2/torrents/info failed", + ), + ( + "fetching https://tracker.example/rss/0123456789abcdef0123456789abcdef", + "fetching https://tracker.example/rss/(removed)", + ), + ( + "https://tracker.example/passkey/AbCdEfGh01234567/get.torrent", + "https://tracker.example/passkey/(removed)/get.torrent", + ), + ( + "POST https://api.telegram.org/bot123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw/sendMessage", + "POST https://api.telegram.org/bot(removed)/sendMessage", + ), + ( + "https://discord.com/api/webhooks/123456789012345678/abcDEF_ghi-JKLmno0123456789", + "https://discord.com/api/webhooks/123456789012345678/(removed)", + ), + ( + "https://hooks.slack.com/services/T0ABC123/B0DEF456/XyZ0123456789abcdefABCDEF", + "https://hooks.slack.com/services/T0ABC123/B0DEF456/(removed)", + ), + # Ids in Shelfmark's own routes and short segments stay readable. + ( + "POST /api/download/0123456789abcdef0123456789abcdef/cancel", + "POST /api/download/0123456789abcdef0123456789abcdef/cancel", + ), + ("https://tracker.example/rss/latest", "https://tracker.example/rss/latest"), + ("contact me at a@b.example, ok", "contact me at a@b.example, ok"), + # The authority ends at "?" or "#", so an address after one is not userinfo. + ( + "https://tracker.example?contact=ops@tracker.example", + "https://tracker.example?contact=ops@tracker.example", + ), + ( + "https://tracker.example#reach-us@tracker.example", + "https://tracker.example#reach-us@tracker.example", + ), + ], +) +def test_redact_secrets_masks_userinfo_and_path_segment_credentials(text, expected): + assert logger_module.redact_secrets(text) == expected + + +@pytest.mark.parametrize( + "module_name", + ["shelfmark.api.websocket", "shelfmark.download.clients"], +) +def test_modules_log_through_the_redacting_formatter(module_name): + """A module on a plain logging.getLogger bypassed redaction (S1).""" + import importlib + + module = importlib.import_module(module_name) + module_logger = getattr(module, "logger", None) or module._logger + + assert module_logger.handlers + assert all( + isinstance(handler.formatter, logger_module.RedactingFormatter) + for handler in module_logger.handlers + ) + + def test_setup_logger_redacts_messages_args_and_tracebacks(monkeypatch): """Task ids built from Prowlarr guids carry tracker API keys (#1486).""" monkeypatch.setattr(logger_module, "ENABLE_LOGGING", False)