fix(logging): mask userinfo and path-segment credentials, route stray loggers through the formatter (#1571)

Logs get pasted into issues, and the formatter from #1488 only masks
`?key=value` query parameters. I went through what else commonly ends up
in a log line and found these unmasked:

- credentials in a URL's userinfo, like a proxy's
`socks5://user:pass@host:1080` or `http://admin:secret@qbittorrent:8080`
- a passkey as a path segment, like `/rss/<key>` or
`/passkey/<key>/get.torrent`, which private trackers use
- the token in a Telegram bot, Discord webhook or Slack webhook URL,
which is what a notification URL looks like when it gets logged

All three are masked now, in `redact_secrets`, so they apply to every
handler. The path rule only fires after a segment that names a secret
(`passkey`, `apikey`, `rsskey`, `authkey`, `token`, `secret`, `rss`,
`torznab`) followed by 16 or more token characters. I left out generic
words like `download` on purpose, because `/api/download/<md5>/cancel`
appears in our own request log and the ids there are useful when
debugging. That is the tradeoff: a passkey in a `/download/<passkey>/`
path still gets through.

Separately, four modules created their logger with plain
`logging.getLogger`, so nothing they logged went through the formatter
at all: `api/websocket.py`, `download/clients/__init__.py`, and the
Jackett, Newznab and Prowlarr package inits. They use `setup_logger` now
like everything else.

Tests: ten cases, seven for the new shapes and three that must stay
readable (an md5 route id, a short `/rss/latest`, an email address),
plus two that check the stray modules now carry the redacting formatter.
Nine of them fail without the change. Full suite passes.

Not covered here: `/api/debug` still zips the existing log files as they
are on disk, so anything logged before this change rides along in a
bundle. That is a separate change in `genDebug.sh`.
This commit is contained in:
splitsec2 authored and GitHub committed 2026-10-24 17:19:53 -06:00
1 parent ef40a38546
commit 47f0acdd31
7 files changed
+115 -12

No files matched your search

+2 -2
View File
@@ -2,12 +2,12 @@
from __future__ import annotations
import logging
import threading
from typing import TYPE_CHECKING, Any
from flask_socketio import SocketIO, join_room, leave_room
from shelfmark.core.logger import setup_logger
from shelfmark.core.viewer_redaction import viewer_download_id
if TYPE_CHECKING:
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
from flask import Flask
logger = logging.getLogger(__name__)
logger = setup_logger(__name__)
class WebSocketManager:
+33 -2
View File
@@ -29,9 +29,40 @@ _SECRET_QUERY_PARAM_PATTERN = re.compile(
)
# Credentials in a URL's userinfo, as in a proxy's socks5://user:pass@host:port. The authority
# ends at the first "/", "?" or "#" (RFC 3986), so an "@" after one of those is not userinfo.
_URL_USERINFO_PATTERN = re.compile(r"(://)[^/?#@\s'\"<>]+@")
# A credential carried as a path segment: a tracker passkey (/rss/<key>), or the token in the
# URL of a chat service's bot or webhook, which notification URLs end up logging. Only segments
# that name a secret are matched, so ids in our own routes (/api/download/<md5>) stay readable.
_SECRET_PATH_SEGMENT_PATTERN = re.compile(
r"(?<=/)((?:passkey|apikey|api[_-]key|rsskey|authkey|token|secret|rss|torznab)/)"
r"[A-Za-z0-9_.~-]{16,}",
re.IGNORECASE,
)
_CHAT_SERVICE_TOKEN_PATTERNS = (
# https://api.telegram.org/bot<id>:<token>/sendMessage
re.compile(r"(/bot)\d+:[A-Za-z0-9_-]{20,}"),
# https://discord.com/api/webhooks/<id>/<token>
re.compile(r"(/webhooks/\d+/)[A-Za-z0-9_-]{20,}"),
# https://hooks.slack.com/services/T000/B000/<token>
re.compile(r"(/services/[A-Z0-9]+/[A-Z0-9]+/)[A-Za-z0-9]{16,}"),
)
def redact_secrets(text: str) -> str:
"""Mask credential query parameters in text, keeping the rest of the URL readable."""
return _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text)
"""Mask credentials in text, keeping the rest of a URL readable.
Covers credential query parameters, userinfo (``user:pass@host``), path segments that
name a secret and the tokens of Telegram, Discord and Slack URLs.
"""
text = _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text)
text = _URL_USERINFO_PATTERN.sub(r"\1(removed)@", text)
text = _SECRET_PATH_SEGMENT_PATTERN.sub(r"\1(removed)", text)
for pattern in _CHAT_SERVICE_TOKEN_PATTERNS:
text = pattern.sub(r"\1(removed)", text)
return text
class RedactingFormatter(logging.Formatter):
+3 -2
View File
@@ -9,7 +9,6 @@ This module provides:
Clients register themselves via the @register_client decorator.
"""
import logging
import os
import random
import time
@@ -23,10 +22,12 @@ from typing import TYPE_CHECKING, TypeVar, cast
import requests
from shelfmark.core.logger import setup_logger
if TYPE_CHECKING:
from collections.abc import Callable
_logger = logging.getLogger(__name__)
_logger = setup_logger(__name__)
# Type variable for generic return type
T = TypeVar("T")
@@ -26,6 +26,6 @@ try:
import_module("shelfmark.download.clients")
import_module("shelfmark.download.clients.settings")
except ImportError as e:
import logging
from shelfmark.core.logger import setup_logger
logging.getLogger(__name__).debug("Download clients not loaded: %s", e)
setup_logger(__name__).debug("Download clients not loaded: %s", e)
@@ -27,6 +27,6 @@ try:
import_module("shelfmark.download.clients")
import_module("shelfmark.download.clients.settings")
except ImportError as e:
import logging
from shelfmark.core.logger import setup_logger
logging.getLogger(__name__).debug("Download clients not loaded: %s", e)
setup_logger(__name__).debug("Download clients not loaded: %s", e)
@@ -27,6 +27,6 @@ try:
import_module("shelfmark.download.clients")
import_module("shelfmark.download.clients.settings")
except ImportError as e:
import logging
from shelfmark.core.logger import setup_logger
logging.getLogger(__name__).debug("Download clients not loaded: %s", e)
setup_logger(__name__).debug("Download clients not loaded: %s", e)
+71
View File
@@ -91,6 +91,77 @@ def test_redact_secrets_masks_credential_query_params(text, expected):
assert logger_module.redact_secrets(text) == expected
@pytest.mark.parametrize(
("text", "expected"),
[
(
"proxy socks5://alice:p4ss@proxy.example:1080 unreachable",
"proxy socks5://(removed)@proxy.example:1080 unreachable",
),
(
"GET http://admin:secret@qbittorrent:8080/api/v2/torrents/info failed",
"GET http://(removed)@qbittorrent:8080/api/v2/torrents/info failed",
),
(
"fetching https://tracker.example/rss/0123456789abcdef0123456789abcdef",
"fetching https://tracker.example/rss/(removed)",
),
(
"https://tracker.example/passkey/AbCdEfGh01234567/get.torrent",
"https://tracker.example/passkey/(removed)/get.torrent",
),
(
"POST https://api.telegram.org/bot123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw/sendMessage",
"POST https://api.telegram.org/bot(removed)/sendMessage",
),
(
"https://discord.com/api/webhooks/123456789012345678/abcDEF_ghi-JKLmno0123456789",
"https://discord.com/api/webhooks/123456789012345678/(removed)",
),
(
"https://hooks.slack.com/services/T0ABC123/B0DEF456/XyZ0123456789abcdefABCDEF",
"https://hooks.slack.com/services/T0ABC123/B0DEF456/(removed)",
),
# Ids in Shelfmark's own routes and short segments stay readable.
(
"POST /api/download/0123456789abcdef0123456789abcdef/cancel",
"POST /api/download/0123456789abcdef0123456789abcdef/cancel",
),
("https://tracker.example/rss/latest", "https://tracker.example/rss/latest"),
("contact me at a@b.example, ok", "contact me at a@b.example, ok"),
# The authority ends at "?" or "#", so an address after one is not userinfo.
(
"https://tracker.example?contact=ops@tracker.example",
"https://tracker.example?contact=ops@tracker.example",
),
(
"https://tracker.example#reach-us@tracker.example",
"https://tracker.example#reach-us@tracker.example",
),
],
)
def test_redact_secrets_masks_userinfo_and_path_segment_credentials(text, expected):
assert logger_module.redact_secrets(text) == expected
@pytest.mark.parametrize(
"module_name",
["shelfmark.api.websocket", "shelfmark.download.clients"],
)
def test_modules_log_through_the_redacting_formatter(module_name):
"""A module on a plain logging.getLogger bypassed redaction (S1)."""
import importlib
module = importlib.import_module(module_name)
module_logger = getattr(module, "logger", None) or module._logger
assert module_logger.handlers
assert all(
isinstance(handler.formatter, logger_module.RedactingFormatter)
for handler in module_logger.handlers
)
def test_setup_logger_redacts_messages_args_and_tracebacks(monkeypatch):
"""Task ids built from Prowlarr guids carry tracker API keys (#1486)."""
monkeypatch.setattr(logger_module, "ENABLE_LOGGING", False)