mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-17 17:50:33 +00:00
fix(logging): mask userinfo and path-segment credentials, route stray loggers through the formatter (#1571)
Logs get pasted into issues, and the formatter from #1488 only masks `?key=value` query parameters. I went through what else commonly ends up in a log line and found these unmasked: - credentials in a URL's userinfo, like a proxy's `socks5://user:pass@host:1080` or `http://admin:secret@qbittorrent:8080` - a passkey as a path segment, like `/rss/<key>` or `/passkey/<key>/get.torrent`, which private trackers use - the token in a Telegram bot, Discord webhook or Slack webhook URL, which is what a notification URL looks like when it gets logged All three are masked now, in `redact_secrets`, so they apply to every handler. The path rule only fires after a segment that names a secret (`passkey`, `apikey`, `rsskey`, `authkey`, `token`, `secret`, `rss`, `torznab`) followed by 16 or more token characters. I left out generic words like `download` on purpose, because `/api/download/<md5>/cancel` appears in our own request log and the ids there are useful when debugging. That is the tradeoff: a passkey in a `/download/<passkey>/` path still gets through. Separately, four modules created their logger with plain `logging.getLogger`, so nothing they logged went through the formatter at all: `api/websocket.py`, `download/clients/__init__.py`, and the Jackett, Newznab and Prowlarr package inits. They use `setup_logger` now like everything else. Tests: ten cases, seven for the new shapes and three that must stay readable (an md5 route id, a short `/rss/latest`, an email address), plus two that check the stray modules now carry the redacting formatter. Nine of them fail without the change. Full suite passes. Not covered here: `/api/debug` still zips the existing log files as they are on disk, so anything logged before this change rides along in a bundle. That is a separate change in `genDebug.sh`.
This commit is contained in:
1 parent
ef40a38546
commit
47f0acdd31
7 files changed
+115
-12
No files matched your search
@@ -2,12 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from flask_socketio import SocketIO, join_room, leave_room
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
from shelfmark.core.viewer_redaction import viewer_download_id
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from flask import Flask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
class WebSocketManager:
|
||||
|
||||
@@ -29,9 +29,40 @@ _SECRET_QUERY_PARAM_PATTERN = re.compile(
|
||||
)
|
||||
|
||||
|
||||
# Credentials in a URL's userinfo, as in a proxy's socks5://user:pass@host:port. The authority
|
||||
# ends at the first "/", "?" or "#" (RFC 3986), so an "@" after one of those is not userinfo.
|
||||
_URL_USERINFO_PATTERN = re.compile(r"(://)[^/?#@\s'\"<>]+@")
|
||||
|
||||
# A credential carried as a path segment: a tracker passkey (/rss/<key>), or the token in the
|
||||
# URL of a chat service's bot or webhook, which notification URLs end up logging. Only segments
|
||||
# that name a secret are matched, so ids in our own routes (/api/download/<md5>) stay readable.
|
||||
_SECRET_PATH_SEGMENT_PATTERN = re.compile(
|
||||
r"(?<=/)((?:passkey|apikey|api[_-]key|rsskey|authkey|token|secret|rss|torznab)/)"
|
||||
r"[A-Za-z0-9_.~-]{16,}",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CHAT_SERVICE_TOKEN_PATTERNS = (
|
||||
# https://api.telegram.org/bot<id>:<token>/sendMessage
|
||||
re.compile(r"(/bot)\d+:[A-Za-z0-9_-]{20,}"),
|
||||
# https://discord.com/api/webhooks/<id>/<token>
|
||||
re.compile(r"(/webhooks/\d+/)[A-Za-z0-9_-]{20,}"),
|
||||
# https://hooks.slack.com/services/T000/B000/<token>
|
||||
re.compile(r"(/services/[A-Z0-9]+/[A-Z0-9]+/)[A-Za-z0-9]{16,}"),
|
||||
)
|
||||
|
||||
|
||||
def redact_secrets(text: str) -> str:
|
||||
"""Mask credential query parameters in text, keeping the rest of the URL readable."""
|
||||
return _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text)
|
||||
"""Mask credentials in text, keeping the rest of a URL readable.
|
||||
|
||||
Covers credential query parameters, userinfo (``user:pass@host``), path segments that
|
||||
name a secret and the tokens of Telegram, Discord and Slack URLs.
|
||||
"""
|
||||
text = _SECRET_QUERY_PARAM_PATTERN.sub(r"\1=(removed)", text)
|
||||
text = _URL_USERINFO_PATTERN.sub(r"\1(removed)@", text)
|
||||
text = _SECRET_PATH_SEGMENT_PATTERN.sub(r"\1(removed)", text)
|
||||
for pattern in _CHAT_SERVICE_TOKEN_PATTERNS:
|
||||
text = pattern.sub(r"\1(removed)", text)
|
||||
return text
|
||||
|
||||
|
||||
class RedactingFormatter(logging.Formatter):
|
||||
|
||||
@@ -9,7 +9,6 @@ This module provides:
|
||||
Clients register themselves via the @register_client decorator.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
@@ -23,10 +22,12 @@ from typing import TYPE_CHECKING, TypeVar, cast
|
||||
|
||||
import requests
|
||||
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
_logger = setup_logger(__name__)
|
||||
|
||||
# Type variable for generic return type
|
||||
T = TypeVar("T")
|
||||
|
||||
@@ -26,6 +26,6 @@ try:
|
||||
import_module("shelfmark.download.clients")
|
||||
import_module("shelfmark.download.clients.settings")
|
||||
except ImportError as e:
|
||||
import logging
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logging.getLogger(__name__).debug("Download clients not loaded: %s", e)
|
||||
setup_logger(__name__).debug("Download clients not loaded: %s", e)
|
||||
@@ -27,6 +27,6 @@ try:
|
||||
import_module("shelfmark.download.clients")
|
||||
import_module("shelfmark.download.clients.settings")
|
||||
except ImportError as e:
|
||||
import logging
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logging.getLogger(__name__).debug("Download clients not loaded: %s", e)
|
||||
setup_logger(__name__).debug("Download clients not loaded: %s", e)
|
||||
@@ -27,6 +27,6 @@ try:
|
||||
import_module("shelfmark.download.clients")
|
||||
import_module("shelfmark.download.clients.settings")
|
||||
except ImportError as e:
|
||||
import logging
|
||||
from shelfmark.core.logger import setup_logger
|
||||
|
||||
logging.getLogger(__name__).debug("Download clients not loaded: %s", e)
|
||||
setup_logger(__name__).debug("Download clients not loaded: %s", e)
|
||||
@@ -91,6 +91,77 @@ def test_redact_secrets_masks_credential_query_params(text, expected):
|
||||
assert logger_module.redact_secrets(text) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
(
|
||||
"proxy socks5://alice:p4ss@proxy.example:1080 unreachable",
|
||||
"proxy socks5://(removed)@proxy.example:1080 unreachable",
|
||||
),
|
||||
(
|
||||
"GET http://admin:secret@qbittorrent:8080/api/v2/torrents/info failed",
|
||||
"GET http://(removed)@qbittorrent:8080/api/v2/torrents/info failed",
|
||||
),
|
||||
(
|
||||
"fetching https://tracker.example/rss/0123456789abcdef0123456789abcdef",
|
||||
"fetching https://tracker.example/rss/(removed)",
|
||||
),
|
||||
(
|
||||
"https://tracker.example/passkey/AbCdEfGh01234567/get.torrent",
|
||||
"https://tracker.example/passkey/(removed)/get.torrent",
|
||||
),
|
||||
(
|
||||
"POST https://api.telegram.org/bot123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw/sendMessage",
|
||||
"POST https://api.telegram.org/bot(removed)/sendMessage",
|
||||
),
|
||||
(
|
||||
"https://discord.com/api/webhooks/123456789012345678/abcDEF_ghi-JKLmno0123456789",
|
||||
"https://discord.com/api/webhooks/123456789012345678/(removed)",
|
||||
),
|
||||
(
|
||||
"https://hooks.slack.com/services/T0ABC123/B0DEF456/XyZ0123456789abcdefABCDEF",
|
||||
"https://hooks.slack.com/services/T0ABC123/B0DEF456/(removed)",
|
||||
),
|
||||
# Ids in Shelfmark's own routes and short segments stay readable.
|
||||
(
|
||||
"POST /api/download/0123456789abcdef0123456789abcdef/cancel",
|
||||
"POST /api/download/0123456789abcdef0123456789abcdef/cancel",
|
||||
),
|
||||
("https://tracker.example/rss/latest", "https://tracker.example/rss/latest"),
|
||||
("contact me at a@b.example, ok", "contact me at a@b.example, ok"),
|
||||
# The authority ends at "?" or "#", so an address after one is not userinfo.
|
||||
(
|
||||
"https://tracker.example?contact=ops@tracker.example",
|
||||
"https://tracker.example?contact=ops@tracker.example",
|
||||
),
|
||||
(
|
||||
"https://tracker.example#reach-us@tracker.example",
|
||||
"https://tracker.example#reach-us@tracker.example",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_redact_secrets_masks_userinfo_and_path_segment_credentials(text, expected):
|
||||
assert logger_module.redact_secrets(text) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_name",
|
||||
["shelfmark.api.websocket", "shelfmark.download.clients"],
|
||||
)
|
||||
def test_modules_log_through_the_redacting_formatter(module_name):
|
||||
"""A module on a plain logging.getLogger bypassed redaction (S1)."""
|
||||
import importlib
|
||||
|
||||
module = importlib.import_module(module_name)
|
||||
module_logger = getattr(module, "logger", None) or module._logger
|
||||
|
||||
assert module_logger.handlers
|
||||
assert all(
|
||||
isinstance(handler.formatter, logger_module.RedactingFormatter)
|
||||
for handler in module_logger.handlers
|
||||
)
|
||||
|
||||
|
||||
def test_setup_logger_redacts_messages_args_and_tracebacks(monkeypatch):
|
||||
"""Task ids built from Prowlarr guids carry tracker API keys (#1486)."""
|
||||
monkeypatch.setattr(logger_module, "ENABLE_LOGGING", False)
|
||||
|
||||
Reference in new issue
Block a user