mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-10 12:28:27 +00:00
splitsec2
cfba44b754
Draft until #1571 merges: this is stacked on it, so the first two commits here are that PR's and only the last one is new. I will rebase once #1571 is in. Shelfmark strips credentials and links out of text in five places, written in different PRs with different patterns: the log formatter, `_safe_url` and `_redact_urls_in_text` in `torrent_utils.py`, `redact_status_message` in `viewer_redaction.py`, the `sed` rules in `genDebug.sh`, and the password handling in the settings registry. Between them there were three regexes that find a URL in text, two separate userinfo patterns and no shared list of what counts as a secret. #1571 made the gap visible, because every pattern added to the logger had to be remembered somewhere else too. This moves the text and URL part into `shelfmark/core/redaction.py`, which imports nothing from the app so the logger can depend on it: - `redact_secrets` (mask in place, for logs) moved from `core/logger.py`, patterns unchanged. The logger imports it. - `safe_url` and `redact_urls_in_text` (keep scheme, host and path, drop the query) moved from `torrent_utils.py` and are public now, so `alldebrid.py` no longer imports a private name from another module. - `strip_links` (remove links entirely) is what `redact_status_message` now calls. It and `redact_urls_in_text` share one link pattern, the one the viewer redaction already used. Two small behaviour changes come from sharing that pattern. A closing bracket or quote right after a URL is no longer swallowed into it when scrubbing an exception message. And the scheme-less path that requests prints after `url: ` now keeps only what is before its query in `redact_urls_in_text`, where before it passed through as it was (the log formatter masked the key, but the path and query stayed in the message). Also here: the two `sed` rules in `genDebug.sh` that mask userinfo now stop at `?` and `#` like the logger pattern does, with a test that fails on the old script. Not in this PR: the debug bundle zips the live log directory as it is, so log lines written before #1571 are not redacted in it. Fixing that means copying the logs to a staging folder and running them through this module, which I would do as a separate change. Tests: a new file covers `safe_url`, `redact_urls_in_text` and `strip_links` directly, and two checks that the viewer and the logger really use these. Full suite passes.
216 lines
9.0 KiB
Bash
Executable File
216 lines
9.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Set up log paths
|
|
LOG_ROOT=${LOG_ROOT:-"/var/log"}
|
|
LOG_DIR="$LOG_ROOT/shelfmark"
|
|
OUTPUT_FILE_NAME="shelfmark-debug_BUILD-${BUILD_VERSION:-local}_RELEASE-${RELEASE_VERSION:-NA}_$(date +%Y%m%d-%H%M%S)"
|
|
OUTPUT_FILE="/tmp/$OUTPUT_FILE_NAME.zip"
|
|
|
|
# Create LOG_DIR if it doesn't exist
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Add system information directly to LOG_DIR
|
|
echo "=== System Information ===" > "$LOG_DIR/system_info.txt"
|
|
echo "Date: $(date)" >> "$LOG_DIR/system_info.txt"
|
|
echo "Hostname: $(hostname)" >> "$LOG_DIR/system_info.txt"
|
|
echo "Kernel: $(uname -a)" >> "$LOG_DIR/system_info.txt"
|
|
echo "" >> "$LOG_DIR/system_info.txt"
|
|
|
|
# Add disk usage
|
|
echo "=== Disk Usage ===" >> "$LOG_DIR/system_info.txt"
|
|
df -h >> "$LOG_DIR/system_info.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/system_info.txt"
|
|
|
|
# Add memory info
|
|
echo "=== Memory Info ===" >> "$LOG_DIR/system_info.txt"
|
|
free -h >> "$LOG_DIR/system_info.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/system_info.txt"
|
|
|
|
# Add running processes
|
|
echo "=== Running Processes ===" >> "$LOG_DIR/system_info.txt"
|
|
ps aux >> "$LOG_DIR/system_info.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/system_info.txt"
|
|
|
|
# Add network information using basic commands
|
|
echo "=== Network Information ===" > "$LOG_DIR/network_info.txt"
|
|
|
|
# Try to get basic connectivity information
|
|
echo "=== Basic Connectivity ===" >> "$LOG_DIR/network_info.txt"
|
|
echo "Hostname resolution:" >> "$LOG_DIR/network_info.txt"
|
|
cat /etc/hosts >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
echo "DNS configuration:" >> "$LOG_DIR/network_info.txt"
|
|
cat /etc/resolv.conf >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
# Try to get interface information from /proc
|
|
echo "=== Network Interfaces (/proc) ===" >> "$LOG_DIR/network_info.txt"
|
|
if [ -f "/proc/net/dev" ]; then
|
|
cat /proc/net/dev >> "$LOG_DIR/network_info.txt" 2>&1
|
|
else
|
|
echo "Not available: /proc/net/dev not found" >> "$LOG_DIR/network_info.txt"
|
|
fi
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
# Try connectivity tests
|
|
echo "=== Internet Connectivity ===" >> "$LOG_DIR/network_info.txt"
|
|
ping -c 3 1.1.1.1 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
ping -c 3 one.one.one.one >> "$LOG_DIR/network_info.txt" 2>&1 || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
# Test IPv6 connectivity
|
|
echo "=== IPv6 Connectivity ===" >> "$LOG_DIR/network_info.txt"
|
|
# Check if IPv6 is enabled
|
|
if [ -f "/proc/sys/net/ipv6/conf/all/disable_ipv6" ]; then
|
|
IPV6_DISABLED=$(cat /proc/sys/net/ipv6/conf/all/disable_ipv6)
|
|
if [ "$IPV6_DISABLED" = "0" ]; then
|
|
echo "IPv6 is enabled in the kernel" >> "$LOG_DIR/network_info.txt"
|
|
else
|
|
echo "IPv6 is disabled in the kernel" >> "$LOG_DIR/network_info.txt"
|
|
fi
|
|
else
|
|
echo "Unable to determine IPv6 kernel status" >> "$LOG_DIR/network_info.txt"
|
|
fi
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
# Try IPv6 connectivity test using Cloudflare's IPv6 DNS
|
|
echo "Testing IPv6 connectivity to Cloudflare DNS:" >> "$LOG_DIR/network_info.txt"
|
|
ping6 -c 3 2606:4700:4700::1111 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
# Test SSL connectivity
|
|
echo "=== SSL Connectivity Tests ===" >> "$LOG_DIR/network_info.txt"
|
|
echo "Testing SSL connection to Cloudflare (1.1.1.1):" >> "$LOG_DIR/network_info.txt"
|
|
echo | openssl s_client -connect 1.1.1.1:443 2>&1 | grep -E "Verify return code:|subject=|issuer=" >> "$LOG_DIR/network_info.txt" || echo "SSL test to Cloudflare failed" >> "$LOG_DIR/network_info.txt"
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
echo "Testing SSL connection to cloudflare.com:" >> "$LOG_DIR/network_info.txt"
|
|
echo | openssl s_client -connect cloudflare.com:443 2>&1 | grep -E "Verify return code:|subject=|issuer=" >> "$LOG_DIR/network_info.txt" || echo "SSL test to cloudflare.com failed" >> "$LOG_DIR/network_info.txt"
|
|
echo "" >> "$LOG_DIR/network_info.txt"
|
|
|
|
# Add installed packages
|
|
# The image ships without pip, so ask the app's interpreter directly
|
|
echo "=== Installed Python Packages ===" > "$LOG_DIR/packages.txt"
|
|
python - >> "$LOG_DIR/packages.txt" 2>&1 <<'PY' || echo "Could not list Python packages" >> "$LOG_DIR/packages.txt"
|
|
import importlib.metadata as metadata
|
|
|
|
for dist in sorted(metadata.distributions(), key=lambda dist: (dist.name or "").lower()):
|
|
print(f"{dist.name}=={dist.version}")
|
|
PY
|
|
echo "" >> "$LOG_DIR/packages.txt"
|
|
|
|
# Check Permissions
|
|
echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
|
|
echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
|
|
ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/permissions.txt"
|
|
echo "ls -all ${INGEST_DIR:-/books}" >> "$LOG_DIR/permissions.txt"
|
|
ls -all ${INGEST_DIR:-/books} >> "$LOG_DIR/permissions.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/permissions.txt"
|
|
echo "ls -all /var/log/shelfmark" >> "$LOG_DIR/permissions.txt"
|
|
ls -all /var/log/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/permissions.txt"
|
|
echo "ls -all /tmp/shelfmark" >> "$LOG_DIR/permissions.txt"
|
|
ls -all /tmp/shelfmark >> "$LOG_DIR/permissions.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/permissions.txt"
|
|
|
|
# Check Iptables (NAT)
|
|
echo "=== IPtables NAT Rules ===" > "$LOG_DIR/iptables_nat.txt"
|
|
iptables -t nat -L -v -n >> "$LOG_DIR/iptables_nat.txt" 2>&1
|
|
|
|
# Check DNS Resolution details
|
|
echo "=== DNS Resolution Test ===" > "$LOG_DIR/dns_test.txt"
|
|
echo "Resolving google.com:" >> "$LOG_DIR/dns_test.txt"
|
|
nslookup google.com >> "$LOG_DIR/dns_test.txt" 2>&1
|
|
echo "" >> "$LOG_DIR/dns_test.txt"
|
|
echo "Resolving check.torproject.org:" >> "$LOG_DIR/dns_test.txt"
|
|
nslookup check.torproject.org >> "$LOG_DIR/dns_test.txt" 2>&1
|
|
|
|
|
|
# Check if running in Docker
|
|
echo "=== Container Info ===" > "$LOG_DIR/container_info.txt"
|
|
if [ -f /.dockerenv ]; then
|
|
echo "Running in Docker container (found /.dockerenv)" >> "$LOG_DIR/container_info.txt"
|
|
elif grep -q docker /proc/1/cgroup 2>/dev/null; then
|
|
echo "Running in Docker container (detected from cgroups)" >> "$LOG_DIR/container_info.txt"
|
|
else
|
|
echo "Not running in Docker container" >> "$LOG_DIR/container_info.txt"
|
|
fi
|
|
|
|
# Add environment variables (redacting sensitive info and the credentials in URLs
|
|
# such as a proxy's http://user:pass@host:port)
|
|
env | grep -v -E "^[^=]*(_KEY|_SECRET|_PASSWORD|_TOKEN|_MAM_ID)[^=]*=" | sed -E 's#(://)[^/?#@[:space:]]+@#\1[REDACTED]@#g' | sort > "$LOG_DIR/environment.txt"
|
|
|
|
# Redact sensitive fields (API keys, passwords, tokens, the MAM session id), the
|
|
# credentials in URLs, and notification URLs, which carry their service's token
|
|
redact_config_json() {
|
|
sed -E \
|
|
-e 's/("(AA_DONATOR_KEY|HARDCOVER_API_KEY|[^"]*_KEY|[^"]*_SECRET|[^"]*_PASSWORD|[^"]*_TOKEN|[^"]*_MAM_ID)"[[:space:]]*:[[:space:]]*")[^"]+"/\1[REDACTED]"/g' \
|
|
-e 's#(://)[^/?#@[:space:]"]+@#\1[REDACTED]@#g' \
|
|
-e 's#("url"[[:space:]]*:[[:space:]]*"([[:alnum:]+.-]+://)?)[^"]+"#\1[REDACTED]"#g' \
|
|
"$1"
|
|
}
|
|
|
|
# Add configuration files (redacting sensitive values)
|
|
CONFIG_DIR=${CONFIG_DIR:-"/config"}
|
|
if [ -d "$CONFIG_DIR" ]; then
|
|
mkdir -p "$LOG_DIR/config"
|
|
|
|
# Copy and redact main settings file
|
|
if [ -f "$CONFIG_DIR/settings.json" ]; then
|
|
redact_config_json "$CONFIG_DIR/settings.json" > "$LOG_DIR/config/settings.json" 2>/dev/null
|
|
fi
|
|
|
|
# Copy and redact plugin config files
|
|
if [ -d "$CONFIG_DIR/plugins" ]; then
|
|
mkdir -p "$LOG_DIR/config/plugins"
|
|
for config_file in "$CONFIG_DIR/plugins"/*.json; do
|
|
if [ -f "$config_file" ]; then
|
|
filename=$(basename "$config_file")
|
|
redact_config_json "$config_file" > "$LOG_DIR/config/plugins/$filename" 2>/dev/null
|
|
fi
|
|
done
|
|
fi
|
|
|
|
echo "Configuration files copied (sensitive values redacted)" >> "$LOG_DIR/container_info.txt"
|
|
else
|
|
echo "Config directory not found at $CONFIG_DIR" >> "$LOG_DIR/container_info.txt"
|
|
fi
|
|
|
|
echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt
|
|
curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt 2>&1
|
|
echo "" >> $LOG_DIR/network_info.txt
|
|
echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt
|
|
curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt 2>&1
|
|
echo "" >> $LOG_DIR/network_info.txt
|
|
echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt
|
|
curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt 2>&1
|
|
echo "" >> $LOG_DIR/network_info.txt
|
|
echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt
|
|
curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt 2>&1
|
|
|
|
# Copy Tor logs if they exist
|
|
if [ -f "/var/log/tor/notices.log" ]; then
|
|
cp "/var/log/tor/notices.log" "$LOG_DIR/tor_notices.log"
|
|
fi
|
|
|
|
# Copy Supervisor logs if they exist
|
|
if [ -d "/var/log/supervisor" ]; then
|
|
cp -rf "/var/log/supervisor/" "$LOG_DIR/supervisor/"
|
|
fi
|
|
|
|
# Create the zip file directly from LOG_DIR
|
|
ln -s "$LOG_DIR" /tmp/$OUTPUT_FILE_NAME
|
|
(cd /tmp && zip -r "$OUTPUT_FILE" $OUTPUT_FILE_NAME > /dev/null 2>&1)
|
|
rm -f /tmp/$OUTPUT_FILE_NAME
|
|
|
|
if [ -f "$OUTPUT_FILE" ]; then
|
|
echo "$OUTPUT_FILE"
|
|
exit 0
|
|
else
|
|
echo "Failed to create debug archive"
|
|
exit 1
|
|
fi
|