Fix tor timeout (#349)

Tentative fix for #340
2026-10-04 15:41:12 +01:00 · 2025-12-18 15:53:39 -05:00
parent 0cac541c0b
commit 15a61a5191
3 changed files with 125 additions and 65 deletions
@@ -153,6 +153,8 @@ RUN apt-get update && \
    apt-get install -y --no-install-recommends \
    # --- Tor ---
    tor \
+    # --- Supervisor ---
+    supervisor \
    # --- iptables ---
    iptables && \
    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
@@ -18,17 +18,17 @@ echo "" >> "$LOG_DIR/system_info.txt"

 # Add disk usage
 echo "=== Disk Usage ===" >> "$LOG_DIR/system_info.txt"
-df -h >> "$LOG_DIR/system_info.txt"
+df -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"

 # Add memory info
 echo "=== Memory Info ===" >> "$LOG_DIR/system_info.txt"
-free -h >> "$LOG_DIR/system_info.txt"
+free -h >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"

 # Add running processes
 echo "=== Running Processes ===" >> "$LOG_DIR/system_info.txt"
-ps aux >> "$LOG_DIR/system_info.txt"
+ps aux >> "$LOG_DIR/system_info.txt" 2>&1
 echo "" >> "$LOG_DIR/system_info.txt"

 # Add network information using basic commands
@@ -37,17 +37,17 @@ echo "=== Network Information ===" > "$LOG_DIR/network_info.txt"
 # Try to get basic connectivity information
 echo "=== Basic Connectivity ===" >> "$LOG_DIR/network_info.txt"
 echo "Hostname resolution:" >> "$LOG_DIR/network_info.txt"
-cat /etc/hosts 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
+cat /etc/hosts >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/hosts" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"

 echo "DNS configuration:" >> "$LOG_DIR/network_info.txt"
-cat /etc/resolv.conf 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
+cat /etc/resolv.conf >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Unable to read /etc/resolv.conf" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"

 # Try to get interface information from /proc
 echo "=== Network Interfaces (/proc) ===" >> "$LOG_DIR/network_info.txt"
 if [ -f "/proc/net/dev" ]; then
-  cat /proc/net/dev >> "$LOG_DIR/network_info.txt"
+  cat /proc/net/dev >> "$LOG_DIR/network_info.txt" 2>&1
 else
  echo "Not available: /proc/net/dev not found" >> "$LOG_DIR/network_info.txt"
 fi
@@ -55,9 +55,9 @@ echo "" >> "$LOG_DIR/network_info.txt"

 # Try connectivity tests
 echo "=== Internet Connectivity ===" >> "$LOG_DIR/network_info.txt"
-ping -c 3 1.1.1.1 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
+ping -c 3 1.1.1.1 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "Ping command failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"
-ping -c 3 one.one.one.one 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
+ping -c 3 one.one.one.one >> "$LOG_DIR/network_info.txt" 2>&1 || echo "DNS resolution test failed" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"

 # Test IPv6 connectivity
@@ -77,7 +77,7 @@ echo "" >> "$LOG_DIR/network_info.txt"

 # Try IPv6 connectivity test using Cloudflare's IPv6 DNS
 echo "Testing IPv6 connectivity to Cloudflare DNS:" >> "$LOG_DIR/network_info.txt"
-ping6 -c 3 2606:4700:4700::1111 2>/dev/null >> "$LOG_DIR/network_info.txt" || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
+ping6 -c 3 2606:4700:4700::1111 >> "$LOG_DIR/network_info.txt" 2>&1 || echo "IPv6 ping failed or not available" >> "$LOG_DIR/network_info.txt"
 echo "" >> "$LOG_DIR/network_info.txt"

 # Test SSL connectivity
@@ -92,24 +92,36 @@ echo "" >> "$LOG_DIR/network_info.txt"

 # Add installed packages
 echo "=== Installed Python Packages ===" > "$LOG_DIR/packages.txt"
-pip list 2>/dev/null >> "$LOG_DIR/packages.txt" || echo "pip not found" >> "$LOG_DIR/packages.txt"
+pip list >> "$LOG_DIR/packages.txt" 2>&1 || echo "pip not found" >> "$LOG_DIR/packages.txt"
 echo "" >> "$LOG_DIR/packages.txt"

 # Check Permissions
 echo "=== Permissions ===" > "$LOG_DIR/permissions.txt"
 echo "ls -all /app" >> "$LOG_DIR/permissions.txt"
-ls -all /app >> "$LOG_DIR/permissions.txt"
+ls -all /app >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 echo "ls -all /cwa-book-ingest" >> "$LOG_DIR/permissions.txt"
-ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt"
+ls -all /cwa-book-ingest >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 echo "ls -all /var/log/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+ls -all /var/log/cwa-book-downloader >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"
 echo "ls -all /tmp/cwa-book-downloader" >> "$LOG_DIR/permissions.txt"
-ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt"
+ls -all /tmp/cwa-book-downloader >> "$LOG_DIR/permissions.txt" 2>&1
 echo "" >> "$LOG_DIR/permissions.txt"

+# Check Iptables (NAT)
+echo "=== IPtables NAT Rules ===" > "$LOG_DIR/iptables_nat.txt"
+iptables -t nat -L -v -n >> "$LOG_DIR/iptables_nat.txt" 2>&1
+
+# Check DNS Resolution details
+echo "=== DNS Resolution Test ===" > "$LOG_DIR/dns_test.txt"
+echo "Resolving google.com:" >> "$LOG_DIR/dns_test.txt"
+nslookup google.com >> "$LOG_DIR/dns_test.txt" 2>&1
+echo "" >> "$LOG_DIR/dns_test.txt"
+echo "Resolving check.torproject.org:" >> "$LOG_DIR/dns_test.txt"
+nslookup check.torproject.org >> "$LOG_DIR/dns_test.txt" 2>&1
+

 # Check if running in Docker
 echo "=== Container Info ===" > "$LOG_DIR/container_info.txt"
@@ -125,16 +137,26 @@ fi
 env | grep -v -E "(AA_DONATOR_KEY)" | sort > "$LOG_DIR/environment.txt"

 echo "--- HTTPBin ---" >> $LOG_DIR/network_info.txt
-curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt
+curl -s https://httpbin.org/get >> $LOG_DIR/network_info.txt 2>&1
 echo "" >> $LOG_DIR/network_info.txt
 echo "--- HowsMySSL ---" >> $LOG_DIR/network_info.txt
-curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt
+curl -s https://www.howsmyssl.com/a/check >> $LOG_DIR/network_info.txt 2>&1
 echo "" >> $LOG_DIR/network_info.txt
 echo "--- IPInfo ---" >> $LOG_DIR/network_info.txt
-curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt
+curl -s https://ipinfo.io >> $LOG_DIR/network_info.txt 2>&1
 echo "" >> $LOG_DIR/network_info.txt
 echo "--- Cloudflare Trace ---" >> $LOG_DIR/network_info.txt
-curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt
+curl -s https://1.1.1.1/cdn-cgi/trace >> $LOG_DIR/network_info.txt 2>&1
+
+# Copy Tor logs if they exist
+if [ -f "/var/log/tor/notices.log" ]; then
+  cp "/var/log/tor/notices.log" "$LOG_DIR/tor_notices.log"
+fi
+
+# Copy Supervisor logs if they exist
+if [ -d "/var/log/supervisor" ]; then
+  cp -rf "/var/log/supervisor/" "$LOG_DIR/supervisor/"
+fi

 # Create the zip file directly from LOG_DIR
 ln -s "$LOG_DIR" /tmp/$OUTPUT_FILE_NAME
@@ -72,11 +72,91 @@ EOF

 echo "[*] Setting up DNS..."
 cat <<EOF > /etc/resolv.conf
-127.0.0.1
+nameserver 127.0.0.1
 EOF

 echo "[*] Starting Tor..."
-service tor start
+echo "[*] Configuring Supervisor..."
+mkdir -p /var/log/supervisor
+cat <<EOF > /etc/supervisor/supervisord.conf
+[supervisord]
+nodaemon=false
+logfile=/var/log/supervisor/supervisord.log
+pidfile=/var/run/supervisord.pid
+user=root
+
+[unix_http_server]
+file=/var/run/supervisor.sock   ; (the path to the socket file)
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL  for a unix socket
+
+[program:tor]
+command=/usr/bin/tor -f /etc/tor/torrc
+autostart=true
+autorestart=true
+startretries=100
+stdout_logfile=/var/log/supervisor/tor.log
+stderr_logfile=/var/log/supervisor/tor.err.log
+
+[program:tor-healthcheck]
+command=/app/tor_healthcheck.sh
+autostart=true
+autorestart=true
+stdout_logfile=/var/log/supervisor/healthcheck.log
+stderr_logfile=/var/log/supervisor/healthcheck.err.log
+EOF
+
+# Create healthcheck script
+cat <<'HC' > /app/tor_healthcheck.sh
+#!/bin/bash
+
+# Function to dynamically wait for Tor bootstrap
+wait_for_tor() {
+    echo "$(date): Waiting for Tor to finish bootstrapping..."
+    sleep 30
+    # Reuse the timeout logic from the main script
+    timeout 300 bash -c '
+      while ! grep -q "Bootstrapped 100%" <(tail -n 20 -F /var/log/tor/notices.log 2>/dev/null); do
+        sleep 1
+      done
+    '
+    echo "$(date): Tor seems ready (log message found)."
+}
+
+# Wait for Tor to bootstrap initially
+
+FAIL_COUNT=0
+while true; do
+    # Try to resolve/connect to google.com (timeout 10s)
+    if curl -s --head --max-time 10 https://google.com > /dev/null; then
+        # Success
+        FAIL_COUNT=0
+    else
+        FAIL_COUNT=$((FAIL_COUNT+1))
+        echo "$(date): Healthcheck failed (Count: $FAIL_COUNT)"
+    fi
+
+    # If failed 3 times in a row, restart Tor
+    if [ "$FAIL_COUNT" -ge 3 ]; then
+        echo "$(date): restart trigger - Restarting Tor..."
+        supervisorctl restart tor
+        FAIL_COUNT=0
+        
+        # Wait for it to come back using the dynamic check
+        wait_for_tor
+    fi
+
+    sleep 30
+done
+HC
+chmod +x /app/tor_healthcheck.sh
+
+echo "[*] Starting Tor via Supervisor..."
+/usr/bin/supervisord -c /etc/supervisor/supervisord.conf

 # Wait a bit to ensure Tor has bootstrapped
 echo "[*] Waiting for Tor to finish bootstrapping... (up to 5 minutes)"
@@ -105,14 +185,10 @@ iptables -t nat -A OUTPUT -p tcp --syn -j REDIRECT --to-ports 9040
 # For UDP DNS queries
 iptables -t nat -A OUTPUT -p udp --dport 53 ! -d 127.0.0.1 -j DNAT --to-destination 127.0.0.1:53

+
 # For TCP DNS queries (some DNS queries may use TCP)
 iptables -t nat -A OUTPUT -p tcp --dport 53 ! -d 127.0.0.1 -j DNAT --to-destination 127.0.0.1:53

-# Note: ICMP (ping) is NOT routed through Tor as Tor only supports TCP.
-# ICMP will use default routing. If you need to test connectivity, use:
-# curl -s https://check.torproject.org/api/ip
-# or: curl -s https://icanhazip.com
-
 echo "[✓] Transparent Tor routing enabled."

 sleep 5
@@ -165,45 +241,5 @@ else
    echo "[*] Falling back to container's default timezone: $TZ"
 fi

-# Start a background health check process to monitor Tor
-echo "[*] Starting Tor health check monitor..."
-(
-    check_count=0
-    while true; do
-        sleep 300  # Check every 5 minutes
-        check_count=$((check_count + 1))
-        echo "[*] Tor health check #$check_count at $(date)"
-        
-        # Check if Tor service is running
-        if ! service tor status > /dev/null 2>&1; then
-            echo "[!] $(date): Tor service not running, restarting..."
-            service tor restart
-            sleep 10
-        fi
-        
-        # Test DNS resolution through Tor
-        if ! timeout 10 nslookup google.com 127.0.0.1 > /dev/null 2>&1; then
-            echo "[!] $(date): DNS resolution failed, reloading Tor..."
-            service tor reload
-            sleep 5
-            # Verify DNS works after reload
-            if timeout 10 nslookup google.com 127.0.0.1 > /dev/null 2>&1; then
-                echo "[✓] $(date): DNS resolution restored"
-            else
-                echo "[✗] $(date): DNS still failing after reload, restarting Tor..."
-                service tor restart
-                sleep 10
-            fi
-        fi
-        
-        # Send SIGHUP to Tor to rotate circuits (helps with stale circuits)
-        echo "[*] $(date): Rotating Tor circuits..."
-        pkill -HUP tor || true
-    done
-) >> $LOG_FILE 2>&1 &
-
-TOR_MONITOR_PID=$!
-echo "[✓] Tor health check monitor started in background (PID: $TOR_MONITOR_PID)"
-
 # Run the entrypoint script
 echo "[*] End of tor script"