From 53f37510f392cda1dcc130a978df01e134b4a0fb Mon Sep 17 00:00:00 2001 From: Username Date: Tue, 23 Dec 2025 17:47:12 +0100 Subject: [PATCH] dashboard: add system monitoring and enhanced stats - prominent check type badge in header (SSL/judges/http/irc) - system monitor bar: load, memory, disk, process RSS - anonymity breakdown: elite/anonymous/transparent counts - database health: size, recent activity, dead proxy count - enhanced Tor pool stats: requests, success rate, latency - SQLite ANALYZE/VACUUM functions for query optimization - database statistics API functions --- dbs.py | 77 ++++++++++++++++ httpd.py | 273 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 341 insertions(+), 9 deletions(-) diff --git a/dbs.py b/dbs.py index aed53bd..f02fcfb 100644 --- a/dbs.py +++ b/dbs.py @@ -476,3 +476,80 @@ def get_stats_history(sqlite, hours=24): 'proto_http', 'proto_socks4', 'proto_socks5'] return [dict(zip(cols, row)) for row in rows] + + +def analyze_database(sqlite): + """Run ANALYZE to update SQLite query planner statistics. + + Should be called periodically (e.g., hourly) for optimal query performance. + Also enables stat4 for better index statistics on complex queries. + + Args: + sqlite: Database connection + """ + try: + # Enable advanced statistics (persists in database) + sqlite.execute('PRAGMA analysis_limit=1000') + # Run ANALYZE on all tables and indexes + sqlite.execute('ANALYZE') + sqlite.commit() + _log('database ANALYZE completed', 'debug') + except Exception as e: + _log('database ANALYZE failed: %s' % str(e), 'warn') + + +def vacuum_database(sqlite): + """Run VACUUM to reclaim unused space and defragment database. + + Should be called infrequently (e.g., daily or weekly) as it's expensive. + Requires no active transactions. + + Args: + sqlite: Database connection + """ + try: + sqlite.execute('VACUUM') + _log('database VACUUM completed', 'info') + except Exception as e: + _log('database VACUUM failed: %s' % str(e), 'warn') + + +def get_database_stats(sqlite): + """Get database statistics for monitoring. + + Args: + sqlite: Database connection + + Returns: + Dict with database statistics + """ + stats = {} + try: + row = sqlite.execute('PRAGMA page_count').fetchone() + stats['page_count'] = row[0] if row else 0 + + row = sqlite.execute('PRAGMA page_size').fetchone() + stats['page_size'] = row[0] if row else 4096 + + row = sqlite.execute('PRAGMA freelist_count').fetchone() + stats['freelist_count'] = row[0] if row else 0 + + # Calculate sizes + stats['total_size'] = stats['page_count'] * stats['page_size'] + stats['free_size'] = stats['freelist_count'] * stats['page_size'] + stats['used_size'] = stats['total_size'] - stats['free_size'] + + # Table row counts + row = sqlite.execute('SELECT COUNT(*) FROM proxylist').fetchone() + stats['proxy_count'] = row[0] if row else 0 + + row = sqlite.execute('SELECT COUNT(*) FROM proxylist WHERE failed=0').fetchone() + stats['working_count'] = row[0] if row else 0 + + row = sqlite.execute('SELECT COUNT(*) FROM uris').fetchone() + stats['uri_count'] = row[0] if row else 0 + + except Exception: + pass + + return stats diff --git a/httpd.py b/httpd.py index d4a5a52..6a77bde 100644 --- a/httpd.py +++ b/httpd.py @@ -6,9 +6,137 @@ import BaseHTTPServer import json import threading import time +import os import mysqlite from misc import _log + +def get_system_stats(): + """Collect system resource statistics.""" + stats = {} + + # Load average (1, 5, 15 min) + try: + load = os.getloadavg() + stats['load_1m'] = round(load[0], 2) + stats['load_5m'] = round(load[1], 2) + stats['load_15m'] = round(load[2], 2) + except (OSError, AttributeError): + stats['load_1m'] = stats['load_5m'] = stats['load_15m'] = 0 + + # CPU count + try: + stats['cpu_count'] = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + stats['cpu_count'] = 1 + + # Memory from /proc/meminfo (Linux) + try: + with open('/proc/meminfo', 'r') as f: + meminfo = {} + for line in f: + parts = line.split() + if len(parts) >= 2: + meminfo[parts[0].rstrip(':')] = int(parts[1]) * 1024 # KB to bytes + total = meminfo.get('MemTotal', 0) + available = meminfo.get('MemAvailable', meminfo.get('MemFree', 0)) + stats['mem_total'] = total + stats['mem_available'] = available + stats['mem_used'] = total - available + stats['mem_pct'] = round((total - available) / total * 100, 1) if total > 0 else 0 + except (IOError, KeyError, ZeroDivisionError): + stats['mem_total'] = stats['mem_available'] = stats['mem_used'] = 0 + stats['mem_pct'] = 0 + + # Disk usage for data directory + try: + st = os.statvfs('data' if os.path.exists('data') else '.') + total = st.f_blocks * st.f_frsize + free = st.f_bavail * st.f_frsize + used = total - free + stats['disk_total'] = total + stats['disk_free'] = free + stats['disk_used'] = used + stats['disk_pct'] = round(used / total * 100, 1) if total > 0 else 0 + except (OSError, ZeroDivisionError): + stats['disk_total'] = stats['disk_free'] = stats['disk_used'] = 0 + stats['disk_pct'] = 0 + + # Process stats from /proc/self/status + try: + with open('/proc/self/status', 'r') as f: + for line in f: + if line.startswith('VmRSS:'): + stats['proc_rss'] = int(line.split()[1]) * 1024 # KB to bytes + elif line.startswith('Threads:'): + stats['proc_threads'] = int(line.split()[1]) + except (IOError, ValueError, IndexError): + stats['proc_rss'] = 0 + stats['proc_threads'] = 0 + + return stats + + +def get_db_health(db): + """Get database health and statistics.""" + stats = {} + try: + # Database file size + db_path = db.path if hasattr(db, 'path') else 'data/proxies.sqlite' + if os.path.exists(db_path): + stats['db_size'] = os.path.getsize(db_path) + else: + stats['db_size'] = 0 + + # Page stats from pragma + row = db.execute('PRAGMA page_count').fetchone() + stats['page_count'] = row[0] if row else 0 + row = db.execute('PRAGMA page_size').fetchone() + stats['page_size'] = row[0] if row else 0 + row = db.execute('PRAGMA freelist_count').fetchone() + stats['freelist_count'] = row[0] if row else 0 + + # Anonymity breakdown + rows = db.execute( + 'SELECT anonymity, COUNT(*) FROM proxylist WHERE failed=0 GROUP BY anonymity' + ).fetchall() + stats['anonymity'] = {r[0] or 'unknown': r[1] for r in rows} + + # Latency stats + row = db.execute( + 'SELECT AVG(avg_latency), MIN(avg_latency), MAX(avg_latency) ' + 'FROM proxylist WHERE failed=0 AND avg_latency > 0' + ).fetchone() + if row and row[0]: + stats['db_avg_latency'] = round(row[0], 1) + stats['db_min_latency'] = round(row[1], 1) + stats['db_max_latency'] = round(row[2], 1) + else: + stats['db_avg_latency'] = stats['db_min_latency'] = stats['db_max_latency'] = 0 + + # Recent activity + now = int(time.time()) + row = db.execute( + 'SELECT COUNT(*) FROM proxylist WHERE tested > ?', (now - 3600,) + ).fetchone() + stats['tested_last_hour'] = row[0] if row else 0 + + row = db.execute( + 'SELECT COUNT(*) FROM proxylist WHERE added > ?', (now - 86400,) + ).fetchone() + stats['added_last_day'] = row[0] if row else 0 + + # Dead proxies count + row = db.execute( + 'SELECT COUNT(*) FROM proxylist WHERE failed > 0' + ).fetchone() + stats['dead_count'] = row[0] if row else 0 + + except Exception: + pass + + return stats + # Detect if gevent has monkey-patched the environment try: from gevent import monkey @@ -61,6 +189,19 @@ body { .dot { width: 6px; height: 6px; border-radius: 50%; background: var(--green); animation: pulse 2s infinite; } .dot.err { background: var(--red); animation: none; } @keyframes pulse { 50% { opacity: 0.5; } } +.mode-badge { padding: 4px 10px; border-radius: 4px; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; } +.mode-ssl { background: rgba(63,185,80,0.2); color: var(--green); border: 1px solid var(--green); } +.mode-judges { background: rgba(88,166,255,0.2); color: var(--blue); border: 1px solid var(--blue); } +.mode-http { background: rgba(210,153,34,0.2); color: var(--yellow); border: 1px solid var(--yellow); } +.mode-irc { background: rgba(163,113,247,0.2); color: var(--purple); border: 1px solid var(--purple); } + +/* System monitor bar */ +.sysbar { display: flex; gap: 16px; padding: 8px 12px; background: var(--card); border: 1px solid var(--border); border-radius: 6px; margin-bottom: 16px; font-size: 11px; } +.sysbar-item { display: flex; align-items: center; gap: 6px; } +.sysbar-lbl { color: var(--dim); } +.sysbar-val { font-weight: 600; font-feature-settings: "tnum"; } +.sysbar-bar { width: 50px; height: 4px; background: var(--border); border-radius: 2px; overflow: hidden; } +.sysbar-fill { height: 100%; border-radius: 2px; transition: width 0.3s; } /* Grid */ .g { display: grid; gap: 12px; margin-bottom: 16px; } @@ -187,6 +328,14 @@ var fmt = function(n) { return n == null ? '-' : n.toLocaleString(); }; var fmtDec = function(n, d) { return n == null ? '-' : n.toFixed(d || 1); }; var pct = function(n, t) { return t > 0 ? ((n / t) * 100).toFixed(1) : '0.0'; }; +function fmtBytes(b) { + if (!b || b <= 0) return '-'; + var units = ['B', 'KB', 'MB', 'GB', 'TB']; + var i = 0; + while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; } + return b.toFixed(i > 0 ? 1 : 0) + ' ' + units[i]; +} + function fmtTime(s) { if (!s) return '-'; var d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60); @@ -195,6 +344,12 @@ function fmtTime(s) { return m + 'm ' + Math.floor(s % 60) + 's'; } +function setBarColor(el, pct) { + if (pct > 90) return 'background:var(--red)'; + if (pct > 70) return 'background:var(--yellow)'; + return 'background:var(--green)'; +} + function fmtMs(ms) { if (!ms || ms <= 0) return '-'; if (ms < 1000) return Math.round(ms) + 'ms'; @@ -258,6 +413,27 @@ function renderLeaderboard(id, data, nameKey, valKey, limit) { function update(d) { $('dot').className = 'dot'; $('statusTxt').textContent = 'Live'; + // Check type badge (prominent display) + var ct = d.checktype || 'unknown'; + var ctBadge = $('checktypeBadge'); + if (ctBadge) { + ctBadge.textContent = ct.toUpperCase(); + ctBadge.className = 'mode-badge mode-' + ct; + } + + // System monitor bar + var sys = d.system || {}; + $('sysLoad').textContent = (sys.load_1m || 0) + ' / ' + (sys.cpu_count || 1); + $('sysMemVal').textContent = fmtBytes(sys.mem_used) + ' / ' + fmtBytes(sys.mem_total); + $('sysMemPct').textContent = (sys.mem_pct || 0) + '%'; + var memFill = $('sysMemFill'); + if (memFill) { memFill.style.width = (sys.mem_pct || 0) + '%'; memFill.style.cssText = 'width:' + (sys.mem_pct || 0) + '%;' + setBarColor(memFill, sys.mem_pct || 0); } + $('sysDiskVal').textContent = fmtBytes(sys.disk_used) + ' / ' + fmtBytes(sys.disk_total); + $('sysDiskPct').textContent = (sys.disk_pct || 0) + '%'; + var diskFill = $('sysDiskFill'); + if (diskFill) { diskFill.style.width = (sys.disk_pct || 0) + '%'; diskFill.style.cssText = 'width:' + (sys.disk_pct || 0) + '%;' + setBarColor(diskFill, sys.disk_pct || 0); } + $('sysProcMem').textContent = fmtBytes(sys.proc_rss); + // Main stats - db stats are nested under d.db var db = d.db || {}; $('working').textContent = fmt(db.working); @@ -298,7 +474,6 @@ function update(d) { setBar('threadBar', d.threads, d.max_threads, 'blu'); $('queue').textContent = fmt(d.queue_size); $('uptime').textContent = fmtTime(d.uptime_seconds); - $('checktype').textContent = d.checktype || '-'; // Charts renderLineChart('rateChart', d.rate_history, '#58a6ff', d.peak_rate * 1.1); @@ -429,6 +604,41 @@ function update(d) { $('certErrors').className = 'stat-val ' + (ssl.cert_errors > 0 ? 'yel' : 'grn'); } + // Anonymity breakdown + var dbh = d.db_health || {}; + if (dbh.anonymity) { + var anonHtml = ''; + var anonColors = {elite: 'grn', anonymous: 'blu', transparent: 'yel', unknown: 'dim'}; + var anonOrder = ['elite', 'anonymous', 'transparent', 'unknown']; + anonOrder.forEach(function(level) { + var count = dbh.anonymity[level] || 0; + if (count > 0) { + anonHtml += '
' + level.charAt(0).toUpperCase() + level.slice(1) + ''; + anonHtml += '' + fmt(count) + '
'; + } + }); + $('anonBreakdown').innerHTML = anonHtml || '
No data
'; + } + + // Database health + if (dbh.db_size) { + $('dbSize').textContent = fmtBytes(dbh.db_size); + $('dbTestedHour').textContent = fmt(dbh.tested_last_hour); + $('dbAddedDay').textContent = fmt(dbh.added_last_day); + $('dbDead').textContent = fmt(dbh.dead_count); + } + + // Tor pool enhanced stats + if (d.tor_pool) { + var tp = d.tor_pool; + $('torTotal').textContent = fmt(tp.total_requests || 0); + $('torSuccess').textContent = fmtDec(tp.success_rate || 0, 1) + '%'; + $('torHealthy').textContent = (tp.healthy_count || 0) + '/' + (tp.total_count || 0); + if (tp.avg_latency) { + $('torLatency').textContent = fmtMs(tp.avg_latency); + } + } + $('lastUpdate').textContent = new Date().toLocaleTimeString(); } @@ -456,11 +666,24 @@ DASHBOARD_HTML = '''

PPF Dashboard

+ -
Connecting
Updated: -
+ +
+
Load:-
+
Memory:- +
+ -
+
Disk:- +
+ -
+
Process:-
+
+
@@ -598,11 +821,10 @@ DASHBOARD_HTML = '''
-
System
-
Threads-
+
Worker Pool
+
Active Threads-
-
Queue Size-
-
Check Type-
+
Job Queue-
Judge Services
@@ -613,10 +835,22 @@ DASHBOARD_HTML = '''
- -
-
Tor Pool
-
+ +
+
+
Tor Pool
+
Total Requests-
+
Success Rate-
+
Healthy Nodes-
+
Avg Latency-
+
Exit Nodes
+
+
+
+
Anonymity Levels
+
+
Elite = no headers, Anonymous = adds headers, Transparent = reveals IP
+
@@ -643,6 +877,24 @@ DASHBOARD_HTML = '''
Database Overview
+
+
+
Database Size
+
-
+
+
+
Tested (1h)
+
-
+
+
+
Added (24h)
+
-
+
+
+
Dead Proxies
+
-
+
+
Working by Protocol
@@ -972,10 +1224,13 @@ class ProxyAPIServer(threading.Thread): stats = {} if self.stats_provider: stats = self.stats_provider() + # Add system stats + stats['system'] = get_system_stats() # Add database stats try: db = mysqlite.mysqlite(self.database, str) stats['db'] = self._get_db_stats(db) + stats['db_health'] = get_db_health(db) except Exception: pass return json.dumps(stats, indent=2), 'application/json', 200