dashboard: add system monitoring and enhanced stats

- prominent check type badge in header (SSL/judges/http/irc)
- system monitor bar: load, memory, disk, process RSS
- anonymity breakdown: elite/anonymous/transparent counts
- database health: size, recent activity, dead proxy count
- enhanced Tor pool stats: requests, success rate, latency
- SQLite ANALYZE/VACUUM functions for query optimization
- database statistics API functions
This commit is contained in:
Username
2025-12-23 17:47:12 +01:00
parent 20fc1b01fd
commit 53f37510f3
2 changed files with 341 additions and 9 deletions

77
dbs.py
View File

@@ -476,3 +476,80 @@ def get_stats_history(sqlite, hours=24):
'proto_http', 'proto_socks4', 'proto_socks5']
return [dict(zip(cols, row)) for row in rows]
def analyze_database(sqlite):
"""Run ANALYZE to update SQLite query planner statistics.
Should be called periodically (e.g., hourly) for optimal query performance.
Also enables stat4 for better index statistics on complex queries.
Args:
sqlite: Database connection
"""
try:
# Enable advanced statistics (persists in database)
sqlite.execute('PRAGMA analysis_limit=1000')
# Run ANALYZE on all tables and indexes
sqlite.execute('ANALYZE')
sqlite.commit()
_log('database ANALYZE completed', 'debug')
except Exception as e:
_log('database ANALYZE failed: %s' % str(e), 'warn')
def vacuum_database(sqlite):
"""Run VACUUM to reclaim unused space and defragment database.
Should be called infrequently (e.g., daily or weekly) as it's expensive.
Requires no active transactions.
Args:
sqlite: Database connection
"""
try:
sqlite.execute('VACUUM')
_log('database VACUUM completed', 'info')
except Exception as e:
_log('database VACUUM failed: %s' % str(e), 'warn')
def get_database_stats(sqlite):
"""Get database statistics for monitoring.
Args:
sqlite: Database connection
Returns:
Dict with database statistics
"""
stats = {}
try:
row = sqlite.execute('PRAGMA page_count').fetchone()
stats['page_count'] = row[0] if row else 0
row = sqlite.execute('PRAGMA page_size').fetchone()
stats['page_size'] = row[0] if row else 4096
row = sqlite.execute('PRAGMA freelist_count').fetchone()
stats['freelist_count'] = row[0] if row else 0
# Calculate sizes
stats['total_size'] = stats['page_count'] * stats['page_size']
stats['free_size'] = stats['freelist_count'] * stats['page_size']
stats['used_size'] = stats['total_size'] - stats['free_size']
# Table row counts
row = sqlite.execute('SELECT COUNT(*) FROM proxylist').fetchone()
stats['proxy_count'] = row[0] if row else 0
row = sqlite.execute('SELECT COUNT(*) FROM proxylist WHERE failed=0').fetchone()
stats['working_count'] = row[0] if row else 0
row = sqlite.execute('SELECT COUNT(*) FROM uris').fetchone()
stats['uri_count'] = row[0] if row else 0
except Exception:
pass
return stats

273
httpd.py
View File

@@ -6,9 +6,137 @@ import BaseHTTPServer
import json
import threading
import time
import os
import mysqlite
from misc import _log
def get_system_stats():
"""Collect system resource statistics."""
stats = {}
# Load average (1, 5, 15 min)
try:
load = os.getloadavg()
stats['load_1m'] = round(load[0], 2)
stats['load_5m'] = round(load[1], 2)
stats['load_15m'] = round(load[2], 2)
except (OSError, AttributeError):
stats['load_1m'] = stats['load_5m'] = stats['load_15m'] = 0
# CPU count
try:
stats['cpu_count'] = os.sysconf('SC_NPROCESSORS_ONLN')
except (ValueError, OSError, AttributeError):
stats['cpu_count'] = 1
# Memory from /proc/meminfo (Linux)
try:
with open('/proc/meminfo', 'r') as f:
meminfo = {}
for line in f:
parts = line.split()
if len(parts) >= 2:
meminfo[parts[0].rstrip(':')] = int(parts[1]) * 1024 # KB to bytes
total = meminfo.get('MemTotal', 0)
available = meminfo.get('MemAvailable', meminfo.get('MemFree', 0))
stats['mem_total'] = total
stats['mem_available'] = available
stats['mem_used'] = total - available
stats['mem_pct'] = round((total - available) / total * 100, 1) if total > 0 else 0
except (IOError, KeyError, ZeroDivisionError):
stats['mem_total'] = stats['mem_available'] = stats['mem_used'] = 0
stats['mem_pct'] = 0
# Disk usage for data directory
try:
st = os.statvfs('data' if os.path.exists('data') else '.')
total = st.f_blocks * st.f_frsize
free = st.f_bavail * st.f_frsize
used = total - free
stats['disk_total'] = total
stats['disk_free'] = free
stats['disk_used'] = used
stats['disk_pct'] = round(used / total * 100, 1) if total > 0 else 0
except (OSError, ZeroDivisionError):
stats['disk_total'] = stats['disk_free'] = stats['disk_used'] = 0
stats['disk_pct'] = 0
# Process stats from /proc/self/status
try:
with open('/proc/self/status', 'r') as f:
for line in f:
if line.startswith('VmRSS:'):
stats['proc_rss'] = int(line.split()[1]) * 1024 # KB to bytes
elif line.startswith('Threads:'):
stats['proc_threads'] = int(line.split()[1])
except (IOError, ValueError, IndexError):
stats['proc_rss'] = 0
stats['proc_threads'] = 0
return stats
def get_db_health(db):
"""Get database health and statistics."""
stats = {}
try:
# Database file size
db_path = db.path if hasattr(db, 'path') else 'data/proxies.sqlite'
if os.path.exists(db_path):
stats['db_size'] = os.path.getsize(db_path)
else:
stats['db_size'] = 0
# Page stats from pragma
row = db.execute('PRAGMA page_count').fetchone()
stats['page_count'] = row[0] if row else 0
row = db.execute('PRAGMA page_size').fetchone()
stats['page_size'] = row[0] if row else 0
row = db.execute('PRAGMA freelist_count').fetchone()
stats['freelist_count'] = row[0] if row else 0
# Anonymity breakdown
rows = db.execute(
'SELECT anonymity, COUNT(*) FROM proxylist WHERE failed=0 GROUP BY anonymity'
).fetchall()
stats['anonymity'] = {r[0] or 'unknown': r[1] for r in rows}
# Latency stats
row = db.execute(
'SELECT AVG(avg_latency), MIN(avg_latency), MAX(avg_latency) '
'FROM proxylist WHERE failed=0 AND avg_latency > 0'
).fetchone()
if row and row[0]:
stats['db_avg_latency'] = round(row[0], 1)
stats['db_min_latency'] = round(row[1], 1)
stats['db_max_latency'] = round(row[2], 1)
else:
stats['db_avg_latency'] = stats['db_min_latency'] = stats['db_max_latency'] = 0
# Recent activity
now = int(time.time())
row = db.execute(
'SELECT COUNT(*) FROM proxylist WHERE tested > ?', (now - 3600,)
).fetchone()
stats['tested_last_hour'] = row[0] if row else 0
row = db.execute(
'SELECT COUNT(*) FROM proxylist WHERE added > ?', (now - 86400,)
).fetchone()
stats['added_last_day'] = row[0] if row else 0
# Dead proxies count
row = db.execute(
'SELECT COUNT(*) FROM proxylist WHERE failed > 0'
).fetchone()
stats['dead_count'] = row[0] if row else 0
except Exception:
pass
return stats
# Detect if gevent has monkey-patched the environment
try:
from gevent import monkey
@@ -61,6 +189,19 @@ body {
.dot { width: 6px; height: 6px; border-radius: 50%; background: var(--green); animation: pulse 2s infinite; }
.dot.err { background: var(--red); animation: none; }
@keyframes pulse { 50% { opacity: 0.5; } }
.mode-badge { padding: 4px 10px; border-radius: 4px; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; }
.mode-ssl { background: rgba(63,185,80,0.2); color: var(--green); border: 1px solid var(--green); }
.mode-judges { background: rgba(88,166,255,0.2); color: var(--blue); border: 1px solid var(--blue); }
.mode-http { background: rgba(210,153,34,0.2); color: var(--yellow); border: 1px solid var(--yellow); }
.mode-irc { background: rgba(163,113,247,0.2); color: var(--purple); border: 1px solid var(--purple); }
/* System monitor bar */
.sysbar { display: flex; gap: 16px; padding: 8px 12px; background: var(--card); border: 1px solid var(--border); border-radius: 6px; margin-bottom: 16px; font-size: 11px; }
.sysbar-item { display: flex; align-items: center; gap: 6px; }
.sysbar-lbl { color: var(--dim); }
.sysbar-val { font-weight: 600; font-feature-settings: "tnum"; }
.sysbar-bar { width: 50px; height: 4px; background: var(--border); border-radius: 2px; overflow: hidden; }
.sysbar-fill { height: 100%; border-radius: 2px; transition: width 0.3s; }
/* Grid */
.g { display: grid; gap: 12px; margin-bottom: 16px; }
@@ -187,6 +328,14 @@ var fmt = function(n) { return n == null ? '-' : n.toLocaleString(); };
var fmtDec = function(n, d) { return n == null ? '-' : n.toFixed(d || 1); };
var pct = function(n, t) { return t > 0 ? ((n / t) * 100).toFixed(1) : '0.0'; };
function fmtBytes(b) {
if (!b || b <= 0) return '-';
var units = ['B', 'KB', 'MB', 'GB', 'TB'];
var i = 0;
while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
return b.toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
}
function fmtTime(s) {
if (!s) return '-';
var d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
@@ -195,6 +344,12 @@ function fmtTime(s) {
return m + 'm ' + Math.floor(s % 60) + 's';
}
function setBarColor(el, pct) {
if (pct > 90) return 'background:var(--red)';
if (pct > 70) return 'background:var(--yellow)';
return 'background:var(--green)';
}
function fmtMs(ms) {
if (!ms || ms <= 0) return '-';
if (ms < 1000) return Math.round(ms) + 'ms';
@@ -258,6 +413,27 @@ function renderLeaderboard(id, data, nameKey, valKey, limit) {
function update(d) {
$('dot').className = 'dot'; $('statusTxt').textContent = 'Live';
// Check type badge (prominent display)
var ct = d.checktype || 'unknown';
var ctBadge = $('checktypeBadge');
if (ctBadge) {
ctBadge.textContent = ct.toUpperCase();
ctBadge.className = 'mode-badge mode-' + ct;
}
// System monitor bar
var sys = d.system || {};
$('sysLoad').textContent = (sys.load_1m || 0) + ' / ' + (sys.cpu_count || 1);
$('sysMemVal').textContent = fmtBytes(sys.mem_used) + ' / ' + fmtBytes(sys.mem_total);
$('sysMemPct').textContent = (sys.mem_pct || 0) + '%';
var memFill = $('sysMemFill');
if (memFill) { memFill.style.width = (sys.mem_pct || 0) + '%'; memFill.style.cssText = 'width:' + (sys.mem_pct || 0) + '%;' + setBarColor(memFill, sys.mem_pct || 0); }
$('sysDiskVal').textContent = fmtBytes(sys.disk_used) + ' / ' + fmtBytes(sys.disk_total);
$('sysDiskPct').textContent = (sys.disk_pct || 0) + '%';
var diskFill = $('sysDiskFill');
if (diskFill) { diskFill.style.width = (sys.disk_pct || 0) + '%'; diskFill.style.cssText = 'width:' + (sys.disk_pct || 0) + '%;' + setBarColor(diskFill, sys.disk_pct || 0); }
$('sysProcMem').textContent = fmtBytes(sys.proc_rss);
// Main stats - db stats are nested under d.db
var db = d.db || {};
$('working').textContent = fmt(db.working);
@@ -298,7 +474,6 @@ function update(d) {
setBar('threadBar', d.threads, d.max_threads, 'blu');
$('queue').textContent = fmt(d.queue_size);
$('uptime').textContent = fmtTime(d.uptime_seconds);
$('checktype').textContent = d.checktype || '-';
// Charts
renderLineChart('rateChart', d.rate_history, '#58a6ff', d.peak_rate * 1.1);
@@ -429,6 +604,41 @@ function update(d) {
$('certErrors').className = 'stat-val ' + (ssl.cert_errors > 0 ? 'yel' : 'grn');
}
// Anonymity breakdown
var dbh = d.db_health || {};
if (dbh.anonymity) {
var anonHtml = '';
var anonColors = {elite: 'grn', anonymous: 'blu', transparent: 'yel', unknown: 'dim'};
var anonOrder = ['elite', 'anonymous', 'transparent', 'unknown'];
anonOrder.forEach(function(level) {
var count = dbh.anonymity[level] || 0;
if (count > 0) {
anonHtml += '<div class="stat-row"><span class="stat-lbl">' + level.charAt(0).toUpperCase() + level.slice(1) + '</span>';
anonHtml += '<span class="stat-val ' + anonColors[level] + '">' + fmt(count) + '</span></div>';
}
});
$('anonBreakdown').innerHTML = anonHtml || '<div style="color:var(--dim)">No data</div>';
}
// Database health
if (dbh.db_size) {
$('dbSize').textContent = fmtBytes(dbh.db_size);
$('dbTestedHour').textContent = fmt(dbh.tested_last_hour);
$('dbAddedDay').textContent = fmt(dbh.added_last_day);
$('dbDead').textContent = fmt(dbh.dead_count);
}
// Tor pool enhanced stats
if (d.tor_pool) {
var tp = d.tor_pool;
$('torTotal').textContent = fmt(tp.total_requests || 0);
$('torSuccess').textContent = fmtDec(tp.success_rate || 0, 1) + '%';
$('torHealthy').textContent = (tp.healthy_count || 0) + '/' + (tp.total_count || 0);
if (tp.avg_latency) {
$('torLatency').textContent = fmtMs(tp.avg_latency);
}
}
$('lastUpdate').textContent = new Date().toLocaleTimeString();
}
@@ -456,11 +666,24 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
<div class="hdr">
<h1>PPF Dashboard</h1>
<div class="status">
<span class="mode-badge mode-ssl" id="checktypeBadge">-</span>
<div class="status-item"><div class="dot" id="dot"></div><span id="statusTxt">Connecting</span></div>
<div class="status-item">Updated: <span id="lastUpdate">-</span></div>
</div>
</div>
<!-- System Monitor Bar -->
<div class="sysbar">
<div class="sysbar-item"><span class="sysbar-lbl">Load:</span><span class="sysbar-val" id="sysLoad">-</span></div>
<div class="sysbar-item"><span class="sysbar-lbl">Memory:</span><span class="sysbar-val" id="sysMemVal">-</span>
<div class="sysbar-bar"><div class="sysbar-fill" id="sysMemFill" style="width:0"></div></div>
<span class="sysbar-val" id="sysMemPct">-</span></div>
<div class="sysbar-item"><span class="sysbar-lbl">Disk:</span><span class="sysbar-val" id="sysDiskVal">-</span>
<div class="sysbar-bar"><div class="sysbar-fill" id="sysDiskFill" style="width:0"></div></div>
<span class="sysbar-val" id="sysDiskPct">-</span></div>
<div class="sysbar-item"><span class="sysbar-lbl">Process:</span><span class="sysbar-val" id="sysProcMem">-</span></div>
</div>
<!-- Primary Stats Row -->
<div class="g g5">
<div class="c">
@@ -598,11 +821,10 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
<!-- System & Infrastructure -->
<div class="g g2">
<div class="c">
<div class="sec-hdr" style="margin-top:0">System</div>
<div class="stat-row"><span class="stat-lbl">Threads</span><span class="stat-val" id="threads">-</span></div>
<div class="sec-hdr" style="margin-top:0">Worker Pool</div>
<div class="stat-row"><span class="stat-lbl">Active Threads</span><span class="stat-val" id="threads">-</span></div>
<div class="bar-wrap"><div class="bar blu" id="threadBar" style="width:0"></div></div>
<div class="stat-row" style="margin-top:8px"><span class="stat-lbl">Queue Size</span><span class="stat-val yel" id="queue">-</span></div>
<div class="stat-row"><span class="stat-lbl">Check Type</span><span class="stat-val" id="checktype">-</span></div>
<div class="stat-row" style="margin-top:8px"><span class="stat-lbl">Job Queue</span><span class="stat-val yel" id="queue">-</span></div>
</div>
<div class="c">
<div class="sec-hdr" style="margin-top:0">Judge Services</div>
@@ -613,10 +835,22 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
</div>
</div>
<!-- Tor Pool -->
<div class="sec">
<div class="sec-hdr">Tor Pool</div>
<div class="g g3" id="torPool"></div>
<!-- Tor Pool & Anonymity -->
<div class="g g2">
<div class="c">
<div class="sec-hdr" style="margin-top:0">Tor Pool</div>
<div class="stat-row"><span class="stat-lbl">Total Requests</span><span class="stat-val" id="torTotal">-</span></div>
<div class="stat-row"><span class="stat-lbl">Success Rate</span><span class="stat-val grn" id="torSuccess">-</span></div>
<div class="stat-row"><span class="stat-lbl">Healthy Nodes</span><span class="stat-val" id="torHealthy">-</span></div>
<div class="stat-row"><span class="stat-lbl">Avg Latency</span><span class="stat-val cyn" id="torLatency">-</span></div>
<div class="lbl" style="margin-top:10px">Exit Nodes</div>
<div class="g g3" id="torPool" style="margin-top:8px"></div>
</div>
<div class="c">
<div class="sec-hdr" style="margin-top:0">Anonymity Levels</div>
<div id="anonBreakdown"></div>
<div class="sub" style="margin-top:8px;font-size:10px">Elite = no headers, Anonymous = adds headers, Transparent = reveals IP</div>
</div>
</div>
<!-- Scraper & SSL Stats -->
@@ -643,6 +877,24 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
<!-- Database Stats -->
<div class="sec">
<div class="sec-hdr">Database Overview</div>
<div class="g g4">
<div class="c c-sm">
<div class="lbl">Database Size</div>
<div class="val-sm cyn" id="dbSize">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Tested (1h)</div>
<div class="val-sm blu" id="dbTestedHour">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Added (24h)</div>
<div class="val-sm grn" id="dbAddedDay">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Dead Proxies</div>
<div class="val-sm red" id="dbDead">-</div>
</div>
</div>
<div class="g g2">
<div class="c">
<div class="lbl">Working by Protocol</div>
@@ -972,10 +1224,13 @@ class ProxyAPIServer(threading.Thread):
stats = {}
if self.stats_provider:
stats = self.stats_provider()
# Add system stats
stats['system'] = get_system_stats()
# Add database stats
try:
db = mysqlite.mysqlite(self.database, str)
stats['db'] = self._get_db_stats(db)
stats['db_health'] = get_db_health(db)
except Exception:
pass
return json.dumps(stats, indent=2), 'application/json', 200