dashboard: add system monitoring and enhanced stats

- prominent check type badge in header (SSL/judges/http/irc)
- system monitor bar: load, memory, disk, process RSS
- anonymity breakdown: elite/anonymous/transparent counts
- database health: size, recent activity, dead proxy count
- enhanced Tor pool stats: requests, success rate, latency
- SQLite ANALYZE/VACUUM functions for query optimization
- database statistics API functions

dbs.py

@@ -476,3 +476,80 @@ def get_stats_history(sqlite, hours=24):
             'proto_http', 'proto_socks4', 'proto_socks5']
 
     return [dict(zip(cols, row)) for row in rows]
+
+
+def analyze_database(sqlite):
+    """Run ANALYZE to update SQLite query planner statistics.
+
+    Should be called periodically (e.g., hourly) for optimal query performance.
+    Also enables stat4 for better index statistics on complex queries.
+
+    Args:
+        sqlite: Database connection
+    """
+    try:
+        # Enable advanced statistics (persists in database)
+        sqlite.execute('PRAGMA analysis_limit=1000')
+        # Run ANALYZE on all tables and indexes
+        sqlite.execute('ANALYZE')
+        sqlite.commit()
+        _log('database ANALYZE completed', 'debug')
+    except Exception as e:
+        _log('database ANALYZE failed: %s' % str(e), 'warn')
+
+
+def vacuum_database(sqlite):
+    """Run VACUUM to reclaim unused space and defragment database.
+
+    Should be called infrequently (e.g., daily or weekly) as it's expensive.
+    Requires no active transactions.
+
+    Args:
+        sqlite: Database connection
+    """
+    try:
+        sqlite.execute('VACUUM')
+        _log('database VACUUM completed', 'info')
+    except Exception as e:
+        _log('database VACUUM failed: %s' % str(e), 'warn')
+
+
+def get_database_stats(sqlite):
+    """Get database statistics for monitoring.
+
+    Args:
+        sqlite: Database connection
+
+    Returns:
+        Dict with database statistics
+    """
+    stats = {}
+    try:
+        row = sqlite.execute('PRAGMA page_count').fetchone()
+        stats['page_count'] = row[0] if row else 0
+
+        row = sqlite.execute('PRAGMA page_size').fetchone()
+        stats['page_size'] = row[0] if row else 4096
+
+        row = sqlite.execute('PRAGMA freelist_count').fetchone()
+        stats['freelist_count'] = row[0] if row else 0
+
+        # Calculate sizes
+        stats['total_size'] = stats['page_count'] * stats['page_size']
+        stats['free_size'] = stats['freelist_count'] * stats['page_size']
+        stats['used_size'] = stats['total_size'] - stats['free_size']
+
+        # Table row counts
+        row = sqlite.execute('SELECT COUNT(*) FROM proxylist').fetchone()
+        stats['proxy_count'] = row[0] if row else 0
+
+        row = sqlite.execute('SELECT COUNT(*) FROM proxylist WHERE failed=0').fetchone()
+        stats['working_count'] = row[0] if row else 0
+
+        row = sqlite.execute('SELECT COUNT(*) FROM uris').fetchone()
+        stats['uri_count'] = row[0] if row else 0
+
+    except Exception:
+        pass
+
+    return stats

httpd.py

@@ -6,9 +6,137 @@ import BaseHTTPServer
 import json
 import threading
 import time
+import os
 import mysqlite
 from misc import _log
 
+
+def get_system_stats():
+    """Collect system resource statistics."""
+    stats = {}
+
+    # Load average (1, 5, 15 min)
+    try:
+        load = os.getloadavg()
+        stats['load_1m'] = round(load[0], 2)
+        stats['load_5m'] = round(load[1], 2)
+        stats['load_15m'] = round(load[2], 2)
+    except (OSError, AttributeError):
+        stats['load_1m'] = stats['load_5m'] = stats['load_15m'] = 0
+
+    # CPU count
+    try:
+        stats['cpu_count'] = os.sysconf('SC_NPROCESSORS_ONLN')
+    except (ValueError, OSError, AttributeError):
+        stats['cpu_count'] = 1
+
+    # Memory from /proc/meminfo (Linux)
+    try:
+        with open('/proc/meminfo', 'r') as f:
+            meminfo = {}
+            for line in f:
+                parts = line.split()
+                if len(parts) >= 2:
+                    meminfo[parts[0].rstrip(':')] = int(parts[1]) * 1024  # KB to bytes
+            total = meminfo.get('MemTotal', 0)
+            available = meminfo.get('MemAvailable', meminfo.get('MemFree', 0))
+            stats['mem_total'] = total
+            stats['mem_available'] = available
+            stats['mem_used'] = total - available
+            stats['mem_pct'] = round((total - available) / total * 100, 1) if total > 0 else 0
+    except (IOError, KeyError, ZeroDivisionError):
+        stats['mem_total'] = stats['mem_available'] = stats['mem_used'] = 0
+        stats['mem_pct'] = 0
+
+    # Disk usage for data directory
+    try:
+        st = os.statvfs('data' if os.path.exists('data') else '.')
+        total = st.f_blocks * st.f_frsize
+        free = st.f_bavail * st.f_frsize
+        used = total - free
+        stats['disk_total'] = total
+        stats['disk_free'] = free
+        stats['disk_used'] = used
+        stats['disk_pct'] = round(used / total * 100, 1) if total > 0 else 0
+    except (OSError, ZeroDivisionError):
+        stats['disk_total'] = stats['disk_free'] = stats['disk_used'] = 0
+        stats['disk_pct'] = 0
+
+    # Process stats from /proc/self/status
+    try:
+        with open('/proc/self/status', 'r') as f:
+            for line in f:
+                if line.startswith('VmRSS:'):
+                    stats['proc_rss'] = int(line.split()[1]) * 1024  # KB to bytes
+                elif line.startswith('Threads:'):
+                    stats['proc_threads'] = int(line.split()[1])
+    except (IOError, ValueError, IndexError):
+        stats['proc_rss'] = 0
+        stats['proc_threads'] = 0
+
+    return stats
+
+
+def get_db_health(db):
+    """Get database health and statistics."""
+    stats = {}
+    try:
+        # Database file size
+        db_path = db.path if hasattr(db, 'path') else 'data/proxies.sqlite'
+        if os.path.exists(db_path):
+            stats['db_size'] = os.path.getsize(db_path)
+        else:
+            stats['db_size'] = 0
+
+        # Page stats from pragma
+        row = db.execute('PRAGMA page_count').fetchone()
+        stats['page_count'] = row[0] if row else 0
+        row = db.execute('PRAGMA page_size').fetchone()
+        stats['page_size'] = row[0] if row else 0
+        row = db.execute('PRAGMA freelist_count').fetchone()
+        stats['freelist_count'] = row[0] if row else 0
+
+        # Anonymity breakdown
+        rows = db.execute(
+            'SELECT anonymity, COUNT(*) FROM proxylist WHERE failed=0 GROUP BY anonymity'
+        ).fetchall()
+        stats['anonymity'] = {r[0] or 'unknown': r[1] for r in rows}
+
+        # Latency stats
+        row = db.execute(
+            'SELECT AVG(avg_latency), MIN(avg_latency), MAX(avg_latency) '
+            'FROM proxylist WHERE failed=0 AND avg_latency > 0'
+        ).fetchone()
+        if row and row[0]:
+            stats['db_avg_latency'] = round(row[0], 1)
+            stats['db_min_latency'] = round(row[1], 1)
+            stats['db_max_latency'] = round(row[2], 1)
+        else:
+            stats['db_avg_latency'] = stats['db_min_latency'] = stats['db_max_latency'] = 0
+
+        # Recent activity
+        now = int(time.time())
+        row = db.execute(
+            'SELECT COUNT(*) FROM proxylist WHERE tested > ?', (now - 3600,)
+        ).fetchone()
+        stats['tested_last_hour'] = row[0] if row else 0
+
+        row = db.execute(
+            'SELECT COUNT(*) FROM proxylist WHERE added > ?', (now - 86400,)
+        ).fetchone()
+        stats['added_last_day'] = row[0] if row else 0
+
+        # Dead proxies count
+        row = db.execute(
+            'SELECT COUNT(*) FROM proxylist WHERE failed > 0'
+        ).fetchone()
+        stats['dead_count'] = row[0] if row else 0
+
+    except Exception:
+        pass
+
+    return stats
+
 # Detect if gevent has monkey-patched the environment
 try:
     from gevent import monkey
@@ -61,6 +189,19 @@ body {
 .dot { width: 6px; height: 6px; border-radius: 50%; background: var(--green); animation: pulse 2s infinite; }
 .dot.err { background: var(--red); animation: none; }
 @keyframes pulse { 50% { opacity: 0.5; } }
+.mode-badge { padding: 4px 10px; border-radius: 4px; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; }
+.mode-ssl { background: rgba(63,185,80,0.2); color: var(--green); border: 1px solid var(--green); }
+.mode-judges { background: rgba(88,166,255,0.2); color: var(--blue); border: 1px solid var(--blue); }
+.mode-http { background: rgba(210,153,34,0.2); color: var(--yellow); border: 1px solid var(--yellow); }
+.mode-irc { background: rgba(163,113,247,0.2); color: var(--purple); border: 1px solid var(--purple); }
+
+/* System monitor bar */
+.sysbar { display: flex; gap: 16px; padding: 8px 12px; background: var(--card); border: 1px solid var(--border); border-radius: 6px; margin-bottom: 16px; font-size: 11px; }
+.sysbar-item { display: flex; align-items: center; gap: 6px; }
+.sysbar-lbl { color: var(--dim); }
+.sysbar-val { font-weight: 600; font-feature-settings: "tnum"; }
+.sysbar-bar { width: 50px; height: 4px; background: var(--border); border-radius: 2px; overflow: hidden; }
+.sysbar-fill { height: 100%; border-radius: 2px; transition: width 0.3s; }
 
 /* Grid */
 .g { display: grid; gap: 12px; margin-bottom: 16px; }
@@ -187,6 +328,14 @@ var fmt = function(n) { return n == null ? '-' : n.toLocaleString(); };
 var fmtDec = function(n, d) { return n == null ? '-' : n.toFixed(d || 1); };
 var pct = function(n, t) { return t > 0 ? ((n / t) * 100).toFixed(1) : '0.0'; };
 
+function fmtBytes(b) {
+    if (!b || b <= 0) return '-';
+    var units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    var i = 0;
+    while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
+    return b.toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
+}
+
 function fmtTime(s) {
     if (!s) return '-';
     var d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
@@ -195,6 +344,12 @@ function fmtTime(s) {
     return m + 'm ' + Math.floor(s % 60) + 's';
 }
 
+function setBarColor(el, pct) {
+    if (pct > 90) return 'background:var(--red)';
+    if (pct > 70) return 'background:var(--yellow)';
+    return 'background:var(--green)';
+}
+
 function fmtMs(ms) {
     if (!ms || ms <= 0) return '-';
     if (ms < 1000) return Math.round(ms) + 'ms';
@@ -258,6 +413,27 @@ function renderLeaderboard(id, data, nameKey, valKey, limit) {
 function update(d) {
     $('dot').className = 'dot'; $('statusTxt').textContent = 'Live';
 
+    // Check type badge (prominent display)
+    var ct = d.checktype || 'unknown';
+    var ctBadge = $('checktypeBadge');
+    if (ctBadge) {
+        ctBadge.textContent = ct.toUpperCase();
+        ctBadge.className = 'mode-badge mode-' + ct;
+    }
+
+    // System monitor bar
+    var sys = d.system || {};
+    $('sysLoad').textContent = (sys.load_1m || 0) + ' / ' + (sys.cpu_count || 1);
+    $('sysMemVal').textContent = fmtBytes(sys.mem_used) + ' / ' + fmtBytes(sys.mem_total);
+    $('sysMemPct').textContent = (sys.mem_pct || 0) + '%';
+    var memFill = $('sysMemFill');
+    if (memFill) { memFill.style.width = (sys.mem_pct || 0) + '%'; memFill.style.cssText = 'width:' + (sys.mem_pct || 0) + '%;' + setBarColor(memFill, sys.mem_pct || 0); }
+    $('sysDiskVal').textContent = fmtBytes(sys.disk_used) + ' / ' + fmtBytes(sys.disk_total);
+    $('sysDiskPct').textContent = (sys.disk_pct || 0) + '%';
+    var diskFill = $('sysDiskFill');
+    if (diskFill) { diskFill.style.width = (sys.disk_pct || 0) + '%'; diskFill.style.cssText = 'width:' + (sys.disk_pct || 0) + '%;' + setBarColor(diskFill, sys.disk_pct || 0); }
+    $('sysProcMem').textContent = fmtBytes(sys.proc_rss);
+
     // Main stats - db stats are nested under d.db
     var db = d.db || {};
     $('working').textContent = fmt(db.working);
@@ -298,7 +474,6 @@ function update(d) {
     setBar('threadBar', d.threads, d.max_threads, 'blu');
     $('queue').textContent = fmt(d.queue_size);
     $('uptime').textContent = fmtTime(d.uptime_seconds);
-    $('checktype').textContent = d.checktype || '-';
 
     // Charts
     renderLineChart('rateChart', d.rate_history, '#58a6ff', d.peak_rate * 1.1);
@@ -429,6 +604,41 @@ function update(d) {
         $('certErrors').className = 'stat-val ' + (ssl.cert_errors > 0 ? 'yel' : 'grn');
     }
 
+    // Anonymity breakdown
+    var dbh = d.db_health || {};
+    if (dbh.anonymity) {
+        var anonHtml = '';
+        var anonColors = {elite: 'grn', anonymous: 'blu', transparent: 'yel', unknown: 'dim'};
+        var anonOrder = ['elite', 'anonymous', 'transparent', 'unknown'];
+        anonOrder.forEach(function(level) {
+            var count = dbh.anonymity[level] || 0;
+            if (count > 0) {
+                anonHtml += '<div class="stat-row"><span class="stat-lbl">' + level.charAt(0).toUpperCase() + level.slice(1) + '</span>';
+                anonHtml += '<span class="stat-val ' + anonColors[level] + '">' + fmt(count) + '</span></div>';
+            }
+        });
+        $('anonBreakdown').innerHTML = anonHtml || '<div style="color:var(--dim)">No data</div>';
+    }
+
+    // Database health
+    if (dbh.db_size) {
+        $('dbSize').textContent = fmtBytes(dbh.db_size);
+        $('dbTestedHour').textContent = fmt(dbh.tested_last_hour);
+        $('dbAddedDay').textContent = fmt(dbh.added_last_day);
+        $('dbDead').textContent = fmt(dbh.dead_count);
+    }
+
+    // Tor pool enhanced stats
+    if (d.tor_pool) {
+        var tp = d.tor_pool;
+        $('torTotal').textContent = fmt(tp.total_requests || 0);
+        $('torSuccess').textContent = fmtDec(tp.success_rate || 0, 1) + '%';
+        $('torHealthy').textContent = (tp.healthy_count || 0) + '/' + (tp.total_count || 0);
+        if (tp.avg_latency) {
+            $('torLatency').textContent = fmtMs(tp.avg_latency);
+        }
+    }
+
     $('lastUpdate').textContent = new Date().toLocaleTimeString();
 }
 
@@ -456,11 +666,24 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
     <div class="hdr">
         <h1>PPF Dashboard</h1>
         <div class="status">
+            <span class="mode-badge mode-ssl" id="checktypeBadge">-</span>
             <div class="status-item"><div class="dot" id="dot"></div><span id="statusTxt">Connecting</span></div>
             <div class="status-item">Updated: <span id="lastUpdate">-</span></div>
         </div>
     </div>
 
+    <!-- System Monitor Bar -->
+    <div class="sysbar">
+        <div class="sysbar-item"><span class="sysbar-lbl">Load:</span><span class="sysbar-val" id="sysLoad">-</span></div>
+        <div class="sysbar-item"><span class="sysbar-lbl">Memory:</span><span class="sysbar-val" id="sysMemVal">-</span>
+            <div class="sysbar-bar"><div class="sysbar-fill" id="sysMemFill" style="width:0"></div></div>
+            <span class="sysbar-val" id="sysMemPct">-</span></div>
+        <div class="sysbar-item"><span class="sysbar-lbl">Disk:</span><span class="sysbar-val" id="sysDiskVal">-</span>
+            <div class="sysbar-bar"><div class="sysbar-fill" id="sysDiskFill" style="width:0"></div></div>
+            <span class="sysbar-val" id="sysDiskPct">-</span></div>
+        <div class="sysbar-item"><span class="sysbar-lbl">Process:</span><span class="sysbar-val" id="sysProcMem">-</span></div>
+    </div>
+
     <!-- Primary Stats Row -->
     <div class="g g5">
         <div class="c">
@@ -598,11 +821,10 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
     <!-- System & Infrastructure -->
     <div class="g g2">
         <div class="c">
-            <div class="sec-hdr" style="margin-top:0">System</div>
+            <div class="sec-hdr" style="margin-top:0">Worker Pool</div>
-            <div class="stat-row"><span class="stat-lbl">Threads</span><span class="stat-val" id="threads">-</span></div>
+            <div class="stat-row"><span class="stat-lbl">Active Threads</span><span class="stat-val" id="threads">-</span></div>
             <div class="bar-wrap"><div class="bar blu" id="threadBar" style="width:0"></div></div>
-            <div class="stat-row" style="margin-top:8px"><span class="stat-lbl">Queue Size</span><span class="stat-val yel" id="queue">-</span></div>
+            <div class="stat-row" style="margin-top:8px"><span class="stat-lbl">Job Queue</span><span class="stat-val yel" id="queue">-</span></div>
-            <div class="stat-row"><span class="stat-lbl">Check Type</span><span class="stat-val" id="checktype">-</span></div>
         </div>
         <div class="c">
             <div class="sec-hdr" style="margin-top:0">Judge Services</div>
@@ -613,10 +835,22 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
         </div>
     </div>
 
-    <!-- Tor Pool -->
+    <!-- Tor Pool & Anonymity -->
-    <div class="sec">
+    <div class="g g2">
-        <div class="sec-hdr">Tor Pool</div>
+        <div class="c">
-        <div class="g g3" id="torPool"></div>
+            <div class="sec-hdr" style="margin-top:0">Tor Pool</div>
+            <div class="stat-row"><span class="stat-lbl">Total Requests</span><span class="stat-val" id="torTotal">-</span></div>
+            <div class="stat-row"><span class="stat-lbl">Success Rate</span><span class="stat-val grn" id="torSuccess">-</span></div>
+            <div class="stat-row"><span class="stat-lbl">Healthy Nodes</span><span class="stat-val" id="torHealthy">-</span></div>
+            <div class="stat-row"><span class="stat-lbl">Avg Latency</span><span class="stat-val cyn" id="torLatency">-</span></div>
+            <div class="lbl" style="margin-top:10px">Exit Nodes</div>
+            <div class="g g3" id="torPool" style="margin-top:8px"></div>
+        </div>
+        <div class="c">
+            <div class="sec-hdr" style="margin-top:0">Anonymity Levels</div>
+            <div id="anonBreakdown"></div>
+            <div class="sub" style="margin-top:8px;font-size:10px">Elite = no headers, Anonymous = adds headers, Transparent = reveals IP</div>
+        </div>
     </div>
 
     <!-- Scraper & SSL Stats -->
@@ -643,6 +877,24 @@ DASHBOARD_HTML = '''<!DOCTYPE html>
     <!-- Database Stats -->
     <div class="sec">
         <div class="sec-hdr">Database Overview</div>
+        <div class="g g4">
+            <div class="c c-sm">
+                <div class="lbl">Database Size</div>
+                <div class="val-sm cyn" id="dbSize">-</div>
+            </div>
+            <div class="c c-sm">
+                <div class="lbl">Tested (1h)</div>
+                <div class="val-sm blu" id="dbTestedHour">-</div>
+            </div>
+            <div class="c c-sm">
+                <div class="lbl">Added (24h)</div>
+                <div class="val-sm grn" id="dbAddedDay">-</div>
+            </div>
+            <div class="c c-sm">
+                <div class="lbl">Dead Proxies</div>
+                <div class="val-sm red" id="dbDead">-</div>
+            </div>
+        </div>
         <div class="g g2">
             <div class="c">
                 <div class="lbl">Working by Protocol</div>
@@ -972,10 +1224,13 @@ class ProxyAPIServer(threading.Thread):
             stats = {}
             if self.stats_provider:
                 stats = self.stats_provider()
+            # Add system stats
+            stats['system'] = get_system_stats()
             # Add database stats
             try:
                 db = mysqlite.mysqlite(self.database, str)
                 stats['db'] = self._get_db_stats(db)
+                stats['db_health'] = get_db_health(db)
             except Exception:
                 pass
             return json.dumps(stats, indent=2), 'application/json', 200