f83733dd46
- Add DEAD_PROXY=-1 constant for permanently dead proxies - Mark proxy dead when: failed >= max_fail*2, or max_fail with fatal error - Fatal errors: refused, unreachable, auth (proxy definitely not working) - Dead proxies excluded from testing (failed >= 0 query) - Cleanup_stale also removes old dead proxies - Dashboard shows separate dead vs failing counts
1285 lines
56 KiB
Python
1285 lines
56 KiB
Python
#!/usr/bin/env python2
|
|
# -*- coding: utf-8 -*-
|
|
"""HTTP API server with advanced web dashboard for PPF."""
|
|
|
|
import BaseHTTPServer
|
|
import json
|
|
import threading
|
|
import time
|
|
import os
|
|
import mysqlite
|
|
from misc import _log
|
|
|
|
|
|
def get_system_stats():
|
|
"""Collect system resource statistics."""
|
|
stats = {}
|
|
|
|
# Load average (1, 5, 15 min)
|
|
try:
|
|
load = os.getloadavg()
|
|
stats['load_1m'] = round(load[0], 2)
|
|
stats['load_5m'] = round(load[1], 2)
|
|
stats['load_15m'] = round(load[2], 2)
|
|
except (OSError, AttributeError):
|
|
stats['load_1m'] = stats['load_5m'] = stats['load_15m'] = 0
|
|
|
|
# CPU count
|
|
try:
|
|
stats['cpu_count'] = os.sysconf('SC_NPROCESSORS_ONLN')
|
|
except (ValueError, OSError, AttributeError):
|
|
stats['cpu_count'] = 1
|
|
|
|
# Memory from /proc/meminfo (Linux)
|
|
try:
|
|
with open('/proc/meminfo', 'r') as f:
|
|
meminfo = {}
|
|
for line in f:
|
|
parts = line.split()
|
|
if len(parts) >= 2:
|
|
meminfo[parts[0].rstrip(':')] = int(parts[1]) * 1024 # KB to bytes
|
|
total = meminfo.get('MemTotal', 0)
|
|
available = meminfo.get('MemAvailable', meminfo.get('MemFree', 0))
|
|
stats['mem_total'] = total
|
|
stats['mem_available'] = available
|
|
stats['mem_used'] = total - available
|
|
stats['mem_pct'] = round((total - available) / total * 100, 1) if total > 0 else 0
|
|
except (IOError, KeyError, ZeroDivisionError):
|
|
stats['mem_total'] = stats['mem_available'] = stats['mem_used'] = 0
|
|
stats['mem_pct'] = 0
|
|
|
|
# Disk usage for data directory
|
|
try:
|
|
st = os.statvfs('data' if os.path.exists('data') else '.')
|
|
total = st.f_blocks * st.f_frsize
|
|
free = st.f_bavail * st.f_frsize
|
|
used = total - free
|
|
stats['disk_total'] = total
|
|
stats['disk_free'] = free
|
|
stats['disk_used'] = used
|
|
stats['disk_pct'] = round(used / total * 100, 1) if total > 0 else 0
|
|
except (OSError, ZeroDivisionError):
|
|
stats['disk_total'] = stats['disk_free'] = stats['disk_used'] = 0
|
|
stats['disk_pct'] = 0
|
|
|
|
# Process stats from /proc/self/status
|
|
try:
|
|
with open('/proc/self/status', 'r') as f:
|
|
for line in f:
|
|
if line.startswith('VmRSS:'):
|
|
stats['proc_rss'] = int(line.split()[1]) * 1024 # KB to bytes
|
|
elif line.startswith('Threads:'):
|
|
stats['proc_threads'] = int(line.split()[1])
|
|
except (IOError, ValueError, IndexError):
|
|
stats['proc_rss'] = 0
|
|
stats['proc_threads'] = 0
|
|
|
|
return stats
|
|
|
|
|
|
def get_db_health(db):
|
|
"""Get database health and statistics."""
|
|
stats = {}
|
|
try:
|
|
# Database file size
|
|
db_path = db.path if hasattr(db, 'path') else 'data/proxies.sqlite'
|
|
if os.path.exists(db_path):
|
|
stats['db_size'] = os.path.getsize(db_path)
|
|
else:
|
|
stats['db_size'] = 0
|
|
|
|
# Page stats from pragma
|
|
row = db.execute('PRAGMA page_count').fetchone()
|
|
stats['page_count'] = row[0] if row else 0
|
|
row = db.execute('PRAGMA page_size').fetchone()
|
|
stats['page_size'] = row[0] if row else 0
|
|
row = db.execute('PRAGMA freelist_count').fetchone()
|
|
stats['freelist_count'] = row[0] if row else 0
|
|
|
|
# Anonymity breakdown
|
|
rows = db.execute(
|
|
'SELECT anonymity, COUNT(*) FROM proxylist WHERE failed=0 GROUP BY anonymity'
|
|
).fetchall()
|
|
stats['anonymity'] = {r[0] or 'unknown': r[1] for r in rows}
|
|
|
|
# Latency stats
|
|
row = db.execute(
|
|
'SELECT AVG(avg_latency), MIN(avg_latency), MAX(avg_latency) '
|
|
'FROM proxylist WHERE failed=0 AND avg_latency > 0'
|
|
).fetchone()
|
|
if row and row[0]:
|
|
stats['db_avg_latency'] = round(row[0], 1)
|
|
stats['db_min_latency'] = round(row[1], 1)
|
|
stats['db_max_latency'] = round(row[2], 1)
|
|
else:
|
|
stats['db_avg_latency'] = stats['db_min_latency'] = stats['db_max_latency'] = 0
|
|
|
|
# Recent activity
|
|
now = int(time.time())
|
|
row = db.execute(
|
|
'SELECT COUNT(*) FROM proxylist WHERE tested > ?', (now - 3600,)
|
|
).fetchone()
|
|
stats['tested_last_hour'] = row[0] if row else 0
|
|
|
|
row = db.execute(
|
|
'SELECT COUNT(*) FROM proxylist WHERE added > ?', (now - 86400,)
|
|
).fetchone()
|
|
stats['added_last_day'] = row[0] if row else 0
|
|
|
|
# Dead proxies count (permanently dead = -1, failing = positive)
|
|
row = db.execute(
|
|
'SELECT COUNT(*) FROM proxylist WHERE failed = -1'
|
|
).fetchone()
|
|
stats['dead_count'] = row[0] if row else 0
|
|
|
|
# Failing proxies count (positive fail count but not permanently dead)
|
|
row = db.execute(
|
|
'SELECT COUNT(*) FROM proxylist WHERE failed > 0'
|
|
).fetchone()
|
|
stats['failing_count'] = row[0] if row else 0
|
|
|
|
except Exception:
|
|
pass
|
|
|
|
return stats
|
|
|
|
# Detect if gevent has monkey-patched the environment
|
|
try:
|
|
from gevent import monkey
|
|
GEVENT_PATCHED = monkey.is_module_patched('socket')
|
|
except ImportError:
|
|
GEVENT_PATCHED = False
|
|
|
|
if GEVENT_PATCHED:
|
|
from gevent.pywsgi import WSGIServer
|
|
|
|
# Theme colors - modern dark palette
|
|
THEME = {
|
|
'bg': '#0d1117',
|
|
'card': '#161b22',
|
|
'card_alt': '#1c2128',
|
|
'border': '#30363d',
|
|
'text': '#e6edf3',
|
|
'dim': '#7d8590',
|
|
'green': '#3fb950',
|
|
'red': '#f85149',
|
|
'yellow': '#d29922',
|
|
'blue': '#58a6ff',
|
|
'purple': '#a371f7',
|
|
'cyan': '#39c5cf',
|
|
'orange': '#db6d28',
|
|
'pink': '#db61a2',
|
|
}
|
|
|
|
DASHBOARD_CSS = '''
|
|
:root {
|
|
--bg: {bg}; --card: {card}; --card-alt: {card_alt}; --border: {border};
|
|
--text: {text}; --dim: {dim}; --green: {green}; --red: {red};
|
|
--yellow: {yellow}; --blue: {blue}; --purple: {purple};
|
|
--cyan: {cyan}; --orange: {orange}; --pink: {pink};
|
|
}
|
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
|
|
font-size: 13px; background: var(--bg); color: var(--text);
|
|
padding: 16px; min-height: 100vh; line-height: 1.4;
|
|
}
|
|
.container { max-width: 1400px; margin: 0 auto; }
|
|
|
|
/* Header */
|
|
.hdr { display: flex; justify-content: space-between; align-items: center; margin-bottom: 16px; padding-bottom: 12px; border-bottom: 1px solid var(--border); }
|
|
.hdr h1 { font-size: 18px; font-weight: 600; display: flex; align-items: center; gap: 10px; }
|
|
.hdr h1::before { content: ""; width: 10px; height: 10px; background: var(--green); border-radius: 50%; box-shadow: 0 0 8px var(--green); }
|
|
.status { display: flex; align-items: center; gap: 12px; font-size: 12px; color: var(--dim); }
|
|
.status-item { display: flex; align-items: center; gap: 4px; }
|
|
.dot { width: 6px; height: 6px; border-radius: 50%; background: var(--green); animation: pulse 2s infinite; }
|
|
.dot.err { background: var(--red); animation: none; }
|
|
@keyframes pulse { 50% { opacity: 0.5; } }
|
|
.mode-badge { padding: 4px 10px; border-radius: 4px; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; }
|
|
.mode-ssl { background: rgba(63,185,80,0.2); color: var(--green); border: 1px solid var(--green); }
|
|
.mode-judges { background: rgba(88,166,255,0.2); color: var(--blue); border: 1px solid var(--blue); }
|
|
.mode-http { background: rgba(210,153,34,0.2); color: var(--yellow); border: 1px solid var(--yellow); }
|
|
.mode-irc { background: rgba(163,113,247,0.2); color: var(--purple); border: 1px solid var(--purple); }
|
|
|
|
/* System monitor bar */
|
|
.sysbar { display: flex; gap: 16px; padding: 8px 12px; background: var(--card); border: 1px solid var(--border); border-radius: 6px; margin-bottom: 16px; font-size: 11px; }
|
|
.sysbar-item { display: flex; align-items: center; gap: 6px; }
|
|
.sysbar-lbl { color: var(--dim); }
|
|
.sysbar-val { font-weight: 600; font-feature-settings: "tnum"; }
|
|
.sysbar-bar { width: 50px; height: 4px; background: var(--border); border-radius: 2px; overflow: hidden; }
|
|
.sysbar-fill { height: 100%; border-radius: 2px; transition: width 0.3s; }
|
|
|
|
/* Grid */
|
|
.g { display: grid; gap: 12px; margin-bottom: 16px; }
|
|
.g2 { grid-template-columns: repeat(2, 1fr); }
|
|
.g3 { grid-template-columns: repeat(3, 1fr); }
|
|
.g4 { grid-template-columns: repeat(4, 1fr); }
|
|
.g5 { grid-template-columns: repeat(5, 1fr); }
|
|
.g6 { grid-template-columns: repeat(6, 1fr); }
|
|
@media (max-width: 1200px) { .g5, .g6 { grid-template-columns: repeat(4, 1fr); } }
|
|
@media (max-width: 900px) { .g3, .g4, .g5, .g6 { grid-template-columns: repeat(2, 1fr); } }
|
|
@media (max-width: 600px) { .g2, .g3, .g4, .g5, .g6 { grid-template-columns: 1fr; } }
|
|
|
|
/* Cards */
|
|
.c { background: var(--card); border: 1px solid var(--border); border-radius: 8px; padding: 14px; }
|
|
.c-lg { padding: 16px 18px; }
|
|
.c-sm { padding: 10px 12px; }
|
|
.lbl { font-size: 10px; color: var(--dim); text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 6px; font-weight: 500; }
|
|
.val { font-size: 26px; font-weight: 700; font-feature-settings: "tnum"; letter-spacing: -0.5px; }
|
|
.val-md { font-size: 20px; }
|
|
.val-sm { font-size: 16px; }
|
|
.sub { font-size: 11px; color: var(--dim); margin-top: 4px; }
|
|
.grn { color: var(--green); } .red { color: var(--red); } .yel { color: var(--yellow); }
|
|
.blu { color: var(--blue); } .pur { color: var(--purple); } .cyn { color: var(--cyan); }
|
|
.org { color: var(--orange); } .pnk { color: var(--pink); }
|
|
|
|
/* Section headers */
|
|
.sec { margin-bottom: 16px; }
|
|
.sec-hdr { font-size: 11px; font-weight: 600; color: var(--dim); text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 10px; display: flex; align-items: center; gap: 8px; }
|
|
.sec-hdr::before { content: ""; width: 3px; height: 12px; background: var(--blue); border-radius: 2px; }
|
|
|
|
/* Progress bars */
|
|
.bar-wrap { height: 6px; background: var(--border); border-radius: 3px; margin-top: 8px; overflow: hidden; }
|
|
.bar { height: 100%; border-radius: 3px; transition: width 0.4s ease; }
|
|
.bar.grn { background: linear-gradient(90deg, #238636, #3fb950); }
|
|
.bar.red { background: linear-gradient(90deg, #da3633, #f85149); }
|
|
.bar.yel { background: linear-gradient(90deg, #9e6a03, #d29922); }
|
|
.bar.blu { background: linear-gradient(90deg, #1f6feb, #58a6ff); }
|
|
|
|
/* Charts */
|
|
.chart { width: 100%; height: 80px; margin-top: 8px; }
|
|
.chart-lg { height: 120px; }
|
|
.chart svg { width: 100%; height: 100%; }
|
|
.chart-line { fill: none; stroke-width: 1.5; stroke-linecap: round; stroke-linejoin: round; }
|
|
.chart-area { opacity: 0.15; }
|
|
.chart-grid { stroke: var(--border); stroke-width: 0.5; }
|
|
.chart-label { font-size: 9px; fill: var(--dim); }
|
|
|
|
/* Histogram bars */
|
|
.histo { display: flex; align-items: flex-end; gap: 2px; height: 60px; margin-top: 8px; }
|
|
.histo-bar { flex: 1; background: var(--blue); border-radius: 2px 2px 0 0; min-height: 2px; transition: height 0.3s; position: relative; }
|
|
.histo-bar:hover { opacity: 0.8; }
|
|
.histo-bar::after { content: attr(data-label); position: absolute; bottom: -16px; left: 50%; transform: translateX(-50%); font-size: 8px; color: var(--dim); white-space: nowrap; }
|
|
.histo-labels { display: flex; justify-content: space-between; margin-top: 20px; font-size: 9px; color: var(--dim); }
|
|
|
|
/* Stat rows */
|
|
.stat-row { display: flex; justify-content: space-between; align-items: center; padding: 6px 0; font-size: 12px; }
|
|
.stat-row + .stat-row { border-top: 1px solid rgba(48,54,61,0.5); }
|
|
.stat-lbl { color: var(--dim); display: flex; align-items: center; gap: 6px; }
|
|
.stat-val { font-weight: 600; font-feature-settings: "tnum"; }
|
|
.stat-bar { width: 60px; height: 4px; background: var(--border); border-radius: 2px; margin-left: 8px; overflow: hidden; }
|
|
.stat-bar-fill { height: 100%; border-radius: 2px; }
|
|
|
|
/* Leaderboard */
|
|
.lb { font-size: 12px; }
|
|
.lb-item { display: flex; align-items: center; gap: 8px; padding: 5px 0; }
|
|
.lb-item + .lb-item { border-top: 1px solid rgba(48,54,61,0.3); }
|
|
.lb-rank { width: 18px; height: 18px; border-radius: 4px; background: var(--card-alt); display: flex; align-items: center; justify-content: center; font-size: 10px; font-weight: 600; color: var(--dim); }
|
|
.lb-rank.top { background: var(--yellow); color: var(--bg); }
|
|
.lb-name { flex: 1; font-family: ui-monospace, monospace; color: var(--text); }
|
|
.lb-val { font-weight: 600; font-feature-settings: "tnum"; }
|
|
|
|
/* Tags */
|
|
.tag { display: inline-flex; align-items: center; gap: 4px; padding: 3px 8px; border-radius: 4px; font-size: 10px; font-weight: 600; }
|
|
.tag-ok { background: rgba(63,185,80,0.15); color: var(--green); }
|
|
.tag-err { background: rgba(248,81,73,0.15); color: var(--red); }
|
|
.tag-warn { background: rgba(210,153,34,0.15); color: var(--yellow); }
|
|
.tag-info { background: rgba(88,166,255,0.15); color: var(--blue); }
|
|
|
|
/* Mini stats */
|
|
.mini { display: flex; gap: 16px; flex-wrap: wrap; margin-top: 8px; }
|
|
.mini-item { display: flex; align-items: baseline; gap: 4px; }
|
|
.mini-val { font-size: 14px; font-weight: 600; font-feature-settings: "tnum"; }
|
|
.mini-lbl { font-size: 10px; color: var(--dim); }
|
|
|
|
/* Proto cards */
|
|
.proto-card { text-align: center; }
|
|
.proto-icon { font-size: 20px; margin-bottom: 4px; }
|
|
.proto-name { font-size: 10px; color: var(--dim); text-transform: uppercase; letter-spacing: 0.5px; }
|
|
.proto-val { font-size: 18px; font-weight: 700; margin: 4px 0; }
|
|
.proto-rate { font-size: 11px; padding: 2px 6px; border-radius: 3px; display: inline-block; }
|
|
|
|
/* Pie charts */
|
|
.pie-wrap { display: flex; gap: 16px; align-items: center; }
|
|
.pie { width: 90px; height: 90px; border-radius: 50%; flex-shrink: 0; }
|
|
.legend { flex: 1; }
|
|
.legend-item { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 12px; }
|
|
.legend-dot { width: 10px; height: 10px; border-radius: 3px; flex-shrink: 0; }
|
|
.legend-name { flex: 1; color: var(--dim); }
|
|
.legend-val { font-weight: 600; font-feature-settings: "tnum"; }
|
|
|
|
/* Tor/Judge cards */
|
|
.host-card { display: flex; justify-content: space-between; align-items: center; }
|
|
.host-addr { font-family: ui-monospace, monospace; font-size: 12px; }
|
|
.host-stats { font-size: 11px; color: var(--dim); }
|
|
|
|
.judge-item { display: flex; justify-content: space-between; align-items: center; padding: 4px 0; font-size: 11px; }
|
|
.judge-item + .judge-item { border-top: 1px solid rgba(48,54,61,0.3); }
|
|
.judge-name { font-family: ui-monospace, monospace; color: var(--dim); flex: 1; }
|
|
.judge-stats { display: flex; gap: 12px; }
|
|
|
|
/* Percentile badges */
|
|
.pct-badges { display: flex; gap: 8px; margin-top: 8px; }
|
|
.pct-badge { flex: 1; text-align: center; padding: 8px; background: var(--card-alt); border-radius: 6px; }
|
|
.pct-label { font-size: 10px; color: var(--dim); text-transform: uppercase; }
|
|
.pct-value { font-size: 16px; font-weight: 700; margin-top: 2px; }
|
|
|
|
/* Footer */
|
|
.ftr { text-align: center; font-size: 11px; color: var(--dim); padding: 16px 0; margin-top: 8px; border-top: 1px solid var(--border); }
|
|
'''
|
|
|
|
DASHBOARD_JS = '''
|
|
var $ = function(id) { return document.getElementById(id); };
|
|
var fmt = function(n) { return n == null ? '-' : n.toLocaleString(); };
|
|
var fmtDec = function(n, d) { return n == null ? '-' : n.toFixed(d || 1); };
|
|
var pct = function(n, t) { return t > 0 ? ((n / t) * 100).toFixed(1) : '0.0'; };
|
|
|
|
function fmtBytes(b) {
|
|
if (!b || b <= 0) return '-';
|
|
var units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
|
var i = 0;
|
|
while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
|
|
return b.toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
|
|
}
|
|
|
|
function fmtTime(s) {
|
|
if (!s) return '-';
|
|
var d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
|
|
if (d > 0) return d + 'd ' + h + 'h';
|
|
if (h > 0) return h + 'h ' + m + 'm';
|
|
return m + 'm ' + Math.floor(s % 60) + 's';
|
|
}
|
|
|
|
function setBarColor(el, pct) {
|
|
if (pct > 90) return 'background:var(--red)';
|
|
if (pct > 70) return 'background:var(--yellow)';
|
|
return 'background:var(--green)';
|
|
}
|
|
|
|
function fmtMs(ms) {
|
|
if (!ms || ms <= 0) return '-';
|
|
if (ms < 1000) return Math.round(ms) + 'ms';
|
|
return (ms / 1000).toFixed(1) + 's';
|
|
}
|
|
|
|
function setBar(id, val, max, cls) {
|
|
var el = $(id);
|
|
if (el) { el.style.width = Math.min(val / max * 100, 100) + '%'; el.className = 'bar ' + (cls || 'grn'); }
|
|
}
|
|
|
|
function conicGrad(segs) {
|
|
var parts = [], deg = 0;
|
|
segs.forEach(function(s) { parts.push(s.c + ' ' + deg + 'deg ' + (deg + s.d) + 'deg'); deg += s.d; });
|
|
return 'conic-gradient(' + parts.join(', ') + ')';
|
|
}
|
|
|
|
function renderLineChart(id, data, color, maxVal) {
|
|
var el = $(id); if (!el || !data || data.length < 2) return;
|
|
var w = el.clientWidth || 300, h = el.clientHeight || 80;
|
|
var max = maxVal || Math.max.apply(null, data) || 1;
|
|
var padY = 5, padX = 2;
|
|
var points = data.map(function(v, i) {
|
|
var x = padX + (i / (data.length - 1)) * (w - 2 * padX);
|
|
var y = h - padY - ((v / max) * (h - 2 * padY));
|
|
return x + ',' + y;
|
|
});
|
|
var areaPoints = padX + ',' + (h - padY) + ' ' + points.join(' ') + ' ' + (w - padX) + ',' + (h - padY);
|
|
el.innerHTML = '<svg viewBox="0 0 ' + w + ' ' + h + '" preserveAspectRatio="none">' +
|
|
'<polygon class="chart-area" points="' + areaPoints + '" fill="' + color + '"/>' +
|
|
'<polyline class="chart-line" points="' + points.join(' ') + '" stroke="' + color + '"/></svg>';
|
|
}
|
|
|
|
function renderHistogram(id, data) {
|
|
var el = $(id); if (!el || !data || !data.length) return;
|
|
var max = Math.max.apply(null, data.map(function(d) { return d.pct; })) || 1;
|
|
var colors = ['#3fb950', '#3fb950', '#58a6ff', '#58a6ff', '#d29922', '#f85149', '#f85149'];
|
|
var html = '';
|
|
data.forEach(function(d, i) {
|
|
var h = Math.max(4, (d.pct / max) * 100);
|
|
var c = colors[Math.min(i, colors.length - 1)];
|
|
html += '<div class="histo-bar" style="height:' + h + '%;background:' + c + '" data-label="' + d.range + '" title="' + d.range + 'ms: ' + d.count + ' (' + d.pct + '%)"></div>';
|
|
});
|
|
el.innerHTML = html;
|
|
}
|
|
|
|
function renderLeaderboard(id, data, nameKey, valKey, limit) {
|
|
var el = $(id); if (!el) return;
|
|
limit = limit || 8;
|
|
if (!data || !data.length) { el.innerHTML = '<div style="color:var(--dim);font-size:11px;padding:8px 0">No data</div>'; return; }
|
|
var html = '';
|
|
data.slice(0, limit).forEach(function(item, i) {
|
|
var name = Array.isArray(item) ? item[0] : item[nameKey];
|
|
var val = Array.isArray(item) ? item[1] : item[valKey];
|
|
html += '<div class="lb-item"><div class="lb-rank' + (i === 0 ? ' top' : '') + '">' + (i + 1) + '</div>';
|
|
html += '<span class="lb-name">' + name + '</span><span class="lb-val grn">' + fmt(val) + '</span></div>';
|
|
});
|
|
el.innerHTML = html;
|
|
}
|
|
|
|
function update(d) {
|
|
$('dot').className = 'dot'; $('statusTxt').textContent = 'Live';
|
|
|
|
// Check type badge (prominent display)
|
|
var ct = d.checktype || 'unknown';
|
|
var ctBadge = $('checktypeBadge');
|
|
if (ctBadge) {
|
|
ctBadge.textContent = ct.toUpperCase();
|
|
ctBadge.className = 'mode-badge mode-' + ct;
|
|
}
|
|
|
|
// System monitor bar
|
|
var sys = d.system || {};
|
|
$('sysLoad').textContent = (sys.load_1m || 0) + ' / ' + (sys.cpu_count || 1);
|
|
$('sysMemVal').textContent = fmtBytes(sys.mem_used) + ' / ' + fmtBytes(sys.mem_total);
|
|
$('sysMemPct').textContent = (sys.mem_pct || 0) + '%';
|
|
var memFill = $('sysMemFill');
|
|
if (memFill) { memFill.style.width = (sys.mem_pct || 0) + '%'; memFill.style.cssText = 'width:' + (sys.mem_pct || 0) + '%;' + setBarColor(memFill, sys.mem_pct || 0); }
|
|
$('sysDiskVal').textContent = fmtBytes(sys.disk_used) + ' / ' + fmtBytes(sys.disk_total);
|
|
$('sysDiskPct').textContent = (sys.disk_pct || 0) + '%';
|
|
var diskFill = $('sysDiskFill');
|
|
if (diskFill) { diskFill.style.width = (sys.disk_pct || 0) + '%'; diskFill.style.cssText = 'width:' + (sys.disk_pct || 0) + '%;' + setBarColor(diskFill, sys.disk_pct || 0); }
|
|
$('sysProcMem').textContent = fmtBytes(sys.proc_rss);
|
|
|
|
// Main stats - db stats are nested under d.db
|
|
var db = d.db || {};
|
|
$('working').textContent = fmt(db.working);
|
|
$('total').textContent = fmt(db.total);
|
|
$('tested').textContent = fmt(d.tested);
|
|
$('passed').textContent = fmt(d.passed);
|
|
$('failed').textContent = fmt(d.failed);
|
|
|
|
// Success rate
|
|
var sr = d.success_rate || 0;
|
|
$('successRate').textContent = fmtDec(sr, 1) + '%';
|
|
$('successRate').className = 'val-md ' + (sr < 20 ? 'red' : sr < 50 ? 'yel' : 'grn');
|
|
setBar('srBar', sr, 100, sr < 20 ? 'red' : sr < 50 ? 'yel' : 'grn');
|
|
|
|
var rsr = d.recent_success_rate || 0;
|
|
$('recentSuccessRate').textContent = fmtDec(rsr, 1) + '%';
|
|
$('recentSuccessRate').className = 'stat-val ' + (rsr < 20 ? 'red' : rsr < 50 ? 'yel' : 'grn');
|
|
|
|
// Rates
|
|
$('rate').textContent = fmtDec(d.rate, 2);
|
|
$('recentRate').textContent = fmtDec(d.recent_rate, 2);
|
|
$('peakRate').textContent = fmtDec(d.peak_rate, 2);
|
|
$('passRate').textContent = fmtDec(d.pass_rate, 3);
|
|
|
|
// Latency
|
|
var lat = d.avg_latency || 0;
|
|
$('avgLatency').textContent = fmtMs(lat);
|
|
$('minLatency').textContent = fmtMs(d.min_latency);
|
|
$('maxLatency').textContent = fmtMs(d.max_latency);
|
|
|
|
var pctl = d.latency_percentiles || {};
|
|
$('p50').textContent = fmtMs(pctl.p50);
|
|
$('p90').textContent = fmtMs(pctl.p90);
|
|
$('p99').textContent = fmtMs(pctl.p99);
|
|
|
|
// System
|
|
$('threads').textContent = d.threads + '/' + d.max_threads;
|
|
setBar('threadBar', d.threads, d.max_threads, 'blu');
|
|
$('queue').textContent = fmt(d.queue_size);
|
|
$('uptime').textContent = fmtTime(d.uptime_seconds);
|
|
|
|
// Charts
|
|
renderLineChart('rateChart', d.rate_history, '#58a6ff', d.peak_rate * 1.1);
|
|
renderLineChart('srChart', d.success_rate_history, '#3fb950', 100);
|
|
renderHistogram('latencyHisto', d.latency_histogram);
|
|
|
|
// Protocol breakdown
|
|
var ps = d.proto_stats || {};
|
|
['http', 'socks4', 'socks5'].forEach(function(p) {
|
|
var s = ps[p] || {passed: 0, tested: 0, success_rate: 0};
|
|
$(p + 'Passed').textContent = fmt(s.passed);
|
|
$(p + 'Tested').textContent = fmt(s.tested);
|
|
var rateEl = $(p + 'Rate');
|
|
rateEl.textContent = fmtDec(s.success_rate, 0) + '%';
|
|
rateEl.className = 'proto-rate ' + (s.success_rate < 20 ? 'tag-err' : s.success_rate < 50 ? 'tag-warn' : 'tag-ok');
|
|
});
|
|
|
|
// Results pie
|
|
var passed = d.passed || 0, failed = d.failed || 0, total = passed + failed;
|
|
if (total > 0) {
|
|
var passedDeg = (passed / total) * 360;
|
|
$('resultsPie').style.background = conicGrad([{c:'#3fb950',d:passedDeg},{c:'#f85149',d:360-passedDeg}]);
|
|
}
|
|
$('passedLeg').textContent = fmt(passed);
|
|
$('passedPct').textContent = pct(passed, total) + '%';
|
|
$('failedLeg').textContent = fmt(failed);
|
|
$('failedPct').textContent = pct(failed, total) + '%';
|
|
|
|
// Failures breakdown
|
|
var fhtml = '', colors = ['#f85149','#db6d28','#d29922','#58a6ff','#a371f7','#39c5cf','#db61a2','#7d8590'];
|
|
if (d.failures && Object.keys(d.failures).length > 0) {
|
|
var cats = Object.keys(d.failures).sort(function(a,b) { return d.failures[b] - d.failures[a]; });
|
|
var failTotal = cats.reduce(function(s, c) { return s + d.failures[c]; }, 0);
|
|
var segs = [];
|
|
cats.forEach(function(cat, i) {
|
|
var n = d.failures[cat], col = colors[i % colors.length];
|
|
segs.push({c: col, d: (n / failTotal) * 360});
|
|
fhtml += '<div class="legend-item"><div class="legend-dot" style="background:' + col + '"></div>';
|
|
fhtml += '<span class="legend-name">' + cat + '</span><span class="legend-val">' + n + '</span></div>';
|
|
});
|
|
$('failPie').style.background = conicGrad(segs);
|
|
} else {
|
|
$('failPie').style.background = 'var(--border)';
|
|
fhtml = '<div style="color:var(--dim);font-size:11px">No failures yet</div>';
|
|
}
|
|
$('failLegend').innerHTML = fhtml;
|
|
|
|
// Leaderboards (session data)
|
|
renderLeaderboard('topCountries', d.top_countries_session, 'code', 'count');
|
|
renderLeaderboard('topAsns', d.top_asns_session, 'asn', 'count');
|
|
|
|
// Tor pool
|
|
var thtml = '';
|
|
if (d.tor_pool && d.tor_pool.hosts) {
|
|
d.tor_pool.hosts.forEach(function(h) {
|
|
thtml += '<div class="c c-sm"><div class="host-card">';
|
|
thtml += '<span class="host-addr">' + h.address + '</span>';
|
|
thtml += '<span class="tag ' + (h.healthy ? 'tag-ok' : 'tag-err') + '">' + (h.healthy ? 'OK' : 'DOWN') + '</span>';
|
|
thtml += '</div><div class="host-stats">' + fmtMs(h.latency_ms) + ' / ' + fmtDec(h.success_rate, 0) + '% success</div></div>';
|
|
});
|
|
}
|
|
$('torPool').innerHTML = thtml || '<div class="c c-sm" style="color:var(--dim)">No Tor hosts</div>';
|
|
|
|
// Judges
|
|
if (d.judges) {
|
|
$('judgesAvail').textContent = d.judges.available + '/' + d.judges.total;
|
|
$('judgesCooldown').textContent = d.judges.in_cooldown;
|
|
var jhtml = '';
|
|
if (d.judges.top_judges) {
|
|
d.judges.top_judges.slice(0, 6).forEach(function(j) {
|
|
jhtml += '<div class="judge-item"><span class="judge-name">' + j.judge + '</span>';
|
|
jhtml += '<div class="judge-stats"><span class="grn">' + j.success + '/' + j.tests + '</span>';
|
|
jhtml += '<span style="color:var(--dim)">' + fmtDec(j.rate, 0) + '%</span></div></div>';
|
|
});
|
|
}
|
|
$('topJudges').innerHTML = jhtml || '<div style="color:var(--dim)">No data</div>';
|
|
}
|
|
|
|
// Database stats
|
|
if (d.db) {
|
|
var dbs = d.db;
|
|
$('dbByProto').innerHTML = ['http', 'socks4', 'socks5'].map(function(p) {
|
|
var c = dbs.by_proto && dbs.by_proto[p] || 0;
|
|
return '<div class="stat-row"><span class="stat-lbl">' + p.toUpperCase() + '</span><span class="stat-val">' + fmt(c) + '</span></div>';
|
|
}).join('');
|
|
|
|
var chtml = '';
|
|
if (dbs.top_countries) {
|
|
dbs.top_countries.slice(0, 5).forEach(function(c, i) {
|
|
var name = c.code || c[0], cnt = c.count || c[1];
|
|
chtml += '<div class="lb-item"><div class="lb-rank' + (i === 0 ? ' top' : '') + '">' + (i + 1) + '</div>';
|
|
chtml += '<span class="lb-name">' + name + '</span><span class="lb-val blu">' + fmt(cnt) + '</span></div>';
|
|
});
|
|
}
|
|
$('dbCountries').innerHTML = chtml || '<div style="color:var(--dim)">No data</div>';
|
|
}
|
|
|
|
// Scraper/Engine stats
|
|
$('engAvail').textContent = fmt(d.engines_available);
|
|
$('engBackoff').textContent = fmt(d.engines_backoff);
|
|
$('engTotal').textContent = fmt(d.engines_total);
|
|
if (d.scraper && d.scraper.engines) {
|
|
var ehtml = '';
|
|
d.scraper.engines.slice(0, 5).forEach(function(e, i) {
|
|
var statusCls = e.available ? 'tag-ok' : 'tag-warn';
|
|
var statusTxt = e.available ? 'OK' : (e.backoff_remaining > 0 ? e.backoff_remaining + 's' : 'OFF');
|
|
ehtml += '<div class="lb-item"><div class="lb-rank' + (i === 0 ? ' top' : '') + '">' + (i + 1) + '</div>';
|
|
ehtml += '<span class="lb-name">' + e.name + '</span>';
|
|
ehtml += '<span class="tag ' + statusCls + '">' + statusTxt + '</span>';
|
|
ehtml += '<span class="lb-val grn">' + fmt(e.successes) + '</span></div>';
|
|
});
|
|
$('topEngines').innerHTML = ehtml || '<div style="color:var(--dim)">No engines</div>';
|
|
} else {
|
|
$('topEngines').innerHTML = '<div style="color:var(--dim)">Scraper disabled</div>';
|
|
}
|
|
|
|
// SSL/TLS stats
|
|
if (d.ssl) {
|
|
var ssl = d.ssl;
|
|
$('sslTested').textContent = fmt(ssl.tested);
|
|
$('sslPassed').textContent = fmt(ssl.passed);
|
|
$('sslFailed').textContent = fmt(ssl.failed);
|
|
var sslRate = ssl.success_rate || 0;
|
|
setBar('sslBar', sslRate, 100, sslRate < 50 ? 'red' : sslRate < 80 ? 'yel' : 'grn');
|
|
$('mitmDetected').textContent = fmt(ssl.mitm_detected);
|
|
$('mitmDetected').className = 'stat-val ' + (ssl.mitm_detected > 0 ? 'red' : 'grn');
|
|
$('certErrors').textContent = fmt(ssl.cert_errors);
|
|
$('certErrors').className = 'stat-val ' + (ssl.cert_errors > 0 ? 'yel' : 'grn');
|
|
}
|
|
|
|
// Anonymity breakdown
|
|
var dbh = d.db_health || {};
|
|
if (dbh.anonymity) {
|
|
var anonHtml = '';
|
|
var anonColors = {elite: 'grn', anonymous: 'blu', transparent: 'yel', unknown: 'dim'};
|
|
var anonOrder = ['elite', 'anonymous', 'transparent', 'unknown'];
|
|
anonOrder.forEach(function(level) {
|
|
var count = dbh.anonymity[level] || 0;
|
|
if (count > 0) {
|
|
anonHtml += '<div class="stat-row"><span class="stat-lbl">' + level.charAt(0).toUpperCase() + level.slice(1) + '</span>';
|
|
anonHtml += '<span class="stat-val ' + anonColors[level] + '">' + fmt(count) + '</span></div>';
|
|
}
|
|
});
|
|
$('anonBreakdown').innerHTML = anonHtml || '<div style="color:var(--dim)">No data</div>';
|
|
}
|
|
|
|
// Database health
|
|
if (dbh.db_size) {
|
|
$('dbSize').textContent = fmtBytes(dbh.db_size);
|
|
$('dbTestedHour').textContent = fmt(dbh.tested_last_hour);
|
|
$('dbAddedDay').textContent = fmt(dbh.added_last_day);
|
|
$('dbDead').textContent = fmt(dbh.dead_count);
|
|
}
|
|
|
|
// Tor pool enhanced stats
|
|
if (d.tor_pool) {
|
|
var tp = d.tor_pool;
|
|
$('torTotal').textContent = fmt(tp.total_requests || 0);
|
|
$('torSuccess').textContent = fmtDec(tp.success_rate || 0, 1) + '%';
|
|
$('torHealthy').textContent = (tp.healthy_count || 0) + '/' + (tp.total_count || 0);
|
|
if (tp.avg_latency) {
|
|
$('torLatency').textContent = fmtMs(tp.avg_latency);
|
|
}
|
|
}
|
|
|
|
$('lastUpdate').textContent = new Date().toLocaleTimeString();
|
|
}
|
|
|
|
function fetchStats() {
|
|
fetch('/api/stats')
|
|
.then(function(r) { return r.json(); })
|
|
.then(update)
|
|
.catch(function(e) { $('dot').className = 'dot err'; $('statusTxt').textContent = 'Error'; });
|
|
}
|
|
|
|
fetchStats();
|
|
setInterval(fetchStats, 3000);
|
|
'''
|
|
|
|
DASHBOARD_HTML = '''<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>PPF Dashboard</title>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<link rel="stylesheet" href="/static/style.css">
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<div class="hdr">
|
|
<h1>PPF Dashboard</h1>
|
|
<div class="status">
|
|
<span class="mode-badge mode-ssl" id="checktypeBadge">-</span>
|
|
<div class="status-item"><div class="dot" id="dot"></div><span id="statusTxt">Connecting</span></div>
|
|
<div class="status-item">Updated: <span id="lastUpdate">-</span></div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- System Monitor Bar -->
|
|
<div class="sysbar">
|
|
<div class="sysbar-item"><span class="sysbar-lbl">Load:</span><span class="sysbar-val" id="sysLoad">-</span></div>
|
|
<div class="sysbar-item"><span class="sysbar-lbl">Memory:</span><span class="sysbar-val" id="sysMemVal">-</span>
|
|
<div class="sysbar-bar"><div class="sysbar-fill" id="sysMemFill" style="width:0"></div></div>
|
|
<span class="sysbar-val" id="sysMemPct">-</span></div>
|
|
<div class="sysbar-item"><span class="sysbar-lbl">Disk:</span><span class="sysbar-val" id="sysDiskVal">-</span>
|
|
<div class="sysbar-bar"><div class="sysbar-fill" id="sysDiskFill" style="width:0"></div></div>
|
|
<span class="sysbar-val" id="sysDiskPct">-</span></div>
|
|
<div class="sysbar-item"><span class="sysbar-lbl">Process:</span><span class="sysbar-val" id="sysProcMem">-</span></div>
|
|
</div>
|
|
|
|
<!-- Primary Stats Row -->
|
|
<div class="g g5">
|
|
<div class="c">
|
|
<div class="lbl">Working Proxies</div>
|
|
<div class="val grn" id="working">-</div>
|
|
<div class="sub">of <span id="total">-</span> in database</div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="lbl">Tests This Session</div>
|
|
<div class="val" id="tested">-</div>
|
|
<div class="sub"><span class="grn" id="passed">-</span> passed / <span class="red" id="failed">-</span> failed</div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="lbl">Success Rate</div>
|
|
<div class="val-md grn" id="successRate">-</div>
|
|
<div class="bar-wrap"><div class="bar grn" id="srBar" style="width:0"></div></div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="lbl">Test Rate</div>
|
|
<div class="val-md blu" id="rate">-</div>
|
|
<div class="sub">tests/sec average</div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="lbl">Uptime</div>
|
|
<div class="val-md" id="uptime">-</div>
|
|
<div class="sub">session duration</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Rate & Success Charts -->
|
|
<div class="g g2">
|
|
<div class="c c-lg">
|
|
<div class="lbl">Test Rate History (10 min)</div>
|
|
<div class="mini">
|
|
<div class="mini-item"><span class="mini-val blu" id="recentRate">-</span><span class="mini-lbl">current</span></div>
|
|
<div class="mini-item"><span class="mini-val yel" id="peakRate">-</span><span class="mini-lbl">peak</span></div>
|
|
<div class="mini-item"><span class="mini-val grn" id="passRate">-</span><span class="mini-lbl">pass/s</span></div>
|
|
</div>
|
|
<div class="chart chart-lg" id="rateChart"></div>
|
|
</div>
|
|
<div class="c c-lg">
|
|
<div class="lbl">Success Rate History</div>
|
|
<div class="mini">
|
|
<div class="mini-item"><span class="mini-val" id="recentSuccessRate">-</span><span class="mini-lbl">recent</span></div>
|
|
</div>
|
|
<div class="chart chart-lg" id="srChart"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Latency Section -->
|
|
<div class="sec">
|
|
<div class="sec-hdr">Latency Analysis</div>
|
|
<div class="g g2">
|
|
<div class="c">
|
|
<div class="stat-row"><span class="stat-lbl">Average</span><span class="stat-val cyn" id="avgLatency">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Min</span><span class="stat-val grn" id="minLatency">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Max</span><span class="stat-val red" id="maxLatency">-</span></div>
|
|
<div class="pct-badges">
|
|
<div class="pct-badge"><div class="pct-label">P50</div><div class="pct-value cyn" id="p50">-</div></div>
|
|
<div class="pct-badge"><div class="pct-label">P90</div><div class="pct-value yel" id="p90">-</div></div>
|
|
<div class="pct-badge"><div class="pct-label">P99</div><div class="pct-value org" id="p99">-</div></div>
|
|
</div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="lbl">Response Time Distribution</div>
|
|
<div class="histo" id="latencyHisto"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Protocol Breakdown -->
|
|
<div class="sec">
|
|
<div class="sec-hdr">Protocol Performance</div>
|
|
<div class="g g3">
|
|
<div class="c proto-card">
|
|
<div class="proto-icon">🌐</div>
|
|
<div class="proto-name">HTTP</div>
|
|
<div class="proto-val grn" id="httpPassed">-</div>
|
|
<div class="sub">of <span id="httpTested">-</span> tested</div>
|
|
<div class="proto-rate tag-ok" id="httpRate">-</div>
|
|
</div>
|
|
<div class="c proto-card">
|
|
<div class="proto-icon">🔌</div>
|
|
<div class="proto-name">SOCKS4</div>
|
|
<div class="proto-val blu" id="socks4Passed">-</div>
|
|
<div class="sub">of <span id="socks4Tested">-</span> tested</div>
|
|
<div class="proto-rate tag-ok" id="socks4Rate">-</div>
|
|
</div>
|
|
<div class="c proto-card">
|
|
<div class="proto-icon">🔒</div>
|
|
<div class="proto-name">SOCKS5</div>
|
|
<div class="proto-val pur" id="socks5Passed">-</div>
|
|
<div class="sub">of <span id="socks5Tested">-</span> tested</div>
|
|
<div class="proto-rate tag-ok" id="socks5Rate">-</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Results & Failures -->
|
|
<div class="g g2">
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">Test Results</div>
|
|
<div class="pie-wrap">
|
|
<div class="pie" id="resultsPie"></div>
|
|
<div class="legend">
|
|
<div class="legend-item"><div class="legend-dot" style="background:#3fb950"></div><span class="legend-name">Passed</span><span class="legend-val grn" id="passedLeg">-</span><span class="sub" id="passedPct">-</span></div>
|
|
<div class="legend-item"><div class="legend-dot" style="background:#f85149"></div><span class="legend-name">Failed</span><span class="legend-val red" id="failedLeg">-</span><span class="sub" id="failedPct">-</span></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">Failure Breakdown</div>
|
|
<div class="pie-wrap">
|
|
<div class="pie" id="failPie"></div>
|
|
<div class="legend" id="failLegend"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Geographic Distribution -->
|
|
<div class="sec">
|
|
<div class="sec-hdr">Geographic Distribution (Session)</div>
|
|
<div class="g g2">
|
|
<div class="c">
|
|
<div class="lbl">Top Countries</div>
|
|
<div class="lb" id="topCountries"></div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="lbl">Top ASNs</div>
|
|
<div class="lb" id="topAsns"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- System & Infrastructure -->
|
|
<div class="g g2">
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">Worker Pool</div>
|
|
<div class="stat-row"><span class="stat-lbl">Active Threads</span><span class="stat-val" id="threads">-</span></div>
|
|
<div class="bar-wrap"><div class="bar blu" id="threadBar" style="width:0"></div></div>
|
|
<div class="stat-row" style="margin-top:8px"><span class="stat-lbl">Job Queue</span><span class="stat-val yel" id="queue">-</span></div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">Judge Services</div>
|
|
<div class="stat-row"><span class="stat-lbl">Available</span><span class="stat-val grn" id="judgesAvail">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">In Cooldown</span><span class="stat-val yel" id="judgesCooldown">-</span></div>
|
|
<div class="lbl" style="margin-top:10px">Top Performers</div>
|
|
<div id="topJudges"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Tor Pool & Anonymity -->
|
|
<div class="g g2">
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">Tor Pool</div>
|
|
<div class="stat-row"><span class="stat-lbl">Total Requests</span><span class="stat-val" id="torTotal">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Success Rate</span><span class="stat-val grn" id="torSuccess">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Healthy Nodes</span><span class="stat-val" id="torHealthy">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Avg Latency</span><span class="stat-val cyn" id="torLatency">-</span></div>
|
|
<div class="lbl" style="margin-top:10px">Exit Nodes</div>
|
|
<div class="g g3" id="torPool" style="margin-top:8px"></div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">Anonymity Levels</div>
|
|
<div id="anonBreakdown"></div>
|
|
<div class="sub" style="margin-top:8px;font-size:10px">Elite = no headers, Anonymous = adds headers, Transparent = reveals IP</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Scraper & SSL Stats -->
|
|
<div class="g g2">
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">Search Engines</div>
|
|
<div class="stat-row"><span class="stat-lbl">Available</span><span class="stat-val grn" id="engAvail">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">In Backoff</span><span class="stat-val yel" id="engBackoff">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Total</span><span class="stat-val" id="engTotal">-</span></div>
|
|
<div class="lbl" style="margin-top:10px">Top Engines</div>
|
|
<div class="lb" id="topEngines"></div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="sec-hdr" style="margin-top:0">SSL/TLS Security</div>
|
|
<div class="stat-row"><span class="stat-lbl">SSL Tests</span><span class="stat-val" id="sslTested">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Passed</span><span class="stat-val grn" id="sslPassed">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Failed</span><span class="stat-val red" id="sslFailed">-</span></div>
|
|
<div class="bar-wrap" style="margin:8px 0"><div class="bar grn" id="sslBar" style="width:0"></div></div>
|
|
<div class="stat-row"><span class="stat-lbl">MITM Detected</span><span class="stat-val red" id="mitmDetected">-</span></div>
|
|
<div class="stat-row"><span class="stat-lbl">Cert Errors</span><span class="stat-val yel" id="certErrors">-</span></div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Database Stats -->
|
|
<div class="sec">
|
|
<div class="sec-hdr">Database Overview</div>
|
|
<div class="g g4">
|
|
<div class="c c-sm">
|
|
<div class="lbl">Database Size</div>
|
|
<div class="val-sm cyn" id="dbSize">-</div>
|
|
</div>
|
|
<div class="c c-sm">
|
|
<div class="lbl">Tested (1h)</div>
|
|
<div class="val-sm blu" id="dbTestedHour">-</div>
|
|
</div>
|
|
<div class="c c-sm">
|
|
<div class="lbl">Added (24h)</div>
|
|
<div class="val-sm grn" id="dbAddedDay">-</div>
|
|
</div>
|
|
<div class="c c-sm">
|
|
<div class="lbl">Dead Proxies</div>
|
|
<div class="val-sm red" id="dbDead">-</div>
|
|
</div>
|
|
</div>
|
|
<div class="g g2">
|
|
<div class="c">
|
|
<div class="lbl">Working by Protocol</div>
|
|
<div id="dbByProto"></div>
|
|
</div>
|
|
<div class="c">
|
|
<div class="lbl">Top Countries (All Time)</div>
|
|
<div class="lb" id="dbCountries"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="ftr">PPF Proxy Fetcher</div>
|
|
</div>
|
|
<script src="/static/dashboard.js"></script>
|
|
</body>
|
|
</html>'''
|
|
|
|
|
|
class ProxyAPIHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
|
"""HTTP request handler for proxy API."""
|
|
|
|
database = None
|
|
stats_provider = None
|
|
|
|
def log_message(self, format, *args):
|
|
pass
|
|
|
|
def send_response_body(self, body, content_type, status=200):
|
|
self.send_response(status)
|
|
self.send_header('Content-Type', content_type)
|
|
self.send_header('Content-Length', len(body))
|
|
self.send_header('Cache-Control', 'no-cache')
|
|
if content_type == 'application/json':
|
|
self.send_header('Access-Control-Allow-Origin', '*')
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def send_json(self, data, status=200):
|
|
self.send_response_body(json.dumps(data, indent=2), 'application/json', status)
|
|
|
|
def send_text(self, text, status=200):
|
|
self.send_response_body(text, 'text/plain', status)
|
|
|
|
def send_html(self, html, status=200):
|
|
self.send_response_body(html, 'text/html; charset=utf-8', status)
|
|
|
|
def send_css(self, css, status=200):
|
|
self.send_response_body(css, 'text/css; charset=utf-8', status)
|
|
|
|
def send_js(self, js, status=200):
|
|
self.send_response_body(js, 'application/javascript; charset=utf-8', status)
|
|
|
|
def do_GET(self):
|
|
path = self.path.split('?')[0]
|
|
routes = {
|
|
'/': self.handle_index,
|
|
'/dashboard': self.handle_dashboard,
|
|
'/static/style.css': self.handle_css,
|
|
'/static/dashboard.js': self.handle_js,
|
|
'/api/stats': self.handle_stats,
|
|
'/proxies': self.handle_proxies,
|
|
'/proxies/count': self.handle_count,
|
|
'/health': self.handle_health,
|
|
}
|
|
handler = routes.get(path)
|
|
if handler:
|
|
handler()
|
|
else:
|
|
self.send_json({'error': 'not found'}, 404)
|
|
|
|
def handle_index(self):
|
|
self.send_json({
|
|
'endpoints': {
|
|
'/dashboard': 'web dashboard (HTML)',
|
|
'/api/stats': 'runtime statistics (JSON)',
|
|
'/proxies': 'list working proxies (params: limit, proto, country, asn)',
|
|
'/proxies/count': 'count working proxies',
|
|
'/health': 'health check',
|
|
}
|
|
})
|
|
|
|
def handle_dashboard(self):
|
|
self.send_html(DASHBOARD_HTML)
|
|
|
|
def handle_css(self):
|
|
self.send_css(DASHBOARD_CSS)
|
|
|
|
def handle_js(self):
|
|
self.send_js(DASHBOARD_JS)
|
|
|
|
def get_db_stats(self):
|
|
"""Get statistics from database."""
|
|
try:
|
|
db = mysqlite.mysqlite(self.database, str)
|
|
stats = {}
|
|
|
|
# Total counts
|
|
row = db.execute('SELECT COUNT(*) FROM proxylist WHERE failed=0').fetchone()
|
|
stats['working'] = row[0] if row else 0
|
|
row = db.execute('SELECT COUNT(*) FROM proxylist').fetchone()
|
|
stats['total'] = row[0] if row else 0
|
|
|
|
# By protocol
|
|
rows = db.execute(
|
|
'SELECT proto, COUNT(*) FROM proxylist WHERE failed=0 GROUP BY proto'
|
|
).fetchall()
|
|
stats['by_proto'] = {r[0] or 'unknown': r[1] for r in rows}
|
|
|
|
# Top countries
|
|
rows = db.execute(
|
|
'SELECT country, COUNT(*) as c FROM proxylist WHERE failed=0 AND country IS NOT NULL '
|
|
'GROUP BY country ORDER BY c DESC LIMIT 10'
|
|
).fetchall()
|
|
stats['top_countries'] = [{'code': r[0], 'count': r[1]} for r in rows]
|
|
|
|
# Top ASNs
|
|
rows = db.execute(
|
|
'SELECT asn, COUNT(*) as c FROM proxylist WHERE failed=0 AND asn IS NOT NULL '
|
|
'GROUP BY asn ORDER BY c DESC LIMIT 10'
|
|
).fetchall()
|
|
stats['top_asns'] = [(r[0], r[1]) for r in rows]
|
|
|
|
return stats
|
|
except Exception as e:
|
|
return {'error': str(e)}
|
|
|
|
def handle_stats(self):
|
|
stats = {}
|
|
|
|
# Runtime stats from provider
|
|
if self.stats_provider:
|
|
try:
|
|
stats.update(self.stats_provider())
|
|
except Exception as e:
|
|
_log('stats_provider error: %s' % str(e), 'error')
|
|
|
|
# Add system stats
|
|
stats['system'] = get_system_stats()
|
|
|
|
# Add database stats
|
|
try:
|
|
db = mysqlite.mysqlite(self.database, str)
|
|
stats['db'] = self.get_db_stats()
|
|
stats['db_health'] = get_db_health(db)
|
|
except Exception:
|
|
pass
|
|
|
|
self.send_json(stats)
|
|
|
|
def handle_proxies(self):
|
|
params = {}
|
|
if '?' in self.path:
|
|
for pair in self.path.split('?')[1].split('&'):
|
|
if '=' in pair:
|
|
k, v = pair.split('=', 1)
|
|
params[k] = v
|
|
|
|
limit = min(int(params.get('limit', 100)), 1000)
|
|
proto = params.get('proto', '')
|
|
country = params.get('country', '')
|
|
asn = params.get('asn', '')
|
|
fmt = params.get('format', 'json')
|
|
|
|
sql = 'SELECT ip, port, proto, country, asn, avg_latency FROM proxylist WHERE failed=0'
|
|
args = []
|
|
|
|
if proto:
|
|
sql += ' AND proto=?'
|
|
args.append(proto)
|
|
if country:
|
|
sql += ' AND country=?'
|
|
args.append(country.upper())
|
|
if asn:
|
|
sql += ' AND asn=?'
|
|
args.append(int(asn))
|
|
|
|
sql += ' ORDER BY avg_latency ASC, tested DESC LIMIT ?'
|
|
args.append(limit)
|
|
|
|
try:
|
|
db = mysqlite.mysqlite(self.database, str)
|
|
rows = db.execute(sql, args).fetchall()
|
|
|
|
if fmt == 'plain':
|
|
self.send_text('\n'.join('%s:%s' % (r[0], r[1]) for r in rows))
|
|
else:
|
|
proxies = [{
|
|
'ip': r[0], 'port': r[1], 'proto': r[2],
|
|
'country': r[3], 'asn': r[4], 'latency': r[5]
|
|
} for r in rows]
|
|
self.send_json({'count': len(proxies), 'proxies': proxies})
|
|
except Exception as e:
|
|
self.send_json({'error': str(e)}, 500)
|
|
|
|
def handle_count(self):
|
|
try:
|
|
db = mysqlite.mysqlite(self.database, str)
|
|
row = db.execute('SELECT COUNT(*) FROM proxylist WHERE failed=0').fetchone()
|
|
self.send_json({'count': row[0] if row else 0})
|
|
except Exception as e:
|
|
self.send_json({'error': str(e)}, 500)
|
|
|
|
def handle_health(self):
|
|
self.send_json({'status': 'ok', 'timestamp': int(time.time())})
|
|
|
|
|
|
class ProxyAPIServer(threading.Thread):
|
|
"""Threaded HTTP API server.
|
|
|
|
Uses gevent's WSGIServer when running in a gevent-patched environment,
|
|
otherwise falls back to standard BaseHTTPServer.
|
|
"""
|
|
|
|
def __init__(self, host, port, database, stats_provider=None):
|
|
threading.Thread.__init__(self)
|
|
self.host = host
|
|
self.port = port
|
|
self.database = database
|
|
self.stats_provider = stats_provider
|
|
self.daemon = True
|
|
self.server = None
|
|
self._stop_event = threading.Event() if not GEVENT_PATCHED else None
|
|
|
|
def _wsgi_app(self, environ, start_response):
|
|
"""WSGI application wrapper for gevent."""
|
|
path = environ.get('PATH_INFO', '/').split('?')[0]
|
|
method = environ.get('REQUEST_METHOD', 'GET')
|
|
|
|
if method != 'GET':
|
|
start_response('405 Method Not Allowed', [('Content-Type', 'text/plain')])
|
|
return [b'Method not allowed']
|
|
|
|
# Route handling
|
|
try:
|
|
response_body, content_type, status = self._handle_route(path)
|
|
status_line = '%d %s' % (status, 'OK' if status == 200 else 'Error')
|
|
headers = [
|
|
('Content-Type', content_type),
|
|
('Content-Length', str(len(response_body))),
|
|
('Cache-Control', 'no-cache'),
|
|
]
|
|
if content_type == 'application/json':
|
|
headers.append(('Access-Control-Allow-Origin', '*'))
|
|
start_response(status_line, headers)
|
|
return [response_body.encode('utf-8') if isinstance(response_body, unicode) else response_body]
|
|
except Exception as e:
|
|
error_body = json.dumps({'error': str(e)})
|
|
start_response('500 Internal Server Error', [
|
|
('Content-Type', 'application/json'),
|
|
('Content-Length', str(len(error_body))),
|
|
])
|
|
return [error_body]
|
|
|
|
def _handle_route(self, path):
|
|
"""Handle route and return (body, content_type, status)."""
|
|
if path == '/':
|
|
body = json.dumps({
|
|
'endpoints': {
|
|
'/dashboard': 'web dashboard (HTML)',
|
|
'/api/stats': 'runtime statistics (JSON)',
|
|
'/proxies': 'list working proxies (params: limit, proto, country, asn)',
|
|
'/proxies/count': 'count working proxies',
|
|
'/health': 'health check',
|
|
}
|
|
}, indent=2)
|
|
return body, 'application/json', 200
|
|
elif path == '/dashboard':
|
|
return DASHBOARD_HTML, 'text/html; charset=utf-8', 200
|
|
elif path == '/static/style.css':
|
|
# Use str.format() instead of % to avoid issues with % escaping
|
|
css = DASHBOARD_CSS
|
|
for key, val in THEME.items():
|
|
css = css.replace('{' + key + '}', val)
|
|
return css, 'text/css; charset=utf-8', 200
|
|
elif path == '/static/dashboard.js':
|
|
return DASHBOARD_JS, 'application/javascript; charset=utf-8', 200
|
|
elif path == '/api/stats':
|
|
stats = {}
|
|
if self.stats_provider:
|
|
stats = self.stats_provider()
|
|
# Add system stats
|
|
stats['system'] = get_system_stats()
|
|
# Add database stats
|
|
try:
|
|
db = mysqlite.mysqlite(self.database, str)
|
|
stats['db'] = self._get_db_stats(db)
|
|
stats['db_health'] = get_db_health(db)
|
|
except Exception:
|
|
pass
|
|
return json.dumps(stats, indent=2), 'application/json', 200
|
|
elif path == '/proxies':
|
|
try:
|
|
db = mysqlite.mysqlite(self.database, str)
|
|
rows = db.execute(
|
|
'SELECT proxy, proto, country, asn FROM proxylist WHERE failed=0 LIMIT 100'
|
|
).fetchall()
|
|
proxies = [{'proxy': r[0], 'proto': r[1], 'country': r[2], 'asn': r[3]} for r in rows]
|
|
return json.dumps({'proxies': proxies}, indent=2), 'application/json', 200
|
|
except Exception as e:
|
|
return json.dumps({'error': str(e)}), 'application/json', 500
|
|
elif path == '/proxies/count':
|
|
try:
|
|
db = mysqlite.mysqlite(self.database, str)
|
|
row = db.execute('SELECT COUNT(*) FROM proxylist WHERE failed=0').fetchone()
|
|
return json.dumps({'count': row[0] if row else 0}), 'application/json', 200
|
|
except Exception as e:
|
|
return json.dumps({'error': str(e)}), 'application/json', 500
|
|
elif path == '/health':
|
|
return json.dumps({'status': 'ok', 'timestamp': int(time.time())}), 'application/json', 200
|
|
else:
|
|
return json.dumps({'error': 'not found'}), 'application/json', 404
|
|
|
|
def _get_db_stats(self, db):
|
|
"""Get database statistics."""
|
|
stats = {}
|
|
try:
|
|
# By protocol
|
|
rows = db.execute(
|
|
'SELECT proto, COUNT(*) FROM proxylist WHERE failed=0 GROUP BY proto'
|
|
).fetchall()
|
|
stats['by_proto'] = {r[0]: r[1] for r in rows if r[0]}
|
|
|
|
# Top countries
|
|
rows = db.execute(
|
|
'SELECT country, COUNT(*) as cnt FROM proxylist WHERE failed=0 AND country IS NOT NULL '
|
|
'GROUP BY country ORDER BY cnt DESC LIMIT 10'
|
|
).fetchall()
|
|
stats['top_countries'] = [{'code': r[0], 'count': r[1]} for r in rows]
|
|
|
|
# Total counts
|
|
row = db.execute('SELECT COUNT(*) FROM proxylist WHERE failed=0').fetchone()
|
|
stats['working'] = row[0] if row else 0
|
|
row = db.execute('SELECT COUNT(*) FROM proxylist').fetchone()
|
|
stats['total'] = row[0] if row else 0
|
|
except Exception:
|
|
pass
|
|
return stats
|
|
|
|
def run(self):
|
|
ProxyAPIHandler.database = self.database
|
|
ProxyAPIHandler.stats_provider = self.stats_provider
|
|
|
|
if GEVENT_PATCHED:
|
|
# Use gevent's WSGIServer for proper async handling
|
|
self.server = WSGIServer((self.host, self.port), self._wsgi_app, log=None)
|
|
_log('httpd listening on %s:%d (gevent)' % (self.host, self.port), 'info')
|
|
self.server.serve_forever()
|
|
else:
|
|
# Standard BaseHTTPServer for non-gevent environments
|
|
self.server = BaseHTTPServer.HTTPServer((self.host, self.port), ProxyAPIHandler)
|
|
_log('httpd listening on %s:%d' % (self.host, self.port), 'info')
|
|
self.server.serve_forever()
|
|
|
|
def stop(self):
|
|
if self.server:
|
|
if GEVENT_PATCHED:
|
|
self.server.stop()
|
|
else:
|
|
self.server.shutdown()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import sys
|
|
host = '127.0.0.1'
|
|
port = 8081
|
|
database = 'data/proxies.sqlite'
|
|
|
|
if len(sys.argv) > 1:
|
|
database = sys.argv[1]
|
|
|
|
_log('starting test server on %s:%d (db: %s)' % (host, port, database), 'info')
|
|
server = ProxyAPIServer(host, port, database)
|
|
server.start()
|
|
|
|
try:
|
|
while True:
|
|
time.sleep(1)
|
|
except KeyboardInterrupt:
|
|
server.stop()
|
|
_log('server stopped', 'info')
|