diff --git a/httpd.py b/httpd.py
index b689d0e..d4a5a52 100644
--- a/httpd.py
+++ b/httpd.py
@@ -1,75 +1,850 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
-"""Simple HTTP API server for querying working proxies."""
+"""HTTP API server with advanced web dashboard for PPF."""
import BaseHTTPServer
import json
import threading
+import time
import mysqlite
from misc import _log
+# Detect if gevent has monkey-patched the environment
+try:
+ from gevent import monkey
+ GEVENT_PATCHED = monkey.is_module_patched('socket')
+except ImportError:
+ GEVENT_PATCHED = False
+
+if GEVENT_PATCHED:
+ from gevent.pywsgi import WSGIServer
+
+# Theme colors - modern dark palette
+THEME = {
+ 'bg': '#0d1117',
+ 'card': '#161b22',
+ 'card_alt': '#1c2128',
+ 'border': '#30363d',
+ 'text': '#e6edf3',
+ 'dim': '#7d8590',
+ 'green': '#3fb950',
+ 'red': '#f85149',
+ 'yellow': '#d29922',
+ 'blue': '#58a6ff',
+ 'purple': '#a371f7',
+ 'cyan': '#39c5cf',
+ 'orange': '#db6d28',
+ 'pink': '#db61a2',
+}
+
+DASHBOARD_CSS = '''
+:root {
+ --bg: {bg}; --card: {card}; --card-alt: {card_alt}; --border: {border};
+ --text: {text}; --dim: {dim}; --green: {green}; --red: {red};
+ --yellow: {yellow}; --blue: {blue}; --purple: {purple};
+ --cyan: {cyan}; --orange: {orange}; --pink: {pink};
+}
+* { box-sizing: border-box; margin: 0; padding: 0; }
+body {
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
+ font-size: 13px; background: var(--bg); color: var(--text);
+ padding: 16px; min-height: 100vh; line-height: 1.4;
+}
+.container { max-width: 1400px; margin: 0 auto; }
+
+/* Header */
+.hdr { display: flex; justify-content: space-between; align-items: center; margin-bottom: 16px; padding-bottom: 12px; border-bottom: 1px solid var(--border); }
+.hdr h1 { font-size: 18px; font-weight: 600; display: flex; align-items: center; gap: 10px; }
+.hdr h1::before { content: ""; width: 10px; height: 10px; background: var(--green); border-radius: 50%; box-shadow: 0 0 8px var(--green); }
+.status { display: flex; align-items: center; gap: 12px; font-size: 12px; color: var(--dim); }
+.status-item { display: flex; align-items: center; gap: 4px; }
+.dot { width: 6px; height: 6px; border-radius: 50%; background: var(--green); animation: pulse 2s infinite; }
+.dot.err { background: var(--red); animation: none; }
+@keyframes pulse { 50% { opacity: 0.5; } }
+
+/* Grid */
+.g { display: grid; gap: 12px; margin-bottom: 16px; }
+.g2 { grid-template-columns: repeat(2, 1fr); }
+.g3 { grid-template-columns: repeat(3, 1fr); }
+.g4 { grid-template-columns: repeat(4, 1fr); }
+.g5 { grid-template-columns: repeat(5, 1fr); }
+.g6 { grid-template-columns: repeat(6, 1fr); }
+@media (max-width: 1200px) { .g5, .g6 { grid-template-columns: repeat(4, 1fr); } }
+@media (max-width: 900px) { .g3, .g4, .g5, .g6 { grid-template-columns: repeat(2, 1fr); } }
+@media (max-width: 600px) { .g2, .g3, .g4, .g5, .g6 { grid-template-columns: 1fr; } }
+
+/* Cards */
+.c { background: var(--card); border: 1px solid var(--border); border-radius: 8px; padding: 14px; }
+.c-lg { padding: 16px 18px; }
+.c-sm { padding: 10px 12px; }
+.lbl { font-size: 10px; color: var(--dim); text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 6px; font-weight: 500; }
+.val { font-size: 26px; font-weight: 700; font-feature-settings: "tnum"; letter-spacing: -0.5px; }
+.val-md { font-size: 20px; }
+.val-sm { font-size: 16px; }
+.sub { font-size: 11px; color: var(--dim); margin-top: 4px; }
+.grn { color: var(--green); } .red { color: var(--red); } .yel { color: var(--yellow); }
+.blu { color: var(--blue); } .pur { color: var(--purple); } .cyn { color: var(--cyan); }
+.org { color: var(--orange); } .pnk { color: var(--pink); }
+
+/* Section headers */
+.sec { margin-bottom: 16px; }
+.sec-hdr { font-size: 11px; font-weight: 600; color: var(--dim); text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 10px; display: flex; align-items: center; gap: 8px; }
+.sec-hdr::before { content: ""; width: 3px; height: 12px; background: var(--blue); border-radius: 2px; }
+
+/* Progress bars */
+.bar-wrap { height: 6px; background: var(--border); border-radius: 3px; margin-top: 8px; overflow: hidden; }
+.bar { height: 100%; border-radius: 3px; transition: width 0.4s ease; }
+.bar.grn { background: linear-gradient(90deg, #238636, #3fb950); }
+.bar.red { background: linear-gradient(90deg, #da3633, #f85149); }
+.bar.yel { background: linear-gradient(90deg, #9e6a03, #d29922); }
+.bar.blu { background: linear-gradient(90deg, #1f6feb, #58a6ff); }
+
+/* Charts */
+.chart { width: 100%; height: 80px; margin-top: 8px; }
+.chart-lg { height: 120px; }
+.chart svg { width: 100%; height: 100%; }
+.chart-line { fill: none; stroke-width: 1.5; stroke-linecap: round; stroke-linejoin: round; }
+.chart-area { opacity: 0.15; }
+.chart-grid { stroke: var(--border); stroke-width: 0.5; }
+.chart-label { font-size: 9px; fill: var(--dim); }
+
+/* Histogram bars */
+.histo { display: flex; align-items: flex-end; gap: 2px; height: 60px; margin-top: 8px; }
+.histo-bar { flex: 1; background: var(--blue); border-radius: 2px 2px 0 0; min-height: 2px; transition: height 0.3s; position: relative; }
+.histo-bar:hover { opacity: 0.8; }
+.histo-bar::after { content: attr(data-label); position: absolute; bottom: -16px; left: 50%; transform: translateX(-50%); font-size: 8px; color: var(--dim); white-space: nowrap; }
+.histo-labels { display: flex; justify-content: space-between; margin-top: 20px; font-size: 9px; color: var(--dim); }
+
+/* Stat rows */
+.stat-row { display: flex; justify-content: space-between; align-items: center; padding: 6px 0; font-size: 12px; }
+.stat-row + .stat-row { border-top: 1px solid rgba(48,54,61,0.5); }
+.stat-lbl { color: var(--dim); display: flex; align-items: center; gap: 6px; }
+.stat-val { font-weight: 600; font-feature-settings: "tnum"; }
+.stat-bar { width: 60px; height: 4px; background: var(--border); border-radius: 2px; margin-left: 8px; overflow: hidden; }
+.stat-bar-fill { height: 100%; border-radius: 2px; }
+
+/* Leaderboard */
+.lb { font-size: 12px; }
+.lb-item { display: flex; align-items: center; gap: 8px; padding: 5px 0; }
+.lb-item + .lb-item { border-top: 1px solid rgba(48,54,61,0.3); }
+.lb-rank { width: 18px; height: 18px; border-radius: 4px; background: var(--card-alt); display: flex; align-items: center; justify-content: center; font-size: 10px; font-weight: 600; color: var(--dim); }
+.lb-rank.top { background: var(--yellow); color: var(--bg); }
+.lb-name { flex: 1; font-family: ui-monospace, monospace; color: var(--text); }
+.lb-val { font-weight: 600; font-feature-settings: "tnum"; }
+
+/* Tags */
+.tag { display: inline-flex; align-items: center; gap: 4px; padding: 3px 8px; border-radius: 4px; font-size: 10px; font-weight: 600; }
+.tag-ok { background: rgba(63,185,80,0.15); color: var(--green); }
+.tag-err { background: rgba(248,81,73,0.15); color: var(--red); }
+.tag-warn { background: rgba(210,153,34,0.15); color: var(--yellow); }
+.tag-info { background: rgba(88,166,255,0.15); color: var(--blue); }
+
+/* Mini stats */
+.mini { display: flex; gap: 16px; flex-wrap: wrap; margin-top: 8px; }
+.mini-item { display: flex; align-items: baseline; gap: 4px; }
+.mini-val { font-size: 14px; font-weight: 600; font-feature-settings: "tnum"; }
+.mini-lbl { font-size: 10px; color: var(--dim); }
+
+/* Proto cards */
+.proto-card { text-align: center; }
+.proto-icon { font-size: 20px; margin-bottom: 4px; }
+.proto-name { font-size: 10px; color: var(--dim); text-transform: uppercase; letter-spacing: 0.5px; }
+.proto-val { font-size: 18px; font-weight: 700; margin: 4px 0; }
+.proto-rate { font-size: 11px; padding: 2px 6px; border-radius: 3px; display: inline-block; }
+
+/* Pie charts */
+.pie-wrap { display: flex; gap: 16px; align-items: center; }
+.pie { width: 90px; height: 90px; border-radius: 50%; flex-shrink: 0; }
+.legend { flex: 1; }
+.legend-item { display: flex; align-items: center; gap: 8px; padding: 3px 0; font-size: 12px; }
+.legend-dot { width: 10px; height: 10px; border-radius: 3px; flex-shrink: 0; }
+.legend-name { flex: 1; color: var(--dim); }
+.legend-val { font-weight: 600; font-feature-settings: "tnum"; }
+
+/* Tor/Judge cards */
+.host-card { display: flex; justify-content: space-between; align-items: center; }
+.host-addr { font-family: ui-monospace, monospace; font-size: 12px; }
+.host-stats { font-size: 11px; color: var(--dim); }
+
+.judge-item { display: flex; justify-content: space-between; align-items: center; padding: 4px 0; font-size: 11px; }
+.judge-item + .judge-item { border-top: 1px solid rgba(48,54,61,0.3); }
+.judge-name { font-family: ui-monospace, monospace; color: var(--dim); flex: 1; }
+.judge-stats { display: flex; gap: 12px; }
+
+/* Percentile badges */
+.pct-badges { display: flex; gap: 8px; margin-top: 8px; }
+.pct-badge { flex: 1; text-align: center; padding: 8px; background: var(--card-alt); border-radius: 6px; }
+.pct-label { font-size: 10px; color: var(--dim); text-transform: uppercase; }
+.pct-value { font-size: 16px; font-weight: 700; margin-top: 2px; }
+
+/* Footer */
+.ftr { text-align: center; font-size: 11px; color: var(--dim); padding: 16px 0; margin-top: 8px; border-top: 1px solid var(--border); }
+'''
+
+DASHBOARD_JS = '''
+var $ = function(id) { return document.getElementById(id); };
+var fmt = function(n) { return n == null ? '-' : n.toLocaleString(); };
+var fmtDec = function(n, d) { return n == null ? '-' : n.toFixed(d || 1); };
+var pct = function(n, t) { return t > 0 ? ((n / t) * 100).toFixed(1) : '0.0'; };
+
+function fmtTime(s) {
+ if (!s) return '-';
+ var d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
+ if (d > 0) return d + 'd ' + h + 'h';
+ if (h > 0) return h + 'h ' + m + 'm';
+ return m + 'm ' + Math.floor(s % 60) + 's';
+}
+
+function fmtMs(ms) {
+ if (!ms || ms <= 0) return '-';
+ if (ms < 1000) return Math.round(ms) + 'ms';
+ return (ms / 1000).toFixed(1) + 's';
+}
+
+function setBar(id, val, max, cls) {
+ var el = $(id);
+ if (el) { el.style.width = Math.min(val / max * 100, 100) + '%'; el.className = 'bar ' + (cls || 'grn'); }
+}
+
+function conicGrad(segs) {
+ var parts = [], deg = 0;
+ segs.forEach(function(s) { parts.push(s.c + ' ' + deg + 'deg ' + (deg + s.d) + 'deg'); deg += s.d; });
+ return 'conic-gradient(' + parts.join(', ') + ')';
+}
+
+function renderLineChart(id, data, color, maxVal) {
+ var el = $(id); if (!el || !data || data.length < 2) return;
+ var w = el.clientWidth || 300, h = el.clientHeight || 80;
+ var max = maxVal || Math.max.apply(null, data) || 1;
+ var padY = 5, padX = 2;
+ var points = data.map(function(v, i) {
+ var x = padX + (i / (data.length - 1)) * (w - 2 * padX);
+ var y = h - padY - ((v / max) * (h - 2 * padY));
+ return x + ',' + y;
+ });
+ var areaPoints = padX + ',' + (h - padY) + ' ' + points.join(' ') + ' ' + (w - padX) + ',' + (h - padY);
+ el.innerHTML = '' +
+ ' ';
+}
+
+function renderHistogram(id, data) {
+ var el = $(id); if (!el || !data || !data.length) return;
+ var max = Math.max.apply(null, data.map(function(d) { return d.pct; })) || 1;
+ var colors = ['#3fb950', '#3fb950', '#58a6ff', '#58a6ff', '#d29922', '#f85149', '#f85149'];
+ var html = '';
+ data.forEach(function(d, i) {
+ var h = Math.max(4, (d.pct / max) * 100);
+ var c = colors[Math.min(i, colors.length - 1)];
+ html += '
';
+ });
+ el.innerHTML = html;
+}
+
+function renderLeaderboard(id, data, nameKey, valKey, limit) {
+ var el = $(id); if (!el) return;
+ limit = limit || 8;
+ if (!data || !data.length) { el.innerHTML = 'No data
'; return; }
+ var html = '';
+ data.slice(0, limit).forEach(function(item, i) {
+ var name = Array.isArray(item) ? item[0] : item[nameKey];
+ var val = Array.isArray(item) ? item[1] : item[valKey];
+ html += '' + (i + 1) + '
';
+ html += '
' + name + ' ' + fmt(val) + '
';
+ fhtml += '
' + cat + ' ' + n + ' No failures yet
';
+ }
+ $('failLegend').innerHTML = fhtml;
+
+ // Leaderboards (session data)
+ renderLeaderboard('topCountries', d.top_countries_session, 'code', 'count');
+ renderLeaderboard('topAsns', d.top_asns_session, 'asn', 'count');
+
+ // Tor pool
+ var thtml = '';
+ if (d.tor_pool && d.tor_pool.hosts) {
+ d.tor_pool.hosts.forEach(function(h) {
+ thtml += '';
+ thtml += '' + h.address + ' ';
+ thtml += '' + (h.healthy ? 'OK' : 'DOWN') + ' ';
+ thtml += '
' + fmtMs(h.latency_ms) + ' / ' + fmtDec(h.success_rate, 0) + '% success
No Tor hosts
';
+
+ // Judges
+ if (d.judges) {
+ $('judgesAvail').textContent = d.judges.available + '/' + d.judges.total;
+ $('judgesCooldown').textContent = d.judges.in_cooldown;
+ var jhtml = '';
+ if (d.judges.top_judges) {
+ d.judges.top_judges.slice(0, 6).forEach(function(j) {
+ jhtml += '' + j.judge + ' ';
+ jhtml += '
' + j.success + '/' + j.tests + ' ';
+ jhtml += '' + fmtDec(j.rate, 0) + '%
No data
';
+ }
+
+ // Database stats
+ if (d.db) {
+ var dbs = d.db;
+ $('dbByProto').innerHTML = ['http', 'socks4', 'socks5'].map(function(p) {
+ var c = dbs.by_proto && dbs.by_proto[p] || 0;
+ return '' + p.toUpperCase() + ' ' + fmt(c) + '
';
+ }).join('');
+
+ var chtml = '';
+ if (dbs.top_countries) {
+ dbs.top_countries.slice(0, 5).forEach(function(c, i) {
+ var name = c.code || c[0], cnt = c.count || c[1];
+ chtml += '' + (i + 1) + '
';
+ chtml += '
' + name + ' ' + fmt(cnt) + ' No data
';
+ }
+
+ // Scraper/Engine stats
+ $('engAvail').textContent = fmt(d.engines_available);
+ $('engBackoff').textContent = fmt(d.engines_backoff);
+ $('engTotal').textContent = fmt(d.engines_total);
+ if (d.scraper && d.scraper.engines) {
+ var ehtml = '';
+ d.scraper.engines.slice(0, 5).forEach(function(e, i) {
+ var statusCls = e.available ? 'tag-ok' : 'tag-warn';
+ var statusTxt = e.available ? 'OK' : (e.backoff_remaining > 0 ? e.backoff_remaining + 's' : 'OFF');
+ ehtml += '' + (i + 1) + '
';
+ ehtml += '
' + e.name + ' ';
+ ehtml += '
' + statusTxt + ' ';
+ ehtml += '
' + fmt(e.successes) + ' No engines
';
+ } else {
+ $('topEngines').innerHTML = 'Scraper disabled
';
+ }
+
+ // SSL/TLS stats
+ if (d.ssl) {
+ var ssl = d.ssl;
+ $('sslTested').textContent = fmt(ssl.tested);
+ $('sslPassed').textContent = fmt(ssl.passed);
+ $('sslFailed').textContent = fmt(ssl.failed);
+ var sslRate = ssl.success_rate || 0;
+ setBar('sslBar', sslRate, 100, sslRate < 50 ? 'red' : sslRate < 80 ? 'yel' : 'grn');
+ $('mitmDetected').textContent = fmt(ssl.mitm_detected);
+ $('mitmDetected').className = 'stat-val ' + (ssl.mitm_detected > 0 ? 'red' : 'grn');
+ $('certErrors').textContent = fmt(ssl.cert_errors);
+ $('certErrors').className = 'stat-val ' + (ssl.cert_errors > 0 ? 'yel' : 'grn');
+ }
+
+ $('lastUpdate').textContent = new Date().toLocaleTimeString();
+}
+
+function fetchStats() {
+ fetch('/api/stats')
+ .then(function(r) { return r.json(); })
+ .then(update)
+ .catch(function(e) { $('dot').className = 'dot err'; $('statusTxt').textContent = 'Error'; });
+}
+
+fetchStats();
+setInterval(fetchStats, 3000);
+'''
+
+DASHBOARD_HTML = '''
+
+
+ PPF Dashboard
+
+
+
+
+
+
+
Working Proxies
+
-
+
of - in database
+
+
+
Tests This Session
+
-
+
- passed / - failed
+
+
+
+
Test Rate
+
-
+
tests/sec average
+
+
+
Uptime
+
-
+
session duration
+
+
+
+
+
+
+
Test Rate History (10 min)
+
+
- current
+
- peak
+
- pass/s
+
+
+
+
+
Success Rate History
+
+
+
+
+
+
+
+
Latency Analysis
+
+
+
Average -
+
Min -
+
Max -
+
+
+
+
Response Time Distribution
+
+
+
+
+
+
+
+
Protocol Performance
+
+
+
🌐
+
HTTP
+
-
+
of - tested
+
-
+
+
+
🔌
+
SOCKS4
+
-
+
of - tested
+
-
+
+
+
🔒
+
SOCKS5
+
-
+
of - tested
+
-
+
+
+
+
+
+
+
+
+
+
Geographic Distribution (Session)
+
+
+
+
+
+
+
System
+
Threads -
+
+
Queue Size -
+
Check Type -
+
+
+
Judge Services
+
Available -
+
In Cooldown -
+
Top Performers
+
+
+
+
+
+
+
+
+
+
+
Search Engines
+
Available -
+
In Backoff -
+
Total -
+
Top Engines
+
+
+
+
SSL/TLS Security
+
SSL Tests -
+
Passed -
+
Failed -
+
+
MITM Detected -
+
Cert Errors -
+
+
+
+
+
+
Database Overview
+
+
+
Working by Protocol
+
+
+
+
Top Countries (All Time)
+
+
+
+
+
+
PPF Proxy Fetcher
+