username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dashboard: refactor layout and add worker stats
All checks were successful
CI / syntax-check (push) Successful in 3s
Details
CI / memory-leak-check (push) Successful in 11s
Details

Browse Source
This commit is contained in:
Username
2025-12-28 15:19:50 +01:00
parent e89db20f5b
commit 35f24bb8b0
3 changed files with 205 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
static/dashboard.html
View File

@@ -33,57 +33,45 @@
<div class="sysbar-bar"><div class="sysbar-fill" id="sysDiskFill" style="width:0"></div></div>
<span class="sysbar-val" id="sysDiskPct">-</span></div>
<div class="sysbar-item"><span class="sysbar-lbl">Process:</span><span class="sysbar-val" id="sysProcMem">-</span></div>
<div class="sysbar-item sysbar-net"><span class="sysbar-lbl">Scrape:</span><span class="sysbar-val net-tx" id="netScrapeTx">-</span><span class="sysbar-val net-rx" id="netScrapeRx">-</span></div>
<div class="sysbar-item sysbar-net"><span class="sysbar-lbl">Proxy:</span><span class="sysbar-val net-tx" id="netProxyTx">-</span><span class="sysbar-val net-rx" id="netProxyRx">-</span></div>
</div>
<!-- Primary Stats Row -->
<div class="g g5">
<!-- Distributed Workers (Primary) -->
<div class="g g3" style="margin-bottom:16px">
<div class="c">
<div class="lbl">Working Proxies</div>
<div class="val grn" id="working">-</div>
<div class="sub">of <span id="total">-</span> in database</div>
</div>
<div class="c">
<div class="lbl">Tests (Cumulative)</div>
<div class="val" id="tested">-</div>
<div class="sub"><span class="grn" id="passed">-</span> passed / <span class="red" id="failed">-</span> failed</div>
<div class="lbl">Active Workers</div>
<div class="val cyn" id="dwActive">-</div>
<div class="sub">distributed nodes</div>
</div>
<div class="c">
<div class="lbl">Success Rate</div>
<div class="val-md grn" id="successRate">-</div>
<div class="bar-wrap"><div class="bar grn" id="srBar" style="width:0"></div></div>
</div>
<div class="c">
<div class="lbl">Test Rate</div>
<div class="val-md blu" id="rate">-</div>
<div class="sub">tests/sec (60s)</div>
</div>
<div class="c">
<div class="lbl">Uptime</div>
<div class="val-md" id="uptime">-</div>
<div class="sub">session duration</div>
<div class="lbl">Combined Rate</div>
<div class="val blu" id="dwRate">-</div>
<div class="sub">tests/sec (all workers)</div>
</div>
</div>
<!-- Worker Pool (always visible) -->
<!-- Worker Stats & Scraper Stats -->
<div class="g g2" style="margin-bottom:16px">
<div class="c">
<div class="sec-hdr" style="margin-top:0">Worker Pool</div>
<div class="sec-hdr" style="margin-top:0">Worker Testing</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">Active Threads</span><span class="stat-val" id="threads">-</span></div>
<div class="bar-wrap"><div class="bar blu" id="threadBar" style="width:0"></div></div>
<div class="stat-row" style="margin-top:8px"><span class="stat-lbl">Job Queue</span><span class="stat-val yel" id="queue">-</span></div>
<div class="stat-row"><span class="stat-lbl">Queue ETA</span><span class="stat-val cyn" id="queueEta">-</span></div>
<div class="stat-row"><span class="stat-lbl">Tests (Session)</span><span class="stat-val" id="dwTested">-</span></div>
<div class="stat-row"><span class="stat-lbl">Working Found</span><span class="stat-val grn" id="dwWorking">-</span></div>
<div class="stat-row"><span class="stat-lbl">Success Rate</span><span class="stat-val" id="successRate">-</span></div>
<div class="stat-row"><span class="stat-lbl">Uptime</span><span class="stat-val" id="uptime">-</span></div>
</div>
</div>
<div class="c">
<div class="sec-hdr" style="margin-top:0">Tor Pool</div>
<div class="sec-hdr" style="margin-top:0">Scraper Network</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">Total Requests</span><span class="stat-val" id="torTotal">-</span></div>
<div class="stat-row"><span class="stat-lbl">Success Rate</span><span class="stat-val grn" id="torSuccess">-</span></div>
<div class="stat-row"><span class="stat-lbl">Healthy Nodes</span><span class="stat-val" id="torHealthy">-</span></div>
<div class="stat-row"><span class="stat-lbl">Avg Latency</span><span class="stat-val cyn" id="torLatency">-</span></div>
<div class="stat-row"><span class="stat-lbl">Tor Requests</span><span class="stat-val" id="torTotal">-</span></div>
<div class="stat-row"><span class="stat-lbl">Tor Success</span><span class="stat-val grn" id="torSuccess">-</span></div>
<div class="stat-row"><span class="stat-lbl">Engines Active</span><span class="stat-val cyn" id="engActive">-</span></div>
<div class="stat-row"><span class="stat-lbl">Proxies Added</span><span class="stat-val yel" id="proxiesAdded">-</span></div>
</div>
</div>
</div>
@@ -91,17 +79,69 @@
<!-- Tab Navigation -->
<div class="tabs">
<div class="tab-nav">
<button class="tab-btn active" data-tab="perf">Performance</button>
<button class="tab-btn active" data-tab="workers">Workers</button>
<button class="tab-btn" data-tab="perf">Performance</button>
<button class="tab-btn" data-tab="proto">Protocols</button>
<button class="tab-btn" data-tab="geo">Geography</button>
<button class="tab-btn" data-tab="infra">Infrastructure</button>
<button class="tab-btn" data-tab="scraper">Scraper</button>
<button class="tab-btn" data-tab="mitm">MITM</button>
<button class="tab-btn" data-tab="db">Database</button>
</div>
</div>
<!-- Workers Tab (default) -->
<div id="tab-workers" class="tab-content active">
<div class="sec">
<div class="sec-hdr">Distributed Worker Nodes</div>
<div class="g g4">
<div class="c c-sm">
<div class="lbl">Active</div>
<div class="val-sm cyn" id="wkActive">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Total Tested</div>
<div class="val-sm blu" id="wkTested">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Working Found</div>
<div class="val-sm grn" id="wkWorking">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Success Rate</div>
<div class="val-sm yel" id="wkSuccessRate">-</div>
</div>
</div>
</div>
<div class="sec">
<div class="sec-hdr">Worker Status</div>
<div class="g g2" id="workerCards">
<div class="c" style="text-align:center;color:var(--dim);padding:40px">
<div style="font-size:24px;margin-bottom:8px">No workers connected</div>
<div style="font-size:12px">Add workers with: <code>python ppf.py --worker --server URL</code></div>
</div>
</div>
</div>
<div class="sec">
<div class="sec-hdr">Queue Status</div>
<div class="g g3">
<div class="c c-sm">
<div class="lbl">Pending Tests</div>
<div class="val-sm" id="queuePending">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Claimed by Workers</div>
<div class="val-sm cyn" id="queueClaimed">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Due for Retest</div>
<div class="val-sm yel" id="queueDue">-</div>
</div>
</div>
</div>
</div>
<!-- Performance Tab -->
<div id="tab-perf" class="tab-content active">
<div id="tab-perf" class="tab-content">
<!-- Rate & Success Charts -->
<div class="g g2">
<div class="c c-lg">
@@ -230,74 +270,9 @@
</div>
<!-- Infrastructure Tab -->
<div id="tab-infra" class="tab-content">
<!-- Judge Services & Anonymity -->
<div class="g g2">
<div class="c">
<div class="sec-hdr" style="margin-top:0">Judge Services</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">Available</span><span class="stat-val grn" id="judgesAvail">-</span></div>
<div class="stat-row"><span class="stat-lbl">In Cooldown</span><span class="stat-val yel" id="judgesCooldown">-</span></div>
</div>
<div class="lbl" style="margin-top:10px">Top Performers</div>
<div class="lb-wrap">
<div id="topJudges"></div>
</div>
</div>
<div class="c">
<div class="sec-hdr" style="margin-top:0">Anonymity Levels</div>
<div class="stats-wrap">
<div id="anonBreakdown"></div>
</div>
<div class="sub" style="margin-top:8px;font-size:10px">Elite = no headers, Anonymous = adds headers, Transparent = reveals IP</div>
</div>
</div>
<!-- Tor Exit Nodes -->
<div class="sec">
<div class="sec-hdr">Tor Exit Nodes</div>
<div class="g g3" id="torPool"></div>
</div>
<!-- Network Usage -->
<div class="sec">
<div class="sec-hdr">Network Usage</div>
<div class="g g3">
<div class="c c-sm">
<div class="lbl">Total RX</div>
<div class="val-sm grn" id="netRx">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Total TX</div>
<div class="val-sm blu" id="netTx">-</div>
</div>
<div class="c c-sm">
<div class="lbl">Total</div>
<div class="val-sm cyn" id="netTotal">-</div>
</div>
</div>
<div class="g g2" style="margin-top:12px">
<div class="c">
<div class="lbl" style="text-align:center;margin-bottom:8px">By Category</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">Proxy Testing</span><span class="stat-val cyn" id="netProxy">-</span></div>
<div class="stat-row"><span class="stat-lbl">Scraping</span><span class="stat-val yel" id="netScraper">-</span></div>
</div>
</div>
<div class="c">
<div class="lbl" style="text-align:center;margin-bottom:8px">Rates (avg)</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">RX Rate</span><span class="stat-val grn" id="netRxRate">-</span></div>
<div class="stat-row"><span class="stat-lbl">TX Rate</span><span class="stat-val blu" id="netTxRate">-</span></div>
</div>
</div>
</div>
<div class="lbl" style="margin-top:12px">Per Tor Node</div>
<div class="g g3" id="netTorNodes"></div>
</div>
<!-- Scraper & SSL Stats -->
<!-- Scraper Tab -->
<div id="tab-scraper" class="tab-content">
<!-- Search Engines -->
<div class="g g2">
<div class="c">
<div class="sec-hdr" style="margin-top:0">Search Engines</div>
@@ -312,7 +287,31 @@
</div>
</div>
<div class="c">
<div class="sec-hdr" style="margin-top:0">SSL/TLS Security</div>
<div class="sec-hdr" style="margin-top:0">Tor Network (Scraping)</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">Healthy Nodes</span><span class="stat-val grn" id="torHealthy">-</span></div>
<div class="stat-row"><span class="stat-lbl">Avg Latency</span><span class="stat-val cyn" id="torLatency">-</span></div>
</div>
<div class="lbl" style="margin-top:10px">Tor Exit Nodes</div>
<div class="g g2" id="torPool" style="margin-top:8px"></div>
</div>
</div>
<!-- Judge Services -->
<div class="g g2">
<div class="c">
<div class="sec-hdr" style="margin-top:0">Judge Services</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">Available</span><span class="stat-val grn" id="judgesAvail">-</span></div>
<div class="stat-row"><span class="stat-lbl">In Cooldown</span><span class="stat-val yel" id="judgesCooldown">-</span></div>
</div>
<div class="lbl" style="margin-top:10px">Top Performers</div>
<div class="lb-wrap">
<div id="topJudges"></div>
</div>
</div>
<div class="c">
<div class="sec-hdr" style="margin-top:0">SSL/TLS Testing</div>
<div class="stats-wrap">
<div class="stat-row"><span class="stat-lbl">SSL Tests</span><span class="stat-val" id="sslTested">-</span></div>
<div class="stat-row"><span class="stat-lbl">Passed</span><span class="stat-val grn" id="sslPassed">-</span></div>

187
static/dashboard.js
View File

@@ -9,10 +9,6 @@ var uplotCharts = {};
// Chart.js instances (persistent)
var chartJsInstances = {};
// Network rate tracking (for real-time speed calculation)
var prevNet = null;
var prevNetTime = null;
// Tab switching
function initTabs() {
$$('.tab-btn').forEach(function(btn) {
@@ -86,13 +82,6 @@ function fmtBytes(b) {
return b.toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
}
function fmtRate(bps) {
if (bps < 1) return '0';
if (bps < 1024) return bps.toFixed(0) + 'B';
if (bps < 1024 * 1024) return (bps / 1024).toFixed(1) + 'K';
return (bps / (1024 * 1024)).toFixed(1) + 'M';
}
function fmtTime(s) {
if (!s) return '-';
var d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
@@ -301,46 +290,32 @@ function update(d) {
var memGrowthStr = memGrowth > 0 ? ' (+' + fmtBytes(memGrowth) + ')' : '';
$('sysProcMem').textContent = fmtBytes(sys.proc_rss) + memGrowthStr;
// Network speed (current rate, not average)
var net = d.network || {};
var now = Date.now();
if (prevNet && prevNetTime) {
var dt = (now - prevNetTime) / 1000;
if (dt > 0) {
var s = net.scraper || {}, ps = prevNet.scraper || {};
var p = net.proxy || {}, pp = prevNet.proxy || {};
$('netScrapeTx').textContent = fmtRate((s.bytes_tx - (ps.bytes_tx || 0)) / dt);
$('netScrapeRx').textContent = fmtRate((s.bytes_rx - (ps.bytes_rx || 0)) / dt);
$('netProxyTx').textContent = fmtRate((p.bytes_tx - (pp.bytes_tx || 0)) / dt);
$('netProxyRx').textContent = fmtRate((p.bytes_rx - (pp.bytes_rx || 0)) / dt);
}
}
prevNet = net;
prevNetTime = now;
// Main stats - db stats are nested under d.db
var db = d.db || {};
$('working').textContent = fmt(db.working);
$('total').textContent = fmt(db.total);
$('tested').textContent = fmt(d.tested);
$('passed').textContent = fmt(d.passed);
$('failed').textContent = fmt(d.failed);
// Success rate
// Success rate (displayed in Worker Testing section)
var sr = d.success_rate || 0;
$('successRate').textContent = fmtDec(sr, 1) + '%';
$('successRate').className = 'val-md ' + (sr < 20 ? 'red' : sr < 50 ? 'yel' : 'grn');
setBar('srBar', sr, 100, sr < 20 ? 'red' : sr < 50 ? 'yel' : 'grn');
$('successRate').className = 'stat-val ' + (sr < 20 ? 'red' : sr < 50 ? 'yel' : 'grn');
var rsr = d.recent_success_rate || 0;
$('recentSuccessRate').textContent = fmtDec(rsr, 1) + '%';
$('recentSuccessRate').className = 'stat-val ' + (rsr < 20 ? 'red' : rsr < 50 ? 'yel' : 'grn');
if ($('recentSuccessRate')) {
$('recentSuccessRate').textContent = fmtDec(rsr, 1) + '%';
$('recentSuccessRate').className = 'stat-val ' + (rsr < 20 ? 'red' : rsr < 50 ? 'yel' : 'grn');
}
// Rates
$('rate').textContent = fmtDec(d.recent_rate, 2);
$('recentRate').textContent = fmtDec(d.recent_rate, 2) + '/s';
$('peakRate').textContent = fmtDec(d.peak_rate, 2) + '/s';
$('passRate').textContent = fmtDec(d.pass_rate, 3);
// Rates (Performance tab)
if ($('recentRate')) $('recentRate').textContent = fmtDec(d.recent_rate, 2) + '/s';
if ($('peakRate')) $('peakRate').textContent = fmtDec(d.peak_rate, 2) + '/s';
if ($('passRate')) $('passRate').textContent = fmtDec(d.pass_rate, 3);
// Scraper stats
var engAvail = d.engines_available || 0;
var engTotal = d.engines_total || 0;
if ($('engActive')) $('engActive').textContent = engAvail + '/' + engTotal;
if ($('proxiesAdded')) $('proxiesAdded').textContent = fmt(db.added_last_day || d.proxies_added || 0);
// Latency
var lat = d.avg_latency || 0;
@@ -354,18 +329,6 @@ function update(d) {
$('p99').textContent = fmtMs(pctl.p99);
// System
$('threads').textContent = d.threads + '/' + d.max_threads;
setBar('threadBar', d.threads, d.max_threads, 'blu');
$('queue').textContent = fmt(d.queue_size);
// Calculate queue ETA: queue_size / tests_per_second
var queueEta = '-';
if (d.queue_size > 0 && d.recent_rate > 0.01) {
var etaSecs = d.queue_size / d.recent_rate;
queueEta = fmtTime(etaSecs);
} else if (d.queue_size === 0) {
queueEta = 'empty';
}
$('queueEta').textContent = queueEta;
$('uptime').textContent = fmtTime(d.uptime_seconds);
// Charts
@@ -621,37 +584,6 @@ function update(d) {
$('mitmRecent').innerHTML = recentHtml;
}
// Network usage stats
if (d.network) {
var net = d.network;
$('netRx').textContent = fmtBytes(net.bytes_rx || 0);
$('netTx').textContent = fmtBytes(net.bytes_tx || 0);
$('netTotal').textContent = fmtBytes(net.bytes_total || 0);
$('netRxRate').textContent = fmtBytes(net.rx_rate || 0) + '/s';
$('netTxRate').textContent = fmtBytes(net.tx_rate || 0) + '/s';
if (net.proxy) {
$('netProxy').textContent = fmtBytes(net.proxy.bytes_total || 0);
}
if (net.scraper) {
$('netScraper').textContent = fmtBytes(net.scraper.bytes_total || 0);
}
// Per-tor-node stats
var torContainer = $('netTorNodes');
if (torContainer && net.tor_nodes) {
var html = '';
var nodes = Object.keys(net.tor_nodes).sort();
nodes.forEach(function(node) {
var s = net.tor_nodes[node];
html += '<div class="c c-sm">';
html += '<div class="lbl" style="font-size:10px">' + node + '</div>';
html += '<div class="val-sm cyn">' + fmtBytes(s.rx + s.tx) + '</div>';
html += '<div class="sub" style="font-size:9px">' + fmt(s.requests || 0) + ' req</div>';
html += '</div>';
});
torContainer.innerHTML = html;
}
}
$('lastUpdate').textContent = new Date().toLocaleTimeString();
}
@@ -660,6 +592,93 @@ function fetchStats() {
.then(function(r) { return r.json(); })
.then(update)
.catch(function(e) { $('dot').className = 'dot err'; $('statusTxt').textContent = 'Error'; });
// Also fetch worker stats
fetchWorkers();
}
function fetchWorkers() {
fetch('/api/workers')
.then(function(r) { return r.json(); })
.then(updateWorkers)
.catch(function(e) { console.error('Failed to fetch workers:', e); });
}
function updateWorkers(data) {
if (!data) return;
// Update summary stats (both main panel and Workers tab)
var active = data.active || 0;
var total = data.total || 0;
if ($('wkActive')) $('wkActive').textContent = active + '/' + total;
if ($('dwActive')) $('dwActive').textContent = active + '/' + total;
if (data.summary) {
if ($('wkTested')) $('wkTested').textContent = fmt(data.summary.total_tested);
if ($('wkWorking')) $('wkWorking').textContent = fmt(data.summary.total_working);
if ($('wkSuccessRate')) $('wkSuccessRate').textContent = data.summary.overall_success_rate.toFixed(1) + '%';
// Main panel distributed workers
if ($('dwTested')) $('dwTested').textContent = fmt(data.summary.total_tested);
if ($('dwWorking')) $('dwWorking').textContent = fmt(data.summary.total_working);
// Calculate combined rate from worker stats
var combinedRate = 0;
if (data.workers) {
data.workers.forEach(function(w) {
if (w.active && w.test_rate) combinedRate += w.test_rate;
});
}
if ($('dwRate')) $('dwRate').textContent = combinedRate > 0 ? combinedRate.toFixed(1) + '/s' : '-';
}
// Queue status
if (data.queue) {
if ($('queuePending')) $('queuePending').textContent = fmt(data.queue.pending || 0);
if ($('queueClaimed')) $('queueClaimed').textContent = fmt(data.queue.claimed || 0);
if ($('queueDue')) $('queueDue').textContent = fmt(data.queue.due || 0);
}
// Update worker cards
var container = $('workerCards');
if (!container) return;
if (!data.workers || data.workers.length === 0) {
container.innerHTML = '<div class="c" style="text-align:center;color:var(--dim);padding:40px">' +
'<div style="font-size:24px;margin-bottom:8px">No workers connected</div>' +
'<div style="font-size:12px">Add workers with: <code>python ppf.py --register --server URL</code></div></div>';
return;
}
var html = '';
data.workers.forEach(function(w) {
var statusClass = w.active ? 'grn' : 'red';
var statusText = w.active ? 'ACTIVE' : 'OFFLINE';
var successRate = w.success_rate || 0;
var rateClass = successRate >= 50 ? 'grn' : (successRate >= 20 ? 'yel' : 'red');
var profBadge = w.profiling ? '<span class="tag tag-warn" style="margin-left:4px;font-size:9px">PROF</span>' : '';
html += '<div class="c">' +
'<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:8px">' +
'<span style="font-size:1.1em;font-weight:600;color:var(--cyn)">' + w.name + '</span>' +
'<span><span class="tag tag-' + (w.active ? 'ok' : 'err') + '">' + statusText + '</span>' + profBadge + '</span>' +
'</div>' +
'<div class="stats-wrap" style="margin:0">' +
'<div class="stat-row"><span class="stat-lbl">Tested</span><span class="stat-val">' + fmt(w.proxies_tested) + '</span></div>' +
'<div class="stat-row"><span class="stat-lbl">Working</span><span class="stat-val grn">' + fmt(w.proxies_working) + '</span></div>' +
'<div class="stat-row"><span class="stat-lbl">Success</span><span class="stat-val ' + rateClass + '">' + successRate.toFixed(1) + '%</span></div>' +
'<div class="stat-row"><span class="stat-lbl">Jobs</span><span class="stat-val">' + fmt(w.jobs_completed) + '</span></div>' +
'</div>' +
'<div style="font-size:0.75em;color:var(--dim);margin-top:8px">' +
'IP: ' + w.ip + ' | Last: ' + formatAge(w.age) +
'</div>' +
'</div>';
});
container.innerHTML = html;
}
function formatAge(seconds) {
if (seconds < 60) return seconds + 's ago';
if (seconds < 3600) return Math.floor(seconds / 60) + 'm ago';
if (seconds < 86400) return Math.floor(seconds / 3600) + 'h ago';
return Math.floor(seconds / 86400) + 'd ago';
}
// Visibility-aware polling - pause when tab is hidden

3
static/style.css
View File

@@ -102,9 +102,6 @@ h3 { font-size: 13px; font-weight: 600; color: var(--dim); margin-bottom: 8px; }
.sysbar-val { font-weight: 600; font-feature-settings: "tnum"; }
.sysbar-bar { width: 50px; height: 4px; background: var(--border); border-radius: 2px; overflow: hidden; }
.sysbar-fill { height: 100%; border-radius: 2px; transition: width 0.3s; }
.sysbar-net .net-tx::before { content: '↑'; opacity: 0.5; margin-right: 1px; }
.sysbar-net .net-rx::before { content: '↓'; opacity: 0.5; margin-right: 1px; }
.sysbar-net .sysbar-val { min-width: 42px; }
/* Grid */
.g { display: grid; gap: 12px; margin-bottom: 16px; }