diff --git a/httpd.py b/httpd.py index 0a75807..bd48bbf 100644 --- a/httpd.py +++ b/httpd.py @@ -126,11 +126,17 @@ def get_db_health(db): ).fetchone() stats['added_last_day'] = row[0] if row else 0 - # Dead proxies count + # Dead proxies count (permanently dead = -1, failing = positive) + row = db.execute( + 'SELECT COUNT(*) FROM proxylist WHERE failed = -1' + ).fetchone() + stats['dead_count'] = row[0] if row else 0 + + # Failing proxies count (positive fail count but not permanently dead) row = db.execute( 'SELECT COUNT(*) FROM proxylist WHERE failed > 0' ).fetchone() - stats['dead_count'] = row[0] if row else 0 + stats['failing_count'] = row[0] if row else 0 except Exception: pass diff --git a/proxywatchd.py b/proxywatchd.py index babccde..e8ad806 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -65,6 +65,11 @@ IP_PATTERN = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}' # Pattern for header echo - if X-Forwarded-For or Via present, proxy reveals chain HEADER_REVEAL_PATTERN = r'(X-Forwarded-For|Via|X-Real-Ip|Forwarded)' +# Dead proxy marker - proxies with failed=-1 are permanently dead and never retested +DEAD_PROXY = -1 +# Error categories that indicate proxy is definitely dead (not temporary failure) +FATAL_ERROR_CATEGORIES = ('refused', 'unreachable', 'auth') + # Patterns indicating judge is blocking the proxy (not a proxy failure) # These should NOT count as proxy failures - retry with different judge JUDGE_BLOCK_PATTERNS = [ @@ -839,6 +844,8 @@ class ProxyTestState(): self.last_latency_ms = None # average latency from successful tests self.exit_ip = None # IP seen by target server (for anonymity detection) self.reveals_headers = None # True if proxy adds X-Forwarded-For/Via headers + self.last_fail_category = None # failure category from last test (for dead detection) + self.original_failcount = failcount # failcount before this test cycle # SSL/TLS tracking self.had_ssl_test = False self.ssl_success = False @@ -981,6 +988,7 @@ class ProxyTestState(): # Real proxy failure self.failcount += 1 self.consecutive_success = 0 + self.last_fail_category = fail_category return (False, fail_category) @@ -1526,14 +1534,27 @@ class Proxywatchd(): if len(self.collected) == 0: return True sc = 0 + dead_count = 0 args = [] latency_updates = [] # (proxy, latency_ms) for successful tests + max_fail = config.watchd.max_fail for job in self.collected: if job.failcount == 0: sc += 1 if job.last_latency_ms is not None: latency_updates.append((job.proxy, job.last_latency_ms)) - args.append((job.failcount, job.checktime, 1, job.country, job.proto, + # Check if proxy should be marked as permanently dead + effective_failcount = job.failcount + if job.failcount > 0: + # Mark dead if: exceeded max_fail*2, OR reached max_fail with fatal error + is_fatal = job.last_fail_category in FATAL_ERROR_CATEGORIES + if job.failcount >= max_fail * 2: + effective_failcount = DEAD_PROXY + dead_count += 1 + elif job.failcount >= max_fail and is_fatal: + effective_failcount = DEAD_PROXY + dead_count += 1 + args.append((effective_failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.mitm, job.consecutive_success, job.asn, job.proxy)) @@ -1556,7 +1577,8 @@ class Proxywatchd(): now = time.time() if (now - self.last_update_log) >= self.log_interval or success_rate > 0: - _log("updating %d DB entries (success rate: %.2f%%)" % (len(self.collected), success_rate), 'watchd') + dead_msg = ', %d marked dead' % dead_count if dead_count > 0 else '' + _log("updating %d DB entries (success rate: %.2f%%%s)" % (len(self.collected), success_rate, dead_msg), 'watchd') self.last_update_log = now self._prep_db() query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=?,mitm=?,consecutive_success=?,asn=? WHERE proxy=?' @@ -1583,10 +1605,10 @@ class Proxywatchd(): stale_seconds = config.watchd.stale_days * 86400 cutoff = int(time.time()) - stale_seconds self._prep_db() - # delete proxies that: failed >= max_fail AND last tested before cutoff + # delete proxies that: (failed >= max_fail OR permanently dead) AND last tested before cutoff result = self.mysqlite.execute( - 'DELETE FROM proxylist WHERE failed >= ? AND tested < ?', - (config.watchd.max_fail, cutoff) + 'DELETE FROM proxylist WHERE (failed >= ? OR failed = ?) AND tested < ?', + (config.watchd.max_fail, DEAD_PROXY, cutoff) ) count = result.rowcount if hasattr(result, 'rowcount') else 0 self.mysqlite.commit()