add latency tracking and dynamic thread scaling

- dbs.py: add avg_latency, latency_samples columns with migration
- dbs.py: update_proxy_latency() with exponential moving average
- proxywatchd.py: ThreadScaler class for dynamic thread count
- proxywatchd.py: calculate/record latency for successful proxies
- proxywatchd.py: _spawn_thread(), _remove_thread(), _adjust_threads()
- scaler reports status alongside periodic stats
This commit is contained in:
Username
2025-12-21 00:08:19 +01:00
parent 1e43f50aa6
commit 79475c2bff
3 changed files with 221 additions and 79 deletions

46
dbs.py
View File

@@ -6,6 +6,46 @@ import time
from misc import _log
def _migrate_latency_columns(sqlite):
"""Add latency columns to existing databases."""
try:
sqlite.execute('SELECT avg_latency FROM proxylist LIMIT 1')
except Exception:
sqlite.execute('ALTER TABLE proxylist ADD COLUMN avg_latency REAL DEFAULT 0')
sqlite.execute('ALTER TABLE proxylist ADD COLUMN latency_samples INT DEFAULT 0')
sqlite.commit()
def update_proxy_latency(sqlite, proxy, latency_ms):
"""Update rolling average latency for a proxy.
Args:
sqlite: Database connection
proxy: Proxy address (ip:port)
latency_ms: Response latency in milliseconds
"""
row = sqlite.execute(
'SELECT avg_latency, latency_samples FROM proxylist WHERE proxy=?',
(proxy,)
).fetchone()
if row:
old_avg, samples = row[0] or 0, row[1] or 0
# Exponential moving average, capped at 100 samples
new_samples = min(samples + 1, 100)
if samples == 0:
new_avg = latency_ms
else:
# Weight recent samples more heavily
alpha = 2.0 / (new_samples + 1)
new_avg = alpha * latency_ms + (1 - alpha) * old_avg
sqlite.execute(
'UPDATE proxylist SET avg_latency=?, latency_samples=? WHERE proxy=?',
(new_avg, new_samples, proxy)
)
def create_table_if_not_exists(sqlite, dbname):
"""Create database table with indexes if it doesn't exist."""
if dbname == 'proxylist':
@@ -22,11 +62,15 @@ def create_table_if_not_exists(sqlite, dbname):
ip TEXT,
port INT,
consecutive_success INT,
total_duration INT)""")
total_duration INT,
avg_latency REAL DEFAULT 0,
latency_samples INT DEFAULT 0)""")
# Indexes for common query patterns
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_failed ON proxylist(failed)')
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_tested ON proxylist(tested)')
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_proto ON proxylist(proto)')
# Migration: add latency columns if missing
_migrate_latency_columns(sqlite)
elif dbname == 'uris':
sqlite.execute("""CREATE TABLE IF NOT EXISTS uris (