add latency tracking and dynamic thread scaling
- dbs.py: add avg_latency, latency_samples columns with migration - dbs.py: update_proxy_latency() with exponential moving average - proxywatchd.py: ThreadScaler class for dynamic thread count - proxywatchd.py: calculate/record latency for successful proxies - proxywatchd.py: _spawn_thread(), _remove_thread(), _adjust_threads() - scaler reports status alongside periodic stats
This commit is contained in:
46
dbs.py
46
dbs.py
@@ -6,6 +6,46 @@ import time
|
||||
from misc import _log
|
||||
|
||||
|
||||
def _migrate_latency_columns(sqlite):
|
||||
"""Add latency columns to existing databases."""
|
||||
try:
|
||||
sqlite.execute('SELECT avg_latency FROM proxylist LIMIT 1')
|
||||
except Exception:
|
||||
sqlite.execute('ALTER TABLE proxylist ADD COLUMN avg_latency REAL DEFAULT 0')
|
||||
sqlite.execute('ALTER TABLE proxylist ADD COLUMN latency_samples INT DEFAULT 0')
|
||||
sqlite.commit()
|
||||
|
||||
|
||||
def update_proxy_latency(sqlite, proxy, latency_ms):
|
||||
"""Update rolling average latency for a proxy.
|
||||
|
||||
Args:
|
||||
sqlite: Database connection
|
||||
proxy: Proxy address (ip:port)
|
||||
latency_ms: Response latency in milliseconds
|
||||
"""
|
||||
row = sqlite.execute(
|
||||
'SELECT avg_latency, latency_samples FROM proxylist WHERE proxy=?',
|
||||
(proxy,)
|
||||
).fetchone()
|
||||
|
||||
if row:
|
||||
old_avg, samples = row[0] or 0, row[1] or 0
|
||||
# Exponential moving average, capped at 100 samples
|
||||
new_samples = min(samples + 1, 100)
|
||||
if samples == 0:
|
||||
new_avg = latency_ms
|
||||
else:
|
||||
# Weight recent samples more heavily
|
||||
alpha = 2.0 / (new_samples + 1)
|
||||
new_avg = alpha * latency_ms + (1 - alpha) * old_avg
|
||||
|
||||
sqlite.execute(
|
||||
'UPDATE proxylist SET avg_latency=?, latency_samples=? WHERE proxy=?',
|
||||
(new_avg, new_samples, proxy)
|
||||
)
|
||||
|
||||
|
||||
def create_table_if_not_exists(sqlite, dbname):
|
||||
"""Create database table with indexes if it doesn't exist."""
|
||||
if dbname == 'proxylist':
|
||||
@@ -22,11 +62,15 @@ def create_table_if_not_exists(sqlite, dbname):
|
||||
ip TEXT,
|
||||
port INT,
|
||||
consecutive_success INT,
|
||||
total_duration INT)""")
|
||||
total_duration INT,
|
||||
avg_latency REAL DEFAULT 0,
|
||||
latency_samples INT DEFAULT 0)""")
|
||||
# Indexes for common query patterns
|
||||
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_failed ON proxylist(failed)')
|
||||
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_tested ON proxylist(tested)')
|
||||
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_proto ON proxylist(proto)')
|
||||
# Migration: add latency columns if missing
|
||||
_migrate_latency_columns(sqlite)
|
||||
|
||||
elif dbname == 'uris':
|
||||
sqlite.execute("""CREATE TABLE IF NOT EXISTS uris (
|
||||
|
||||
Reference in New Issue
Block a user