httpd: pass url database to api server
This commit is contained in:
Username
53
httpd.py
53
httpd.py
@@ -86,8 +86,8 @@ _worker_test_history_lock = threading.Lock()
|
||||
_test_history_window = 120 # seconds to keep test history for rate calculation
|
||||
|
||||
# Fair distribution settings
|
||||
_min_batch_size = 100 # minimum proxies per batch
|
||||
_max_batch_size = 1000 # maximum proxies per batch
|
||||
_min_batch_size = 1 # minimum proxies per batch
|
||||
_max_batch_size = 10000 # maximum proxies per batch
|
||||
_worker_timeout = 120 # seconds before worker considered inactive
|
||||
|
||||
# Session tracking
|
||||
@@ -170,19 +170,22 @@ def get_due_proxy_count(db):
|
||||
|
||||
|
||||
def calculate_fair_batch_size(db, worker_id):
|
||||
"""Calculate fair batch size based on active workers and queue size."""
|
||||
"""Calculate fair batch size based on active workers and queue size.
|
||||
|
||||
Divides due work evenly among active workers. No artificial floor —
|
||||
if only 6 proxies are due with 3 workers, each gets 2.
|
||||
"""
|
||||
active_workers = max(1, get_active_worker_count())
|
||||
due_count = get_due_proxy_count(db)
|
||||
|
||||
if due_count == 0:
|
||||
return _min_batch_size
|
||||
return 0
|
||||
|
||||
# Fair share: divide due work among active workers
|
||||
# Add 20% buffer for speed variations between workers
|
||||
fair_share = int((due_count / active_workers) * 1.2)
|
||||
# Fair share: divide due work evenly among active workers
|
||||
fair_share = max(1, int(due_count / active_workers))
|
||||
|
||||
# Clamp to bounds
|
||||
batch_size = max(_min_batch_size, min(fair_share, _max_batch_size))
|
||||
# Clamp to upper bound only
|
||||
batch_size = min(fair_share, _max_batch_size)
|
||||
|
||||
_log('fair_batch: due=%d workers=%d share=%d batch=%d' % (
|
||||
due_count, active_workers, fair_share, batch_size), 'debug')
|
||||
@@ -381,7 +384,7 @@ def claim_work(db, worker_id, count=100):
|
||||
priority_params = [now_int, _working_checktime, now_int, _fail_retry_interval]
|
||||
|
||||
query = '''
|
||||
SELECT ip, port, proto, failed,
|
||||
SELECT ip, port, proto, failed, source_proto,
|
||||
CASE
|
||||
WHEN tested IS NULL THEN 0
|
||||
WHEN (%s) > 3600 THEN 1
|
||||
@@ -413,6 +416,7 @@ def claim_work(db, worker_id, count=100):
|
||||
'port': row[1],
|
||||
'proto': row[2],
|
||||
'failed': row[3],
|
||||
'source_proto': row[4],
|
||||
})
|
||||
|
||||
if claimed:
|
||||
@@ -1078,7 +1082,7 @@ class ProxyAPIHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
asn = params.get('asn', '')
|
||||
fmt = params.get('format', 'json')
|
||||
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency, protos_working FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
args = []
|
||||
|
||||
if proto:
|
||||
@@ -1103,7 +1107,8 @@ class ProxyAPIHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
else:
|
||||
proxies = [{
|
||||
'ip': r[0], 'port': r[1], 'proto': r[2],
|
||||
'country': r[3], 'asn': r[4], 'latency': r[5]
|
||||
'country': r[3], 'asn': r[4], 'latency': r[5],
|
||||
'protos': r[6].split(',') if r[6] else [r[2]]
|
||||
} for r in rows]
|
||||
self.send_json({'count': len(proxies), 'proxies': proxies})
|
||||
except Exception as e:
|
||||
@@ -1122,7 +1127,7 @@ class ProxyAPIHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
asn = params.get('asn', '')
|
||||
fmt = params.get('format', 'json')
|
||||
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency, protos_working FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
args = []
|
||||
|
||||
if proto:
|
||||
@@ -1146,7 +1151,8 @@ class ProxyAPIHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
else:
|
||||
proxies = [{
|
||||
'ip': r[0], 'port': r[1], 'proto': r[2],
|
||||
'country': r[3], 'asn': r[4], 'latency': r[5]
|
||||
'country': r[3], 'asn': r[4], 'latency': r[5],
|
||||
'protos': r[6].split(',') if r[6] else [r[2]]
|
||||
} for r in rows]
|
||||
self.send_json({'count': len(proxies), 'proxies': proxies})
|
||||
except Exception as e:
|
||||
@@ -1171,11 +1177,12 @@ class ProxyAPIServer(threading.Thread):
|
||||
otherwise falls back to standard BaseHTTPServer.
|
||||
"""
|
||||
|
||||
def __init__(self, host, port, database, stats_provider=None, profiling=False):
|
||||
def __init__(self, host, port, database, stats_provider=None, profiling=False, url_database=None):
|
||||
threading.Thread.__init__(self)
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.database = database
|
||||
self.url_database = url_database
|
||||
self.stats_provider = stats_provider
|
||||
self.profiling = profiling
|
||||
self.daemon = True
|
||||
@@ -1444,7 +1451,7 @@ class ProxyAPIServer(threading.Thread):
|
||||
asn = query_params.get('asn', '')
|
||||
fmt = query_params.get('format', 'json')
|
||||
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency, protos_working FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
args = []
|
||||
if proto:
|
||||
sql += ' AND proto=?'
|
||||
@@ -1463,7 +1470,11 @@ class ProxyAPIServer(threading.Thread):
|
||||
|
||||
if fmt == 'plain':
|
||||
return '\n'.join('%s:%s' % (r[0], r[1]) for r in rows), 'text/plain', 200
|
||||
proxies = [{'proxy': '%s:%s' % (r[0], r[1]), 'proto': r[2], 'country': r[3], 'asn': r[4], 'latency': r[5]} for r in rows]
|
||||
proxies = [{
|
||||
'proxy': '%s:%s' % (r[0], r[1]), 'proto': r[2], 'country': r[3],
|
||||
'asn': r[4], 'latency': r[5],
|
||||
'protos': r[6].split(',') if r[6] else [r[2]]
|
||||
} for r in rows]
|
||||
return json.dumps({'count': len(proxies), 'proxies': proxies}, indent=2), 'application/json', 200
|
||||
except Exception as e:
|
||||
return json.dumps({'error': str(e)}), 'application/json', 500
|
||||
@@ -1474,7 +1485,7 @@ class ProxyAPIServer(threading.Thread):
|
||||
asn = query_params.get('asn', '')
|
||||
fmt = query_params.get('format', 'json')
|
||||
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
sql = 'SELECT ip, port, proto, country, asn, avg_latency, protos_working FROM proxylist WHERE failed=0 AND proto IS NOT NULL'
|
||||
args = []
|
||||
if proto:
|
||||
sql += ' AND proto=?'
|
||||
@@ -1492,7 +1503,11 @@ class ProxyAPIServer(threading.Thread):
|
||||
|
||||
if fmt == 'plain':
|
||||
return '\n'.join('%s://%s:%s' % (r[2] or 'http', r[0], r[1]) for r in rows), 'text/plain', 200
|
||||
proxies = [{'proxy': '%s:%s' % (r[0], r[1]), 'proto': r[2], 'country': r[3], 'asn': r[4], 'latency': r[5]} for r in rows]
|
||||
proxies = [{
|
||||
'proxy': '%s:%s' % (r[0], r[1]), 'proto': r[2], 'country': r[3],
|
||||
'asn': r[4], 'latency': r[5],
|
||||
'protos': r[6].split(',') if r[6] else [r[2]]
|
||||
} for r in rows]
|
||||
return json.dumps({'count': len(proxies), 'proxies': proxies}, indent=2), 'application/json', 200
|
||||
except Exception as e:
|
||||
return json.dumps({'error': str(e)}), 'application/json', 500
|
||||
|
||||
Reference in New Issue
Block a user