From d1e22a388c263c5e1bc2eebf3d93c0f76292cf88 Mon Sep 17 00:00:00 2001 From: Username Date: Sun, 22 Feb 2026 11:18:51 +0100 Subject: [PATCH] httpd: add ASN enrichment for worker-reported proxies Load pyasn database in httpd and look up ASN when workers report working proxies. Previously ASN was only populated by proxywatchd which doesn't run independently on the master node, leaving all worker-reported proxies with asn=null. --- httpd.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/httpd.py b/httpd.py index 60d7017..6b9eb2b 100644 --- a/httpd.py +++ b/httpd.py @@ -31,6 +31,13 @@ except (ImportError, IOError, ValueError): _geodb = None _geolite = False +# ASN lookup (optional) +try: + import pyasn + _asndb = pyasn.pyasn(os.path.join("data", "ipasn.dat")) +except (ImportError, IOError): + _asndb = None + # Rate limiting configuration _rate_limits = defaultdict(list) _rate_lock = threading.Lock() @@ -604,7 +611,7 @@ def submit_proxy_reports(db, worker_id, proxies): ''', (proxy_key, ip, port, proto, now_int, now_int, latency, now_int, checktype, target)) - # Geolocate if IP2Location available + # Geolocate and ASN lookup if _geolite and _geodb: try: rec = _geodb.get_all(ip) @@ -614,6 +621,15 @@ def submit_proxy_reports(db, worker_id, proxies): (rec.country_short, proxy_key)) except Exception: pass + if _asndb: + try: + asn_result = _asndb.lookup(ip) + if asn_result and asn_result[0]: + db.execute( + 'UPDATE proxylist SET asn=? WHERE proxy=?', + (asn_result[0], proxy_key)) + except Exception: + pass # Track per-URL working count for working_ratio if source_url: