httpd: add ASN enrichment for worker-reported proxies
All checks were successful
CI / validate (push) Successful in 21s

Load pyasn database in httpd and look up ASN when workers report
working proxies. Previously ASN was only populated by proxywatchd
which doesn't run independently on the master node, leaving all
worker-reported proxies with asn=null.
This commit is contained in:
Username
2026-02-22 11:18:51 +01:00
parent 7ae0ac0c26
commit d1e22a388c

View File

@@ -31,6 +31,13 @@ except (ImportError, IOError, ValueError):
_geodb = None _geodb = None
_geolite = False _geolite = False
# ASN lookup (optional)
try:
import pyasn
_asndb = pyasn.pyasn(os.path.join("data", "ipasn.dat"))
except (ImportError, IOError):
_asndb = None
# Rate limiting configuration # Rate limiting configuration
_rate_limits = defaultdict(list) _rate_limits = defaultdict(list)
_rate_lock = threading.Lock() _rate_lock = threading.Lock()
@@ -604,7 +611,7 @@ def submit_proxy_reports(db, worker_id, proxies):
''', (proxy_key, ip, port, proto, now_int, now_int, latency, now_int, ''', (proxy_key, ip, port, proto, now_int, now_int, latency, now_int,
checktype, target)) checktype, target))
# Geolocate if IP2Location available # Geolocate and ASN lookup
if _geolite and _geodb: if _geolite and _geodb:
try: try:
rec = _geodb.get_all(ip) rec = _geodb.get_all(ip)
@@ -614,6 +621,15 @@ def submit_proxy_reports(db, worker_id, proxies):
(rec.country_short, proxy_key)) (rec.country_short, proxy_key))
except Exception: except Exception:
pass pass
if _asndb:
try:
asn_result = _asndb.lookup(ip)
if asn_result and asn_result[0]:
db.execute(
'UPDATE proxylist SET asn=? WHERE proxy=?',
(asn_result[0], proxy_key))
except Exception:
pass
# Track per-URL working count for working_ratio # Track per-URL working count for working_ratio
if source_url: if source_url: