httpd: add ASN enrichment for worker-reported proxies
All checks were successful
CI / validate (push) Successful in 21s
All checks were successful
CI / validate (push) Successful in 21s
Load pyasn database in httpd and look up ASN when workers report working proxies. Previously ASN was only populated by proxywatchd which doesn't run independently on the master node, leaving all worker-reported proxies with asn=null.
This commit is contained in:
18
httpd.py
18
httpd.py
@@ -31,6 +31,13 @@ except (ImportError, IOError, ValueError):
|
||||
_geodb = None
|
||||
_geolite = False
|
||||
|
||||
# ASN lookup (optional)
|
||||
try:
|
||||
import pyasn
|
||||
_asndb = pyasn.pyasn(os.path.join("data", "ipasn.dat"))
|
||||
except (ImportError, IOError):
|
||||
_asndb = None
|
||||
|
||||
# Rate limiting configuration
|
||||
_rate_limits = defaultdict(list)
|
||||
_rate_lock = threading.Lock()
|
||||
@@ -604,7 +611,7 @@ def submit_proxy_reports(db, worker_id, proxies):
|
||||
''', (proxy_key, ip, port, proto, now_int, now_int, latency, now_int,
|
||||
checktype, target))
|
||||
|
||||
# Geolocate if IP2Location available
|
||||
# Geolocate and ASN lookup
|
||||
if _geolite and _geodb:
|
||||
try:
|
||||
rec = _geodb.get_all(ip)
|
||||
@@ -614,6 +621,15 @@ def submit_proxy_reports(db, worker_id, proxies):
|
||||
(rec.country_short, proxy_key))
|
||||
except Exception:
|
||||
pass
|
||||
if _asndb:
|
||||
try:
|
||||
asn_result = _asndb.lookup(ip)
|
||||
if asn_result and asn_result[0]:
|
||||
db.execute(
|
||||
'UPDATE proxylist SET asn=? WHERE proxy=?',
|
||||
(asn_result[0], proxy_key))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Track per-URL working count for working_ratio
|
||||
if source_url:
|
||||
|
||||
Reference in New Issue
Block a user