diff --git a/httpd.py b/httpd.py index 60d7017..6b9eb2b 100644 --- a/httpd.py +++ b/httpd.py @@ -31,6 +31,13 @@ except (ImportError, IOError, ValueError): _geodb = None _geolite = False +# ASN lookup (optional) +try: + import pyasn + _asndb = pyasn.pyasn(os.path.join("data", "ipasn.dat")) +except (ImportError, IOError): + _asndb = None + # Rate limiting configuration _rate_limits = defaultdict(list) _rate_lock = threading.Lock() @@ -604,7 +611,7 @@ def submit_proxy_reports(db, worker_id, proxies): ''', (proxy_key, ip, port, proto, now_int, now_int, latency, now_int, checktype, target)) - # Geolocate if IP2Location available + # Geolocate and ASN lookup if _geolite and _geodb: try: rec = _geodb.get_all(ip) @@ -614,6 +621,15 @@ def submit_proxy_reports(db, worker_id, proxies): (rec.country_short, proxy_key)) except Exception: pass + if _asndb: + try: + asn_result = _asndb.lookup(ip) + if asn_result and asn_result[0]: + db.execute( + 'UPDATE proxylist SET asn=? WHERE proxy=?', + (asn_result[0], proxy_key)) + except Exception: + pass # Track per-URL working count for working_ratio if source_url: