username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use geoloc

Browse Source 
now saves proxy's country in db
This commit is contained in:
Mickaël Serneels
2019-05-17 22:59:32 +02:00
parent eeedf9d0a1
commit f179080cca
1 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
proxywatchd.py
+12 -12
View File
@@ -2,7 +2,7 @@
import threading import threading
import time, random, string, re, copy import time, random, string, re, copy
#from geoip import geolite2 from geoip import geolite2
from config import Config from config import Config
@@ -44,13 +44,14 @@ def socks4_resolve(srvname, server_port):
class WorkerJob(): class WorkerJob():
def __init__(self, proxy, proto, failcount, success_count, total_duration): def __init__(self, proxy, proto, failcount, success_count, total_duration, country):
self.proxy = proxy self.proxy = proxy
self.proto = proto self.proto = proto
self.failcount = failcount self.failcount = failcount
self.checktime = None self.checktime = None
self.success_count = success_count self.success_count = success_count
self.total_duration = total_duration self.total_duration = total_duration
self.country = country
def connect_socket(self): def connect_socket(self):
srvname = random.choice(config.servers).strip() srvname = random.choice(config.servers).strip()
@@ -128,19 +129,18 @@ class WorkerJob():
if re.match('^(:|NOTICE)', recv, re.IGNORECASE): if re.match('^(:|NOTICE)', recv, re.IGNORECASE):
duration = (time.time() - duration) duration = (time.time() - duration)
#match = geolite2.lookup(proxy[0].split(':')[0]) if not self.country or self.country == 'unknown' or self.country == 'N/A':
match = None match = geolite2.lookup(self.proxy.split(':')[0])
if match is not None: match = match.country if match is not None: self.country = match.country
else: match = 'unknown' else: self.country = 'N/A'
self.proto = proto self.proto = proto
self.failcount = 0 self.failcount = 0
self.success_count = self.success_count + 1 self.success_count = self.success_count + 1
self.total_duration += int(duration*1000) self.total_duration += int(duration*1000)
cstats = "" if match == 'unknown' else ' c: %s;'%match
torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor
recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv]) recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv])
_log('%s://%s;%s d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recvstats), 'xxxxx') _log('%s://%s (%s) d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, self.country, duration, torstats, srv, recvstats), 'xxxxx')
except KeyboardInterrupt as e: except KeyboardInterrupt as e:
raise e raise e
except rocksock.RocksockException as e: except rocksock.RocksockException as e:
@@ -264,7 +264,7 @@ class Proxywatchd():
} }
def fetch_rows(self): def fetch_rows(self):
q = 'SELECT proxy,proto,failed,success_count,total_duration FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()' q = 'SELECT proxy,proto,failed,success_count,total_duration,country FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()'
rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall() rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall()
# check oldies ? # check oldies ?
if len(rows) < config.watchd.threads and config.watchd.oldies: if len(rows) < config.watchd.threads and config.watchd.oldies:
@@ -280,7 +280,7 @@ class Proxywatchd():
self.tor_safeguard = config.watchd.tor_safeguard self.tor_safeguard = config.watchd.tor_safeguard
rows = self.fetch_rows() rows = self.fetch_rows()
for row in rows: for row in rows:
job = WorkerJob(row[0], row[1], row[2], row[3], row[4]) job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5])
self.jobs.append(job) self.jobs.append(job)
self._close_db() self._close_db()
@@ -301,7 +301,7 @@ class Proxywatchd():
args = [] args = []
for job in self.collected: for job in self.collected:
if job.failcount == 0: sc += 1 if job.failcount == 0: sc += 1
args.append( (job.failcount, job.checktime, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.proxy) ) args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
success_rate = (float(sc) / len(self.collected)) * 100 success_rate = (float(sc) / len(self.collected)) * 100
ret = True ret = True
@@ -311,7 +311,7 @@ class Proxywatchd():
args = [] args = []
for job in self.collected: for job in self.collected:
if job.failcount == 0: if job.failcount == 0:
args.append( (job.failcount, job.checktime, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.proxy) ) args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
ret = False ret = False
_log("updating %d DB entries (success rate: %.2f%%)"%(len(self.collected), success_rate), 'watchd') _log("updating %d DB entries (success rate: %.2f%%)"%(len(self.collected), success_rate), 'watchd')