From f179080cca7647149f52481e8e54d6891822f9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Serneels?= Date: Fri, 17 May 2019 22:59:32 +0200 Subject: [PATCH] use geoloc now saves proxy's country in db --- proxywatchd.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/proxywatchd.py b/proxywatchd.py index fd1c73f..7d5ac04 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -2,7 +2,7 @@ import threading import time, random, string, re, copy -#from geoip import geolite2 +from geoip import geolite2 from config import Config @@ -44,13 +44,14 @@ def socks4_resolve(srvname, server_port): class WorkerJob(): - def __init__(self, proxy, proto, failcount, success_count, total_duration): + def __init__(self, proxy, proto, failcount, success_count, total_duration, country): self.proxy = proxy self.proto = proto self.failcount = failcount self.checktime = None self.success_count = success_count self.total_duration = total_duration + self.country = country def connect_socket(self): srvname = random.choice(config.servers).strip() @@ -128,19 +129,18 @@ class WorkerJob(): if re.match('^(:|NOTICE)', recv, re.IGNORECASE): duration = (time.time() - duration) - #match = geolite2.lookup(proxy[0].split(':')[0]) - match = None - if match is not None: match = match.country - else: match = 'unknown' + if not self.country or self.country == 'unknown' or self.country == 'N/A': + match = geolite2.lookup(self.proxy.split(':')[0]) + if match is not None: self.country = match.country + else: self.country = 'N/A' self.proto = proto self.failcount = 0 self.success_count = self.success_count + 1 self.total_duration += int(duration*1000) - cstats = "" if match == 'unknown' else ' c: %s;'%match torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv]) - _log('%s://%s;%s d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recvstats), 'xxxxx') + _log('%s://%s (%s) d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, self.country, duration, torstats, srv, recvstats), 'xxxxx') except KeyboardInterrupt as e: raise e except rocksock.RocksockException as e: @@ -264,7 +264,7 @@ class Proxywatchd(): } def fetch_rows(self): - q = 'SELECT proxy,proto,failed,success_count,total_duration FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()' + q = 'SELECT proxy,proto,failed,success_count,total_duration,country FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()' rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall() # check oldies ? if len(rows) < config.watchd.threads and config.watchd.oldies: @@ -280,7 +280,7 @@ class Proxywatchd(): self.tor_safeguard = config.watchd.tor_safeguard rows = self.fetch_rows() for row in rows: - job = WorkerJob(row[0], row[1], row[2], row[3], row[4]) + job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5]) self.jobs.append(job) self._close_db() @@ -301,7 +301,7 @@ class Proxywatchd(): args = [] for job in self.collected: if job.failcount == 0: sc += 1 - args.append( (job.failcount, job.checktime, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.proxy) ) + args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) ) success_rate = (float(sc) / len(self.collected)) * 100 ret = True @@ -311,7 +311,7 @@ class Proxywatchd(): args = [] for job in self.collected: if job.failcount == 0: - args.append( (job.failcount, job.checktime, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.proxy) ) + args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) ) ret = False _log("updating %d DB entries (success rate: %.2f%%)"%(len(self.collected), success_rate), 'watchd')