diff --git a/dbs.py b/dbs.py index 24fdc49..0308155 100644 --- a/dbs.py +++ b/dbs.py @@ -34,9 +34,9 @@ def insert_proxies(proxydb, proxies, url): new = [] for p in proxies: - new.append((timestamp,p,3,0,0,0,0)) + new.append((timestamp,p,3,0,0,0,0,0)) - proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,failed,tested,success_count,total_duration,mitm) VALUES (?,?,?,?,?,?,?)', new) + proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,failed,tested,success_count,total_duration,mitm,consecutive_success) VALUES (?,?,?,?,?,?,?,?)', new) proxydb.commit() _log('+%d proxy/ies from %s' % (len(proxies), url), 'added') diff --git a/proxywatchd.py b/proxywatchd.py index e80165e..3ce90f1 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -48,7 +48,7 @@ def socks4_resolve(srvname, server_port): class WorkerJob(): - def __init__(self, proxy, proto, failcount, success_count, total_duration, country, oldies = False): + def __init__(self, proxy, proto, failcount, success_count, total_duration, country, mitm, consecutive_success, oldies = False): self.proxy = proxy self.proto = proto self.failcount = failcount @@ -56,16 +56,21 @@ class WorkerJob(): self.success_count = success_count self.total_duration = total_duration self.country = country + self.mitm = mitm + self.consecutive_success = consecutive_success self.isoldies = oldies def connect_socket(self, checktype): + if checktype == 'irc': srvname = random.choice(config.servers).strip() use_ssl = random.choice([0,1]) if config.watchd.use_ssl == 2 else config.watchd.use_ssl + if (self.consecutive_success % 3) == 0: use_ssl = 1 server_port = 6697 if use_ssl else 6667 elif checktype == 'http': srvname = random.choice( ['www.facebook.com', 'www.reddit.com', 'www.twitter.com'] ) use_ssl = random.choice([0,1]) if config.watchd.use_ssl == 2 else config.watchd.use_ssl + if (self.consecutive_success % 3) == 0: use_ssl = 1 server_port = 443 if use_ssl else 80 protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto] @@ -124,6 +129,10 @@ class WorkerJob(): _log("could not resolve connection target %s"%srvname, "ERROR") break + elif err == rocksock.RS_E_SSL_CERTIFICATE_ERROR: + fail_inc = 0 + self.mitm = 1 + except KeyboardInterrupt as e: raise(e) @@ -153,6 +162,8 @@ class WorkerJob(): self.proto = proto self.failcount = 0 + if (self.consecutive_success %3) == 0: self.mitm = 0 + self.consecutive_success = self.consecutive_success + 1 self.success_count = self.success_count + 1 self.total_duration += int(duration*1000) torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor @@ -160,10 +171,12 @@ class WorkerJob(): _log('%s://%s (%s) d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, self.country, duration, torstats, srv, recvstats), 'xxxxx') else: self.failcount += 1 + self.consecutive_success = 0 except KeyboardInterrupt as e: raise e except rocksock.RocksockException as e: self.failcount += 1 + self.consecutive_success = 0 finally: sock.disconnect() @@ -284,7 +297,7 @@ class Proxywatchd(): def fetch_rows(self): self.isoldies = False - q = 'SELECT proxy,proto,failed,success_count,total_duration,country FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()' + q = 'SELECT proxy,proto,failed,success_count,total_duration,country,mitm,consecutive_success FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()' rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall() # check oldies ? if len(rows) < config.watchd.threads: @@ -303,7 +316,7 @@ class Proxywatchd(): rows = self.fetch_rows() #print('preparing jobbs, oldies: %s' % str(self.isoldies)) for row in rows: - job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5], self.isoldies) + job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], self.isoldies) self.jobs.append(job) self._close_db() @@ -324,7 +337,7 @@ class Proxywatchd(): args = [] for job in self.collected: if job.failcount == 0: sc += 1 - args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) ) + args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.mitm, job.consecutive_success, job.proxy) ) success_rate = (float(sc) / len(self.collected)) * 100 ret = True @@ -334,12 +347,12 @@ class Proxywatchd(): args = [] for job in self.collected: if job.failcount == 0: - args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) ) + args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.mitm, job.consecutive_success, job.proxy) ) ret = False _log("updating %d DB entries (success rate: %.2f%%)"%(len(self.collected), success_rate), 'watchd') self._prep_db() - query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=? WHERE proxy=?' + query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=?,mitm=?,consecutive_success=? WHERE proxy=?' self.mysqlite.executemany(query, args) self.mysqlite.commit() self._close_db()