diff --git a/config.py b/config.py index 3204a09..421c064 100644 --- a/config.py +++ b/config.py @@ -26,3 +26,7 @@ def load(): args = aparse.parse_args() watchd_threads = args.watchd_threads + + global servers + with open('servers.txt', 'r') as handle: + servers = handle.read().split('\n') diff --git a/ppf.py b/ppf.py index 59deb86..a7e308e 100755 --- a/ppf.py +++ b/ppf.py @@ -149,7 +149,6 @@ def proxyleech(sqlite, rows): if __name__ == '__main__': config.load() - print repr(config.torhosts) proxies={'http':'socks4://%s' % random.choice(config.torhosts),'https':'socks4://%s' % random.choice(config.torhosts)} sqlite = mysqlite.mysqlite(config.database, str) @@ -172,7 +171,11 @@ if __name__ == '__main__': empty = [ urignore.append(i.split('/')[2]) for i in searx_instances ] # start proxy watcher - watcherd = proxywatchd.Proxywatchd() if config.watchd_threads > 0 else None + if config.watchd_threads > 0: + watcherd = proxywatchd.Proxywatchd() + watcherd.run_background() + else: + watcherd = None while True: try: @@ -184,9 +187,12 @@ if __name__ == '__main__': ## sleep else: time.sleep(10) - except KeyboardInterrupt: break + except KeyboardInterrupt: + print "XXXXXX" + if watcherd: watcherd.stop() + break print '\r', # stop things - if watcherd: watcherd.stop() + #if watcherd: watcherd.stop() diff --git a/proxywatchd.py b/proxywatchd.py index 1af99e0..ae4aef7 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -1,8 +1,7 @@ #!/usr/bin/env python -from threading import Thread -import threading, commands -import socket, time, random, sys, string, re +import threading +import time, random, string, re, copy import requests #from geoip import geolite2 @@ -12,50 +11,13 @@ import mysqlite from misc import _log import rocksock -class Proxywatchd(Thread): - - def stop(self): - _log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()])) - self.running = 0 - - def __init__(self): - Thread.__init__(self) - config.load() - self.threads = [] - self.running = 1 - - # create table if needed - self.mysqlite = mysqlite.mysqlite(config.database, str) - self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)') - self.mysqlite.commit() - self.echoise = time.time() - 3600; - self.ticks = time.time() - 3600; - - with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n') - - self.start() - - def run(self): - _log('Starting proxywatchd..', 'notice') - - threads = [] - self.mysqlite = mysqlite.mysqlite(config.database, str) - - while self.running: - - if len(threads) < config.watchd_threads: - t = threading.Thread(target=self.daemon, args=(self.servers,)) - t.start() - threads.append(t) - time.sleep( random.choice( xrange(1,3))) - - else: time.sleep(1) - - if (time.time() - self.echoise) >= 180: - _log('Proxywatchd threads: %d/%d' % (len(threads), config.watchd_threads)) - self.echoise = time.time() - - self.mysqlite.close() +class WorkerJob(): + def __init__(self, proxy, proto, failcount): + self.proxy = proxy + self.proto = proto + self.failcount = failcount + self.nextcheck = None + self.duration = None def is_drone_bl(self, proxy): p = proxy.split(':')[0] @@ -64,85 +26,196 @@ class Proxywatchd(Thread): if 'No incidents regarding' in resp.text: return 0 else: return 1 - def connect_socket(self, proxy, servers, proto = None): - protos = ['http', 'socks5', 'socks4'] if proto is None else proto + def connect_socket(self): + protos = ['http', 'socks5', 'socks4'] if self.proto is None else self.proto for proto in protos: torhost = random.choice(config.torhosts) duration = time.time() - proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost), - rocksock.RocksockProxyFromURL('%s://%s' % (proto, proxy[0])), - ] + proxies = [ + rocksock.RocksockProxyFromURL('socks4://%s' % torhost), + rocksock.RocksockProxyFromURL('%s://%s' % (proto, self.proxy)), + ] - srv = random.choice(servers).strip() + srv = random.choice(config.servers).strip() try: sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=config.timeout) sock.connect() sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT'])) return sock, proto, duration, torhost, srv - + except KeyboardInterrupt as e: + raise(e) except: sock.disconnect() return None, None, None, None, None - def daemon(self, servers): - sqlite = mysqlite.mysqlite(config.database, str) - threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] ) + def run(self): + self.nextcheck = (time.time() + 1800 + ((1+int(self.failcount)) * 3600)) - q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed 1: + wt.start_thread() + self.threads.append(wt) + + while True: + + if self.stopping.is_set(): + for wt in self.threads: + wt.stop() + for wt in self.threads: + wt.term() + self.collect_work() + self.submit_collected() + break + + if len(self.jobs) == 0: + self.prepare_jobs() + while len(self.jobs): + tid = self.find_best_thread() + self.threads[tid].add_jobs([self.jobs.pop()]) + + if config.watchd_threads == 1: # single_thread scenario + self.threads[0].workloop() + + self.collect_work() + + if len(self.collected) > 50: + self.submit_collected() + + time.sleep(1) + + self.mysqlite.close() + +if __name__ == '__main__': + w = Proxywatchd() + try: + w.run() + except KeyboardInterrupt as e: + w.stop()