From ffbe450aee43ca592d68f0e146976ced22314662 Mon Sep 17 00:00:00 2001 From: rofl0r Date: Sat, 5 Jan 2019 03:47:03 +0000 Subject: [PATCH] outsource configuration to external module --- config.py | 21 +++++++++++++++++++++ ppf.py | 22 ++++++++-------------- proxywatchd.py | 34 ++++++++++++---------------------- 3 files changed, 41 insertions(+), 36 deletions(-) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..61c921b --- /dev/null +++ b/config.py @@ -0,0 +1,21 @@ +from ConfigParser import SafeConfigParser + +_loaded = False + +def load(): + if _loaded: return + global database, maxfail, search, torhosts, watchd_threads, checktime, timeout, read_timeout + + ## read the config files + parser = SafeConfigParser() + parser.read('config.ini') + + database = parser.get('global', 'database') + maxfail = parser.getint('global', 'proxy_max_fail') + + search = parser.getboolean('proxyfind', 'search') + + torhosts = [ str(i).strip() for i in parser.get('global', 'tor_host').split(',') ] + watchd_threads = parser.getint('watcherd', 'threads') + timeout = parser.getint('watcherd', 'timeout') + diff --git a/ppf.py b/ppf.py index a91a4de..59deb86 100755 --- a/ppf.py +++ b/ppf.py @@ -6,13 +6,13 @@ import random, time import re import urllib import hashlib -from ConfigParser import SafeConfigParser from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) import mysqlite import proxywatchd from misc import _log from soup_parser import soupify +import config base_header = { 'Accept-Language':'en-US,en;q=0.8', @@ -24,7 +24,6 @@ base_header = { searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' ) retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded') -CONFIG = 'config.ini' def cleanhtml(raw_html): cleanr = re.compile('<.*?>') @@ -69,7 +68,7 @@ def insert_proxies(proxies, uri, sqlite): def proxyfind(sqlite = None): #print('entering proxyfind...') - if not sqlite: sqlite = mysqlite.mysqlite(database,str) + if not sqlite: sqlite = mysqlite.mysqlite(config.database,str) uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ] @@ -149,16 +148,11 @@ def proxyleech(sqlite, rows): if __name__ == '__main__': - ## read the config files - parser = SafeConfigParser() - parser.read(CONFIG) + config.load() + print repr(config.torhosts) + proxies={'http':'socks4://%s' % random.choice(config.torhosts),'https':'socks4://%s' % random.choice(config.torhosts)} - database = parser.get('global', 'database') - search = parser.getboolean('proxyfind', 'search') - tor_hosts = parser.get('global', 'tor_host').split(',') - proxies={'http':'socks4://%s' % random.choice(tor_hosts),'https':'socks4://%s' % random.choice(tor_hosts)} - - sqlite = mysqlite.mysqlite(database, str) + sqlite = mysqlite.mysqlite(config.database, str) ## create dbs if required sqlite.execute('CREATE TABLE IF NOT EXISTS uris (added INT, url TEXT, check_time INT, error INT, driver INT, hash TEXT)') @@ -178,7 +172,7 @@ if __name__ == '__main__': empty = [ urignore.append(i.split('/')[2]) for i in searx_instances ] # start proxy watcher - watcherd = proxywatchd.Proxywatchd(CONFIG) if parser.getboolean('watcherd', 'enabled') else None + watcherd = proxywatchd.Proxywatchd() if config.watchd_threads > 0 else None while True: try: @@ -186,7 +180,7 @@ if __name__ == '__main__': rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time= 180: - _log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads)) + _log('Proxywatchd threads: %d/%d' % (len(threads), config.watchd_threads)) self.echoise = time.time() self.mysqlite.close() def is_drone_bl(self, proxy): p = proxy.split(':')[0] - proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))} + proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(config.torhosts))} resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies) if 'No incidents regarding' in resp.text: return 0 else: return 1 @@ -78,7 +68,7 @@ class Proxywatchd(Thread): protos = ['http', 'socks5', 'socks4'] if proto is None else proto for proto in protos: - torhost = random.choice(self.torhosts) + torhost = random.choice(config.torhosts) duration = time.time() proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost), rocksock.RocksockProxyFromURL('%s://%s' % (proto, proxy[0])), @@ -86,7 +76,7 @@ class Proxywatchd(Thread): srv = random.choice(servers).strip() try: - sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=self.timeout) + sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=config.timeout) sock.connect() sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT'])) return sock, proto, duration, torhost, srv @@ -96,14 +86,14 @@ class Proxywatchd(Thread): return None, None, None, None, None def daemon(self, servers): - sqlite = mysqlite.mysqlite(self.database, str) + sqlite = mysqlite.mysqlite(config.database, str) threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] ) q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed