spaces2tabs
This commit is contained in:
mickael
@@ -7,12 +7,12 @@ import rocksock
|
|||||||
|
|
||||||
""" return formatted timestamp """
|
""" return formatted timestamp """
|
||||||
def timestamp():
|
def timestamp():
|
||||||
return time.strftime('%H:%M:%S', time.gmtime())
|
return time.strftime('%H:%M:%S', time.gmtime())
|
||||||
|
|
||||||
""" return some random string """
|
""" return some random string """
|
||||||
def random_string(strlen=20):
|
def random_string(strlen=20):
|
||||||
return ''.join([random.choice(string.letters) for x in xrange(strlen)])
|
return ''.join([random.choice(string.letters) for x in xrange(strlen)])
|
||||||
|
|
||||||
def _log(strng, level='info'):
|
def _log(strng, level='info'):
|
||||||
print '%s/%s\t%s' % (timestamp(), level, strng)
|
print '%s/%s\t%s' % (timestamp(), level, strng)
|
||||||
|
|
||||||
|
|||||||
@@ -3,31 +3,31 @@ import sqlite3 as lite
|
|||||||
|
|
||||||
class mysqlite:
|
class mysqlite:
|
||||||
|
|
||||||
def execute(self, query, args = None, rmin = 1.5, rmax = 7.0):
|
def execute(self, query, args = None, rmin = 1.5, rmax = 7.0):
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try: return self.cursor.execute(query,args) if args else self.cursor.execute(query)
|
try: return self.cursor.execute(query,args) if args else self.cursor.execute(query)
|
||||||
except:
|
except:
|
||||||
print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
|
print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
|
||||||
time.sleep(random.uniform(rmin, rmax))
|
time.sleep(random.uniform(rmin, rmax))
|
||||||
|
|
||||||
def executemany(self, query, args, rmin = 1.5, rmax = 7.0):
|
def executemany(self, query, args, rmin = 1.5, rmax = 7.0):
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try: return self.cursor.executemany(query,args)
|
try: return self.cursor.executemany(query,args)
|
||||||
except:
|
except:
|
||||||
print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
|
print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
|
||||||
time.sleep(random.uniform(rmin, rmax))
|
time.sleep(random.uniform(rmin, rmax))
|
||||||
|
|
||||||
def commit(self, rmin = 1.5, rmax = 7.0):
|
def commit(self, rmin = 1.5, rmax = 7.0):
|
||||||
while True:
|
while True:
|
||||||
try: return self.handle.commit()
|
try: return self.handle.commit()
|
||||||
except: time.sleep(random.uniform(rmin, rmax))
|
except: time.sleep(random.uniform(rmin, rmax))
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
self.handle.close()
|
self.handle.close()
|
||||||
|
|
||||||
def __init__(self, database, factory = None):
|
def __init__(self, database, factory = None):
|
||||||
self.handle = lite.connect(database)
|
self.handle = lite.connect(database)
|
||||||
if factory: self.handle.text_factory = factory
|
if factory: self.handle.text_factory = factory
|
||||||
self.cursor = self.handle.cursor()
|
self.cursor = self.handle.cursor()
|
||||||
|
|||||||
@@ -15,145 +15,145 @@ import rocksock
|
|||||||
|
|
||||||
class Proxywatchd(Thread):
|
class Proxywatchd(Thread):
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
_log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()]))
|
_log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()]))
|
||||||
self.running = 0
|
self.running = 0
|
||||||
|
|
||||||
def __init__(self, config_file):
|
def __init__(self, config_file):
|
||||||
Thread.__init__(self)
|
Thread.__init__(self)
|
||||||
|
|
||||||
self.threads = []
|
self.threads = []
|
||||||
self.running = 1
|
self.running = 1
|
||||||
self.parser = SafeConfigParser()
|
self.parser = SafeConfigParser()
|
||||||
self.parser.read(config_file)
|
self.parser.read(config_file)
|
||||||
|
|
||||||
self.maxfail = self.parser.getint('global', 'proxy_max_fail')
|
self.maxfail = self.parser.getint('global', 'proxy_max_fail')
|
||||||
self.maxthreads = self.parser.getint('watcherd', 'threads')
|
self.maxthreads = self.parser.getint('watcherd', 'threads')
|
||||||
self.checktime = self.parser.getint('watcherd', 'checktime')
|
self.checktime = self.parser.getint('watcherd', 'checktime')
|
||||||
self.timeout = self.parser.getint('watcherd', 'timeout')
|
self.timeout = self.parser.getint('watcherd', 'timeout')
|
||||||
self.database = self.parser.get('global', 'database')
|
self.database = self.parser.get('global', 'database')
|
||||||
self.torhosts = [ str(i).strip() for i in self.parser.get('global', 'tor_host').split(',') ]
|
self.torhosts = [ str(i).strip() for i in self.parser.get('global', 'tor_host').split(',') ]
|
||||||
self.read_timeout = self.parser.getint('watcherd', 'read_timeout')
|
self.read_timeout = self.parser.getint('watcherd', 'read_timeout')
|
||||||
|
|
||||||
# create table if needed
|
# create table if needed
|
||||||
self.mysqlite = mysqlite.mysqlite(self.database, str)
|
self.mysqlite = mysqlite.mysqlite(self.database, str)
|
||||||
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
|
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
|
||||||
self.mysqlite.commit()
|
self.mysqlite.commit()
|
||||||
self.echoise = time.time() - 3600;
|
self.echoise = time.time() - 3600;
|
||||||
self.ticks = time.time() - 3600;
|
self.ticks = time.time() - 3600;
|
||||||
|
|
||||||
with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n')
|
with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n')
|
||||||
|
|
||||||
self.start()
|
self.start()
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
_log('Starting proxywatchd..', 'notice')
|
_log('Starting proxywatchd..', 'notice')
|
||||||
|
|
||||||
threads = []
|
threads = []
|
||||||
self.mysqlite = mysqlite.mysqlite(self.database, str)
|
self.mysqlite = mysqlite.mysqlite(self.database, str)
|
||||||
|
|
||||||
while self.running:
|
while self.running:
|
||||||
|
|
||||||
if len(threads) < self.maxthreads:
|
if len(threads) < self.maxthreads:
|
||||||
t = threading.Thread(target=self.daemon, args=(self.servers,))
|
t = threading.Thread(target=self.daemon, args=(self.servers,))
|
||||||
t.start()
|
t.start()
|
||||||
threads.append(t)
|
threads.append(t)
|
||||||
time.sleep( random.choice( xrange(1,3)))
|
time.sleep( random.choice( xrange(1,3)))
|
||||||
|
|
||||||
else: time.sleep(1)
|
else: time.sleep(1)
|
||||||
|
|
||||||
if (time.time() - self.echoise) >= 180:
|
if (time.time() - self.echoise) >= 180:
|
||||||
_log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads))
|
_log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads))
|
||||||
self.echoise = time.time()
|
self.echoise = time.time()
|
||||||
|
|
||||||
self.mysqlite.close()
|
self.mysqlite.close()
|
||||||
|
|
||||||
def is_drone_bl(self, proxy):
|
def is_drone_bl(self, proxy):
|
||||||
p = proxy.split(':')[0]
|
p = proxy.split(':')[0]
|
||||||
proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))}
|
proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))}
|
||||||
resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies)
|
resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies)
|
||||||
if 'No incidents regarding' in resp.text: return 0
|
if 'No incidents regarding' in resp.text: return 0
|
||||||
else: return 1
|
else: return 1
|
||||||
|
|
||||||
def connect_socket(self, proxy, servers, proto = None):
|
def connect_socket(self, proxy, servers, proto = None):
|
||||||
protos = ['http', 'socks5', 'socks4']
|
protos = ['http', 'socks5', 'socks4']
|
||||||
|
|
||||||
for proto in protos:
|
for proto in protos:
|
||||||
torhost = random.choice(self.torhosts)
|
torhost = random.choice(self.torhosts)
|
||||||
duration = time.time()
|
duration = time.time()
|
||||||
proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
|
proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
|
||||||
rocksock.RocksockProxyFromURL('%s://%s' % (proto, proxy[0])),
|
rocksock.RocksockProxyFromURL('%s://%s' % (proto, proxy[0])),
|
||||||
]
|
]
|
||||||
|
|
||||||
srv = random.choice(servers).strip()
|
srv = random.choice(servers).strip()
|
||||||
try:
|
try:
|
||||||
sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=self.timeout)
|
sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=self.timeout)
|
||||||
sock.connect()
|
sock.connect()
|
||||||
sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT']))
|
sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT']))
|
||||||
return sock, proto, duration, torhost, srv
|
return sock, proto, duration, torhost, srv
|
||||||
|
|
||||||
except: sock.disconnect()
|
except: sock.disconnect()
|
||||||
|
|
||||||
return False, False, False, False, False
|
return False, False, False, False, False
|
||||||
|
|
||||||
def daemon(self, servers):
|
def daemon(self, servers):
|
||||||
sqlite = mysqlite.mysqlite(self.database, str)
|
sqlite = mysqlite.mysqlite(self.database, str)
|
||||||
threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
|
threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
|
||||||
|
|
||||||
q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM() LIMIT ?'
|
q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM() LIMIT ?'
|
||||||
|
|
||||||
while self.running:
|
while self.running:
|
||||||
sqlite_requests = []
|
sqlite_requests = []
|
||||||
rows = sqlite.execute(q, (self.maxfail, time.time(), random.randint(10,20))).fetchall()
|
rows = sqlite.execute(q, (self.maxfail, time.time(), random.randint(10,20))).fetchall()
|
||||||
if not len(rows):
|
if not len(rows):
|
||||||
time.sleep(random.randint(10,20))
|
time.sleep(random.randint(10,20))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
abc = ' OR proxy='.join( [ '?' for x in xrange(0, len(rows)) ] )
|
abc = ' OR proxy='.join( [ '?' for x in xrange(0, len(rows)) ] )
|
||||||
args = [ (time.time() + 180) ]
|
args = [ (time.time() + 180) ]
|
||||||
e = [ args.append(i[0]) for i in rows ]
|
e = [ args.append(i[0]) for i in rows ]
|
||||||
sqlite.executemany('UPDATE proxylist SET tested=? WHERE proxy=%s' % abc, (args,))
|
sqlite.executemany('UPDATE proxylist SET tested=? WHERE proxy=%s' % abc, (args,))
|
||||||
sqlite.commit()
|
sqlite.commit()
|
||||||
|
|
||||||
for proxy in rows:
|
for proxy in rows:
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
nextcheck = (time.time() + 1800 + ((1+int(proxy[1])) * 3600))
|
nextcheck = (time.time() + 1800 + ((1+int(proxy[1])) * 3600))
|
||||||
|
|
||||||
sock, proto, duration, tor, srv = self.connect_socket(proxy, servers, proto=proxy[3])
|
sock, proto, duration, tor, srv = self.connect_socket(proxy, servers, proto=proxy[3])
|
||||||
if not sock:
|
if not sock:
|
||||||
sqlite_requests.append(((proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
sqlite_requests.append(((proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
recv = sock.recv(6)
|
recv = sock.recv(6)
|
||||||
|
|
||||||
# good data
|
# good data
|
||||||
if re.match('^(:|ERROR|PING|PONG|NOTICE|\*\*\*)', recv, re.IGNORECASE):
|
if re.match('^(:|ERROR|PING|PONG|NOTICE|\*\*\*)', recv, re.IGNORECASE):
|
||||||
duration = (time.time() - duration)
|
duration = (time.time() - duration)
|
||||||
nextcheck = (time.time() + 1800)
|
nextcheck = (time.time() + 1800)
|
||||||
|
|
||||||
#match = geolite2.lookup(proxy[0].split(':')[0])
|
#match = geolite2.lookup(proxy[0].split(':')[0])
|
||||||
match = None
|
match = None
|
||||||
if match is not None: match = match.country
|
if match is not None: match = match.country
|
||||||
else: match = 'unknown'
|
else: match = 'unknown'
|
||||||
|
|
||||||
#dronebl = self.is_drone_bl(proxy[0])
|
#dronebl = self.is_drone_bl(proxy[0])
|
||||||
sqlite_requests.append( (0, nextcheck, 1, match, proto, duration, proxy[0],))
|
sqlite_requests.append( (0, nextcheck, 1, match, proto, duration, proxy[0],))
|
||||||
_log('%s://%s; c: %s; d: %d sec(s); tor: %s; srv: %s; recv: %s' % (proto, proxy[0], match, duration, tor, srv, recv), threadid)
|
_log('%s://%s; c: %s; d: %d sec(s); tor: %s; srv: %s; recv: %s' % (proto, proxy[0], match, duration, tor, srv, recv), threadid)
|
||||||
|
|
||||||
# bad data
|
# bad data
|
||||||
else:
|
else:
|
||||||
sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
||||||
|
|
||||||
# also bad
|
# also bad
|
||||||
except:
|
except:
|
||||||
sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
sock.disconnect()
|
sock.disconnect()
|
||||||
|
|
||||||
for r in sqlite_requests:
|
for r in sqlite_requests:
|
||||||
sqlite.execute('UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?', r)
|
sqlite.execute('UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?', r)
|
||||||
sqlite.commit()
|
sqlite.commit()
|
||||||
|
|
||||||
sqlite.close()
|
sqlite.close()
|
||||||
|
|||||||
Reference in New Issue
Block a user