move includes in root directory

proxywatchd.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+
+from threading import Thread
+import threading, commands
+import socket, time, random, sys, string, re
+import requests
+#from geoip import geolite2
+
+from ConfigParser import SafeConfigParser
+
+import mysqlite
+from misc import _log
+import rocksock
+
+class Proxywatchd(Thread):
+
+	def stop(self):
+		_log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()]))
+		self.running = 0
+
+	def __init__(self, config_file):
+		Thread.__init__(self)
+
+		self.threads = []
+		self.running = 1
+		self.parser = SafeConfigParser()
+		self.parser.read(config_file)
+
+		self.maxfail = self.parser.getint('global', 'proxy_max_fail')
+		self.maxthreads = self.parser.getint('watcherd', 'threads')
+		self.checktime = self.parser.getint('watcherd', 'checktime')
+		self.timeout = self.parser.getint('watcherd', 'timeout')
+		self.database = self.parser.get('global', 'database')
+		self.torhosts = [ str(i).strip() for i in self.parser.get('global', 'tor_host').split(',') ]
+		self.read_timeout = self.parser.getint('watcherd', 'read_timeout')
+
+		# create table if needed
+		self.mysqlite = mysqlite.mysqlite(self.database, str)
+		self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
+		self.mysqlite.commit()
+		self.echoise = time.time() - 3600;
+		self.ticks = time.time() - 3600;
+
+		with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n')
+
+		self.start()
+
+	def run(self):
+		_log('Starting proxywatchd..', 'notice')
+
+		threads = []
+		self.mysqlite = mysqlite.mysqlite(self.database, str)
+
+		while self.running:
+
+			if len(threads) < self.maxthreads:
+				t = threading.Thread(target=self.daemon, args=(self.servers,))
+				t.start()
+				threads.append(t)
+				time.sleep( random.choice( xrange(1,3)))
+
+			else: time.sleep(1)
+
+			if (time.time() - self.echoise) >= 180:
+				_log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads))
+				self.echoise = time.time()
+
+		self.mysqlite.close()
+
+	def is_drone_bl(self, proxy):
+		p = proxy.split(':')[0]
+		proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))}
+		resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies)
+		if 'No incidents regarding' in resp.text: return 0
+		else: return 1
+
+	def connect_socket(self, proxy, servers, proto = None):
+		protos = ['http', 'socks5', 'socks4']
+
+		for proto in protos:
+			torhost = random.choice(self.torhosts)
+			duration = time.time()
+			proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
+									rocksock.RocksockProxyFromURL('%s://%s' % (proto, proxy[0])),
+								]
+
+			srv = random.choice(servers).strip()
+			try:
+				sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=self.timeout)
+				sock.connect()
+				sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT']))
+				return sock, proto, duration, torhost, srv
+
+			except: sock.disconnect()
+
+		return False, False, False, False, False
+
+	def daemon(self, servers):
+		sqlite = mysqlite.mysqlite(self.database, str)
+		threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
+
+		q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM() LIMIT ?'
+
+		while self.running:
+			sqlite_requests = []
+			rows = sqlite.execute(q, (self.maxfail, time.time(), random.randint(10,20))).fetchall()
+			if not len(rows):
+				time.sleep(random.randint(10,20))
+				continue
+
+			abc = ' OR proxy='.join( [ '?' for x in xrange(0, len(rows)) ] )
+			args = [ (time.time() + 180) ]
+			e = [ args.append(i[0]) for i in rows ]
+			sqlite.executemany('UPDATE proxylist SET tested=? WHERE proxy=%s' % abc, (args,))
+			sqlite.commit()
+
+			for proxy in rows:
+				time.sleep(0.1)
+				nextcheck = (time.time() + 1800 + ((1+int(proxy[1])) * 3600))
+
+				sock, proto, duration, tor, srv = self.connect_socket(proxy, servers, proto=proxy[3])
+				if not sock:
+					sqlite_requests.append(((proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
+					continue
+
+				try:
+					recv = sock.recv(6)
+
+					# good data
+					if re.match('^(:|ERROR|PING|PONG|NOTICE|\*\*\*)', recv, re.IGNORECASE):
+						duration = (time.time() - duration)
+						nextcheck = (time.time() + 1800)
+
+						#match = geolite2.lookup(proxy[0].split(':')[0])
+						match = None
+						if match is not None: match = match.country
+						else: match = 'unknown'
+
+						#dronebl = self.is_drone_bl(proxy[0])
+						sqlite_requests.append( (0, nextcheck, 1, match, proto, duration, proxy[0],))
+						_log('%s://%s; c: %s; d: %d sec(s); tor: %s; srv: %s; recv: %s' % (proto, proxy[0], match, duration, tor, srv, recv), threadid)
+
+					# bad data
+					else:
+						sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
+
+				# also bad
+				except:
+					sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
+
+				finally:
+					sock.disconnect()
+
+			for r in sqlite_requests:
+				sqlite.execute('UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?', r)
+			sqlite.commit()
+
+		sqlite.close()