ppf/proxywatchd.py

#!/usr/bin/env python

from threading import Thread
import threading, commands
import socket, time, random, sys, string, re
import requests
#from geoip import geolite2

from ConfigParser import SafeConfigParser

import mysqlite
from misc import _log
import rocksock

class Proxywatchd(Thread):

	def stop(self):
		_log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()]))
		self.running = 0

	def __init__(self, config_file):
		Thread.__init__(self)

		self.threads = []
		self.running = 1
		self.parser = SafeConfigParser()
		self.parser.read(config_file)

		self.maxfail = self.parser.getint('global', 'proxy_max_fail')
		self.maxthreads = self.parser.getint('watcherd', 'threads')
		self.checktime = self.parser.getint('watcherd', 'checktime')
		self.timeout = self.parser.getint('watcherd', 'timeout')
		self.database = self.parser.get('global', 'database')
		self.torhosts = [ str(i).strip() for i in self.parser.get('global', 'tor_host').split(',') ]
		self.read_timeout = self.parser.getint('watcherd', 'read_timeout')

		# create table if needed
		self.mysqlite = mysqlite.mysqlite(self.database, str)
		self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
		self.mysqlite.commit()
		self.echoise = time.time() - 3600;
		self.ticks = time.time() - 3600;

		with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n')

		self.start()

	def run(self):
		_log('Starting proxywatchd..', 'notice')

		threads = []
		self.mysqlite = mysqlite.mysqlite(self.database, str)

		while self.running:

			if len(threads) < self.maxthreads:
				t = threading.Thread(target=self.daemon, args=(self.servers,))
				t.start()
				threads.append(t)
				time.sleep( random.choice( xrange(1,3)))

			else: time.sleep(1)

			if (time.time() - self.echoise) >= 180:
				_log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads))
				self.echoise = time.time()

		self.mysqlite.close()

	def is_drone_bl(self, proxy):
		p = proxy.split(':')[0]
		proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))}
		resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies)
		if 'No incidents regarding' in resp.text: return 0
		else: return 1

	def connect_socket(self, proxy, servers, proto = None):
		protos = ['http', 'socks5', 'socks4'] if proto is None else proto

		for proto in protos:
			torhost = random.choice(self.torhosts)
			duration = time.time()
			proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
									rocksock.RocksockProxyFromURL('%s://%s' % (proto, proxy[0])),
								]

			srv = random.choice(servers).strip()
			try:
				sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=self.timeout)
				sock.connect()
				sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT']))
				return sock, proto, duration, torhost, srv

			except: sock.disconnect()

		return None, None, None, None, None

	def daemon(self, servers):
		sqlite = mysqlite.mysqlite(self.database, str)
		threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )

		q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM() LIMIT ?'

		while self.running:
			sqlite_requests = []
			rows = sqlite.execute(q, (self.maxfail, time.time(), random.randint(10,20))).fetchall()
			if not len(rows):
				time.sleep(random.randint(10,20))
				continue

			abc = ' OR proxy='.join( [ '?' for x in xrange(0, len(rows)) ] )
			args = [ (time.time() + 180) ]
			e = [ args.append(i[0]) for i in rows ]
			sqlite.executemany('UPDATE proxylist SET tested=? WHERE proxy=%s' % abc, (args,))
			sqlite.commit()

			for proxy in rows:
				time.sleep(0.1)
				nextcheck = (time.time() + 1800 + ((1+int(proxy[1])) * 3600))

				sock, proto, duration, tor, srv = self.connect_socket(proxy, servers, proto=proxy[3])
				if not sock:
					sqlite_requests.append(((proxy[1]+1), nextcheck, 1, 'unknown', None, 0, proxy[0],))
					continue

				try:
					recv = sock.recv(6)

					# good data
					if re.match('^(:|ERROR|PING|PONG|NOTICE|\*\*\*)', recv, re.IGNORECASE):
						duration = (time.time() - duration)
						nextcheck = (time.time() + 1800)

						#match = geolite2.lookup(proxy[0].split(':')[0])
						match = None
						if match is not None: match = match.country
						else: match = 'unknown'

						#dronebl = self.is_drone_bl(proxy[0])
						sqlite_requests.append( (0, nextcheck, 1, match, proto, duration, proxy[0],))
						_log('%s://%s; c: %s; d: %d sec(s); tor: %s; srv: %s; recv: %s' % (proto, proxy[0], match, duration, tor, srv, recv), threadid)

					# bad data
					else:
						sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', None, 0, proxy[0],))

				# also bad
				except:
					sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', None, 0, proxy[0],))

				finally:
					sock.disconnect()

			for r in sqlite_requests:
				sqlite.execute('UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?', r)
			sqlite.commit()

		sqlite.close()