ppf/includes/proxywatchd.py

#!/usr/bin/env python

from threading import Thread
import threading, commands
import socket, time, random, sys, string, re
import requests
#from geoip import geolite2

from ConfigParser import SafeConfigParser

sys.path.append('./includes')
import mysqlite
from misc import _log
import rocksock

class Proxywatchd(Thread):

  def stop(self):
    _log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()]))
    self.running = 0

  def __init__(self, config_file):
    Thread.__init__(self)

    self.threads = []
    self.running = 1
    self.parser = SafeConfigParser()
    self.parser.read(config_file)

    self.maxfail = self.parser.getint('global', 'proxy_max_fail')
    self.maxthreads = self.parser.getint('watcherd', 'threads')
    self.checktime = self.parser.getint('watcherd', 'checktime')
    self.timeout = self.parser.getint('watcherd', 'timeout')
    self.database = self.parser.get('global', 'database')
    self.torhosts = [ str(i).strip() for i in self.parser.get('global', 'tor_host').split(',') ]
    self.read_timeout = self.parser.getint('watcherd', 'read_timeout')

    # create table if needed
    self.mysqlite = mysqlite.mysqlite(self.database, str)
    self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
    self.mysqlite.commit()
    self.echoise = time.time() - 3600;
    self.ticks = time.time() - 3600;

    with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n')

    self.start()

  def run(self):
    _log('Starting proxywatchd..', 'notice')

    threads = []
    self.mysqlite = mysqlite.mysqlite(self.database, str)

    while self.running:

      if len(threads) < self.maxthreads:
        t = threading.Thread(target=self.daemon, args=(self.servers,))
        t.start()
        threads.append(t)
        time.sleep( random.choice( xrange(1,3)))

      else: time.sleep(1)

      if (time.time() - self.echoise) >= 180:
        _log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads))
        self.echoise = time.time()

    self.mysqlite.close()

  def is_drone_bl(self, proxy):
    p = proxy.split(':')[0]
    proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))}
    resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies)
    if 'No incidents regarding' in resp.text: return 0
    else: return 1

  def connect_socket(self, proxy, servers, proto = None):
    protos = ['http', 'socks5', 'socks4']

    for proto in protos:
      torhost = random.choice(self.torhosts)
      duration = time.time()
      proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
                  rocksock.RocksockProxyFromURL('%s://%s' % (proto, proxy[0])),
                ]

      srv = random.choice(servers).strip()
      try:
        sock = rocksock.Rocksock(host=srv, port=6697, ssl=True, proxies=proxies, timeout=self.timeout)
        sock.connect()
        sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT']))
        return sock, proto, duration, torhost, srv

      except: sock.disconnect()

    return False, False, False, False, False

  def daemon(self, servers):
    sqlite = mysqlite.mysqlite(self.database, str)
    threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )

    q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM() LIMIT ?'

    while self.running:

      time.sleep(0.1)
      sqlite_requests = []
      rows = sqlite.execute(q, (self.maxfail, time.time(), random.randint(10,20))).fetchall()

      if not len(rows):
        time.sleep(random.randint(10,20))
        continue

      abc = ' OR proxy='.join( [ '?' for x in xrange(0, len(rows)) ] )
      args = [ (time.time() + 180) ]
      e = [ args.append(i[0]) for i in rows ]
      sqlite.executemany('UPDATE proxylist SET tested=? WHERE proxy=%s' % abc, (args,))
      sqlite.commit()

      for proxy in rows:
        time.sleep(0.1)
        nextcheck = (time.time() + 1800 + ((1+int(proxy[1])) * 3600))

        sock, proto, duration, tor, srv = self.connect_socket(proxy, servers, proto=proxy[3])
        if not sock:
          sqlite_requests.append(((proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
          continue

        try:
          recv = sock.recv(6)

          # good data
          if re.match('^(:|ERROR|PING|PONG|NOTICE|\*\*\*)', recv, re.IGNORECASE):
            duration = (time.time() - duration)
            nextcheck = (time.time() + 1800)

            #match = geolite2.lookup(proxy[0].split(':')[0])
            match = None
            if match is not None: match = match.country
            else: match = 'unknown'

            #dronebl = self.is_drone_bl(proxy[0])
            sqlite_requests.append( (0, nextcheck, 1, match, proto, duration, proxy[0],))
            _log('%s://%s; c: %s; d: %d sec(s); tor: %s; srv: %s; recv: %s' % (proto, proxy[0], match, duration, tor, srv, recv), threadid)

          # bad data
          else:
            sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))

        # also bad
        except:
          sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))

        finally:
          sock.disconnect()

      for r in sqlite_requests:
        sqlite.execute('UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?', r)
      sqlite.commit()

    sqlite.close()