ppf/ppf.py

#!/usr/bin/env python

import dbs
import time
import mysqlite
import proxywatchd
from misc import _log
from config import Config
import fetch
import sys
from bs4 import BeautifulSoup
import re

config = Config()

def import_from_file(fn, sqlite):
	with open(fn, 'r') as f:
		for u in f.read().split('\n'):
			if not len(u): continue
			exists = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE url=?',(u,)).fetchall() ]
			if exists: continue
			print('adding "%s"' % u)
			sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count,proxies_added,retrievals) VALUES (?,?,?,?,?,?,?)', (int(time.time()),u,0,0,0,0,0))
		sqlite.commit()


def get_content_type(url):
	hdr = fetch.fetch_contents(url, head=True)

	for h in hdr.split('\n'):
		if h.lower().startswith('content-type: '): return h.lower().split(':')[1].strip()

	return ''

def is_good_content_type(string):
	allowed_ct = [ 'text/html', 'text/plain', 'atom+xml' ]
	for ct in allowed_ct:
		if ct.lower() in string.lower(): return True
	return False

def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added, content_type):
	if not content_type: content_type = get_content_type(url)

	if is_good_content_type(content_type):
		try: content = fetch.fetch_contents(url)
		except KeyboardInterrupt as e: raise e
		except: content = ''
	else:
		content = ''

	unique_count, new = fetch.extract_proxies(content, proxydb)

	if retrievals == 0:  # new site
		if content != '' and unique_count == 0: # site works but has zero proxy addresses
			error = 99999
	else:
		if len(new) == 0:
			stale_count += 1
		else:
			stale_count = 0
		if content == '':
			error += 1
		else:
			retrievals += 1
			error = 0
			extract_urls(content, url)

	urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url))
	urldb.commit()

	if not len(new): return

	dbs.insert_proxies(proxydb, new, url)

def is_bad_url(uri):
        for u in urignore:
                if re.findall(u, uri): return True
        return False

def extract_urls(html, url):
        mytime = int(time.time())
        proto = url.split(':')[0]
        domain = url.split('/')[2]
        urls = []

        soup = BeautifulSoup(html, features='lxml')

        for a in soup.find_all('a', href=True):
                item = a['href'].encode('utf-8') if isinstance(a['href'], unicode) else a['href']

                if is_bad_url(item):
                        continue
                elif item.startswith('www.'):
                        item = 'http://%s' % item
                elif not item.startswith('http'):
                        if not item.startswith('/'): item = '/%s' % item
                        item = '%s://%s%s' % (proto,domain,item)

                if not item in urls: urls.append(item)
                if len(urls) < 200: continue
                insert_if_not_exists(urls)
                urls = []

        if len(urls): insert_if_not_exists(urls)

def insert_if_not_exists(urls):
        mytime = int(time.time())
        query = 'SELECT url FROM uris WHERE %s' % ' OR '.join( [ 'url=?' for u in urls ] )
        known = [ item[0] for item in urldb.execute(query, urls) ]
        args = [ [mytime, u, (mytime - 3600), 1, 0,0,0] for u in urls if not u in known ]
        if len(args):
                print('new items: %s' % args)
                urldb.executemany('INSERT OR IGNORE INTO uris (added,url,check_time,error,stale_count,proxies_added,retrievals) VALUES (?,?,?,?,?,?,?)', args)
                urldb.commit()


def import_proxies_from_file(proxydb, fn):
	content = open(fn, 'r').read()
	unique_count, new = fetch.extract_proxies(content, proxydb)
	if len(new):
		dbs.insert_proxies(proxydb, new, fn)
		return 0
	return 1

if __name__ == '__main__':
	config.load()
	fetch.set_config(config)

	proxydb = mysqlite.mysqlite(config.watchd.database, str)
	dbs.create_table_if_not_exists(proxydb, 'proxylist')

	with open('urignore.txt', 'r') as f:
		urignore = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]

	urldb = mysqlite.mysqlite(config.ppf.database, str)
	dbs.create_table_if_not_exists(urldb, 'uris')
	import_from_file('import.txt', urldb)
	if len(sys.argv) == 3 and sys.argv[1] == "--file":
		sys.exit(import_proxies_from_file(proxydb, sys.argv[2]))

	# start proxy watcher
	if config.watchd.threads > 0:
		watcherd = proxywatchd.Proxywatchd()
		watcherd.start()
	else:
		watcherd = None

	while True:
		try:
			## any site that needs to be checked ?
			rows = urldb.execute('SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()

			if not len(rows): time.sleep(10)

			for row in rows:
				proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4], row[5])

			#time.sleep(10)

		except KeyboardInterrupt:
			if watcherd:
				watcherd.stop()
				watcherd.finish()
			break

	print '\r',