#!/usr/bin/env python

import dbs
import time
import mysqlite
import proxywatchd
from misc import _log
from config import Config
import fetch
import sys

config = Config()

def import_from_file(fn, sqlite):
	with open(fn, 'r') as f:
		for u in f.read().split('\n'):
			if not len(u): continue
			exists = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE url=?',(u,)).fetchall() ]
			if exists: continue
			print('adding "%s"' % u)
			sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count,proxies_added,retrievals) VALUES (?,?,?,?,?,?,?)', (int(time.time()),u,0,0,0,0,0))
		sqlite.commit()

def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added):
	try: content = fetch.fetch_contents(url)
	except KeyboardInterrupt as e: raise e
	except: content = ''

	unique_count, new = fetch.extract_proxies(content, proxydb)

	if retrievals == 0:  # new site
		if content != '' and unique_count == 0: # site works but has zero proxy addresses
			error = 99999
	else:
		if len(new) == 0:
			stale_count += 1
		else:
			stale_count = 0
		if content == '':
			error += 1
		else:
			retrievals += 1
			error = 0

	urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=? where url=?', (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), url))
	urldb.commit()

	if not len(new): return

	dbs.insert_proxies(proxydb, new, url)


def import_proxies_from_file(proxydb, fn):
	content = open(fn, 'r').read()
	unique_count, new = fetch.extract_proxies(content, proxydb)
	if len(new):
		dbs.insert_proxies(proxydb, new, fn)
		return 0
	return 1

if __name__ == '__main__':
	config.load()
	fetch.set_config(config)

	proxydb = mysqlite.mysqlite(config.watchd.database, str)
	dbs.create_table_if_not_exists(proxydb, 'proxylist')

	urldb = mysqlite.mysqlite(config.ppf.database, str)
	dbs.create_table_if_not_exists(urldb, 'uris')
	import_from_file('import.txt', urldb)
	if len(sys.argv) == 3 and sys.argv[1] == "--file":
		sys.exit(import_proxies_from_file(proxydb, sys.argv[2]))

	# start proxy watcher
	if config.watchd.threads > 0:
		watcherd = proxywatchd.Proxywatchd()
		watcherd.start()
	else:
		watcherd = None

	while True:
		try:
			## any site that needs to be checked ?
			rows = urldb.execute('SELECT url,stale_count,error,retrievals,proxies_added FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()

			for row in rows:
				proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4])

			time.sleep(10)

		except KeyboardInterrupt:
			if watcherd:
				watcherd.stop()
				watcherd.finish()
			break

	print '\r',