#!/usr/bin/env python2 import dbs import time import mysqlite import proxywatchd from misc import _log from config import Config import fetch import sys from soup_parser import soupify, set_nobs import re import threading import random config = Config() def import_from_file(fn, sqlite): with open(fn, 'r') as f: urls = [ url for url in f.read().split('\n') if url ] cinc = 0 while True: chunk = urls[cinc:cinc+200] if chunk: dbs.insert_urls(chunk, 'import.txt', urldb) else: break cinc = cinc + 200 def get_content_type(url, proxy): hdr = fetch.fetch_contents(url, head=True, proxy=proxy) for h in hdr.split('\n'): if h.lower().startswith('content-type: '): return h.lower().split(':')[1].strip() return '' def is_good_content_type(string): allowed_ct = [ 'text/html', 'text/plain', 'atom+xml' ] for ct in allowed_ct: if ct.lower() in string.lower(): return True return False def is_bad_url(uri, domain=None, samedomain=False): # if uri needs to be from same domain and domains missmatch if samedomain and str(uri.split('/')[2]).lower() != str(domain).lower(): return True for u in urignore: if re.findall(u, uri): return True return False def extract_urls(html, url): mytime = int(time.time()) proto = url.split(':')[0] domain = url.split('/')[2] urls = [] soup = soupify(html, nohtml=True) for a in soup.find_all('a', href=True): item = a['href'].encode('utf-8') if isinstance(a['href'], unicode) else a['href'] item = item.strip() if item.startswith('www.'): item = 'http://%s' % item elif not item.startswith('http'): if not item.startswith('/'): item = '/%s' % item item = '%s://%s%s' % (proto,domain,item) elif is_bad_url(item, domain=domain, samedomain=config.ppf.extract_samedomain): continue if not item in urls: urls.append(item) if urls: dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls) def import_proxies_from_file(proxydb, fn): content = open(fn, 'r').read() unique_count, new = fetch.extract_proxies(content, proxydb) if new: dbs.insert_proxies(proxydb, new, fn) return 0 return 1 class Leechered(threading.Thread): def __init__(self, url, stale_count, error, retrievals, proxies_added, content_type, proxy): self.status = 'nok' self.proxylist = [] self.running = True self.url = url self.stale_count = stale_count self.error = error self.retrievals = retrievals self.proxies_added = proxies_added self.content_type = content_type self.proxy = proxy self.execute = '' threading.Thread.__init__(self) def retrieve(self): return self.url, self.proxylist, self.stale_count, self.error, self.retrievals, self.content_type, self.proxies_added, self.execute def status(self): return self.status def run(self): self.status = 'nok' if not self.content_type: self.content_type = get_content_type(self.url, self.proxy) if is_good_content_type(self.content_type): try: content = fetch.fetch_contents(self.url, proxy=self.proxy) except KeyboardInterrupt as e: raise e except Exception as e: _log('%s: fetch error: %s' % (self.url.split('/')[2], str(e)), 'error') content = '' else: content = '' unique = fetch.extract_proxies(content, filter_known=False) self.proxylist = [ proxy for proxy in unique if not fetch.is_known_proxy(proxy) ] proxy_count = len(self.proxylist) if self.retrievals == 0: # new site if content and not self.proxylist: # site works but has zero proxy addresses self.error += 1 self.stale_count += 1 elif proxy_count: self.error = 0 self.stale_count = 0 else: self.error += 2 self.stale_count += 2 else: # not a new site # proxylist is empty if not proxy_count: self.stale_count += 1 # proxylist is not empty: site is working else: self.stale_count = 0 self.error = 0 # site has no content if not content: self.error += 1 self.stale_count += 1 #else: # self.retrievals += 1 # self.error = 0 # self.stale_count = 0 # site has proxies if proxy_count: self.error = 0 self.stale_count = 0 extract_urls(content, self.url) self.execute = (self.error, self.stale_count, int(time.time()), self.retrievals, self.proxies_added+len(self.proxylist), self.content_type, self.url) self.status = 'ok' if __name__ == '__main__': config.load() errors = config.validate() if errors: for e in errors: _log(e, 'error') sys.exit(1) fetch.set_config(config) # handle --nobs flag args = config.aparser.parse_args() if args.nobs: set_nobs(True) proxydb = mysqlite.mysqlite(config.watchd.database, str) dbs.create_table_if_not_exists(proxydb, 'proxylist') fetch.init_known_proxies(proxydb) with open('urignore.txt', 'r') as f: urignore = [ i.strip() for i in f.read().split('\n') if i.strip() ] urldb = mysqlite.mysqlite(config.ppf.database, str) dbs.create_table_if_not_exists(urldb, 'uris') import_from_file('import.txt', urldb) if len(sys.argv) == 3 and sys.argv[1] == "--file": sys.exit(import_proxies_from_file(proxydb, sys.argv[2])) # start proxy watcher if config.watchd.threads > 0: watcherd = proxywatchd.Proxywatchd() watcherd.start() else: watcherd = None qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) 180: _log('running %d thread(s) over %d' % (len(threads), config.ppf.threads), 'ppf') statusmsg = time.time() if not rows: if (time.time() - reqtime) > 3: rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() reqtime = time.time() if len(rows) < config.ppf.threads: time.sleep(60) rows = [] else: _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf') _proxylist = [ '%s://%s' % (p[0], p[1]) for p in proxydb.execute('SELECT proto,proxy from proxylist where failed=0').fetchall() ] if not _proxylist: _proxylist = None for thread in threads: if thread.status == 'ok': url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve() new = [ p for p in proxylist if not fetch.is_known_proxy(p) ] if new: fetch.add_known_proxies(new) execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url) urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute) urldb.commit() if new: dbs.insert_proxies(proxydb, new, url) threads = [ thread for thread in threads if thread.is_alive() ] if len(threads) < config.ppf.threads and rows: p = random.sample(_proxylist, 5) if _proxylist is not None else None row = random.choice(rows) urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0])) urldb.commit() rows.remove(row) t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5], p) threads.append(t) t.start() except KeyboardInterrupt: if watcherd: watcherd.stop() watcherd.finish() break _log('ppf stopped', 'info')