style: normalize indentation and improve code style
- convert tabs to 4-space indentation
- add docstrings to modules and classes
- remove unused import (copy)
- use explicit object inheritance
- use 'while True' over 'while 1'
- use 'while args' over 'while len(args)'
- use '{}' over 'dict()'
- consistent string formatting
- Python 2/3 compatible Queue import
This commit is contained in:
Username
372
ppf.py
Executable file → Normal file
372
ppf.py
Executable file → Normal file
@@ -16,231 +16,231 @@ import random
|
||||
config = Config()
|
||||
|
||||
def import_from_file(fn, sqlite):
|
||||
with open(fn, 'r') as f:
|
||||
urls = [ url for url in f.read().split('\n') if url ]
|
||||
cinc = 0
|
||||
while True:
|
||||
chunk = urls[cinc:cinc+200]
|
||||
if chunk: dbs.insert_urls(chunk, 'import.txt', urldb)
|
||||
else: break
|
||||
cinc = cinc + 200
|
||||
with open(fn, 'r') as f:
|
||||
urls = [ url for url in f.read().split('\n') if url ]
|
||||
cinc = 0
|
||||
while True:
|
||||
chunk = urls[cinc:cinc+200]
|
||||
if chunk: dbs.insert_urls(chunk, 'import.txt', urldb)
|
||||
else: break
|
||||
cinc = cinc + 200
|
||||
|
||||
|
||||
def get_content_type(url, proxy):
|
||||
hdr = fetch.fetch_contents(url, head=True, proxy=proxy)
|
||||
hdr = fetch.fetch_contents(url, head=True, proxy=proxy)
|
||||
|
||||
for h in hdr.split('\n'):
|
||||
if h.lower().startswith('content-type: '): return h.lower().split(':')[1].strip()
|
||||
for h in hdr.split('\n'):
|
||||
if h.lower().startswith('content-type: '): return h.lower().split(':')[1].strip()
|
||||
|
||||
return ''
|
||||
return ''
|
||||
|
||||
def is_good_content_type(string):
|
||||
allowed_ct = [ 'text/html', 'text/plain', 'atom+xml' ]
|
||||
for ct in allowed_ct:
|
||||
if ct.lower() in string.lower(): return True
|
||||
return False
|
||||
allowed_ct = [ 'text/html', 'text/plain', 'atom+xml' ]
|
||||
for ct in allowed_ct:
|
||||
if ct.lower() in string.lower(): return True
|
||||
return False
|
||||
|
||||
def is_bad_url(uri, domain=None, samedomain=False):
|
||||
# if uri needs to be from same domain and domains missmatch
|
||||
if samedomain and str(uri.split('/')[2]).lower() != str(domain).lower():
|
||||
return True
|
||||
for u in urignore:
|
||||
if re.findall(u, uri): return True
|
||||
return False
|
||||
# if uri needs to be from same domain and domains missmatch
|
||||
if samedomain and str(uri.split('/')[2]).lower() != str(domain).lower():
|
||||
return True
|
||||
for u in urignore:
|
||||
if re.findall(u, uri): return True
|
||||
return False
|
||||
|
||||
def extract_urls(html, url):
|
||||
mytime = int(time.time())
|
||||
proto = url.split(':')[0]
|
||||
domain = url.split('/')[2]
|
||||
urls = []
|
||||
mytime = int(time.time())
|
||||
proto = url.split(':')[0]
|
||||
domain = url.split('/')[2]
|
||||
urls = []
|
||||
|
||||
soup = soupify(html, nohtml=True)
|
||||
soup = soupify(html, nohtml=True)
|
||||
|
||||
for a in soup.find_all('a', href=True):
|
||||
item = a['href'].encode('utf-8') if isinstance(a['href'], unicode) else a['href']
|
||||
item = item.strip()
|
||||
for a in soup.find_all('a', href=True):
|
||||
item = a['href'].encode('utf-8') if isinstance(a['href'], unicode) else a['href']
|
||||
item = item.strip()
|
||||
|
||||
if item.startswith('www.'):
|
||||
item = 'http://%s' % item
|
||||
elif not item.startswith('http'):
|
||||
if not item.startswith('/'): item = '/%s' % item
|
||||
item = '%s://%s%s' % (proto,domain,item)
|
||||
if item.startswith('www.'):
|
||||
item = 'http://%s' % item
|
||||
elif not item.startswith('http'):
|
||||
if not item.startswith('/'): item = '/%s' % item
|
||||
item = '%s://%s%s' % (proto,domain,item)
|
||||
|
||||
elif is_bad_url(item, domain=domain, samedomain=config.ppf.extract_samedomain):
|
||||
continue
|
||||
if not item in urls: urls.append(item)
|
||||
elif is_bad_url(item, domain=domain, samedomain=config.ppf.extract_samedomain):
|
||||
continue
|
||||
if not item in urls: urls.append(item)
|
||||
|
||||
if urls: dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls)
|
||||
if urls: dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls)
|
||||
|
||||
def import_proxies_from_file(proxydb, fn):
|
||||
content = open(fn, 'r').read()
|
||||
unique_count, new = fetch.extract_proxies(content, proxydb)
|
||||
if new:
|
||||
dbs.insert_proxies(proxydb, new, fn)
|
||||
return 0
|
||||
return 1
|
||||
content = open(fn, 'r').read()
|
||||
unique_count, new = fetch.extract_proxies(content, proxydb)
|
||||
if new:
|
||||
dbs.insert_proxies(proxydb, new, fn)
|
||||
return 0
|
||||
return 1
|
||||
|
||||
class Leechered(threading.Thread):
|
||||
def __init__(self, url, stale_count, error, retrievals, proxies_added, content_type, proxy):
|
||||
self.status = 'nok'
|
||||
self.proxylist = []
|
||||
self.running = True
|
||||
self.url = url
|
||||
self.stale_count = stale_count
|
||||
self.error = error
|
||||
self.retrievals = retrievals
|
||||
self.proxies_added = proxies_added
|
||||
self.content_type = content_type
|
||||
self.proxy = proxy
|
||||
self.execute = ''
|
||||
threading.Thread.__init__(self)
|
||||
def __init__(self, url, stale_count, error, retrievals, proxies_added, content_type, proxy):
|
||||
self.status = 'nok'
|
||||
self.proxylist = []
|
||||
self.running = True
|
||||
self.url = url
|
||||
self.stale_count = stale_count
|
||||
self.error = error
|
||||
self.retrievals = retrievals
|
||||
self.proxies_added = proxies_added
|
||||
self.content_type = content_type
|
||||
self.proxy = proxy
|
||||
self.execute = ''
|
||||
threading.Thread.__init__(self)
|
||||
|
||||
def retrieve(self):
|
||||
return self.url, self.proxylist, self.stale_count, self.error, self.retrievals, self.content_type, self.proxies_added, self.execute
|
||||
def status(self):
|
||||
return self.status
|
||||
def retrieve(self):
|
||||
return self.url, self.proxylist, self.stale_count, self.error, self.retrievals, self.content_type, self.proxies_added, self.execute
|
||||
def status(self):
|
||||
return self.status
|
||||
|
||||
def run(self):
|
||||
self.status = 'nok'
|
||||
def run(self):
|
||||
self.status = 'nok'
|
||||
|
||||
if not self.content_type: self.content_type = get_content_type(self.url, self.proxy)
|
||||
if not self.content_type: self.content_type = get_content_type(self.url, self.proxy)
|
||||
|
||||
if is_good_content_type(self.content_type):
|
||||
try:
|
||||
content = fetch.fetch_contents(self.url, proxy=self.proxy)
|
||||
except KeyboardInterrupt as e:
|
||||
raise e
|
||||
except Exception as e:
|
||||
_log('%s: fetch error: %s' % (self.url.split('/')[2], str(e)), 'error')
|
||||
content = ''
|
||||
else:
|
||||
content = ''
|
||||
if is_good_content_type(self.content_type):
|
||||
try:
|
||||
content = fetch.fetch_contents(self.url, proxy=self.proxy)
|
||||
except KeyboardInterrupt as e:
|
||||
raise e
|
||||
except Exception as e:
|
||||
_log('%s: fetch error: %s' % (self.url.split('/')[2], str(e)), 'error')
|
||||
content = ''
|
||||
else:
|
||||
content = ''
|
||||
|
||||
unique = fetch.extract_proxies(content, filter_known=False)
|
||||
self.proxylist = [ proxy for proxy in unique if not fetch.is_known_proxy(proxy) ]
|
||||
proxy_count = len(self.proxylist)
|
||||
unique = fetch.extract_proxies(content, filter_known=False)
|
||||
self.proxylist = [ proxy for proxy in unique if not fetch.is_known_proxy(proxy) ]
|
||||
proxy_count = len(self.proxylist)
|
||||
|
||||
if self.retrievals == 0: # new site
|
||||
if content and not self.proxylist: # site works but has zero proxy addresses
|
||||
self.error += 1
|
||||
self.stale_count += 1
|
||||
elif proxy_count:
|
||||
self.error = 0
|
||||
self.stale_count = 0
|
||||
else:
|
||||
self.error += 2
|
||||
self.stale_count += 2
|
||||
else: # not a new site
|
||||
# proxylist is empty
|
||||
if not proxy_count:
|
||||
self.stale_count += 1
|
||||
# proxylist is not empty: site is working
|
||||
else:
|
||||
self.stale_count = 0
|
||||
self.error = 0
|
||||
# site has no content
|
||||
if not content:
|
||||
self.error += 1
|
||||
self.stale_count += 1
|
||||
#else:
|
||||
# self.retrievals += 1
|
||||
# self.error = 0
|
||||
# self.stale_count = 0
|
||||
# site has proxies
|
||||
if proxy_count:
|
||||
self.error = 0
|
||||
self.stale_count = 0
|
||||
extract_urls(content, self.url)
|
||||
if self.retrievals == 0: # new site
|
||||
if content and not self.proxylist: # site works but has zero proxy addresses
|
||||
self.error += 1
|
||||
self.stale_count += 1
|
||||
elif proxy_count:
|
||||
self.error = 0
|
||||
self.stale_count = 0
|
||||
else:
|
||||
self.error += 2
|
||||
self.stale_count += 2
|
||||
else: # not a new site
|
||||
# proxylist is empty
|
||||
if not proxy_count:
|
||||
self.stale_count += 1
|
||||
# proxylist is not empty: site is working
|
||||
else:
|
||||
self.stale_count = 0
|
||||
self.error = 0
|
||||
# site has no content
|
||||
if not content:
|
||||
self.error += 1
|
||||
self.stale_count += 1
|
||||
#else:
|
||||
# self.retrievals += 1
|
||||
# self.error = 0
|
||||
# self.stale_count = 0
|
||||
# site has proxies
|
||||
if proxy_count:
|
||||
self.error = 0
|
||||
self.stale_count = 0
|
||||
extract_urls(content, self.url)
|
||||
|
||||
self.execute = (self.error, self.stale_count, int(time.time()), self.retrievals, self.proxies_added+len(self.proxylist), self.content_type, self.url)
|
||||
self.status = 'ok'
|
||||
self.execute = (self.error, self.stale_count, int(time.time()), self.retrievals, self.proxies_added+len(self.proxylist), self.content_type, self.url)
|
||||
self.status = 'ok'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
config.load()
|
||||
errors = config.validate()
|
||||
if errors:
|
||||
for e in errors:
|
||||
_log(e, 'error')
|
||||
sys.exit(1)
|
||||
fetch.set_config(config)
|
||||
config.load()
|
||||
errors = config.validate()
|
||||
if errors:
|
||||
for e in errors:
|
||||
_log(e, 'error')
|
||||
sys.exit(1)
|
||||
fetch.set_config(config)
|
||||
|
||||
# handle --nobs flag
|
||||
args = config.aparser.parse_args()
|
||||
if args.nobs:
|
||||
set_nobs(True)
|
||||
# handle --nobs flag
|
||||
args = config.aparser.parse_args()
|
||||
if args.nobs:
|
||||
set_nobs(True)
|
||||
|
||||
|
||||
proxydb = mysqlite.mysqlite(config.watchd.database, str)
|
||||
dbs.create_table_if_not_exists(proxydb, 'proxylist')
|
||||
fetch.init_known_proxies(proxydb)
|
||||
proxydb = mysqlite.mysqlite(config.watchd.database, str)
|
||||
dbs.create_table_if_not_exists(proxydb, 'proxylist')
|
||||
fetch.init_known_proxies(proxydb)
|
||||
|
||||
with open('urignore.txt', 'r') as f:
|
||||
urignore = [ i.strip() for i in f.read().split('\n') if i.strip() ]
|
||||
with open('urignore.txt', 'r') as f:
|
||||
urignore = [ i.strip() for i in f.read().split('\n') if i.strip() ]
|
||||
|
||||
urldb = mysqlite.mysqlite(config.ppf.database, str)
|
||||
dbs.create_table_if_not_exists(urldb, 'uris')
|
||||
import_from_file('import.txt', urldb)
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--file":
|
||||
sys.exit(import_proxies_from_file(proxydb, sys.argv[2]))
|
||||
urldb = mysqlite.mysqlite(config.ppf.database, str)
|
||||
dbs.create_table_if_not_exists(urldb, 'uris')
|
||||
import_from_file('import.txt', urldb)
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--file":
|
||||
sys.exit(import_proxies_from_file(proxydb, sys.argv[2]))
|
||||
|
||||
# start proxy watcher
|
||||
if config.watchd.threads > 0:
|
||||
watcherd = proxywatchd.Proxywatchd()
|
||||
watcherd.start()
|
||||
else:
|
||||
watcherd = None
|
||||
# start proxy watcher
|
||||
if config.watchd.threads > 0:
|
||||
watcherd = proxywatchd.Proxywatchd()
|
||||
watcherd.start()
|
||||
else:
|
||||
watcherd = None
|
||||
|
||||
qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM()'
|
||||
threads = []
|
||||
rows = []
|
||||
reqtime = time.time() - 3600
|
||||
statusmsg = time.time()
|
||||
while True:
|
||||
try:
|
||||
time.sleep(random.random()/10)
|
||||
if (time.time() - statusmsg) > 180:
|
||||
_log('running %d thread(s) over %d' % (len(threads), config.ppf.threads), 'ppf')
|
||||
statusmsg = time.time()
|
||||
if not rows:
|
||||
if (time.time() - reqtime) > 3:
|
||||
rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
|
||||
reqtime = time.time()
|
||||
if len(rows) < config.ppf.threads:
|
||||
time.sleep(60)
|
||||
rows = []
|
||||
else:
|
||||
_log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
|
||||
qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM()'
|
||||
threads = []
|
||||
rows = []
|
||||
reqtime = time.time() - 3600
|
||||
statusmsg = time.time()
|
||||
while True:
|
||||
try:
|
||||
time.sleep(random.random()/10)
|
||||
if (time.time() - statusmsg) > 180:
|
||||
_log('running %d thread(s) over %d' % (len(threads), config.ppf.threads), 'ppf')
|
||||
statusmsg = time.time()
|
||||
if not rows:
|
||||
if (time.time() - reqtime) > 3:
|
||||
rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
|
||||
reqtime = time.time()
|
||||
if len(rows) < config.ppf.threads:
|
||||
time.sleep(60)
|
||||
rows = []
|
||||
else:
|
||||
_log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
|
||||
|
||||
_proxylist = [ '%s://%s' % (p[0], p[1]) for p in proxydb.execute('SELECT proto,proxy from proxylist where failed=0').fetchall() ]
|
||||
if not _proxylist: _proxylist = None
|
||||
_proxylist = [ '%s://%s' % (p[0], p[1]) for p in proxydb.execute('SELECT proto,proxy from proxylist where failed=0').fetchall() ]
|
||||
if not _proxylist: _proxylist = None
|
||||
|
||||
for thread in threads:
|
||||
if thread.status == 'ok':
|
||||
url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve()
|
||||
new = [ p for p in proxylist if not fetch.is_known_proxy(p) ]
|
||||
if new:
|
||||
fetch.add_known_proxies(new)
|
||||
execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url)
|
||||
urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute)
|
||||
urldb.commit()
|
||||
if new: dbs.insert_proxies(proxydb, new, url)
|
||||
for thread in threads:
|
||||
if thread.status == 'ok':
|
||||
url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve()
|
||||
new = [ p for p in proxylist if not fetch.is_known_proxy(p) ]
|
||||
if new:
|
||||
fetch.add_known_proxies(new)
|
||||
execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url)
|
||||
urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute)
|
||||
urldb.commit()
|
||||
if new: dbs.insert_proxies(proxydb, new, url)
|
||||
|
||||
threads = [ thread for thread in threads if thread.is_alive() ]
|
||||
if len(threads) < config.ppf.threads and rows:
|
||||
p = random.sample(_proxylist, 5) if _proxylist is not None else None
|
||||
row = random.choice(rows)
|
||||
urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0]))
|
||||
urldb.commit()
|
||||
rows.remove(row)
|
||||
t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5], p)
|
||||
threads.append(t)
|
||||
t.start()
|
||||
threads = [ thread for thread in threads if thread.is_alive() ]
|
||||
if len(threads) < config.ppf.threads and rows:
|
||||
p = random.sample(_proxylist, 5) if _proxylist is not None else None
|
||||
row = random.choice(rows)
|
||||
urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0]))
|
||||
urldb.commit()
|
||||
rows.remove(row)
|
||||
t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5], p)
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if watcherd:
|
||||
watcherd.stop()
|
||||
watcherd.finish()
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
if watcherd:
|
||||
watcherd.stop()
|
||||
watcherd.finish()
|
||||
break
|
||||
|
||||
_log('ppf stopped', 'info')
|
||||
_log('ppf stopped', 'info')
|
||||
|
||||
Reference in New Issue
Block a user