ppf: use shared proxy cache from fetch module

ppf.py

@@ -14,15 +14,14 @@ import threading
 import random
 
 config = Config()
-_known_proxies = {}
 
 def import_from_file(fn, sqlite):
 	with open(fn, 'r') as f:
-		urls = [ url for url in f.read().split('\n') if url != '' ]
+		urls = [ url for url in f.read().split('\n') if url ]
 		cinc = 0
 		while True:
 			chunk = urls[cinc:cinc+200]
-			if len(chunk): dbs.insert_urls(chunk, 'import.txt', urldb)
+			if chunk: dbs.insert_urls(chunk, 'import.txt', urldb)
 			else: break
 			cinc = cinc + 200
 
@@ -71,12 +70,12 @@ def extract_urls(html, url):
 			continue
 		if not item in urls: urls.append(item)
 
-	if len(urls): dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls)
+	if urls: dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls)
 
 def import_proxies_from_file(proxydb, fn):
 	content = open(fn, 'r').read()
 	unique_count, new = fetch.extract_proxies(content, proxydb)
-	if len(new):
+	if new:
 		dbs.insert_proxies(proxydb, new, fn)
 		return 0
 	return 1
@@ -126,21 +125,25 @@ class Leechered(threading.Thread):
 		if not self.content_type: self.content_type = get_content_type(self.url, self.proxy)
 
 		if is_good_content_type(self.content_type):
-			try: content = fetch.fetch_contents(self.url, proxy=self.proxy)
+			try:
-			except KeyboardInterrupt as e: raise e
+				content = fetch.fetch_contents(self.url, proxy=self.proxy)
-			except: content = ''
+			except KeyboardInterrupt as e:
+				raise e
+			except Exception as e:
+				_log('%s: fetch error: %s' % (self.url.split('/')[2], str(e)), 'error')
+				content = ''
 		else:
 			content = ''
 
 		unique = extract_proxies(content)
-		self.proxylist = [ proxy for proxy in unique if not proxy in _known_proxies ]
+		self.proxylist = [ proxy for proxy in unique if not fetch.is_known_proxy(proxy) ]
 		proxy_count = len(self.proxylist)
 
 		if self.retrievals == 0:	# new site
-			if content != '' and len(self.proxylist) == 0: # site works but has zero proxy addresses
+			if content and not self.proxylist: # site works but has zero proxy addresses
 				self.error += 1
 				self.stale_count += 1
-			elif proxy_count > 0:
+			elif proxy_count:
 				self.error = 0
 				self.stale_count = 0
 			else:
@@ -148,14 +151,14 @@ class Leechered(threading.Thread):
 				self.stale_count += 2
 		else:										 # not a new site
 			# proxylist is empty
-			if proxy_count == 0:
+			if not proxy_count:
 				self.stale_count += 1
 			# proxylist is not empty: site is working
 			else:
 				self.stale_count = 0
 				self.error = 0
 			# site has no content
-			if content == '':
+			if not content:
 				self.error += 1
 				self.stale_count += 1
 			#else:
@@ -174,6 +177,11 @@ class Leechered(threading.Thread):
 
 if __name__ == '__main__':
 	config.load()
+	errors = config.validate()
+	if errors:
+		for e in errors:
+			_log(e, 'error')
+		sys.exit(1)
 	fetch.set_config(config)
 
 	# handle --nobs flag
@@ -184,12 +192,10 @@ if __name__ == '__main__':
 
 	proxydb = mysqlite.mysqlite(config.watchd.database, str)
 	dbs.create_table_if_not_exists(proxydb, 'proxylist')
-	known = proxydb.execute('SELECT proxy FROM proxylist').fetchall()
+	fetch.init_known_proxies(proxydb)
-	for k in known:
-		_known_proxies[k[0]] = True
 
 	with open('urignore.txt', 'r') as f:
-		urignore = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]
+		urignore = [ i.strip() for i in f.read().split('\n') if i.strip() ]
 
 	urldb = mysqlite.mysqlite(config.ppf.database, str)
 	dbs.create_table_if_not_exists(urldb, 'uris')
@@ -215,7 +221,7 @@ if __name__ == '__main__':
 			if (time.time() - statusmsg) > 180:
 				_log('running %d thread(s) over %d' % (len(threads), config.ppf.threads), 'ppf')
 				statusmsg = time.time()
-			if not len(rows):
+			if not rows:
 				if (time.time() - reqtime) > 3:
 					rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
 					reqtime = time.time()
@@ -226,23 +232,21 @@ if __name__ == '__main__':
 					_log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
 
 			_proxylist = [ '%s://%s' % (p[0], p[1]) for p in proxydb.execute('SELECT proto,proxy from proxylist where failed=0').fetchall() ]
-			if len(_proxylist) == 0: _proxylist = None
+			if not _proxylist: _proxylist = None
 
 			for thread in threads:
 				if thread.status == 'ok':
 					url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve()
-					new = []
+					new = [ p for p in proxylist if not fetch.is_known_proxy(p) ]
-					for p in proxylist:
+					if new:
-						if not p in _known_proxies:
+						fetch.add_known_proxies(new)
-							new.append(p)
-							_known_proxies[p]=1
 					execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url)
 					urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute)
 					urldb.commit()
-					if len(new): dbs.insert_proxies(proxydb, new, url)
+					if new: dbs.insert_proxies(proxydb, new, url)
 
 			threads = [ thread for thread in threads if thread.is_alive() ]
-			if len(threads) < config.ppf.threads and len(rows):
+			if len(threads) < config.ppf.threads and rows:
 				p = random.sample(_proxylist, 5) if _proxylist is not None else None
 				row = random.choice(rows)
 				urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0]))
@@ -258,4 +262,4 @@ if __name__ == '__main__':
 				watcherd.finish()
 			break
 
-	print '\r',
+	_log('ppf stopped', 'info')