ppf: fix cpu hogs

ppf.py

@@ -51,18 +51,22 @@ def fetch_contents(url):
 
 	return res
 
-def insert_proxies(proxies, uri, sqlite):
+_known_proxies = {}
-	time_now = time.time()
+def insert_proxies(proxies, uri, sqlite, timestamp):
+	global _known_proxies
+	if len(_known_proxies) == 0:
+		known = sqlite.execute('SELECT proxy FROM proxylist').fetchall()
+		for k in known:
+			_known_proxies[k[0]] = True
 
-	query = [ 'proxy=?' for p in proxies ]
+	new = [ (timestamp,i,3,0,0,0) for i in proxies if not i in _known_proxies ]
-	known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ]
+	for i in new:
-	new = [ (time_now,i,3,0,0,0) for i in proxies if not i in known ]
+		_known_proxies[i[1]] = True
 
 	if len(new):
 		sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new)
 		sqlite.commit()
 		_log('+%d item(s) from %s' % (len(new), uri), 'added')
-	time.sleep(0.1)
 
 def proxyfind(sqlite = None):
 	if not sqlite: sqlite = mysqlite.mysqlite(config.database,str)
@@ -97,8 +101,12 @@ def proxyfind(sqlite = None):
 	sqlite.commit()
 
 def is_reserved_ipv4(ip):
-	if ( ip.startswith("10.") or ip.startswith("192.168.") or ip.startswith("127.") or ip.startswith("0.")) or \
+	octets = ip.split('.')
-	(ip.startswith("172.") and (int(ip.split(".")[1]) >= 16 and int(ip.split(".")[1]) <= 31)): return True
+	A = int(octets[0])
+	B = int(octets[1])
+	if A == 10 or A == 127 or A == 0 or \
+	(A == 192 and B == 168) or \
+	(A == 172 and B >= 16 and B <= 31): return True
 	return False
 
 def proxyleech(sqlite, rows):
@@ -107,14 +115,15 @@ def proxyleech(sqlite, rows):
 		except KeyboardInterrupt as e: raise e
 		except: content = ''
 
+		matches = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))
+
+		uniques_dict = {}
+		for p in matches:
+			uniques_dict[p] = True
+
 		uniques = []
-		for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))):
+		for p in uniques_dict.keys():
-			if p in uniques: continue
+			if not is_reserved_ipv4(p): uniques.append(p)
-			try:
-				if not is_reserved_ipv4(p.split(':')[0]): uniques.append(p)
-			except KeyboardInterrupt as e: raise e
-			except:
-				pass
 
 		hash = hashlib.md5(''.join(uniques)).hexdigest()
 
@@ -132,12 +141,14 @@ def proxyleech(sqlite, rows):
 		if not row[1] or row[2] > 0: return
 
 		add = []
+		time_now = time.time()
 		for i in uniques:
 			add.append(i)
 			if len(add) > 500:
-				insert_proxies(add, row[0], sqlite)
+				insert_proxies(add, row[0], sqlite, time_now)
 				add = []
-		if len(add): insert_proxies(add, row[0], sqlite)
+		if len(add): insert_proxies(add, row[0], sqlite, time_now)
+
 
 
 if __name__ == '__main__':