Merge branch 'changes7' into 'master'

Changes7 See merge request mserneels/ppf!8
2019-01-07 18:56:23 +00:00
parent 8b15faf84d 9d41e21258
commit f47745c8d6
5 changed files with 69 additions and 42 deletions
--- a/config.py
+++ b/config.py
@@ -33,4 +33,4 @@ def load():

 	global servers
 	with open('servers.txt', 'r') as handle:
-		servers = handle.read().split('\n')
+		servers = [x.strip() for x in handle.readlines() if len(x.strip()) > 0]
--- a/ppf.py
+++ b/ppf.py
@@ -42,7 +42,7 @@ def fetch_contents(url):
 		'Cache-Control: max-age=0',
 	]
 	proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))]
-	http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0')
+	http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=15, max_tries=1, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0')
 	if not http.connect(): return ''
 	hdr, res = http.get(uri, headers)
 	res = res.encode('utf-8') if isinstance(res, unicode) else res
@@ -51,18 +51,22 @@ def fetch_contents(url):

 	return res

-def insert_proxies(proxies, uri, sqlite):
-	time_now = time.time()
+_known_proxies = {}
+def insert_proxies(proxies, uri, sqlite, timestamp):
+	global _known_proxies
+	if len(_known_proxies) == 0:
+		known = sqlite.execute('SELECT proxy FROM proxylist').fetchall()
+		for k in known:
+			_known_proxies[k[0]] = True

-	query = [ 'proxy=?' for p in proxies ]
-	known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ]
-	new = [ (time_now,i,3,0,0,0) for i in proxies if not i in known ]
+	new = [ (timestamp,i,3,0,0,0) for i in proxies if not i in _known_proxies ]
+	for i in new:
+		_known_proxies[i[1]] = True

 	if len(new):
 		sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new)
 		sqlite.commit()
 		_log('+%d item(s) from %s' % (len(new), uri), 'added')
-	time.sleep(0.1)

 def proxyfind(sqlite = None):
 	if not sqlite: sqlite = mysqlite.mysqlite(config.database,str)
@@ -97,8 +101,12 @@ def proxyfind(sqlite = None):
 	sqlite.commit()

 def is_reserved_ipv4(ip):
-	if ( ip.startswith("10.") or ip.startswith("192.168.") or ip.startswith("127.") or ip.startswith("0.")) or \
-	(ip.startswith("172.") and (int(ip.split(".")[1]) >= 16 and int(ip.split(".")[1]) <= 31)): return True
+	octets = ip.split('.')
+	A = int(octets[0])
+	B = int(octets[1])
+	if A == 10 or A == 127 or A == 0 or \
+	(A == 192 and B == 168) or \
+	(A == 172 and B >= 16 and B <= 31): return True
 	return False

 def proxyleech(sqlite, rows):
@@ -107,14 +115,15 @@ def proxyleech(sqlite, rows):
 		except KeyboardInterrupt as e: raise e
 		except: content = ''

+		matches = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))
+
+		uniques_dict = {}
+		for p in matches:
+			uniques_dict[p] = True
+
 		uniques = []
-		for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))):
-			if p in uniques: continue
-			try:
-				if not is_reserved_ipv4(p.split(':')[0]): uniques.append(p)
-			except KeyboardInterrupt as e: raise e
-			except:
-				pass
+		for p in uniques_dict.keys():
+			if not is_reserved_ipv4(p): uniques.append(p)

 		hash = hashlib.md5(''.join(uniques)).hexdigest()

@@ -132,12 +141,14 @@ def proxyleech(sqlite, rows):
 		if not row[1] or row[2] > 0: return

 		add = []
+		time_now = time.time()
 		for i in uniques:
 			add.append(i)
 			if len(add) > 500:
-				insert_proxies(add, row[0], sqlite)
+				insert_proxies(add, row[0], sqlite, time_now)
 				add = []
-		if len(add): insert_proxies(add, row[0], sqlite)
+		if len(add): insert_proxies(add, row[0], sqlite, time_now)
+


 if __name__ == '__main__':
--- a/proxywatchd.py
+++ b/proxywatchd.py
@@ -28,6 +28,8 @@ class WorkerJob():
 		protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto]
 		server_port = 6697 if config.use_ssl else 6667

+		fail_inc = 1
+
 		for proto in protos:
 			torhost = random.choice(config.torhosts)
 			duration = time.time()
@@ -57,12 +59,16 @@ class WorkerJob():
 					err == rocksock.RS_E_HIT_TIMEOUT:
 						# proxy is not online, so don't waste time trying all possible protocols
 						break
+				elif et == rocksock.RS_ET_GAI:
+					fail_inc = 0
+					_log("could not resolve connection target %s"%srv, "ERROR")
+					break

 			except KeyboardInterrupt as e:
 				raise(e)
 			except: sock.disconnect()

-		return None, None, None, None, None, 1
+		return None, None, None, None, None, fail_inc

 	def run(self):
 		self.nextcheck = (time.time() + 1800 + ((1+int(self.failcount)) * 3600))
@@ -90,7 +96,8 @@ class WorkerJob():
 				self.total_duration += int(duration*1000)
 				cstats = "" if match == 'unknown' else ' c: %s;'%match
 				torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor
-				_log('%s://%s;%s d: %.2f sec(s);%s; srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recv), 'xxxxx')
+				recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv])
+				_log('%s://%s;%s d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recvstats), 'xxxxx')
 		except KeyboardInterrupt as e:
 			raise e
 		except:
@@ -112,6 +119,10 @@ class WorkerThread():
 		if self.thread: self.thread.join()
 	def add_jobs(self, jobs):
 		self.workqueue.extend(jobs)
+	def return_jobs(self):
+		jobs = self.workqueue
+		self.workqueue = []
+		return jobs
 	def jobcount(self):
 		return len(self.workqueue)
 	def collect(self):
@@ -157,7 +168,7 @@ class WorkerThread():
 class Proxywatchd():

 	def stop(self):
-		_log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if True]))
+		_log('halting... (%d thread(s))' % len([item for item in self.threads if True]), 'watchd')
 		self.stopping.set()

 	def _cleanup(self):
@@ -209,8 +220,16 @@ class Proxywatchd():
 		for wt in self.threads:
 			self.collected.extend(wt.collect())

+	def collect_unfinished(self):
+		for wt in self.threads:
+			jobs = wt.return_jobs()
+			self.jobs.extend(jobs)
+		if len(self.jobs):
+			_log("collected %d unfinished jobs"%len(self.jobs), "watchd")
+
 	def submit_collected(self):
-		_log("watchd main thread: updating %d entries"%len(self.collected))
+		if len(self.collected) == 0: return
+		_log("updating %d DB entries"%len(self.collected), 'watchd')
 		self._prep_db()
 		query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=? WHERE proxy=?'
 		args = [ (job.failcount, job.nextcheck, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.proxy) for job in self.collected ]
@@ -235,7 +254,7 @@ class Proxywatchd():
 		t.start()

 	def _run(self):
-		_log('Starting proxywatchd..', 'notice')
+		_log('starting...', 'watchd')

 		for i in range(config.watchd_threads):
 			threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
@@ -251,10 +270,20 @@ class Proxywatchd():
 				if self.in_background: self._cleanup()
 				break

+			sleeptime = 1
+
 			if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0:
-				self.prepare_jobs()
+				self.collect_unfinished()
+				if not len(self.jobs):
+					self.collect_work()
+					self.submit_collected()
+					self.prepare_jobs()
+				else:
+					if len(self.jobs) < len(self.threads):
+						# allow threads enough time to consume the jobs
+						sleeptime = 10
 				if len(self.jobs):
-					_log("watchd main: handing out %d jobs and %d thread(s)"% (len(self.jobs), config.watchd_threads))
+					_log("handing out %d jobs to %d thread(s)"% (len(self.jobs), config.watchd_threads), 'watchd')
 					jpt = len(self.jobs)/config.watchd_threads
 					if len(self.jobs)/float(config.watchd_threads) - jpt > 0.0: jpt += 1
 					for tid in range(config.watchd_threads):
@@ -269,7 +298,7 @@ class Proxywatchd():
 			if len(self.collected) > self.submit_after:
 				self.submit_collected()

-			time.sleep(1)
+			time.sleep(sleeptime)


 if __name__ == '__main__':
--- a/rocksock.py
+++ b/rocksock.py
@@ -1,4 +1,4 @@
-import socket, ssl, select
+import socket, ssl, select, copy

 # rs_proxyType
 RS_PT_NONE = 0
@@ -160,7 +160,7 @@ class Rocksock():
 			if not verifycert: self.sslcontext.verify_mode = ssl.CERT_NONE
 		else:
 			self.sslcontext = None
-		self.proxychain = proxies if proxies else []
+		self.proxychain = copy.copy(proxies) if proxies else []
 		target = RocksockProxy(host, port, RS_PT_NONE)
 		self.proxychain.append(target)
 		self.sock = None
--- a/servers.txt
+++ b/servers.txt
@@ -4,8 +4,6 @@ irc.2600.net
 irc.Undernet.Org
 irc.accessirc.net
 irc.afterx.net
-irc.asylo.com
-irc.atheme.org
 irc.atrum.org
 irc.atw-inter.net
 irc.au.dal.net
@@ -14,10 +12,7 @@ irc.azzurra.org
 irc.blitzed.org
 irc.bongster.org
 irc.chat4all.org
-irc.chatcafe.net
-irc.chatnet.org
 irc.chatspike.net
-irc.chattingaway.com
 irc.choopa.net
 irc.coldfront.net
 irc.cyanide-x.net
@@ -67,15 +62,12 @@ irc.ircube.org
 irc.ircworld.nl
 irc.irdsi.net
 irc.kampungchat.org
-irc.kickchat.com
 irc.knightirc.net
 irc.krono.net
 irc.landoleet.org
 irc.langochat.net
 irc.lichtsnel.nl
 irc.maddshark.net
-irc.matrix-network.org
-irc.mmoirc.com
 irc.mozilla.org
 irc.mugglenet.com
 irc.netgamers.org
@@ -98,8 +90,6 @@ irc.serenity-irc.net
 irc.servercentral.net
 irc.shadowfire.org
 irc.shadowworld.net
-irc.simosnap.net
-irc.slacked.org
 irc.slashnet.org
 irc.snt.utwente.nl
 irc.solidirc.com
@@ -110,7 +100,6 @@ irc.starchat.net
 irc.starlink-irc.org
 irc.starlink.org
 irc.staynet.org
-irc.stratics.com
 irc.swiftirc.net
 irc.teksavvy.ca
 irc.teranova.net
@@ -120,9 +109,7 @@ irc.us.gamesurge.net
 irc.utonet.org
 irc.webchat.org
 irc.whatnet.org
-irc.wikkedwire.com
 irc.wondernet.nu
-irc.worldnet.net
 irc.xevion.net
 irc.xs4all.nl
 irc.zerofuzion.net