From c8d93751fe7a53d9286fa9ce1b0d0a0c341447c9 Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 02:53:36 +0000 Subject: [PATCH 1/9] rocksock: bump to latest --- rocksock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rocksock.py b/rocksock.py index c7c2ebf..15732bb 100644 --- a/rocksock.py +++ b/rocksock.py @@ -1,4 +1,4 @@ -import socket, ssl, select +import socket, ssl, select, copy # rs_proxyType RS_PT_NONE = 0 @@ -160,7 +160,7 @@ class Rocksock(): if not verifycert: self.sslcontext.verify_mode = ssl.CERT_NONE else: self.sslcontext = None - self.proxychain = proxies if proxies else [] + self.proxychain = copy.copy(proxies) if proxies else [] target = RocksockProxy(host, port, RS_PT_NONE) self.proxychain.append(target) self.sock = None From 497f8b205095a23fbb7203ce9019d6a05dcecb7f Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 02:54:33 +0000 Subject: [PATCH 2/9] watchd: remove non-printable chars from logged response --- proxywatchd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxywatchd.py b/proxywatchd.py index cae1c9d..1566365 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -90,7 +90,8 @@ class WorkerJob(): self.total_duration += int(duration*1000) cstats = "" if match == 'unknown' else ' c: %s;'%match torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor - _log('%s://%s;%s d: %.2f sec(s);%s; srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recv), 'xxxxx') + recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv]) + _log('%s://%s;%s d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recvstats), 'xxxxx') except KeyboardInterrupt as e: raise e except: From ad7c7fce670f24e8a15d1b7186a56c4101638d24 Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 05:37:44 +0000 Subject: [PATCH 3/9] ppf: use timeout and only 1 try for http --- ppf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ppf.py b/ppf.py index 9760f5a..e33c56d 100755 --- a/ppf.py +++ b/ppf.py @@ -42,7 +42,7 @@ def fetch_contents(url): 'Cache-Control: max-age=0', ] proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))] - http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') + http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=15, max_tries=1, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') if not http.connect(): return '' hdr, res = http.get(uri, headers) res = res.encode('utf-8') if isinstance(res, unicode) else res From 898c8f36eecb40fc6889c25e761c93a2d482f42d Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 05:38:35 +0000 Subject: [PATCH 4/9] ppf: fix cpu hogs --- ppf.py | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/ppf.py b/ppf.py index e33c56d..95c2fd8 100755 --- a/ppf.py +++ b/ppf.py @@ -51,18 +51,22 @@ def fetch_contents(url): return res -def insert_proxies(proxies, uri, sqlite): - time_now = time.time() +_known_proxies = {} +def insert_proxies(proxies, uri, sqlite, timestamp): + global _known_proxies + if len(_known_proxies) == 0: + known = sqlite.execute('SELECT proxy FROM proxylist').fetchall() + for k in known: + _known_proxies[k[0]] = True - query = [ 'proxy=?' for p in proxies ] - known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ] - new = [ (time_now,i,3,0,0,0) for i in proxies if not i in known ] + new = [ (timestamp,i,3,0,0,0) for i in proxies if not i in _known_proxies ] + for i in new: + _known_proxies[i[1]] = True if len(new): sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new) sqlite.commit() _log('+%d item(s) from %s' % (len(new), uri), 'added') - time.sleep(0.1) def proxyfind(sqlite = None): if not sqlite: sqlite = mysqlite.mysqlite(config.database,str) @@ -97,8 +101,12 @@ def proxyfind(sqlite = None): sqlite.commit() def is_reserved_ipv4(ip): - if ( ip.startswith("10.") or ip.startswith("192.168.") or ip.startswith("127.") or ip.startswith("0.")) or \ - (ip.startswith("172.") and (int(ip.split(".")[1]) >= 16 and int(ip.split(".")[1]) <= 31)): return True + octets = ip.split('.') + A = int(octets[0]) + B = int(octets[1]) + if A == 10 or A == 127 or A == 0 or \ + (A == 192 and B == 168) or \ + (A == 172 and B >= 16 and B <= 31): return True return False def proxyleech(sqlite, rows): @@ -107,14 +115,15 @@ def proxyleech(sqlite, rows): except KeyboardInterrupt as e: raise e except: content = '' + matches = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content)) + + uniques_dict = {} + for p in matches: + uniques_dict[p] = True + uniques = [] - for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))): - if p in uniques: continue - try: - if not is_reserved_ipv4(p.split(':')[0]): uniques.append(p) - except KeyboardInterrupt as e: raise e - except: - pass + for p in uniques_dict.keys(): + if not is_reserved_ipv4(p): uniques.append(p) hash = hashlib.md5(''.join(uniques)).hexdigest() @@ -132,12 +141,14 @@ def proxyleech(sqlite, rows): if not row[1] or row[2] > 0: return add = [] + time_now = time.time() for i in uniques: add.append(i) if len(add) > 500: - insert_proxies(add, row[0], sqlite) + insert_proxies(add, row[0], sqlite, time_now) add = [] - if len(add): insert_proxies(add, row[0], sqlite) + if len(add): insert_proxies(add, row[0], sqlite, time_now) + if __name__ == '__main__': From d93f14ad67912297009f0367ed5b58edeb14bf06 Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 05:39:43 +0000 Subject: [PATCH 5/9] watchd: distribute remaining jobs among threads if one is idle --- proxywatchd.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/proxywatchd.py b/proxywatchd.py index 1566365..3faf1b8 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -113,6 +113,10 @@ class WorkerThread(): if self.thread: self.thread.join() def add_jobs(self, jobs): self.workqueue.extend(jobs) + def return_jobs(self): + jobs = self.workqueue + self.workqueue = [] + return jobs def jobcount(self): return len(self.workqueue) def collect(self): @@ -210,7 +214,15 @@ class Proxywatchd(): for wt in self.threads: self.collected.extend(wt.collect()) + def collect_unfinished(self): + for wt in self.threads: + jobs = wt.return_jobs() + self.jobs.extend(jobs) + if len(self.jobs): + _log("collected %d unfinished jobs"%len(self.jobs), "watchd") + def submit_collected(self): + if len(self.collected) == 0: return _log("watchd main thread: updating %d entries"%len(self.collected)) self._prep_db() query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=? WHERE proxy=?' @@ -252,8 +264,18 @@ class Proxywatchd(): if self.in_background: self._cleanup() break + sleeptime = 1 + if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0: - self.prepare_jobs() + self.collect_unfinished() + if not len(self.jobs): + self.collect_work() + self.submit_collected() + self.prepare_jobs() + else: + if len(self.jobs) < len(self.threads): + # allow threads enough time to consume the jobs + sleeptime = 10 if len(self.jobs): _log("watchd main: handing out %d jobs and %d thread(s)"% (len(self.jobs), config.watchd_threads)) jpt = len(self.jobs)/config.watchd_threads @@ -270,7 +292,7 @@ class Proxywatchd(): if len(self.collected) > self.submit_after: self.submit_collected() - time.sleep(1) + time.sleep(sleeptime) if __name__ == '__main__': From be99b8517b766210fa4e425bb0a6d7a520cec7e0 Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 05:45:09 +0000 Subject: [PATCH 6/9] watchd: beautify logging --- proxywatchd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proxywatchd.py b/proxywatchd.py index 3faf1b8..5780f8d 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -162,7 +162,7 @@ class WorkerThread(): class Proxywatchd(): def stop(self): - _log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if True])) + _log('halting... (%d thread(s))' % len([item for item in self.threads if True]), 'watchd') self.stopping.set() def _cleanup(self): @@ -223,7 +223,7 @@ class Proxywatchd(): def submit_collected(self): if len(self.collected) == 0: return - _log("watchd main thread: updating %d entries"%len(self.collected)) + _log("updating %d DB entries"%len(self.collected), 'watchd') self._prep_db() query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=? WHERE proxy=?' args = [ (job.failcount, job.nextcheck, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.proxy) for job in self.collected ] @@ -248,7 +248,7 @@ class Proxywatchd(): t.start() def _run(self): - _log('Starting proxywatchd..', 'notice') + _log('starting...', 'watchd') for i in range(config.watchd_threads): threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] ) @@ -277,7 +277,7 @@ class Proxywatchd(): # allow threads enough time to consume the jobs sleeptime = 10 if len(self.jobs): - _log("watchd main: handing out %d jobs and %d thread(s)"% (len(self.jobs), config.watchd_threads)) + _log("handing out %d jobs to %d thread(s)"% (len(self.jobs), config.watchd_threads), 'watchd') jpt = len(self.jobs)/config.watchd_threads if len(self.jobs)/float(config.watchd_threads) - jpt > 0.0: jpt += 1 for tid in range(config.watchd_threads): From 9e3c08fcf4a20edc28bcf387fe05bfca4731f879 Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 15:57:15 +0000 Subject: [PATCH 7/9] watchd: do not count name resolution errors as proxy fail --- proxywatchd.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/proxywatchd.py b/proxywatchd.py index 5780f8d..615d614 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -28,6 +28,8 @@ class WorkerJob(): protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto] server_port = 6697 if config.use_ssl else 6667 + fail_inc = 1 + for proto in protos: torhost = random.choice(config.torhosts) duration = time.time() @@ -57,12 +59,16 @@ class WorkerJob(): err == rocksock.RS_E_HIT_TIMEOUT: # proxy is not online, so don't waste time trying all possible protocols break + elif et == rocksock.RS_ET_GAI: + fail_inc = 0 + _log("could not resolve connection target %s"%srv, "ERROR") + break except KeyboardInterrupt as e: raise(e) except: sock.disconnect() - return None, None, None, None, None, 1 + return None, None, None, None, None, fail_inc def run(self): self.nextcheck = (time.time() + 1800 + ((1+int(self.failcount)) * 3600)) From 9e8beca4e1e4b2496906309abb63139496a87e5c Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 16:22:06 +0000 Subject: [PATCH 8/9] do not treat empty lines as servers --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 3aa867f..8c7c510 100644 --- a/config.py +++ b/config.py @@ -33,4 +33,4 @@ def load(): global servers with open('servers.txt', 'r') as handle: - servers = handle.read().split('\n') + servers = [x.strip() for x in handle.readlines() if len(x.strip()) > 0] From 9d41e2125805868358cccd99a996979c03d0ea42 Mon Sep 17 00:00:00 2001 From: rofl0r Date: Mon, 7 Jan 2019 16:22:26 +0000 Subject: [PATCH 9/9] remove dead irc servers (DNS NXDOMAIN) --- servers.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/servers.txt b/servers.txt index 64ac83b..b8c123c 100644 --- a/servers.txt +++ b/servers.txt @@ -4,8 +4,6 @@ irc.2600.net irc.Undernet.Org irc.accessirc.net irc.afterx.net -irc.asylo.com -irc.atheme.org irc.atrum.org irc.atw-inter.net irc.au.dal.net @@ -14,10 +12,7 @@ irc.azzurra.org irc.blitzed.org irc.bongster.org irc.chat4all.org -irc.chatcafe.net -irc.chatnet.org irc.chatspike.net -irc.chattingaway.com irc.choopa.net irc.coldfront.net irc.cyanide-x.net @@ -67,15 +62,12 @@ irc.ircube.org irc.ircworld.nl irc.irdsi.net irc.kampungchat.org -irc.kickchat.com irc.knightirc.net irc.krono.net irc.landoleet.org irc.langochat.net irc.lichtsnel.nl irc.maddshark.net -irc.matrix-network.org -irc.mmoirc.com irc.mozilla.org irc.mugglenet.com irc.netgamers.org @@ -98,8 +90,6 @@ irc.serenity-irc.net irc.servercentral.net irc.shadowfire.org irc.shadowworld.net -irc.simosnap.net -irc.slacked.org irc.slashnet.org irc.snt.utwente.nl irc.solidirc.com @@ -110,7 +100,6 @@ irc.starchat.net irc.starlink-irc.org irc.starlink.org irc.staynet.org -irc.stratics.com irc.swiftirc.net irc.teksavvy.ca irc.teranova.net @@ -120,9 +109,7 @@ irc.us.gamesurge.net irc.utonet.org irc.webchat.org irc.whatnet.org -irc.wikkedwire.com irc.wondernet.nu -irc.worldnet.net irc.xevion.net irc.xs4all.nl irc.zerofuzion.net