diff --git a/config.py b/config.py index 3aa867f..8c7c510 100644 --- a/config.py +++ b/config.py @@ -33,4 +33,4 @@ def load(): global servers with open('servers.txt', 'r') as handle: - servers = handle.read().split('\n') + servers = [x.strip() for x in handle.readlines() if len(x.strip()) > 0] diff --git a/ppf.py b/ppf.py index 9760f5a..95c2fd8 100755 --- a/ppf.py +++ b/ppf.py @@ -42,7 +42,7 @@ def fetch_contents(url): 'Cache-Control: max-age=0', ] proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))] - http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') + http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=15, max_tries=1, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') if not http.connect(): return '' hdr, res = http.get(uri, headers) res = res.encode('utf-8') if isinstance(res, unicode) else res @@ -51,18 +51,22 @@ def fetch_contents(url): return res -def insert_proxies(proxies, uri, sqlite): - time_now = time.time() +_known_proxies = {} +def insert_proxies(proxies, uri, sqlite, timestamp): + global _known_proxies + if len(_known_proxies) == 0: + known = sqlite.execute('SELECT proxy FROM proxylist').fetchall() + for k in known: + _known_proxies[k[0]] = True - query = [ 'proxy=?' for p in proxies ] - known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ] - new = [ (time_now,i,3,0,0,0) for i in proxies if not i in known ] + new = [ (timestamp,i,3,0,0,0) for i in proxies if not i in _known_proxies ] + for i in new: + _known_proxies[i[1]] = True if len(new): sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new) sqlite.commit() _log('+%d item(s) from %s' % (len(new), uri), 'added') - time.sleep(0.1) def proxyfind(sqlite = None): if not sqlite: sqlite = mysqlite.mysqlite(config.database,str) @@ -97,8 +101,12 @@ def proxyfind(sqlite = None): sqlite.commit() def is_reserved_ipv4(ip): - if ( ip.startswith("10.") or ip.startswith("192.168.") or ip.startswith("127.") or ip.startswith("0.")) or \ - (ip.startswith("172.") and (int(ip.split(".")[1]) >= 16 and int(ip.split(".")[1]) <= 31)): return True + octets = ip.split('.') + A = int(octets[0]) + B = int(octets[1]) + if A == 10 or A == 127 or A == 0 or \ + (A == 192 and B == 168) or \ + (A == 172 and B >= 16 and B <= 31): return True return False def proxyleech(sqlite, rows): @@ -107,14 +115,15 @@ def proxyleech(sqlite, rows): except KeyboardInterrupt as e: raise e except: content = '' + matches = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content)) + + uniques_dict = {} + for p in matches: + uniques_dict[p] = True + uniques = [] - for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))): - if p in uniques: continue - try: - if not is_reserved_ipv4(p.split(':')[0]): uniques.append(p) - except KeyboardInterrupt as e: raise e - except: - pass + for p in uniques_dict.keys(): + if not is_reserved_ipv4(p): uniques.append(p) hash = hashlib.md5(''.join(uniques)).hexdigest() @@ -132,12 +141,14 @@ def proxyleech(sqlite, rows): if not row[1] or row[2] > 0: return add = [] + time_now = time.time() for i in uniques: add.append(i) if len(add) > 500: - insert_proxies(add, row[0], sqlite) + insert_proxies(add, row[0], sqlite, time_now) add = [] - if len(add): insert_proxies(add, row[0], sqlite) + if len(add): insert_proxies(add, row[0], sqlite, time_now) + if __name__ == '__main__': diff --git a/proxywatchd.py b/proxywatchd.py index cae1c9d..615d614 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -28,6 +28,8 @@ class WorkerJob(): protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto] server_port = 6697 if config.use_ssl else 6667 + fail_inc = 1 + for proto in protos: torhost = random.choice(config.torhosts) duration = time.time() @@ -57,12 +59,16 @@ class WorkerJob(): err == rocksock.RS_E_HIT_TIMEOUT: # proxy is not online, so don't waste time trying all possible protocols break + elif et == rocksock.RS_ET_GAI: + fail_inc = 0 + _log("could not resolve connection target %s"%srv, "ERROR") + break except KeyboardInterrupt as e: raise(e) except: sock.disconnect() - return None, None, None, None, None, 1 + return None, None, None, None, None, fail_inc def run(self): self.nextcheck = (time.time() + 1800 + ((1+int(self.failcount)) * 3600)) @@ -90,7 +96,8 @@ class WorkerJob(): self.total_duration += int(duration*1000) cstats = "" if match == 'unknown' else ' c: %s;'%match torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor - _log('%s://%s;%s d: %.2f sec(s);%s; srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recv), 'xxxxx') + recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv]) + _log('%s://%s;%s d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, cstats, duration, torstats, srv, recvstats), 'xxxxx') except KeyboardInterrupt as e: raise e except: @@ -112,6 +119,10 @@ class WorkerThread(): if self.thread: self.thread.join() def add_jobs(self, jobs): self.workqueue.extend(jobs) + def return_jobs(self): + jobs = self.workqueue + self.workqueue = [] + return jobs def jobcount(self): return len(self.workqueue) def collect(self): @@ -157,7 +168,7 @@ class WorkerThread(): class Proxywatchd(): def stop(self): - _log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if True])) + _log('halting... (%d thread(s))' % len([item for item in self.threads if True]), 'watchd') self.stopping.set() def _cleanup(self): @@ -209,8 +220,16 @@ class Proxywatchd(): for wt in self.threads: self.collected.extend(wt.collect()) + def collect_unfinished(self): + for wt in self.threads: + jobs = wt.return_jobs() + self.jobs.extend(jobs) + if len(self.jobs): + _log("collected %d unfinished jobs"%len(self.jobs), "watchd") + def submit_collected(self): - _log("watchd main thread: updating %d entries"%len(self.collected)) + if len(self.collected) == 0: return + _log("updating %d DB entries"%len(self.collected), 'watchd') self._prep_db() query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=? WHERE proxy=?' args = [ (job.failcount, job.nextcheck, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.proxy) for job in self.collected ] @@ -235,7 +254,7 @@ class Proxywatchd(): t.start() def _run(self): - _log('Starting proxywatchd..', 'notice') + _log('starting...', 'watchd') for i in range(config.watchd_threads): threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] ) @@ -251,10 +270,20 @@ class Proxywatchd(): if self.in_background: self._cleanup() break + sleeptime = 1 + if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0: - self.prepare_jobs() + self.collect_unfinished() + if not len(self.jobs): + self.collect_work() + self.submit_collected() + self.prepare_jobs() + else: + if len(self.jobs) < len(self.threads): + # allow threads enough time to consume the jobs + sleeptime = 10 if len(self.jobs): - _log("watchd main: handing out %d jobs and %d thread(s)"% (len(self.jobs), config.watchd_threads)) + _log("handing out %d jobs to %d thread(s)"% (len(self.jobs), config.watchd_threads), 'watchd') jpt = len(self.jobs)/config.watchd_threads if len(self.jobs)/float(config.watchd_threads) - jpt > 0.0: jpt += 1 for tid in range(config.watchd_threads): @@ -269,7 +298,7 @@ class Proxywatchd(): if len(self.collected) > self.submit_after: self.submit_collected() - time.sleep(1) + time.sleep(sleeptime) if __name__ == '__main__': diff --git a/rocksock.py b/rocksock.py index c7c2ebf..15732bb 100644 --- a/rocksock.py +++ b/rocksock.py @@ -1,4 +1,4 @@ -import socket, ssl, select +import socket, ssl, select, copy # rs_proxyType RS_PT_NONE = 0 @@ -160,7 +160,7 @@ class Rocksock(): if not verifycert: self.sslcontext.verify_mode = ssl.CERT_NONE else: self.sslcontext = None - self.proxychain = proxies if proxies else [] + self.proxychain = copy.copy(proxies) if proxies else [] target = RocksockProxy(host, port, RS_PT_NONE) self.proxychain.append(target) self.sock = None diff --git a/servers.txt b/servers.txt index 64ac83b..b8c123c 100644 --- a/servers.txt +++ b/servers.txt @@ -4,8 +4,6 @@ irc.2600.net irc.Undernet.Org irc.accessirc.net irc.afterx.net -irc.asylo.com -irc.atheme.org irc.atrum.org irc.atw-inter.net irc.au.dal.net @@ -14,10 +12,7 @@ irc.azzurra.org irc.blitzed.org irc.bongster.org irc.chat4all.org -irc.chatcafe.net -irc.chatnet.org irc.chatspike.net -irc.chattingaway.com irc.choopa.net irc.coldfront.net irc.cyanide-x.net @@ -67,15 +62,12 @@ irc.ircube.org irc.ircworld.nl irc.irdsi.net irc.kampungchat.org -irc.kickchat.com irc.knightirc.net irc.krono.net irc.landoleet.org irc.langochat.net irc.lichtsnel.nl irc.maddshark.net -irc.matrix-network.org -irc.mmoirc.com irc.mozilla.org irc.mugglenet.com irc.netgamers.org @@ -98,8 +90,6 @@ irc.serenity-irc.net irc.servercentral.net irc.shadowfire.org irc.shadowworld.net -irc.simosnap.net -irc.slacked.org irc.slashnet.org irc.snt.utwente.nl irc.solidirc.com @@ -110,7 +100,6 @@ irc.starchat.net irc.starlink-irc.org irc.starlink.org irc.staynet.org -irc.stratics.com irc.swiftirc.net irc.teksavvy.ca irc.teranova.net @@ -120,9 +109,7 @@ irc.us.gamesurge.net irc.utonet.org irc.webchat.org irc.whatnet.org -irc.wikkedwire.com irc.wondernet.nu -irc.worldnet.net irc.xevion.net irc.xs4all.nl irc.zerofuzion.net