diff --git a/ppf.py b/ppf.py index e33c56d..95c2fd8 100755 --- a/ppf.py +++ b/ppf.py @@ -51,18 +51,22 @@ def fetch_contents(url): return res -def insert_proxies(proxies, uri, sqlite): - time_now = time.time() +_known_proxies = {} +def insert_proxies(proxies, uri, sqlite, timestamp): + global _known_proxies + if len(_known_proxies) == 0: + known = sqlite.execute('SELECT proxy FROM proxylist').fetchall() + for k in known: + _known_proxies[k[0]] = True - query = [ 'proxy=?' for p in proxies ] - known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ] - new = [ (time_now,i,3,0,0,0) for i in proxies if not i in known ] + new = [ (timestamp,i,3,0,0,0) for i in proxies if not i in _known_proxies ] + for i in new: + _known_proxies[i[1]] = True if len(new): sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new) sqlite.commit() _log('+%d item(s) from %s' % (len(new), uri), 'added') - time.sleep(0.1) def proxyfind(sqlite = None): if not sqlite: sqlite = mysqlite.mysqlite(config.database,str) @@ -97,8 +101,12 @@ def proxyfind(sqlite = None): sqlite.commit() def is_reserved_ipv4(ip): - if ( ip.startswith("10.") or ip.startswith("192.168.") or ip.startswith("127.") or ip.startswith("0.")) or \ - (ip.startswith("172.") and (int(ip.split(".")[1]) >= 16 and int(ip.split(".")[1]) <= 31)): return True + octets = ip.split('.') + A = int(octets[0]) + B = int(octets[1]) + if A == 10 or A == 127 or A == 0 or \ + (A == 192 and B == 168) or \ + (A == 172 and B >= 16 and B <= 31): return True return False def proxyleech(sqlite, rows): @@ -107,14 +115,15 @@ def proxyleech(sqlite, rows): except KeyboardInterrupt as e: raise e except: content = '' + matches = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content)) + + uniques_dict = {} + for p in matches: + uniques_dict[p] = True + uniques = [] - for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))): - if p in uniques: continue - try: - if not is_reserved_ipv4(p.split(':')[0]): uniques.append(p) - except KeyboardInterrupt as e: raise e - except: - pass + for p in uniques_dict.keys(): + if not is_reserved_ipv4(p): uniques.append(p) hash = hashlib.md5(''.join(uniques)).hexdigest() @@ -132,12 +141,14 @@ def proxyleech(sqlite, rows): if not row[1] or row[2] > 0: return add = [] + time_now = time.time() for i in uniques: add.append(i) if len(add) > 500: - insert_proxies(add, row[0], sqlite) + insert_proxies(add, row[0], sqlite, time_now) add = [] - if len(add): insert_proxies(add, row[0], sqlite) + if len(add): insert_proxies(add, row[0], sqlite, time_now) + if __name__ == '__main__':