username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ppf: fix cpu hogs

Browse Source
This commit is contained in:
rofl0r
2019-01-07 05:38:35 +00:00
parent ad7c7fce67
commit 898c8f36ee
1 changed files with 28 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
ppf.py
View File

@@ -51,18 +51,22 @@ def fetch_contents(url):
return res
def insert_proxies(proxies, uri, sqlite):
time_now = time.time()
_known_proxies = {}
def insert_proxies(proxies, uri, sqlite, timestamp):
global _known_proxies
if len(_known_proxies) == 0:
known = sqlite.execute('SELECT proxy FROM proxylist').fetchall()
for k in known:
_known_proxies[k[0]] = True
query = [ 'proxy=?' for p in proxies ]
known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ]
new = [ (time_now,i,3,0,0,0) for i in proxies if not i in known ]
new = [ (timestamp,i,3,0,0,0) for i in proxies if not i in _known_proxies ]
for i in new:
_known_proxies[i[1]] = True
if len(new):
sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new)
sqlite.commit()
_log('+%d item(s) from %s' % (len(new), uri), 'added')
time.sleep(0.1)
def proxyfind(sqlite = None):
if not sqlite: sqlite = mysqlite.mysqlite(config.database,str)
@@ -97,8 +101,12 @@ def proxyfind(sqlite = None):
sqlite.commit()
def is_reserved_ipv4(ip):
if ( ip.startswith("10.") or ip.startswith("192.168.") or ip.startswith("127.") or ip.startswith("0.")) or \
(ip.startswith("172.") and (int(ip.split(".")[1]) >= 16 and int(ip.split(".")[1]) <= 31)): return True
octets = ip.split('.')
A = int(octets[0])
B = int(octets[1])
if A == 10 or A == 127 or A == 0 or \
(A == 192 and B == 168) or \
(A == 172 and B >= 16 and B <= 31): return True
return False
def proxyleech(sqlite, rows):
@@ -107,14 +115,15 @@ def proxyleech(sqlite, rows):
except KeyboardInterrupt as e: raise e
except: content = ''
matches = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))
uniques_dict = {}
for p in matches:
uniques_dict[p] = True
uniques = []
for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))):
if p in uniques: continue
try:
if not is_reserved_ipv4(p.split(':')[0]): uniques.append(p)
except KeyboardInterrupt as e: raise e
except:
pass
for p in uniques_dict.keys():
if not is_reserved_ipv4(p): uniques.append(p)
hash = hashlib.md5(''.join(uniques)).hexdigest()
@@ -132,12 +141,14 @@ def proxyleech(sqlite, rows):
if not row[1] or row[2] > 0: return
add = []
time_now = time.time()
for i in uniques:
add.append(i)
if len(add) > 500:
insert_proxies(add, row[0], sqlite)
insert_proxies(add, row[0], sqlite, time_now)
add = []
if len(add): insert_proxies(add, row[0], sqlite)
if len(add): insert_proxies(add, row[0], sqlite, time_now)
if __name__ == '__main__':