username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ppf: use slice for the 500 rows limitation

Browse Source
This commit is contained in:
rofl0r
2019-01-11 23:07:09 +00:00
committed by mickael
parent d209356a85
commit 2f7a730311
1 changed files with 11 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
ppf.py
View File

@@ -137,25 +137,20 @@ def is_usable_proxy(proxy):
(A == 172 and B >= 16 and B <= 31): return False
return True
def insert_proxies(proxies, sqlite, timestamp):
def insert_proxies(proxydb, proxies, url):
timestamp = int(time.time())
new = []
for p in proxies:
new.append((timestamp,p,3,0,0,0))
if len(new):
sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new)
sqlite.commit()
while len(new):
proxydb.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new[:500])
new = new[500:]
proxydb.commit()
_log('+%d item(s) from %s' % (len(proxies), url), 'added')
def insert_new_proxies(proxydb, new, url):
add = []
time_now = int(time.time())
for i in new:
add.append(i)
if len(add) >= 500:
insert_proxies(add, proxydb, time_now)
add = []
if len(add): insert_proxies(add, proxydb, time_now)
_log('+%d item(s) from %s' % (len(new), url), 'added')
def proxyleech(proxydb, urldb, url, stale_count, error):
try: content = fetch_contents(url)
@@ -182,7 +177,7 @@ def proxyleech(proxydb, urldb, url, stale_count, error):
if not len(new): return
insert_new_proxies(proxydb, new, url)
insert_proxies(proxydb, new, url)
_known_proxies = {}
def extract_proxies(content):
@@ -214,7 +209,7 @@ def import_proxies_from_file(proxydb, fn):
content = open(fn, 'r').read()
unique_count, new = extract_proxies(content)
if len(new):
insert_new_proxies(proxydb, new, fn)
insert_proxies(proxydb, new, fn)
return 0
return 1