diff --git a/ppf.py b/ppf.py index 7a16d47..14fde76 100755 --- a/ppf.py +++ b/ppf.py @@ -137,25 +137,20 @@ def is_usable_proxy(proxy): (A == 172 and B >= 16 and B <= 31): return False return True -def insert_proxies(proxies, sqlite, timestamp): +def insert_proxies(proxydb, proxies, url): + timestamp = int(time.time()) + new = [] for p in proxies: new.append((timestamp,p,3,0,0,0)) - if len(new): - sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new) - sqlite.commit() + while len(new): + proxydb.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new[:500]) + new = new[500:] + proxydb.commit() + + _log('+%d item(s) from %s' % (len(proxies), url), 'added') -def insert_new_proxies(proxydb, new, url): - add = [] - time_now = int(time.time()) - for i in new: - add.append(i) - if len(add) >= 500: - insert_proxies(add, proxydb, time_now) - add = [] - if len(add): insert_proxies(add, proxydb, time_now) - _log('+%d item(s) from %s' % (len(new), url), 'added') def proxyleech(proxydb, urldb, url, stale_count, error): try: content = fetch_contents(url) @@ -182,7 +177,7 @@ def proxyleech(proxydb, urldb, url, stale_count, error): if not len(new): return - insert_new_proxies(proxydb, new, url) + insert_proxies(proxydb, new, url) _known_proxies = {} def extract_proxies(content): @@ -214,7 +209,7 @@ def import_proxies_from_file(proxydb, fn): content = open(fn, 'r').read() unique_count, new = extract_proxies(content) if len(new): - insert_new_proxies(proxydb, new, fn) + insert_proxies(proxydb, new, fn) return 0 return 1