diff --git a/dbs.py b/dbs.py index fd371b8..a644997 100644 --- a/dbs.py +++ b/dbs.py @@ -1,4 +1,5 @@ -import mysqlite +import time +from misc import _log def create_table_if_not_exists(sqlite, dbname): if dbname == 'proxylist': @@ -25,3 +26,29 @@ def create_table_if_not_exists(sqlite, dbname): )""") sqlite.commit() + +def insert_proxies(proxydb, proxies, url): + timestamp = int(time.time()) + + new = [] + for p in proxies: + new.append((timestamp,p,3,0,0,0)) + + while len(new): + proxydb.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new[:500]) + new = new[500:] + proxydb.commit() + + _log('+%d item(s) from %s' % (len(proxies), url), 'added') + + +def insert_urls(urls, search, sqlite): + query = [ 'url=?' for u in urls ] + known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] + time_now = int(time.time()) + new = [ (time_now,i,0,0,0,0,0) for i in urls if not i in known ] + if not len(new): return + sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new) + sqlite.commit() + _log('+%d item(s) from %s' % (len(new), search), 'added') + diff --git a/ppf.py b/ppf.py index fa94426..ce21129 100755 --- a/ppf.py +++ b/ppf.py @@ -44,32 +44,7 @@ def proxyfind(sqlite = None): for x in range(1,10): content = fetch.fetch_contents('%s/?%s&pageno=%d' % (srx,search_arg,x)) if content: urls = fetch.extract_urls(content, urls, urignore) - if len(urls): insert_urls(urls, search_arg, sqlite) - -def insert_urls(urls, search, sqlite): - query = [ 'url=?' for u in urls ] - known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] - time_now = int(time.time()) - new = [ (time_now,i,0,0,0,0,0) for i in urls if not i in known ] - if not len(new): return - sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new) - sqlite.commit() - _log('+%d item(s) from %s' % (len(new), search), 'added') - - -def insert_proxies(proxydb, proxies, url): - timestamp = int(time.time()) - - new = [] - for p in proxies: - new.append((timestamp,p,3,0,0,0)) - - while len(new): - proxydb.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new[:500]) - new = new[500:] - proxydb.commit() - - _log('+%d item(s) from %s' % (len(proxies), url), 'added') + if len(urls): dbs.insert_urls(urls, search_arg, sqlite) def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added): @@ -98,14 +73,14 @@ def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_adde if not len(new): return - insert_proxies(proxydb, new, url) + dbs.insert_proxies(proxydb, new, url) def import_proxies_from_file(proxydb, fn): content = open(fn, 'r').read() unique_count, new = fetch.extract_proxies(content, proxydb) if len(new): - insert_proxies(proxydb, new, fn) + dbs.insert_proxies(proxydb, new, fn) return 0 return 1