diff --git a/dbs.py b/dbs.py index 57a3e42..fd371b8 100644 --- a/dbs.py +++ b/dbs.py @@ -20,6 +20,7 @@ def create_table_if_not_exists(sqlite, dbname): error INT, stale_count INT, retrievals INT, + proxies_added INT, added INT )""") diff --git a/ppf.py b/ppf.py index 6bfc9d2..43aadb2 100755 --- a/ppf.py +++ b/ppf.py @@ -42,7 +42,7 @@ def import_from_file(fn, sqlite): exists = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE url=?',(u,)).fetchall() ] if exists: continue print('adding "%s"' % u) - sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count) VALUES (?,?,?,?,?)', (int(time.time()),u,0,0,0)) + sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count,proxies_added) VALUES (?,?,?,?,?,?)', (int(time.time()),u,0,0,0,0)) sqlite.commit() def fetch_contents(url): @@ -111,9 +111,9 @@ def insert_urls(urls, search, sqlite): query = [ 'url=?' for u in urls ] known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] time_now = int(time.time()) - new = [ (time_now,i,0,0,0,0) for i in urls if not i in known ] + new = [ (time_now,i,0,0,0,0,0) for i in urls if not i in known ] if not len(new): return - sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals) values(?,?,?,?,?,?)', new) + sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new) sqlite.commit() _log('+%d item(s) from %s' % (len(new), search), 'added') @@ -152,7 +152,7 @@ def insert_proxies(proxydb, proxies, url): _log('+%d item(s) from %s' % (len(proxies), url), 'added') -def proxyleech(proxydb, urldb, url, stale_count, error, retrievals): +def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added): try: content = fetch_contents(url) except KeyboardInterrupt as e: raise e except: content = '' @@ -173,7 +173,7 @@ def proxyleech(proxydb, urldb, url, stale_count, error, retrievals): retrievals += 1 error = 0 - urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=? where url=?', (error, stale_count, int(time.time()), retrievals, url)) + urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=? where url=?', (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), url)) urldb.commit() if not len(new): return @@ -248,10 +248,10 @@ if __name__ == '__main__': while True: try: ## any site that needs to be checked ? - rows = urldb.execute('SELECT url,stale_count,error,retrievals FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?)