import time from misc import _log def create_table_if_not_exists(sqlite, dbname): if dbname == 'proxylist': sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist ( proxy BLOB UNIQUE, country BLOB, added INT, failed INT, tested INT, dronebl INT, proto TEXT, mitm INT, success_count INT, total_duration INT)""") elif dbname == 'uris': sqlite.execute("""CREATE TABLE IF NOT EXISTS uris ( url TEXT UNIQUE, content_type TEXT, check_time INT, error INT, stale_count INT, retrievals INT, proxies_added INT, added INT )""") sqlite.commit() def insert_proxies(proxydb, proxies, url): timestamp = int(time.time()) new = [] for p in proxies: new.append((timestamp,p,3,0,0,0,0,0)) proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,failed,tested,success_count,total_duration,mitm,consecutive_success) VALUES (?,?,?,?,?,?,?,?)', new) proxydb.commit() _log('+%d proxy/ies from %s' % (len(proxies), url), 'added') def insert_urls(urls, search, sqlite): query = [ 'url=?' for u in urls ] known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] time_now = int(time.time()) new = [ (time_now,i,0,1,0,0,0) for i in urls if not i in known ] if not len(new): return sqlite.executemany('INSERT OR IGNORE INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new) sqlite.commit() _log('+%d url(s) from %s' % (len(new), search), 'added')