import time from misc import _log def create_table_if_not_exists(sqlite, dbname): if dbname == 'proxylist': sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist ( proxy BLOB UNIQUE, country BLOB, added INT, failed INT, tested INT, dronebl INT, proto TEXT, mitm INT, success_count INT, ip TEXT, port INT, consecutive_success INT, total_duration INT)""") # indexes for common query patterns sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_failed ON proxylist(failed)') sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_tested ON proxylist(tested)') sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_proto ON proxylist(proto)') elif dbname == 'uris': sqlite.execute("""CREATE TABLE IF NOT EXISTS uris ( url TEXT UNIQUE, content_type TEXT, check_time INT, error INT, stale_count INT, retrievals INT, proxies_added INT, added INT )""") # indexes for common query patterns sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)') sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)') sqlite.commit() def insert_proxies(proxydb, proxies, url): if not proxies: return timestamp = int(time.time()) rows = [] for p in proxies: ip, port = p.split(':') rows.append((timestamp,p,ip,port,3,0,0,0,0,0)) proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,ip,port,failed,tested,success_count,total_duration,mitm,consecutive_success) VALUES (?,?,?,?,?,?,?,?,?,?)', rows) proxydb.commit() _log('+%d proxy/ies from %s' % (len(proxies), url), 'added') def insert_urls(urls, search, sqlite): if not urls: return time_now = int(time.time()) rows = [ (time_now,u,0,1,0,0,0) for u in urls ] sqlite.executemany('INSERT OR IGNORE INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', rows) sqlite.commit() _log('+%d url(s) from %s' % (len(urls), search), 'added')