54 lines
1.4 KiB
Python
54 lines
1.4 KiB
Python
import time
|
|
from misc import _log
|
|
|
|
def create_table_if_not_exists(sqlite, dbname):
|
|
if dbname == 'proxylist':
|
|
sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist (
|
|
proxy BLOB UNIQUE,
|
|
country BLOB,
|
|
added INT,
|
|
failed INT,
|
|
tested INT,
|
|
dronebl INT,
|
|
proto TEXT,
|
|
success_count INT,
|
|
total_duration INT)""")
|
|
|
|
elif dbname == 'uris':
|
|
sqlite.execute("""CREATE TABLE IF NOT EXISTS uris (
|
|
url TEXT UNIQUE,
|
|
content_type TEXT,
|
|
check_time INT,
|
|
error INT,
|
|
stale_count INT,
|
|
retrievals INT,
|
|
proxies_added INT,
|
|
added INT
|
|
)""")
|
|
|
|
sqlite.commit()
|
|
|
|
def insert_proxies(proxydb, proxies, url):
|
|
timestamp = int(time.time())
|
|
|
|
new = []
|
|
for p in proxies:
|
|
new.append((timestamp,p,3,0,0,0))
|
|
|
|
proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new)
|
|
proxydb.commit()
|
|
|
|
_log('+%d proxy/ies from %s' % (len(proxies), url), 'added')
|
|
|
|
|
|
def insert_urls(urls, search, sqlite):
|
|
query = [ 'url=?' for u in urls ]
|
|
known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ]
|
|
time_now = int(time.time())
|
|
new = [ (time_now,i,0,1,0,0,0) for i in urls if not i in known ]
|
|
if not len(new): return
|
|
sqlite.executemany('INSERT OR IGNORE INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new)
|
|
sqlite.commit()
|
|
_log('+%d url(s) from %s' % (len(new), search), 'added')
|
|
|