62 lines
1.9 KiB
Python
62 lines
1.9 KiB
Python
import time
|
|
from misc import _log
|
|
|
|
def create_table_if_not_exists(sqlite, dbname):
|
|
if dbname == 'proxylist':
|
|
sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist (
|
|
proxy BLOB UNIQUE,
|
|
country BLOB,
|
|
added INT,
|
|
failed INT,
|
|
tested INT,
|
|
dronebl INT,
|
|
proto TEXT,
|
|
mitm INT,
|
|
success_count INT,
|
|
ip TEXT,
|
|
port INT,
|
|
consecutive_success INT,
|
|
total_duration INT)""")
|
|
# indexes for common query patterns
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_failed ON proxylist(failed)')
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_tested ON proxylist(tested)')
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_proto ON proxylist(proto)')
|
|
|
|
elif dbname == 'uris':
|
|
sqlite.execute("""CREATE TABLE IF NOT EXISTS uris (
|
|
url TEXT UNIQUE,
|
|
content_type TEXT,
|
|
check_time INT,
|
|
error INT,
|
|
stale_count INT,
|
|
retrievals INT,
|
|
proxies_added INT,
|
|
added INT
|
|
)""")
|
|
# indexes for common query patterns
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)')
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)')
|
|
|
|
sqlite.commit()
|
|
|
|
def insert_proxies(proxydb, proxies, url):
|
|
if not proxies: return
|
|
timestamp = int(time.time())
|
|
rows = []
|
|
for p in proxies:
|
|
ip, port = p.split(':')
|
|
rows.append((timestamp,p,ip,port,3,0,0,0,0,0))
|
|
proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,ip,port,failed,tested,success_count,total_duration,mitm,consecutive_success) VALUES (?,?,?,?,?,?,?,?,?,?)', rows)
|
|
proxydb.commit()
|
|
_log('+%d proxy/ies from %s' % (len(proxies), url), 'added')
|
|
|
|
|
|
def insert_urls(urls, search, sqlite):
|
|
if not urls: return
|
|
time_now = int(time.time())
|
|
rows = [ (time_now,u,0,1,0,0,0) for u in urls ]
|
|
sqlite.executemany('INSERT OR IGNORE INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', rows)
|
|
sqlite.commit()
|
|
_log('+%d url(s) from %s' % (len(urls), search), 'added')
|
|
|