e24f68500c
- convert tabs to 4-space indentation
- add docstrings to modules and classes
- remove unused import (copy)
- use explicit object inheritance
- use 'while True' over 'while 1'
- use 'while args' over 'while len(args)'
- use '{}' over 'dict()'
- consistent string formatting
- Python 2/3 compatible Queue import
81 lines
2.6 KiB
Python
81 lines
2.6 KiB
Python
#!/usr/bin/env python2
|
|
# -*- coding: utf-8 -*-
|
|
"""Database table creation and insertion utilities."""
|
|
|
|
import time
|
|
from misc import _log
|
|
|
|
|
|
def create_table_if_not_exists(sqlite, dbname):
|
|
"""Create database table with indexes if it doesn't exist."""
|
|
if dbname == 'proxylist':
|
|
sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist (
|
|
proxy BLOB UNIQUE,
|
|
country BLOB,
|
|
added INT,
|
|
failed INT,
|
|
tested INT,
|
|
dronebl INT,
|
|
proto TEXT,
|
|
mitm INT,
|
|
success_count INT,
|
|
ip TEXT,
|
|
port INT,
|
|
consecutive_success INT,
|
|
total_duration INT)""")
|
|
# Indexes for common query patterns
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_failed ON proxylist(failed)')
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_tested ON proxylist(tested)')
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_proto ON proxylist(proto)')
|
|
|
|
elif dbname == 'uris':
|
|
sqlite.execute("""CREATE TABLE IF NOT EXISTS uris (
|
|
url TEXT UNIQUE,
|
|
content_type TEXT,
|
|
check_time INT,
|
|
error INT,
|
|
stale_count INT,
|
|
retrievals INT,
|
|
proxies_added INT,
|
|
added INT)""")
|
|
# Indexes for common query patterns
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)')
|
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)')
|
|
|
|
sqlite.commit()
|
|
|
|
|
|
def insert_proxies(proxydb, proxies, url):
|
|
"""Insert new proxies into database."""
|
|
if not proxies:
|
|
return
|
|
timestamp = int(time.time())
|
|
rows = []
|
|
for p in proxies:
|
|
ip, port = p.split(':')
|
|
rows.append((timestamp, p, ip, port, 3, 0, 0, 0, 0, 0))
|
|
proxydb.executemany(
|
|
'INSERT OR IGNORE INTO proxylist '
|
|
'(added,proxy,ip,port,failed,tested,success_count,total_duration,mitm,consecutive_success) '
|
|
'VALUES (?,?,?,?,?,?,?,?,?,?)',
|
|
rows
|
|
)
|
|
proxydb.commit()
|
|
_log('+%d proxy/ies from %s' % (len(proxies), url), 'added')
|
|
|
|
|
|
def insert_urls(urls, search, sqlite):
|
|
"""Insert new URLs into database."""
|
|
if not urls:
|
|
return
|
|
timestamp = int(time.time())
|
|
rows = [(timestamp, u, 0, 1, 0, 0, 0) for u in urls]
|
|
sqlite.executemany(
|
|
'INSERT OR IGNORE INTO uris '
|
|
'(added,url,check_time,error,stale_count,retrievals,proxies_added) '
|
|
'VALUES (?,?,?,?,?,?,?)',
|
|
rows
|
|
)
|
|
sqlite.commit()
|
|
_log('+%d url(s) from %s' % (len(urls), search), 'added')
|