98 lines
2.7 KiB
Python
Executable File
98 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import dbs
|
|
import time
|
|
import mysqlite
|
|
import proxywatchd
|
|
from misc import _log
|
|
from config import Config
|
|
import fetch
|
|
import sys
|
|
|
|
config = Config()
|
|
|
|
def import_from_file(fn, sqlite):
|
|
with open(fn, 'r') as f:
|
|
for u in f.read().split('\n'):
|
|
if not len(u): continue
|
|
exists = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE url=?',(u,)).fetchall() ]
|
|
if exists: continue
|
|
print('adding "%s"' % u)
|
|
sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count,proxies_added,retrievals) VALUES (?,?,?,?,?,?,?)', (int(time.time()),u,0,0,0,0,0))
|
|
sqlite.commit()
|
|
|
|
def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added):
|
|
try: content = fetch.fetch_contents(url)
|
|
except KeyboardInterrupt as e: raise e
|
|
except: content = ''
|
|
|
|
unique_count, new = fetch.extract_proxies(content, proxydb)
|
|
|
|
if retrievals == 0: # new site
|
|
if content != '' and unique_count == 0: # site works but has zero proxy addresses
|
|
error = 99999
|
|
else:
|
|
if len(new) == 0:
|
|
stale_count += 1
|
|
else:
|
|
stale_count = 0
|
|
if content == '':
|
|
error += 1
|
|
else:
|
|
retrievals += 1
|
|
error = 0
|
|
|
|
urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=? where url=?', (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), url))
|
|
urldb.commit()
|
|
|
|
if not len(new): return
|
|
|
|
dbs.insert_proxies(proxydb, new, url)
|
|
|
|
|
|
def import_proxies_from_file(proxydb, fn):
|
|
content = open(fn, 'r').read()
|
|
unique_count, new = fetch.extract_proxies(content, proxydb)
|
|
if len(new):
|
|
dbs.insert_proxies(proxydb, new, fn)
|
|
return 0
|
|
return 1
|
|
|
|
if __name__ == '__main__':
|
|
config.load()
|
|
fetch.set_config(config)
|
|
|
|
proxydb = mysqlite.mysqlite(config.watchd.database, str)
|
|
dbs.create_table_if_not_exists(proxydb, 'proxylist')
|
|
|
|
urldb = mysqlite.mysqlite(config.ppf.database, str)
|
|
dbs.create_table_if_not_exists(urldb, 'uris')
|
|
import_from_file('import.txt', urldb)
|
|
if len(sys.argv) == 3 and sys.argv[1] == "--file":
|
|
sys.exit(import_proxies_from_file(proxydb, sys.argv[2]))
|
|
|
|
# start proxy watcher
|
|
if config.watchd.threads > 0:
|
|
watcherd = proxywatchd.Proxywatchd()
|
|
watcherd.start()
|
|
else:
|
|
watcherd = None
|
|
|
|
while True:
|
|
try:
|
|
## any site that needs to be checked ?
|
|
rows = urldb.execute('SELECT url,stale_count,error,retrievals,proxies_added FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
|
|
|
|
for row in rows:
|
|
proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4])
|
|
|
|
time.sleep(10)
|
|
|
|
except KeyboardInterrupt:
|
|
if watcherd:
|
|
watcherd.stop()
|
|
watcherd.finish()
|
|
break
|
|
|
|
print '\r',
|