73 lines
2.1 KiB
Python
Executable File
73 lines
2.1 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import dbs
|
|
import random, time
|
|
import urllib
|
|
import mysqlite
|
|
import proxywatchd
|
|
from misc import _log
|
|
from config import Config
|
|
import fetch
|
|
import sys
|
|
|
|
config = Config()
|
|
|
|
searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' )
|
|
|
|
def proxyfind(sqlite = None, urignore=None):
|
|
if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str)
|
|
|
|
uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
|
|
if len(uris) > 0 and random.random() < random.random():
|
|
search = 'site:%s' % random.choice(uris).split('/')[2]
|
|
else:
|
|
search = random.choice(search_terms)
|
|
|
|
search = '%s -intitle:pdf' % search
|
|
search_args = [ 'category=general', 'time_range=day', 'q=%s' % urllib.quote_plus(search) ]
|
|
for srx in searx_instances:
|
|
urls = []
|
|
random.shuffle(search_args)
|
|
search_arg = '&'.join(search_args)
|
|
for x in range(1,10):
|
|
content = fetch.fetch_contents('%s/?%s&pageno=%d' % (srx,search_arg,x))
|
|
if content: urls = fetch.extract_urls(content, urls, urignore)
|
|
if len(urls): dbs.insert_urls(urls, search_arg, sqlite)
|
|
|
|
|
|
def load_urignore():
|
|
## load bad terms
|
|
with open('urignore.txt', 'r') as f:
|
|
urignore = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]
|
|
## add searx instances as bad terms (avoid loops)
|
|
for i in searx_instances:
|
|
urignore.append(i.split('/')[2])
|
|
return urignore
|
|
|
|
|
|
if __name__ == '__main__':
|
|
config.load()
|
|
fetch.set_config(config)
|
|
|
|
proxydb = mysqlite.mysqlite(config.watchd.database, str)
|
|
dbs.create_table_if_not_exists(proxydb, 'proxylist')
|
|
|
|
urldb = mysqlite.mysqlite(config.ppf.database, str)
|
|
dbs.create_table_if_not_exists(urldb, 'uris')
|
|
|
|
## load search terms
|
|
with open('search_terms.txt', 'r') as f:
|
|
search_terms = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]
|
|
|
|
urignore = load_urignore()
|
|
|
|
while True:
|
|
try:
|
|
proxyfind(urldb, urignore)
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
break
|
|
|
|
print '\r',
|