diff --git a/scraper.py b/scraper.py index 36f48a0..a75e2d9 100755 --- a/scraper.py +++ b/scraper.py @@ -12,15 +12,17 @@ import sys config = Config() -#searx_instances = ['https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net', 'https://searx.serneels.xyz' ] -searx_instances = ['http://searchb5a7tmimez.onion', 'http://nxhhwbbxc4khvvlw.onion','http://searx.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion', 'http://ulrn6sryqaifefld.onion'] +with open('searx.instances') as h: + searx_instances = [ line.strip() for line in h.readlines() if line.lower().startswith('http') ] + print(searx_instances) + def proxyfind(sqlite = None, urignore=None): random.shuffle(searx_instances) proxydb = mysqlite.mysqlite(config.watchd.database,str) proxies = [ i[0] for i in proxydb.execute('SELECT proxy FROM proxylist WHERE failed=0 ORDER BY RANDOM() LIMIT 10').fetchall() ] search = '' if len(proxies) and random.random() < random.random(): - search = ' '.join( random.sample(proxies, random.randint(1,5))) + search = ' '.join( random.sample(proxies, random.randint(1,3))) if not len(search) or random.random() < random.random(): if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str) @@ -36,6 +38,10 @@ def proxyfind(sqlite = None, urignore=None): search_args = [ 'category=general', 'time_range=%s' % random.choice(['day','week','month','year']), 'q=%s' % urllib.quote_plus(search) ] random.shuffle(search_args) search_arg = '&'.join(search_args) + + if config.scraper.debug: + print('search_arg: %s' % search_arg) + for srx in searx_instances: x = 0 while 1: diff --git a/searx.instances b/searx.instances new file mode 100644 index 0000000..fbf5e38 --- /dev/null +++ b/searx.instances @@ -0,0 +1,15 @@ +https://searx.me +https://searx.xyz +https://searx.site +https://searx.win +https://searx.ru +https://stemy.me/searx +https://searx.at +https://listi.me +https://searx.dk +https://searx.laquadrature.net +## hidden services +http://searchb5a7tmimez.onion +http://nxhhwbbxc4khvvlw.onion +http://searx.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion +http://ulrn6sryqaifefld.onion