username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scraper: make query configurable (Proxies, Websites, Search)

Browse Source 
--scraper.query = 'pws'
This commit is contained in:
Mickaël Serneels
2019-04-22 23:26:37 +02:00
parent 15fc29abc4
commit 482cf79676
2 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scraper.py
View File

@@ -17,24 +17,26 @@ with open('searx.instances') as h:
print(searx_instances)
def proxyfind(sqlite = None, urignore=None):
random.shuffle(searx_instances)
proxydb = mysqlite.mysqlite(config.watchd.database,str)
proxies = [ i[0] for i in proxydb.execute('SELECT proxy FROM proxylist WHERE failed=0 ORDER BY RANDOM() LIMIT 10').fetchall() ]
search = ''
if len(proxies) and random.random() < random.random():
search = ' '.join( random.sample(proxies, random.randint(1,3)))
random.shuffle(searx_instances)
if 'p' in config.scraper.query:
proxydb = mysqlite.mysqlite(config.watchd.database,str)
proxies = [ i[0] for i in proxydb.execute('SELECT proxy FROM proxylist WHERE failed=0 ORDER BY RANDOM() LIMIT 10').fetchall() ]
if len(proxies) and random.random() < random.random():
search = ' '.join( random.sample(proxies, random.randint(1,3)))
if not len(search) or random.random() < random.random():
if 'w' in config.scraper.query and not len(search) or random.random() < random.random():
if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str)
uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
if len(uris) > 0 and random.random() < random.random():
if len(search): search = '%s ' % search
search = search + 'site:%s' % random.choice(uris).split('/')[2]
if not len(search) or random.random() < random.random():
if 's' in config.scraper.query and not len(search) or random.random() < random.random():
if len(search): search = '%s ' % search
search = search + random.choice(search_terms)
if not len(search): return
search_args = [ 'category=general', 'time_range=%s' % random.choice(['day','week','month','year']), 'q=%s' % urllib.quote_plus(search) ]
random.shuffle(search_args)
search_arg = '&'.join(search_args)