From 70b62853949d9cabe318f93324edb4e31a0867e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Serneels?= Date: Tue, 23 Apr 2019 00:22:32 +0200 Subject: [PATCH] scraper: more changes --- scraper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scraper.py b/scraper.py index d851166..b8b036b 100755 --- a/scraper.py +++ b/scraper.py @@ -23,17 +23,17 @@ def proxyfind(sqlite = None, urignore=None): proxydb = mysqlite.mysqlite(config.watchd.database,str) proxies = [ i[0] for i in proxydb.execute('SELECT proxy FROM proxylist WHERE failed=0 ORDER BY RANDOM() LIMIT 10').fetchall() ] if len(proxies) and random.random() < random.random(): - search = ' '.join( random.sample(proxies, random.randint(1,3))) + search = ' '.join( random.sample(proxies, random.randint(1,2))) if 'w' in config.scraper.query and not len(search) or random.random() < random.random(): if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str) uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ] if len(uris) > 0 and random.random() < random.random(): - if len(search): search = '%s ' % search + if len(search): search = '%s OR ' % search search = search + 'site:%s' % random.choice(uris).split('/')[2] if 's' in config.scraper.query and not len(search) or random.random() < random.random(): - if len(search): search = '%s ' % search + if len(search): search = '%s OR ' % search search = search + random.choice(search_terms) if not len(search): return