diff --git a/scraper.py b/scraper.py index d851166..b8b036b 100755 --- a/scraper.py +++ b/scraper.py @@ -23,17 +23,17 @@ def proxyfind(sqlite = None, urignore=None): proxydb = mysqlite.mysqlite(config.watchd.database,str) proxies = [ i[0] for i in proxydb.execute('SELECT proxy FROM proxylist WHERE failed=0 ORDER BY RANDOM() LIMIT 10').fetchall() ] if len(proxies) and random.random() < random.random(): - search = ' '.join( random.sample(proxies, random.randint(1,3))) + search = ' '.join( random.sample(proxies, random.randint(1,2))) if 'w' in config.scraper.query and not len(search) or random.random() < random.random(): if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str) uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ] if len(uris) > 0 and random.random() < random.random(): - if len(search): search = '%s ' % search + if len(search): search = '%s OR ' % search search = search + 'site:%s' % random.choice(uris).split('/')[2] if 's' in config.scraper.query and not len(search) or random.random() < random.random(): - if len(search): search = '%s ' % search + if len(search): search = '%s OR ' % search search = search + random.choice(search_terms) if not len(search): return