username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scraper: more changes

Browse Source
This commit is contained in:
Mickaël Serneels
2019-04-23 00:22:32 +02:00
parent 482cf79676
commit 70b6285394
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
scraper.py
View File

@@ -23,17 +23,17 @@ def proxyfind(sqlite = None, urignore=None):
proxydb = mysqlite.mysqlite(config.watchd.database,str)
proxies = [ i[0] for i in proxydb.execute('SELECT proxy FROM proxylist WHERE failed=0 ORDER BY RANDOM() LIMIT 10').fetchall() ]
if len(proxies) and random.random() < random.random():
search = ' '.join( random.sample(proxies, random.randint(1,3)))
search = ' '.join( random.sample(proxies, random.randint(1,2)))
if 'w' in config.scraper.query and not len(search) or random.random() < random.random():
if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str)
uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
if len(uris) > 0 and random.random() < random.random():
if len(search): search = '%s ' % search
if len(search): search = '%s OR ' % search
search = search + 'site:%s' % random.choice(uris).split('/')[2]
if 's' in config.scraper.query and not len(search) or random.random() < random.random():
if len(search): search = '%s ' % search
if len(search): search = '%s OR ' % search
search = search + random.choice(search_terms)
if not len(search): return