username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

searx: loop for 10 pages on each searx instance

Browse Source
This commit is contained in:
mickael
2019-01-09 22:55:55 +00:00
parent 8993727f03
commit e94644a60e
1 changed files with 22 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
ppf.py
View File

@@ -96,8 +96,6 @@ def insert_proxies(proxies, uri, sqlite, timestamp):
def proxyfind(sqlite = None): def proxyfind(sqlite = None):
if not sqlite: sqlite = mysqlite.mysqlite(config.common.database,str) if not sqlite: sqlite = mysqlite.mysqlite(config.common.database,str)
choice = random.choice(searx_instances)
urls = []
uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ] uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
if len(uris) > 0 and random.random() < random.random(): if len(uris) > 0 and random.random() < random.random():
@@ -105,26 +103,33 @@ def proxyfind(sqlite = None):
else: else:
search = random.choice(search_terms) search = random.choice(search_terms)
content = fetch_contents('%s/?q=%s&pageno=%d' % (choice, urllib.quote_plus(search), random.randint(0,10))) search_args = [ 'category=general', 'time_range=day', 'q=%s' % urllib.quote_plus(search) ]
if not content: return for srx in searx_instances:
urls = []
random.shuffle(search_args)
search_arg = '&'.join(search_args)
for x in range(1,10):
content = fetch_contents('%s/?%s&pageno=%d' % (srx,search_arg,x))
if content: urls = extract_urls(content, urls)
if len(urls): insert_urls(urls, search_arg, sqlite)
def extract_urls(content, urls = []):
soup = soupify(content) soup = soupify(content)
for a in soup.body.find_all('a'): for a in soup.body.find_all('a'):
if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue
badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ] badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ]
if not len(badurl): urls.append(a.attrs['href']) if not len(badurl): urls.append(a.attrs['href'])
return urls
if len(urls): def insert_urls(urls, search, sqlite):
query = [ 'url=?' for u in urls ] query = [ 'url=?' for u in urls ]
known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ]
time_now = int(time.time()) time_now = int(time.time())
new = [ (time_now,i,0,5,0) for i in urls if not i in known ] new = [ (time_now,i,0,5,0) for i in urls if not i in known ]
if len(new): if not len(new): return
sqlite.executemany('INSERT INTO uris (added,url,check_time,error,driver) values(?,?,?,?,?)', new) sqlite.executemany('INSERT INTO uris (added,url,check_time,error,driver) values(?,?,?,?,?)', new)
sqlite.commit()
_log('+%d item(s) from %s' % (len(new), search), 'added')
sqlite.commit() sqlite.commit()
_log('+%d item(s) from %s' % (len(new), search), 'added')
def is_usable_proxy(proxy): def is_usable_proxy(proxy):
octets = proxy.split(':')[0].split('.') octets = proxy.split(':')[0].split('.')
@@ -214,11 +219,11 @@ if __name__ == '__main__':
while True: while True:
try: try:
## any site that needs to be checked ? ## any site that needs to be checked ?
rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time+?+(error*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() ] #rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time+?+(error*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() ]
if len(rows): proxyleech(sqlite,rows) #if len(rows): proxyleech(sqlite,rows)
## search for new website during free time ## search for new website during free time
elif config.ppf.search: proxyfind(sqlite) if config.ppf.search: proxyfind(sqlite)
## sleep ## sleep
else: time.sleep(10) else: time.sleep(10)