From 8993727f0316a8fa474bf078b3d63c522ae50c63 Mon Sep 17 00:00:00 2001 From: mickael Date: Wed, 9 Jan 2019 20:07:28 +0000 Subject: [PATCH 1/4] changed regex --- ppf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ppf.py b/ppf.py index d0df5ba..40541bd 100755 --- a/ppf.py +++ b/ppf.py @@ -146,7 +146,7 @@ def proxyleech(sqlite, rows): except KeyboardInterrupt as e: raise e except: content = '' - matches = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content)) + matches = re.findall(r'([0-9]+(?:\.[0-9]+){3}:[0-9]{1,5})[\D$]', cleanhtml(content)) uniques_dict = {} for p in matches: From e94644a60ee46d50fc9ee109bd2f497d5b78d77b Mon Sep 17 00:00:00 2001 From: mickael Date: Wed, 9 Jan 2019 22:55:55 +0000 Subject: [PATCH 2/4] searx: loop for 10 pages on each searx instance --- ppf.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/ppf.py b/ppf.py index 40541bd..eb78c02 100755 --- a/ppf.py +++ b/ppf.py @@ -96,8 +96,6 @@ def insert_proxies(proxies, uri, sqlite, timestamp): def proxyfind(sqlite = None): if not sqlite: sqlite = mysqlite.mysqlite(config.common.database,str) - choice = random.choice(searx_instances) - urls = [] uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ] if len(uris) > 0 and random.random() < random.random(): @@ -105,26 +103,33 @@ def proxyfind(sqlite = None): else: search = random.choice(search_terms) - content = fetch_contents('%s/?q=%s&pageno=%d' % (choice, urllib.quote_plus(search), random.randint(0,10))) - if not content: return + search_args = [ 'category=general', 'time_range=day', 'q=%s' % urllib.quote_plus(search) ] + for srx in searx_instances: + urls = [] + random.shuffle(search_args) + search_arg = '&'.join(search_args) + for x in range(1,10): + content = fetch_contents('%s/?%s&pageno=%d' % (srx,search_arg,x)) + if content: urls = extract_urls(content, urls) + if len(urls): insert_urls(urls, search_arg, sqlite) +def extract_urls(content, urls = []): soup = soupify(content) for a in soup.body.find_all('a'): if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ] if not len(badurl): urls.append(a.attrs['href']) + return urls - if len(urls): - query = [ 'url=?' for u in urls ] - known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] - time_now = int(time.time()) - new = [ (time_now,i,0,5,0) for i in urls if not i in known ] - if len(new): - sqlite.executemany('INSERT INTO uris (added,url,check_time,error,driver) values(?,?,?,?,?)', new) - sqlite.commit() - _log('+%d item(s) from %s' % (len(new), search), 'added') - +def insert_urls(urls, search, sqlite): + query = [ 'url=?' for u in urls ] + known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] + time_now = int(time.time()) + new = [ (time_now,i,0,5,0) for i in urls if not i in known ] + if not len(new): return + sqlite.executemany('INSERT INTO uris (added,url,check_time,error,driver) values(?,?,?,?,?)', new) sqlite.commit() + _log('+%d item(s) from %s' % (len(new), search), 'added') def is_usable_proxy(proxy): octets = proxy.split(':')[0].split('.') @@ -214,11 +219,11 @@ if __name__ == '__main__': while True: try: ## any site that needs to be checked ? - rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time+?+(error*?) Date: Wed, 9 Jan 2019 23:30:55 +0000 Subject: [PATCH 3/4] add -intitle:pdf to searx query --- ppf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ppf.py b/ppf.py index eb78c02..ea7cf60 100755 --- a/ppf.py +++ b/ppf.py @@ -103,6 +103,7 @@ def proxyfind(sqlite = None): else: search = random.choice(search_terms) + search = '%s -intitle:pdf' % search search_args = [ 'category=general', 'time_range=day', 'q=%s' % urllib.quote_plus(search) ] for srx in searx_instances: urls = [] From da4f228479b5a1149565503d5acb89b6ca1d356a Mon Sep 17 00:00:00 2001 From: mickael Date: Wed, 9 Jan 2019 23:38:59 +0000 Subject: [PATCH 4/4] discard urls who fail at first test --- ppf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ppf.py b/ppf.py index ea7cf60..dbbfecf 100755 --- a/ppf.py +++ b/ppf.py @@ -165,7 +165,10 @@ def proxyleech(sqlite, rows): hash = hashlib.md5(''.join(uniques)).hexdigest() ## empty list of proxies: multiply error by two - if not len(uniques): row[2] = (row[2] * 2) + if not len(uniques): + if row[1]: row[2] = (row[2] * 2) + else: row[2] = 99999 + ## same proxy list: increment error by one elif hash == row[1]: row[2] = (row[2] + 1) ## proxylist was updated: error is zero