split ppf into two programs, ppf/scraper

scraper.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+import dbs
+import random, time
+import urllib
+import mysqlite
+import proxywatchd
+from misc import _log
+from config import Config
+import fetch
+import sys
+
+config = Config()
+
+searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' )
+
+def proxyfind(sqlite = None, urignore=None):
+	if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str)
+
+	uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
+	if len(uris) > 0 and random.random() < random.random():
+		search = 'site:%s' % random.choice(uris).split('/')[2]
+	else:
+		search = random.choice(search_terms)
+
+	search = '%s -intitle:pdf' % search
+	search_args = [ 'category=general', 'time_range=day', 'q=%s' % urllib.quote_plus(search) ]
+	for srx in searx_instances:
+		urls = []
+		random.shuffle(search_args)
+		search_arg = '&'.join(search_args)
+		for x in range(1,10):
+			content = fetch.fetch_contents('%s/?%s&pageno=%d' % (srx,search_arg,x))
+			if content: urls = fetch.extract_urls(content, urls, urignore)
+		if len(urls): dbs.insert_urls(urls, search_arg, sqlite)
+
+
+def load_urignore():
+	## load bad terms
+	with open('urignore.txt', 'r') as f:
+		urignore = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]
+	## add searx instances as bad terms (avoid loops)
+	for i in searx_instances:
+		urignore.append(i.split('/')[2])
+	return urignore
+
+
+if __name__ == '__main__':
+	config.load()
+	fetch.set_config(config)
+
+	proxydb = mysqlite.mysqlite(config.watchd.database, str)
+	dbs.create_table_if_not_exists(proxydb, 'proxylist')
+
+	urldb = mysqlite.mysqlite(config.ppf.database, str)
+	dbs.create_table_if_not_exists(urldb, 'uris')
+
+	## load search terms
+	with open('search_terms.txt', 'r') as f:
+		search_terms = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]
+
+	urignore = load_urignore()
+
+	while True:
+		try:
+			proxyfind(urldb, urignore)
+
+		except KeyboardInterrupt:
+
+			break
+
+	print '\r',