username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

outsource configuration to external module

Browse Source
This commit is contained in:
rofl0r
2019-01-05 03:47:03 +00:00
parent 05294186d4
commit ffbe450aee
3 changed files with 41 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
ppf.py
View File

@@ -6,13 +6,13 @@ import random, time
import re
import urllib
import hashlib
from ConfigParser import SafeConfigParser
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
import mysqlite
import proxywatchd
from misc import _log
from soup_parser import soupify
import config
base_header = {
'Accept-Language':'en-US,en;q=0.8',
@@ -24,7 +24,6 @@ base_header = {
searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' )
retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded')
CONFIG = 'config.ini'
def cleanhtml(raw_html):
cleanr = re.compile('<.*?>')
@@ -69,7 +68,7 @@ def insert_proxies(proxies, uri, sqlite):
def proxyfind(sqlite = None):
#print('entering proxyfind...')
if not sqlite: sqlite = mysqlite.mysqlite(database,str)
if not sqlite: sqlite = mysqlite.mysqlite(config.database,str)
uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
@@ -149,16 +148,11 @@ def proxyleech(sqlite, rows):
if __name__ == '__main__':
## read the config files
parser = SafeConfigParser()
parser.read(CONFIG)
config.load()
print repr(config.torhosts)
proxies={'http':'socks4://%s' % random.choice(config.torhosts),'https':'socks4://%s' % random.choice(config.torhosts)}
database = parser.get('global', 'database')
search = parser.getboolean('proxyfind', 'search')
tor_hosts = parser.get('global', 'tor_host').split(',')
proxies={'http':'socks4://%s' % random.choice(tor_hosts),'https':'socks4://%s' % random.choice(tor_hosts)}
sqlite = mysqlite.mysqlite(database, str)
sqlite = mysqlite.mysqlite(config.database, str)
## create dbs if required
sqlite.execute('CREATE TABLE IF NOT EXISTS uris (added INT, url TEXT, check_time INT, error INT, driver INT, hash TEXT)')
@@ -178,7 +172,7 @@ if __name__ == '__main__':
empty = [ urignore.append(i.split('/')[2]) for i in searx_instances ]
# start proxy watcher
watcherd = proxywatchd.Proxywatchd(CONFIG) if parser.getboolean('watcherd', 'enabled') else None
watcherd = proxywatchd.Proxywatchd() if config.watchd_threads > 0 else None
while True:
try:
@@ -186,7 +180,7 @@ if __name__ == '__main__':
rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time<? AND error<?) ORDER BY RANDOM() LIMIT 25', (time.time(), 10)).fetchall() ]
if len(rows): proxyleech(sqlite,rows)
## search for new website during free time
elif search: proxyfind(sqlite)
elif config.search: proxyfind(sqlite)
## sleep
else: time.sleep(10)