username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement combo config parser

Browse Source 
allows all options to be overridden by command line.

e.g.
[watchd]
threads=10
debug=false

--watch.threads=50 --debug=true
This commit is contained in:
rofl0r
2019-01-08 02:17:04 +00:00
parent e7b8d526c0
commit f16f754b0e
5 changed files with 131 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
config.py
View File

@@ -1,48 +1,24 @@
from ConfigParser import SafeConfigParser
from comboparse import ComboParser
_loaded = False
class Config(ComboParser):
def load(self):
super(Config, self).load()
self.torhosts = [ str(i).strip() for i in self.common.tor_hosts.split(',') ]
with open('servers.txt', 'r') as handle:
self.servers = [x.strip() for x in handle.readlines() if len(x.strip()) > 0]
def __init__(self):
super(Config, self).__init__('config.ini')
self.add_item('common', 'tor_hosts', str, '127.0.0.1:9050', 'comma-separated list of tor proxy address(es)', True)
self.add_item('common', 'database', str, 'proxylist.sqlite', 'filename of database', True)
class phantom():
def __init__(self): pass
self.add_item('watchd', 'max_fail', int, 5, 'number of fails after which a proxy is considered dead', False)
self.add_item('watchd', 'threads', int, 10, 'number of threads watchd uses to check proxies', True)
self.add_item('watchd', 'timeout', int, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False)
self.add_item('watchd', 'submit_after', int, 200, 'min. number of tested proxies for DB write', False)
self.add_item('watchd', 'debug', bool, False, 'whether to print additional debug info', False)
self.add_item('watchd', 'use_ssl', bool, False, 'whether to use SSL and port 6697 to connect to targets (slower)', False)
def load():
if _loaded: return
global database, maxfail, search, torhosts, watchd_threads, checktime, timeout, read_timeout, submit_after, use_ssl, url_checktime, url_perfail_checktime
## read the config files
parser = SafeConfigParser()
parser.read('config.ini')
database = parser.get('global', 'database')
#maxfail = parser.getint('global', 'proxy_max_fail')
torhosts = [ str(i).strip() for i in parser.get('global', 'tor_host').split(',') ]
global _watchd
_watchd = phantom()
_watchd.threads = parser.getint('watcherd', 'threads')
_watchd.timeout = parser.getint('watcherd', 'timeout')
_watchd.submit_after = parser.getint('watcherd', 'submit_after')
_watchd.use_ssl = parser.getboolean('watcherd', 'use_ssl')
_watchd.debug = parser.getboolean('watcherd', 'debug')
_watchd.maxfail = parser.getint('watcherd', 'max_fail')
global _leechd
_leechd = phantom()
_leechd.checktime = parser.get('proxyfind', 'checktime')
_leechd.perfail_checktime = parser.get('proxyfind', 'perfail_checktime')
_leechd.search = parser.getboolean('proxyfind', 'search')
global watchd_debug
watchd_debug = parser.getboolean('watcherd', 'debug')
# allow overriding select items from the commandline
import argparse
aparse = argparse.ArgumentParser()
aparse.add_argument('--watchd_threads', help="how many proxy checker threads to spin up, 0==none, default: 10", type=int, default=_watchd.threads, required=False)
args = aparse.parse_args()
_watchd.threads = args.watchd_threads
global servers
with open('servers.txt', 'r') as handle:
servers = [x.strip() for x in handle.readlines() if len(x.strip()) > 0]
self.add_item('ppf', 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False)
self.add_item('ppf', 'timeout', float, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False)
self.add_item('ppf', 'checktime', int, 3600, 'base checking interval for urls in db in seconds', False)
self.add_item('ppf', 'perfail_checktime', int, 3600, 'additional checking interval for urls in db in seconds per experienced failure', False)