username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ppf: add option for number of http retries

Browse Source
This commit is contained in:
rofl0r
2019-01-08 03:30:31 +00:00
parent 115c4a56f5
commit 38d89f5bd9
3 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config.ini.sample
View File

@@ -13,6 +13,7 @@ debug = false
[ppf] [ppf]
search = true search = true
timeout = 30 timeout = 30
http_retries = 1
checktime = 3600 checktime = 3600
perfail_checktime = 3600 perfail_checktime = 3600

1
config.py
View File

@@ -20,5 +20,6 @@ class Config(ComboParser):
self.add_item('ppf', 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False) self.add_item('ppf', 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False)
self.add_item('ppf', 'timeout', float, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False) self.add_item('ppf', 'timeout', float, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False)
self.add_item('ppf', 'http_retries', int, 1, 'number of retries for http connects', False)
self.add_item('ppf', 'checktime', int, 3600, 'base checking interval for urls in db in seconds', False) self.add_item('ppf', 'checktime', int, 3600, 'base checking interval for urls in db in seconds', False)
self.add_item('ppf', 'perfail_checktime', int, 3600, 'additional checking interval for urls in db in seconds per experienced failure', False) self.add_item('ppf', 'perfail_checktime', int, 3600, 'additional checking interval for urls in db in seconds per experienced failure', False)

2
ppf.py
View File

@@ -44,7 +44,7 @@ def fetch_contents(url):
'Cache-Control: max-age=0', 'Cache-Control: max-age=0',
] ]
proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))] proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))]
http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=1, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0')
if not http.connect(): if not http.connect():
_log("failed to connect to %s"%url, "ppf") _log("failed to connect to %s"%url, "ppf")
return '' return ''