diff --git a/config.ini.sample b/config.ini.sample index 0182c75..459837c 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -13,6 +13,7 @@ debug = false [ppf] search = true timeout = 30 +http_retries = 1 checktime = 3600 perfail_checktime = 3600 diff --git a/config.py b/config.py index 4cf79a9..c632d70 100644 --- a/config.py +++ b/config.py @@ -20,5 +20,6 @@ class Config(ComboParser): self.add_item('ppf', 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False) self.add_item('ppf', 'timeout', float, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False) + self.add_item('ppf', 'http_retries', int, 1, 'number of retries for http connects', False) self.add_item('ppf', 'checktime', int, 3600, 'base checking interval for urls in db in seconds', False) self.add_item('ppf', 'perfail_checktime', int, 3600, 'additional checking interval for urls in db in seconds per experienced failure', False) diff --git a/ppf.py b/ppf.py index 3287118..9c43b71 100755 --- a/ppf.py +++ b/ppf.py @@ -44,7 +44,7 @@ def fetch_contents(url): 'Cache-Control: max-age=0', ] proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))] - http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=1, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') + http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') if not http.connect(): _log("failed to connect to %s"%url, "ppf") return ''