fetch: retry with different Tor circuit on failure
This commit is contained in:
26
fetch.py
26
fetch.py
@@ -511,29 +511,19 @@ def _fetch_contents(url, head = False, proxy=None):
|
||||
|
||||
http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0', log_errors=False)
|
||||
if not http.connect():
|
||||
http.disconnect()
|
||||
http = None
|
||||
tor_retries += 1
|
||||
if tor_retries <= max_tor_retries:
|
||||
# Retry once with different circuit
|
||||
time.sleep(1)
|
||||
continue
|
||||
# Log failure after retries exhausted
|
||||
global _last_fail_log
|
||||
now = time.time()
|
||||
if (now - _last_fail_log) >= _fail_log_interval:
|
||||
_log("failed to connect to %s"%url, "ppf")
|
||||
_last_fail_log = now
|
||||
e = http.get_last_rocksock_exception()
|
||||
if not e:
|
||||
return None
|
||||
et = e.get_errortype()
|
||||
ee = e.get_error()
|
||||
ef = e.get_failedproxy()
|
||||
if et == rocksock.RS_ET_OWN and \
|
||||
ee == rocksock.RS_E_TARGET_CONN_REFUSED \
|
||||
and ef == 0:
|
||||
http.disconnect()
|
||||
http = None
|
||||
tor_retries += 1
|
||||
if tor_retries >= max_tor_retries:
|
||||
_log("tor proxy failed after %d retries" % tor_retries, "error")
|
||||
return None
|
||||
_log("tor proxy retry %d/%d" % (tor_retries, max_tor_retries), "warn")
|
||||
time.sleep(5)
|
||||
continue
|
||||
return None
|
||||
break
|
||||
|
||||
|
||||
Reference in New Issue
Block a user