fetch: retry with different Tor circuit on failure
This commit is contained in:
26
fetch.py
26
fetch.py
@@ -511,29 +511,19 @@ def _fetch_contents(url, head = False, proxy=None):
|
|||||||
|
|
||||||
http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0', log_errors=False)
|
http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0', log_errors=False)
|
||||||
if not http.connect():
|
if not http.connect():
|
||||||
|
http.disconnect()
|
||||||
|
http = None
|
||||||
|
tor_retries += 1
|
||||||
|
if tor_retries <= max_tor_retries:
|
||||||
|
# Retry once with different circuit
|
||||||
|
time.sleep(1)
|
||||||
|
continue
|
||||||
|
# Log failure after retries exhausted
|
||||||
global _last_fail_log
|
global _last_fail_log
|
||||||
now = time.time()
|
now = time.time()
|
||||||
if (now - _last_fail_log) >= _fail_log_interval:
|
if (now - _last_fail_log) >= _fail_log_interval:
|
||||||
_log("failed to connect to %s"%url, "ppf")
|
_log("failed to connect to %s"%url, "ppf")
|
||||||
_last_fail_log = now
|
_last_fail_log = now
|
||||||
e = http.get_last_rocksock_exception()
|
|
||||||
if not e:
|
|
||||||
return None
|
|
||||||
et = e.get_errortype()
|
|
||||||
ee = e.get_error()
|
|
||||||
ef = e.get_failedproxy()
|
|
||||||
if et == rocksock.RS_ET_OWN and \
|
|
||||||
ee == rocksock.RS_E_TARGET_CONN_REFUSED \
|
|
||||||
and ef == 0:
|
|
||||||
http.disconnect()
|
|
||||||
http = None
|
|
||||||
tor_retries += 1
|
|
||||||
if tor_retries >= max_tor_retries:
|
|
||||||
_log("tor proxy failed after %d retries" % tor_retries, "error")
|
|
||||||
return None
|
|
||||||
_log("tor proxy retry %d/%d" % (tor_retries, max_tor_retries), "warn")
|
|
||||||
time.sleep(5)
|
|
||||||
continue
|
|
||||||
return None
|
return None
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user