fetch: unify known proxies cache

fetch.py

@@ -5,6 +5,9 @@ from soup_parser import soupify
 from misc import _log
 
 config = None
+_last_fail_log = 0
+_fail_log_interval = 60
+
 def set_config(cfg):
     global config
     config = cfg
@@ -42,13 +45,19 @@ def _fetch_contents(url, head = False, proxy=None):
     ]
     if config.ppf.debug:
         _log("connecting to %s... (header: %s)" % (url, str(head)), "debug")
+    tor_retries = 0
+    max_tor_retries = 1
     while True:
         proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))]
         if proxy: proxies.append( rocksock.RocksockProxyFromURL(proxy))
 
-        http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0')
+        http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0', log_errors=False)
         if not http.connect():
-            _log("failed to connect to %s"%url, "ppf")
+            global _last_fail_log
+            now = time.time()
+            if (now - _last_fail_log) >= _fail_log_interval:
+                _log("failed to connect to %s"%url, "ppf")
+                _last_fail_log = now
             e = http.get_last_rocksock_exception()
             if not e:
                 return None
@@ -58,7 +67,11 @@ def _fetch_contents(url, head = False, proxy=None):
             if et == rocksock.RS_ET_OWN and \
             ee == rocksock.RS_E_TARGET_CONN_REFUSED \
             and ef == 0:
-                _log("could not connect to proxy 0 - check your connection", "error")
+                tor_retries += 1
+                if tor_retries >= max_tor_retries:
+                    _log("tor proxy failed after %d retries" % tor_retries, "error")
+                    return None
+                _log("tor proxy retry %d/%d" % (tor_retries, max_tor_retries), "warn")
                 time.sleep(5)
                 continue
             return None