refactor core modules, integrate network stats

fetch.py

@@ -1,5 +1,7 @@
 import re, random, time
+import threading
 import rocksock
+import network_stats
 from http2 import RsHttp, _parse_url
 from soup_parser import soupify
 from misc import _log
@@ -43,6 +45,7 @@ def fetch_contents(url, head=False, proxy=None):
 
 retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded')
 def _fetch_contents(url, head = False, proxy=None):
+    network_stats.set_category('scraper')
     host, port, ssl, uri = _parse_url(url)
     headers=[
         'Accept-Language: en-US,en;q=0.8',
@@ -52,47 +55,54 @@ def _fetch_contents(url, head = False, proxy=None):
         _log("connecting to %s... (header: %s)" % (url, str(head)), "debug")
     tor_retries = 0
     max_tor_retries = 1
-    while True:
-        proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))]
-        if proxy: proxies.append( rocksock.RocksockProxyFromURL(proxy))
+    http = None
+    try:
+        while True:
+            proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))]
+            if proxy: proxies.append( rocksock.RocksockProxyFromURL(proxy))
 
-        http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0', log_errors=False)
-        if not http.connect():
-            global _last_fail_log
-            now = time.time()
-            if (now - _last_fail_log) >= _fail_log_interval:
-                _log("failed to connect to %s"%url, "ppf")
-                _last_fail_log = now
-            e = http.get_last_rocksock_exception()
-            if not e:
-                return None
-            et = e.get_errortype()
-            ee = e.get_error()
-            ef = e.get_failedproxy()
-            if et == rocksock.RS_ET_OWN and \
-            ee == rocksock.RS_E_TARGET_CONN_REFUSED \
-            and ef == 0:
-                tor_retries += 1
-                if tor_retries >= max_tor_retries:
-                    _log("tor proxy failed after %d retries" % tor_retries, "error")
+            http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0', log_errors=False)
+            if not http.connect():
+                global _last_fail_log
+                now = time.time()
+                if (now - _last_fail_log) >= _fail_log_interval:
+                    _log("failed to connect to %s"%url, "ppf")
+                    _last_fail_log = now
+                e = http.get_last_rocksock_exception()
+                if not e:
                     return None
-                _log("tor proxy retry %d/%d" % (tor_retries, max_tor_retries), "warn")
-                time.sleep(5)
-                continue
-            return None
-        break
+                et = e.get_errortype()
+                ee = e.get_error()
+                ef = e.get_failedproxy()
+                if et == rocksock.RS_ET_OWN and \
+                ee == rocksock.RS_E_TARGET_CONN_REFUSED \
+                and ef == 0:
+                    http.disconnect()
+                    http = None
+                    tor_retries += 1
+                    if tor_retries >= max_tor_retries:
+                        _log("tor proxy failed after %d retries" % tor_retries, "error")
+                        return None
+                    _log("tor proxy retry %d/%d" % (tor_retries, max_tor_retries), "warn")
+                    time.sleep(5)
+                    continue
+                return None
+            break
 
-    ## only request header
-    if head:
-        hdr = http.head(uri, headers)
-        return hdr
+        ## only request header
+        if head:
+            hdr = http.head(uri, headers)
+            return hdr
 
-    hdr, res = http.get(uri, headers)
-    res = res.encode('utf-8') if isinstance(res, unicode) else res
-    for retry_message in retry_messages:
-        if retry_message in res: return None
+        hdr, res = http.get(uri, headers)
+        res = res.encode('utf-8') if isinstance(res, unicode) else res
+        for retry_message in retry_messages:
+            if retry_message in res: return None
 
-    return res
+        return res
+    finally:
+        if http:
+            http.disconnect()
 
 def valid_port(port):
     """Check if port number is valid (1-65535)."""
@@ -176,25 +186,29 @@ def is_usable_proxy(proxy):
         return False
 
 _known_proxies = {}
+_known_proxies_lock = threading.Lock()
 
 def init_known_proxies(proxydb):
     """Initialize known proxies cache from database."""
     global _known_proxies
-    if _known_proxies:
-        return
-    known = proxydb.execute('SELECT proxy FROM proxylist').fetchall()
-    for k in known:
-        _known_proxies[k[0]] = True
+    with _known_proxies_lock:
+        if _known_proxies:
+            return
+        known = proxydb.execute('SELECT proxy FROM proxylist').fetchall()
+        for k in known:
+            _known_proxies[k[0]] = True
 
 def add_known_proxies(proxies):
     """Add proxies to known cache."""
     global _known_proxies
-    for p in proxies:
-        _known_proxies[p] = True
+    with _known_proxies_lock:
+        for p in proxies:
+            _known_proxies[p] = True
 
 def is_known_proxy(proxy):
     """Check if proxy is in known cache."""
-    return proxy in _known_proxies
+    with _known_proxies_lock:
+        return proxy in _known_proxies
 
 def detect_proto_from_path(url):
     """Detect proxy protocol from URL path.