ppf: make scraper use extra proxies if available

fetch.py

@@ -21,8 +21,20 @@ def cleanhtml(raw_html):
 	html = re.sub(cleanhtml_re[2], ':', html)
 	return html
 
+def fetch_contents(url, head=False, proxy=None):
+	content = None
+	if len(proxy):
+		for p in proxy:
+			content = _fetch_contents(url, head=head, proxy=p)
+			if content is not None: break
+
+	else:
+		content = _fetch_contents(url, head=head)
+
+	return content if content is not None else ''
+
 retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded')
-def fetch_contents(url, head = False):
+def _fetch_contents(url, head = False, proxy=None):
 	host, port, ssl, uri = _parse_url(url)
 	headers=[
 		'Accept-Language: en-US,en;q=0.8',
@@ -32,12 +44,14 @@ def fetch_contents(url, head = False):
 		_log("connecting to %s... (header: %s)" % (url, str(head)), "debug")
 	while True:
 		proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))]
+		if proxy: proxies.append( rocksock.RocksockProxyFromURL(proxy))
+
 		http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0')
 		if not http.connect():
 			_log("failed to connect to %s"%url, "ppf")
 			e = http.get_last_rocksock_exception()
 			if not e:
-				return ''
+				return None
 			et = e.get_errortype()
 			ee = e.get_error()
 			ef = e.get_failedproxy()
@@ -47,7 +61,7 @@ def fetch_contents(url, head = False):
 				_log("could not connect to proxy 0 - check your connection", "error")
 				time.sleep(5)
 				continue
-			return ''
+			return None
 		break
 
 	## only request header
@@ -58,7 +72,7 @@ def fetch_contents(url, head = False):
 	hdr, res = http.get(uri, headers)
 	res = res.encode('utf-8') if isinstance(res, unicode) else res
 	for retry_message in retry_messages:
-		if retry_message in res: return ''
+		if retry_message in res: return None
 
 	return res