username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fetch: unify known proxies cache

Browse Source
This commit is contained in:
Username
2025-12-21 23:37:58 +01:00
parent 747e6dd7aa
commit 9e7c8d78b3
1 changed files with 16 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
fetch.py
View File

@@ -5,6 +5,9 @@ from soup_parser import soupify
from misc import _log from misc import _log
config = None config = None
_last_fail_log = 0
_fail_log_interval = 60
def set_config(cfg): def set_config(cfg):
global config global config
config = cfg config = cfg
@@ -42,13 +45,19 @@ def _fetch_contents(url, head = False, proxy=None):
] ]
if config.ppf.debug: if config.ppf.debug:
_log("connecting to %s... (header: %s)" % (url, str(head)), "debug") _log("connecting to %s... (header: %s)" % (url, str(head)), "debug")
tor_retries = 0
max_tor_retries = 1
while True: while True:
proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))] proxies = [rocksock.RocksockProxyFromURL('socks4://%s' % random.choice( config.torhosts ))]
if proxy: proxies.append( rocksock.RocksockProxyFromURL(proxy)) if proxy: proxies.append( rocksock.RocksockProxyFromURL(proxy))
http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0') http = RsHttp(host,ssl=ssl,port=port, keep_alive=True, timeout=config.ppf.timeout, max_tries=config.ppf.http_retries, follow_redirects=True, auto_set_cookies=True, proxies=proxies, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0', log_errors=False)
if not http.connect(): if not http.connect():
_log("failed to connect to %s"%url, "ppf") global _last_fail_log
now = time.time()
if (now - _last_fail_log) >= _fail_log_interval:
_log("failed to connect to %s"%url, "ppf")
_last_fail_log = now
e = http.get_last_rocksock_exception() e = http.get_last_rocksock_exception()
if not e: if not e:
return None return None
@@ -58,7 +67,11 @@ def _fetch_contents(url, head = False, proxy=None):
if et == rocksock.RS_ET_OWN and \ if et == rocksock.RS_ET_OWN and \
ee == rocksock.RS_E_TARGET_CONN_REFUSED \ ee == rocksock.RS_E_TARGET_CONN_REFUSED \
and ef == 0: and ef == 0:
_log("could not connect to proxy 0 - check your connection", "error") tor_retries += 1
if tor_retries >= max_tor_retries:
_log("tor proxy failed after %d retries" % tor_retries, "error")
return None
_log("tor proxy retry %d/%d" % (tor_retries, max_tor_retries), "warn")
time.sleep(5) time.sleep(5)
continue continue
return None return None