fetch: add unified proxy cache functions
This commit is contained in:
20
fetch.py
20
fetch.py
@@ -97,6 +97,26 @@ def is_usable_proxy(proxy):
|
||||
return True
|
||||
|
||||
_known_proxies = {}
|
||||
|
||||
def init_known_proxies(proxydb):
|
||||
"""Initialize known proxies cache from database."""
|
||||
global _known_proxies
|
||||
if _known_proxies:
|
||||
return
|
||||
known = proxydb.execute('SELECT proxy FROM proxylist').fetchall()
|
||||
for k in known:
|
||||
_known_proxies[k[0]] = True
|
||||
|
||||
def add_known_proxies(proxies):
|
||||
"""Add proxies to known cache."""
|
||||
global _known_proxies
|
||||
for p in proxies:
|
||||
_known_proxies[p] = True
|
||||
|
||||
def is_known_proxy(proxy):
|
||||
"""Check if proxy is in known cache."""
|
||||
return proxy in _known_proxies
|
||||
|
||||
def extract_proxies(content, proxydb):
|
||||
matches = re.findall(r'([0-9]+(?:\.[0-9]+){3}:[0-9]{2,5})[\D$]', cleanhtml(content))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user