diff --git a/fetch.py b/fetch.py index 8ac8b67..d85b75f 100644 --- a/fetch.py +++ b/fetch.py @@ -97,6 +97,26 @@ def is_usable_proxy(proxy): return True _known_proxies = {} + +def init_known_proxies(proxydb): + """Initialize known proxies cache from database.""" + global _known_proxies + if _known_proxies: + return + known = proxydb.execute('SELECT proxy FROM proxylist').fetchall() + for k in known: + _known_proxies[k[0]] = True + +def add_known_proxies(proxies): + """Add proxies to known cache.""" + global _known_proxies + for p in proxies: + _known_proxies[p] = True + +def is_known_proxy(proxy): + """Check if proxy is in known cache.""" + return proxy in _known_proxies + def extract_proxies(content, proxydb): matches = re.findall(r'([0-9]+(?:\.[0-9]+){3}:[0-9]{2,5})[\D$]', cleanhtml(content))