From c759f7197edfa40887511a862791d692abdd8a55 Mon Sep 17 00:00:00 2001 From: Username Date: Sat, 20 Dec 2025 22:28:42 +0100 Subject: [PATCH] ppf: use shared proxy cache from fetch module --- ppf.py | 58 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/ppf.py b/ppf.py index caaf411..e8e210d 100755 --- a/ppf.py +++ b/ppf.py @@ -14,15 +14,14 @@ import threading import random config = Config() -_known_proxies = {} def import_from_file(fn, sqlite): with open(fn, 'r') as f: - urls = [ url for url in f.read().split('\n') if url != '' ] + urls = [ url for url in f.read().split('\n') if url ] cinc = 0 while True: chunk = urls[cinc:cinc+200] - if len(chunk): dbs.insert_urls(chunk, 'import.txt', urldb) + if chunk: dbs.insert_urls(chunk, 'import.txt', urldb) else: break cinc = cinc + 200 @@ -71,12 +70,12 @@ def extract_urls(html, url): continue if not item in urls: urls.append(item) - if len(urls): dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls) + if urls: dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls) def import_proxies_from_file(proxydb, fn): content = open(fn, 'r').read() unique_count, new = fetch.extract_proxies(content, proxydb) - if len(new): + if new: dbs.insert_proxies(proxydb, new, fn) return 0 return 1 @@ -126,21 +125,25 @@ class Leechered(threading.Thread): if not self.content_type: self.content_type = get_content_type(self.url, self.proxy) if is_good_content_type(self.content_type): - try: content = fetch.fetch_contents(self.url, proxy=self.proxy) - except KeyboardInterrupt as e: raise e - except: content = '' + try: + content = fetch.fetch_contents(self.url, proxy=self.proxy) + except KeyboardInterrupt as e: + raise e + except Exception as e: + _log('%s: fetch error: %s' % (self.url.split('/')[2], str(e)), 'error') + content = '' else: content = '' unique = extract_proxies(content) - self.proxylist = [ proxy for proxy in unique if not proxy in _known_proxies ] + self.proxylist = [ proxy for proxy in unique if not fetch.is_known_proxy(proxy) ] proxy_count = len(self.proxylist) if self.retrievals == 0: # new site - if content != '' and len(self.proxylist) == 0: # site works but has zero proxy addresses + if content and not self.proxylist: # site works but has zero proxy addresses self.error += 1 self.stale_count += 1 - elif proxy_count > 0: + elif proxy_count: self.error = 0 self.stale_count = 0 else: @@ -148,14 +151,14 @@ class Leechered(threading.Thread): self.stale_count += 2 else: # not a new site # proxylist is empty - if proxy_count == 0: + if not proxy_count: self.stale_count += 1 # proxylist is not empty: site is working else: self.stale_count = 0 self.error = 0 # site has no content - if content == '': + if not content: self.error += 1 self.stale_count += 1 #else: @@ -174,6 +177,11 @@ class Leechered(threading.Thread): if __name__ == '__main__': config.load() + errors = config.validate() + if errors: + for e in errors: + _log(e, 'error') + sys.exit(1) fetch.set_config(config) # handle --nobs flag @@ -184,12 +192,10 @@ if __name__ == '__main__': proxydb = mysqlite.mysqlite(config.watchd.database, str) dbs.create_table_if_not_exists(proxydb, 'proxylist') - known = proxydb.execute('SELECT proxy FROM proxylist').fetchall() - for k in known: - _known_proxies[k[0]] = True + fetch.init_known_proxies(proxydb) with open('urignore.txt', 'r') as f: - urignore = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ] + urignore = [ i.strip() for i in f.read().split('\n') if i.strip() ] urldb = mysqlite.mysqlite(config.ppf.database, str) dbs.create_table_if_not_exists(urldb, 'uris') @@ -215,7 +221,7 @@ if __name__ == '__main__': if (time.time() - statusmsg) > 180: _log('running %d thread(s) over %d' % (len(threads), config.ppf.threads), 'ppf') statusmsg = time.time() - if not len(rows): + if not rows: if (time.time() - reqtime) > 3: rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() reqtime = time.time() @@ -226,23 +232,21 @@ if __name__ == '__main__': _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf') _proxylist = [ '%s://%s' % (p[0], p[1]) for p in proxydb.execute('SELECT proto,proxy from proxylist where failed=0').fetchall() ] - if len(_proxylist) == 0: _proxylist = None + if not _proxylist: _proxylist = None for thread in threads: if thread.status == 'ok': url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve() - new = [] - for p in proxylist: - if not p in _known_proxies: - new.append(p) - _known_proxies[p]=1 + new = [ p for p in proxylist if not fetch.is_known_proxy(p) ] + if new: + fetch.add_known_proxies(new) execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url) urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute) urldb.commit() - if len(new): dbs.insert_proxies(proxydb, new, url) + if new: dbs.insert_proxies(proxydb, new, url) threads = [ thread for thread in threads if thread.is_alive() ] - if len(threads) < config.ppf.threads and len(rows): + if len(threads) < config.ppf.threads and rows: p = random.sample(_proxylist, 5) if _proxylist is not None else None row = random.choice(rows) urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0])) @@ -258,4 +262,4 @@ if __name__ == '__main__': watcherd.finish() break - print '\r', + _log('ppf stopped', 'info')