From b29c7340020104ec5d0054170e3e454defeafa1f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 6 Feb 2021 14:33:44 +0100 Subject: [PATCH] =?UTF-8?q?fix:=20url=20=E2=86=92=20self.url,=20make=20thr?= =?UTF-8?q?ead=20option=20configurable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 1 + ppf.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index d939da4..25dc8d2 100644 --- a/config.py +++ b/config.py @@ -39,6 +39,7 @@ class Config(ComboParser): self.add_item(section, 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False) self.add_item(section, 'timeout', float, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False) self.add_item(section, 'http_retries', int, 1, 'number of retries for http connects', False) + self.add_item(section, 'threads', int, 1, 'number of threads to run (default: 1)', False) self.add_item(section, 'checktime', int, 3600, 'base checking interval for urls in db in seconds', False) self.add_item(section, 'perfail_checktime', int, 3600, 'additional checking interval for urls in db in seconds per resultless check', False) self.add_item(section, 'max_fail', int, 5, 'number of fails after which an url is considered dead', False) diff --git a/ppf.py b/ppf.py index 691f20f..3180dc8 100755 --- a/ppf.py +++ b/ppf.py @@ -241,7 +241,7 @@ class Leechered(threading.Thread): def run(self): self.status = 'nok' - if not self.content_type: self.content_type = get_content_type(url) + if not self.content_type: self.content_type = get_content_type(self.url) if is_good_content_type(self.content_type): try: content = fetch.fetch_contents(self.url) @@ -318,7 +318,7 @@ if __name__ == '__main__': if (time.time() - reqtime) > 3: rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() reqtime = time.time() - if len(rows) < 5: + if len(rows) < config.ppf.threads: rows = [] for thread in threads: @@ -328,7 +328,7 @@ if __name__ == '__main__': if len(proxylist): dbs.insert_proxies(proxydb, proxylist, url) threads = [ thread for thread in threads if thread.is_alive() ] - if len(threads) < 5 and len(rows): + if len(threads) < config.ppf.threads and len(rows): row = random.choice(rows) rows.remove(row) t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5])