username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: url → self.url, make thread option configurable

Browse Source
This commit is contained in:
Your Name
2021-02-06 14:33:44 +01:00
parent abd9b5bb9f
commit b29c734002
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config.py
View File

@@ -39,6 +39,7 @@ class Config(ComboParser):
self.add_item(section, 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False) self.add_item(section, 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False)
self.add_item(section, 'timeout', float, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False) self.add_item(section, 'timeout', float, 15, 'timeout for blocking operations (connect/recv/...) for proxy checks in seconds', False)
self.add_item(section, 'http_retries', int, 1, 'number of retries for http connects', False) self.add_item(section, 'http_retries', int, 1, 'number of retries for http connects', False)
self.add_item(section, 'threads', int, 1, 'number of threads to run (default: 1)', False)
self.add_item(section, 'checktime', int, 3600, 'base checking interval for urls in db in seconds', False) self.add_item(section, 'checktime', int, 3600, 'base checking interval for urls in db in seconds', False)
self.add_item(section, 'perfail_checktime', int, 3600, 'additional checking interval for urls in db in seconds per resultless check', False) self.add_item(section, 'perfail_checktime', int, 3600, 'additional checking interval for urls in db in seconds per resultless check', False)
self.add_item(section, 'max_fail', int, 5, 'number of fails after which an url is considered dead', False) self.add_item(section, 'max_fail', int, 5, 'number of fails after which an url is considered dead', False)

6
ppf.py
View File

@@ -241,7 +241,7 @@ class Leechered(threading.Thread):
def run(self): def run(self):
self.status = 'nok' self.status = 'nok'
if not self.content_type: self.content_type = get_content_type(url) if not self.content_type: self.content_type = get_content_type(self.url)
if is_good_content_type(self.content_type): if is_good_content_type(self.content_type):
try: content = fetch.fetch_contents(self.url) try: content = fetch.fetch_contents(self.url)
@@ -318,7 +318,7 @@ if __name__ == '__main__':
if (time.time() - reqtime) > 3: if (time.time() - reqtime) > 3:
rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
reqtime = time.time() reqtime = time.time()
if len(rows) < 5: if len(rows) < config.ppf.threads:
rows = [] rows = []
for thread in threads: for thread in threads:
@@ -328,7 +328,7 @@ if __name__ == '__main__':
if len(proxylist): dbs.insert_proxies(proxydb, proxylist, url) if len(proxylist): dbs.insert_proxies(proxydb, proxylist, url)
threads = [ thread for thread in threads if thread.is_alive() ] threads = [ thread for thread in threads if thread.is_alive() ]
if len(threads) < 5 and len(rows): if len(threads) < config.ppf.threads and len(rows):
row = random.choice(rows) row = random.choice(rows)
rows.remove(row) rows.remove(row)
t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5]) t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5])