username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

misc changes and fixes and

Browse Source
This commit is contained in:
Your Name
2021-02-06 15:36:14 +01:00
parent b29c734002
commit 68394da9ab
3 changed files with 57 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
ppf.py
View File

@@ -234,7 +234,7 @@ class Leechered(threading.Thread):
threading.Thread.__init__(self)
def retrieve(self):
return self.url, self.proxylist, self.stale_count, self.error, self.retrievals, self.content_type, self.execute
return self.url, self.proxylist, self.stale_count, self.error, self.retrievals, self.content_type, self.proxies_added, self.execute
def status(self):
return self.status
@@ -258,6 +258,8 @@ class Leechered(threading.Thread):
if content != '' and proxy_count == 0: # site works but has zero proxy addresses
#error = 99999
self.error += 2
else:
self.error += 1
else:
if len(self.proxylist) == 0:
self.stale_count += 1
@@ -307,25 +309,40 @@ if __name__ == '__main__':
#start_server(config.httpd.listenip, config.httpd.port)
qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25'
#qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25'
qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM()'
threads = []
rows = []
rinc = 0
reqtime = time.time() - 3600
statusmsg = time.time()
while True:
try:
if (time.time() - statusmsg) > 180:
_log('running %d thread(s) over %d' % (len(threads), config.ppf.threads), 'ppf')
statusmsg = time.time()
time.sleep(0.1)
if not len(rows):
if (time.time() - reqtime) > 3:
rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
reqtime = time.time()
if len(rows) < config.ppf.threads:
rows = []
else:
_log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
for thread in threads:
if thread.status == 'ok':
url, proxylist, stale_count, error, retrievals, content_type, execute = thread.retrieve()
url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve()
new = []
for p in proxylist:
#print(p)
if not p in _known_proxies:
new.append(p)
_known_proxies[p]=1
execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url)
urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute)
if len(proxylist): dbs.insert_proxies(proxydb, proxylist, url)
if len(new): dbs.insert_proxies(proxydb, new, url)
threads = [ thread for thread in threads if thread.is_alive() ]
if len(threads) < config.ppf.threads and len(rows):