username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

take care of old proxies

Browse Source 
test old proxies during free time
This commit is contained in:
mickael
2019-03-03 09:42:49 +00:00
parent 2bacf77c8c
commit 1ad5ca53e5
2 changed files with 20 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config.py
View File

@@ -22,6 +22,9 @@ class Config(ComboParser):
self.add_item(section, 'checktime', int, 1800, 'base checking interval for proxies in db in seconds', False)
self.add_item(section, 'perfail_checktime', int, 3600, 'additional checking interval for proxies in db in seconds per experienced failure', False)
self.add_item(section, 'database', str, 'websites.sqlite', 'filename of database', True)
self.add_item(section, 'oldies', bool, False, 're-test old proxies as well ? (default: False)', False)
self.add_item(section, 'oldies_checktime', int, 43200, 'base checking interval for *old* proxies in seconds (default: 43200)', False)
self.add_item(section, 'oldies_multi', int, 100, 'fetch threads*multi rows when testing oldies (default: 100)', False)
section = 'ppf'
self.add_item(section, 'debug', bool, False, 'whether to print additional debug info', False)

22
proxywatchd.py
View File

@@ -256,10 +256,22 @@ class Proxywatchd():
'success':0,
}
def fetch_rows(self):
q = 'SELECT proxy,proto,failed,success_count,total_duration FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()'
rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall()
# check oldies ?
if len(rows) < config.watchd.threads and config.watchd.oldies:
## disable tor safeguard for old proxies
self.tor_safeguard = False
q += ' LIMIT ?'
rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time(), config.watchd.threads*config.watchd.oldies_multi)).fetchall()
return rows
def prepare_jobs(self):
self._prep_db()
q = 'SELECT proxy,proto,failed,success_count,total_duration FROM proxylist WHERE failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()' # ' LIMIT ?'
rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime , time.time())).fetchall()
## enable tor safeguard by default
self.tor_safeguard = True
rows = self.fetch_rows()
for row in rows:
job = WorkerJob(row[0], row[1], row[2], row[3], row[4])
self.jobs.append(job)
@@ -286,7 +298,7 @@ class Proxywatchd():
success_rate = (float(sc) / len(self.collected)) * 100
ret = True
if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold:
if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold and self.tor_safeguard:
_log("WATCHD %.2f%% SUCCESS RATE - tor circuit blocked? won't submit fails"%success_rate, "ERROR")
if sc == 0: return False
args = []
@@ -351,7 +363,7 @@ class Proxywatchd():
self.collect_unfinished()
if not len(self.jobs):
self.collect_work()
if not self.submit_collected():
if not self.submit_collected() and self.tor_safeguard:
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
self.collect_unfinished()
sleeptime = 1*60
@@ -375,7 +387,7 @@ class Proxywatchd():
self.collect_work()
if len(self.collected) > self.submit_after:
if not self.submit_collected():
if not self.submit_collected() and self.tor_safeguard:
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
self.collect_unfinished()
sleeptime = 1*60