From 1ad5ca53e520b8fea5a6c07436b0043cc277afa1 Mon Sep 17 00:00:00 2001
From: mickael <mickael@serneels.xyz>
Date: Sun, 3 Mar 2019 09:42:49 +0000
Subject: [PATCH] take care of old proxies

test old proxies during free time
---
 config.py      |  3 +++
 proxywatchd.py | 22 +++++++++++++++++-----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/config.py b/config.py
index aab303f..d0ade8d 100644
--- a/config.py
+++ b/config.py
@@ -22,6 +22,9 @@ class Config(ComboParser):
 		self.add_item(section, 'checktime', int, 1800, 'base checking interval for proxies in db in seconds', False)
 		self.add_item(section, 'perfail_checktime', int, 3600, 'additional checking interval for proxies in db in seconds per experienced failure', False)
 		self.add_item(section, 'database', str, 'websites.sqlite', 'filename of database', True)
+		self.add_item(section, 'oldies', bool, False, 're-test old proxies as well ? (default: False)', False)
+		self.add_item(section, 'oldies_checktime', int, 43200, 'base checking interval for *old* proxies in seconds (default: 43200)', False)
+		self.add_item(section, 'oldies_multi', int, 100, 'fetch threads*multi rows when testing oldies (default: 100)', False)
 
 		section = 'ppf'
 		self.add_item(section, 'debug', bool, False, 'whether to print additional debug info', False)
diff --git a/proxywatchd.py b/proxywatchd.py
index d1df82a..48de504 100644
--- a/proxywatchd.py
+++ b/proxywatchd.py
@@ -256,10 +256,22 @@ class Proxywatchd():
 			'success':0,
 		}
 
+	def fetch_rows(self):
+		q = 'SELECT proxy,proto,failed,success_count,total_duration FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()'
+		rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall()
+		# check oldies ?
+		if len(rows) < config.watchd.threads and config.watchd.oldies:
+			## disable tor safeguard for old proxies
+			self.tor_safeguard = False
+			q += ' LIMIT ?'
+			rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time(), config.watchd.threads*config.watchd.oldies_multi)).fetchall()
+		return rows
+
 	def prepare_jobs(self):
 		self._prep_db()
-		q = 'SELECT proxy,proto,failed,success_count,total_duration FROM proxylist WHERE failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()' # ' LIMIT ?'
-		rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime ,  time.time())).fetchall()
+		## enable tor safeguard by default
+		self.tor_safeguard = True
+		rows = self.fetch_rows()
 		for row in rows:
 			job = WorkerJob(row[0], row[1], row[2], row[3], row[4])
 			self.jobs.append(job)
@@ -286,7 +298,7 @@ class Proxywatchd():
 
 		success_rate = (float(sc) / len(self.collected)) * 100
 		ret = True
-		if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold:
+		if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold and self.tor_safeguard:
 			_log("WATCHD %.2f%% SUCCESS RATE - tor circuit blocked? won't submit fails"%success_rate, "ERROR")
 			if sc == 0: return False
 			args = []
@@ -351,7 +363,7 @@ class Proxywatchd():
 				self.collect_unfinished()
 				if not len(self.jobs):
 					self.collect_work()
-					if not self.submit_collected():
+					if not self.submit_collected() and self.tor_safeguard:
 						_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
 						self.collect_unfinished()
 						sleeptime = 1*60
@@ -375,7 +387,7 @@ class Proxywatchd():
 			self.collect_work()
 
 			if len(self.collected) > self.submit_after:
-				if not self.submit_collected():
+				if not self.submit_collected() and self.tor_safeguard:
 					_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
 					self.collect_unfinished()
 					sleeptime = 1*60