From 74d9d965bb3cf7bfdc8d88ab1a8d308b686c6a14 Mon Sep 17 00:00:00 2001
From: rofl0r <retnyg@gmx.net>
Date: Sat, 5 Jan 2019 18:14:41 +0000
Subject: [PATCH 1/2] proxywatchd: always use a new mysql obj

could prevent memleaks, also helps thread-ownership issues.
---
 proxywatchd.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/proxywatchd.py b/proxywatchd.py
index 8c3db48..3970277 100644
--- a/proxywatchd.py
+++ b/proxywatchd.py
@@ -132,13 +132,18 @@ class Proxywatchd():
 			wt.term()
 		self.collect_work()
 		self.submit_collected()
-		self.mysqlite.close()
 		self.stopped.set()
 
 	def finish(self):
 		if not self.in_background: self._cleanup()
 		while not self.stopped.is_set(): time.sleep(0.1)
 
+	def _prep_db(self):
+		self.mysqlite = mysqlite.mysqlite(config.database, str)
+	def _close_db(self):
+		if self.mysqlite:
+			self.mysqlite.close()
+			self.mysqlite = None
 	def __init__(self):
 		config.load()
 		self.in_background = False
@@ -147,11 +152,10 @@ class Proxywatchd():
 		self.stopped = threading.Event()
 
 		# create table if needed
-		self.mysqlite = mysqlite.mysqlite(config.database, str)
+		self._prep_db()
 		self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
 		self.mysqlite.commit()
-		self.mysqlite.close()
-		self.mysqlite = None
+		self._close_db()
 
 		self.submit_after = 200 # number of collected jobs before writing db
 		self.echoise = time.time() - 3600;
@@ -160,21 +164,25 @@ class Proxywatchd():
 		self.collected = []
 
 	def prepare_jobs(self):
+		self._prep_db()
 		q = 'SELECT proxy,proto,failed FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM()' # ' LIMIT ?'
 		rows = self.mysqlite.execute(q, (config.maxfail, time.time())).fetchall()
 		for row in rows:
 			job = WorkerJob(row[0], row[1], row[2])
 			self.jobs.append(job)
+		self._close_db()
 
 	def collect_work(self):
 		for wt in self.threads:
 			self.collected.extend(wt.collect())
 
 	def submit_collected(self):
+		self._prep_db()
 		query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?'
 		args = [ (job.failcount, job.nextcheck, 1, 'unknown', job.proto, job.duration, job.proxy) for job in self.collected ]
 		self.mysqlite.executemany(query, args)
 		self.mysqlite.commit()
+		self._close_db()
 		self.collected = []
 
 	def start(self):
@@ -194,7 +202,6 @@ class Proxywatchd():
 
 	def _run(self):
 		_log('Starting proxywatchd..', 'notice')
-		self.mysqlite = mysqlite.mysqlite(config.database, str)
 
 		for i in range(config.watchd_threads):
 			threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )

From dacbc8177a56fe9142f71f19c4da834ae7c9b0ed Mon Sep 17 00:00:00 2001
From: rofl0r <retnyg@gmx.net>
Date: Sat, 5 Jan 2019 18:31:36 +0000
Subject: [PATCH 2/2] proxywatchd: fix memleak

---
 proxywatchd.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/proxywatchd.py b/proxywatchd.py
index 3970277..6d81897 100644
--- a/proxywatchd.py
+++ b/proxywatchd.py
@@ -216,9 +216,10 @@ class Proxywatchd():
 				if self.in_background: self._cleanup()
 				break
 
-			if len(self.jobs) == 0:
+			if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0:
 				self.prepare_jobs()
 				if len(self.jobs):
+					_log("watchd main: handing out %d jobs"%len(self.jobs))
 					jpt = len(self.jobs)/config.watchd_threads
 					if len(self.jobs)/float(config.watchd_threads) - jpt > 0.0: jpt += 1
 					for tid in range(config.watchd_threads):