proxywatchd: always use a new mysql obj
could prevent memleaks, also helps thread-ownership issues.
This commit is contained in:
@@ -132,13 +132,18 @@ class Proxywatchd():
|
||||
wt.term()
|
||||
self.collect_work()
|
||||
self.submit_collected()
|
||||
self.mysqlite.close()
|
||||
self.stopped.set()
|
||||
|
||||
def finish(self):
|
||||
if not self.in_background: self._cleanup()
|
||||
while not self.stopped.is_set(): time.sleep(0.1)
|
||||
|
||||
def _prep_db(self):
|
||||
self.mysqlite = mysqlite.mysqlite(config.database, str)
|
||||
def _close_db(self):
|
||||
if self.mysqlite:
|
||||
self.mysqlite.close()
|
||||
self.mysqlite = None
|
||||
def __init__(self):
|
||||
config.load()
|
||||
self.in_background = False
|
||||
@@ -147,11 +152,10 @@ class Proxywatchd():
|
||||
self.stopped = threading.Event()
|
||||
|
||||
# create table if needed
|
||||
self.mysqlite = mysqlite.mysqlite(config.database, str)
|
||||
self._prep_db()
|
||||
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
|
||||
self.mysqlite.commit()
|
||||
self.mysqlite.close()
|
||||
self.mysqlite = None
|
||||
self._close_db()
|
||||
|
||||
self.submit_after = 200 # number of collected jobs before writing db
|
||||
self.echoise = time.time() - 3600;
|
||||
@@ -160,21 +164,25 @@ class Proxywatchd():
|
||||
self.collected = []
|
||||
|
||||
def prepare_jobs(self):
|
||||
self._prep_db()
|
||||
q = 'SELECT proxy,proto,failed FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM()' # ' LIMIT ?'
|
||||
rows = self.mysqlite.execute(q, (config.maxfail, time.time())).fetchall()
|
||||
for row in rows:
|
||||
job = WorkerJob(row[0], row[1], row[2])
|
||||
self.jobs.append(job)
|
||||
self._close_db()
|
||||
|
||||
def collect_work(self):
|
||||
for wt in self.threads:
|
||||
self.collected.extend(wt.collect())
|
||||
|
||||
def submit_collected(self):
|
||||
self._prep_db()
|
||||
query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?'
|
||||
args = [ (job.failcount, job.nextcheck, 1, 'unknown', job.proto, job.duration, job.proxy) for job in self.collected ]
|
||||
self.mysqlite.executemany(query, args)
|
||||
self.mysqlite.commit()
|
||||
self._close_db()
|
||||
self.collected = []
|
||||
|
||||
def start(self):
|
||||
@@ -194,7 +202,6 @@ class Proxywatchd():
|
||||
|
||||
def _run(self):
|
||||
_log('Starting proxywatchd..', 'notice')
|
||||
self.mysqlite = mysqlite.mysqlite(config.database, str)
|
||||
|
||||
for i in range(config.watchd_threads):
|
||||
threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
|
||||
|
||||
Reference in New Issue
Block a user