more changes
This commit is contained in:
695
proxywatchd.py
695
proxywatchd.py
@@ -20,405 +20,410 @@ _run_standalone = False
|
|||||||
cached_dns = dict()
|
cached_dns = dict()
|
||||||
|
|
||||||
def try_div(a, b):
|
def try_div(a, b):
|
||||||
if b != 0: return a/float(b)
|
if b != 0: return a/float(b)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
def socks4_resolve(srvname, server_port):
|
def socks4_resolve(srvname, server_port):
|
||||||
srv = srvname
|
srv = srvname
|
||||||
if srv in cached_dns:
|
if srv in cached_dns:
|
||||||
srv = cached_dns[srvname]
|
srv = cached_dns[srvname]
|
||||||
if config.watchd.debug:
|
if config.watchd.debug:
|
||||||
_log("using cached ip (%s) for %s"%(srv, srvname), "debug")
|
_log("using cached ip (%s) for %s"%(srv, srvname), "debug")
|
||||||
else:
|
else:
|
||||||
dns_fail = False
|
dns_fail = False
|
||||||
try:
|
try:
|
||||||
af, sa = rocksock.resolve(rocksock.RocksockHostinfo(srvname, server_port), want_v4=True)
|
af, sa = rocksock.resolve(rocksock.RocksockHostinfo(srvname, server_port), want_v4=True)
|
||||||
if sa is not None:
|
if sa is not None:
|
||||||
cached_dns[srvname] = sa[0]
|
cached_dns[srvname] = sa[0]
|
||||||
srv = sa[0]
|
srv = sa[0]
|
||||||
else: dns_fail = True
|
else: dns_fail = True
|
||||||
except rocksock.RocksockException as e:
|
except rocksock.RocksockException as e:
|
||||||
assert(e.get_errortype() == rocksock.RS_ET_GAI)
|
assert(e.get_errortype() == rocksock.RS_ET_GAI)
|
||||||
dns_fail = True
|
dns_fail = True
|
||||||
if dns_fail:
|
if dns_fail:
|
||||||
fail_inc = 0
|
fail_inc = 0
|
||||||
_log("could not resolve connection target %s"%srvname, "ERROR")
|
_log("could not resolve connection target %s"%srvname, "ERROR")
|
||||||
return False
|
return False
|
||||||
return srv
|
return srv
|
||||||
|
|
||||||
|
|
||||||
class WorkerJob():
|
class WorkerJob():
|
||||||
def __init__(self, proxy, proto, failcount, success_count, total_duration, country):
|
def __init__(self, proxy, proto, failcount, success_count, total_duration, country, oldies = False):
|
||||||
self.proxy = proxy
|
self.proxy = proxy
|
||||||
self.proto = proto
|
self.proto = proto
|
||||||
self.failcount = failcount
|
self.failcount = failcount
|
||||||
self.checktime = None
|
self.checktime = None
|
||||||
self.success_count = success_count
|
self.success_count = success_count
|
||||||
self.total_duration = total_duration
|
self.total_duration = total_duration
|
||||||
self.country = country
|
self.country = country
|
||||||
|
self.isoldies = oldies
|
||||||
|
|
||||||
def connect_socket(self):
|
def connect_socket(self):
|
||||||
srvname = random.choice(config.servers).strip()
|
srvname = random.choice(config.servers).strip()
|
||||||
protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto]
|
protos = ['http', 'socks5', 'socks4'] if self.proto is None else [self.proto]
|
||||||
use_ssl = random.choice([0,1]) if config.watchd.use_ssl == 2 else config.watchd.use_ssl
|
use_ssl = random.choice([0,1]) if config.watchd.use_ssl == 2 else config.watchd.use_ssl
|
||||||
server_port = 6697 if use_ssl else 6667
|
server_port = 6697 if use_ssl else 6667
|
||||||
|
|
||||||
fail_inc = 1
|
fail_inc = 1
|
||||||
|
|
||||||
for proto in protos:
|
for proto in protos:
|
||||||
torhost = random.choice(config.torhosts)
|
torhost = random.choice(config.torhosts)
|
||||||
# socks4 (without 4a) requires a raw ip address
|
# socks4 (without 4a) requires a raw ip address
|
||||||
# rocksock automatically resolves if needed, but it's more
|
# rocksock automatically resolves if needed, but it's more
|
||||||
# efficient to cache the result.
|
# efficient to cache the result.
|
||||||
if proto == 'socks4': srv = socks4_resolve(srvname, server_port)
|
if proto == 'socks4': srv = socks4_resolve(srvname, server_port)
|
||||||
else: srv = srvname
|
else: srv = srvname
|
||||||
## skip socks4 failed resolution
|
## skip socks4 failed resolution
|
||||||
if not srv: continue
|
if not srv: continue
|
||||||
|
|
||||||
duration = time.time()
|
duration = time.time()
|
||||||
proxies = [
|
proxies = [
|
||||||
rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
|
rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
|
||||||
rocksock.RocksockProxyFromURL('%s://%s' % (proto, self.proxy)),
|
rocksock.RocksockProxyFromURL('%s://%s' % (proto, self.proxy)),
|
||||||
]
|
]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
sock = rocksock.Rocksock(host=srv, port=server_port, ssl=use_ssl, proxies=proxies, timeout=config.watchd.timeout)
|
if self.isoldies:
|
||||||
sock.connect()
|
sock = rocksock.Rocksock(host=srv, port=server_port, ssl=use_ssl, proxies=proxies, timeout=config.watchd.timeout - 2)
|
||||||
sock.send('NICK\n')
|
else:
|
||||||
return sock, proto, duration, torhost, srvname, 0
|
sock = rocksock.Rocksock(host=srv, port=server_port, ssl=use_ssl, proxies=proxies, timeout=config.watchd.timeout)
|
||||||
except rocksock.RocksockException as e:
|
sock.connect()
|
||||||
if config.watchd.debug:
|
sock.send('NICK\n')
|
||||||
_log("proxy failed: %s://%s: %s"%(proto, self.proxy, e.get_errormessage()), 'debug')
|
return sock, proto, duration, torhost, srvname, 0
|
||||||
|
except rocksock.RocksockException as e:
|
||||||
|
if config.watchd.debug:
|
||||||
|
_log("proxy failed: %s://%s: %s"%(proto, self.proxy, e.get_errormessage()), 'debug')
|
||||||
|
|
||||||
et = e.get_errortype()
|
et = e.get_errortype()
|
||||||
err = e.get_error()
|
err = e.get_error()
|
||||||
fp = e.get_failedproxy()
|
fp = e.get_failedproxy()
|
||||||
|
|
||||||
sock.disconnect()
|
sock.disconnect()
|
||||||
|
|
||||||
if et == rocksock.RS_ET_OWN:
|
if et == rocksock.RS_ET_OWN:
|
||||||
if fp == 1 and \
|
if fp == 1 and \
|
||||||
err == rocksock.RS_E_REMOTE_DISCONNECTED or \
|
err == rocksock.RS_E_REMOTE_DISCONNECTED or \
|
||||||
err == rocksock.RS_E_HIT_TIMEOUT:
|
err == rocksock.RS_E_HIT_TIMEOUT:
|
||||||
# proxy is not online, so don't waste time trying all possible protocols
|
# proxy is not online, so don't waste time trying all possible protocols
|
||||||
break
|
break
|
||||||
elif fp == 0 and \
|
elif fp == 0 and \
|
||||||
err == rocksock.RS_E_TARGET_CONN_REFUSED:
|
err == rocksock.RS_E_TARGET_CONN_REFUSED:
|
||||||
fail_inc = 0
|
fail_inc = 0
|
||||||
if random.randint(0, (config.watchd.threads-1)/2) == 0:
|
if random.randint(0, (config.watchd.threads-1)/2) == 0:
|
||||||
_log("could not connect to proxy 0, sleep 5s", "ERROR")
|
_log("could not connect to proxy 0, sleep 5s", "ERROR")
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
elif et == rocksock.RS_ET_GAI:
|
elif et == rocksock.RS_ET_GAI:
|
||||||
assert(0)
|
assert(0)
|
||||||
fail_inc = 0
|
fail_inc = 0
|
||||||
_log("could not resolve connection target %s"%srvname, "ERROR")
|
_log("could not resolve connection target %s"%srvname, "ERROR")
|
||||||
break
|
break
|
||||||
|
|
||||||
except KeyboardInterrupt as e:
|
except KeyboardInterrupt as e:
|
||||||
raise(e)
|
raise(e)
|
||||||
|
|
||||||
return None, None, None, None, None, fail_inc
|
return None, None, None, None, None, fail_inc
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.checktime = int(time.time())
|
self.checktime = int(time.time())
|
||||||
|
|
||||||
sock, proto, duration, tor, srv, failinc = self.connect_socket()
|
sock, proto, duration, tor, srv, failinc = self.connect_socket()
|
||||||
if not sock:
|
if not sock:
|
||||||
self.failcount += failinc
|
self.failcount += failinc
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
recv = sock.recv(6)
|
recv = sock.recv(6)
|
||||||
|
|
||||||
# good data
|
# good data
|
||||||
if re.match('^(:|NOTICE)', recv, re.IGNORECASE):
|
if re.match('^(:|NOTICE)', recv, re.IGNORECASE):
|
||||||
duration = (time.time() - duration)
|
duration = (time.time() - duration)
|
||||||
|
|
||||||
if geolite and not self.country or self.country == 'unknown' or self.country == 'N/A':
|
if geolite and not self.country or self.country == 'unknown' or self.country == 'N/A':
|
||||||
match = geolite2.lookup(self.proxy.split(':')[0])
|
match = geolite2.lookup(self.proxy.split(':')[0])
|
||||||
if match is not None: self.country = match.country
|
if match is not None: self.country = match.country
|
||||||
else: self.country = 'N/A'
|
else: self.country = 'N/A'
|
||||||
|
|
||||||
self.proto = proto
|
self.proto = proto
|
||||||
self.failcount = 0
|
self.failcount = 0
|
||||||
self.success_count = self.success_count + 1
|
self.success_count = self.success_count + 1
|
||||||
self.total_duration += int(duration*1000)
|
self.total_duration += int(duration*1000)
|
||||||
torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor
|
torstats = "" if len(config.torhosts)==1 else ' tor: %s;'%tor
|
||||||
recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv])
|
recvstats = "".join([x if x in string.printable and ord(x) > 32 else '.' for x in recv])
|
||||||
_log('%s://%s (%s) d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, self.country, duration, torstats, srv, recvstats), 'xxxxx')
|
_log('%s://%s (%s) d: %.2f sec(s);%s srv: %s; recv: %s' % (proto, self.proxy, self.country, duration, torstats, srv, recvstats), 'xxxxx')
|
||||||
except KeyboardInterrupt as e:
|
except KeyboardInterrupt as e:
|
||||||
raise e
|
raise e
|
||||||
except rocksock.RocksockException as e:
|
except rocksock.RocksockException as e:
|
||||||
self.failcount += 1
|
self.failcount += 1
|
||||||
finally:
|
finally:
|
||||||
sock.disconnect()
|
sock.disconnect()
|
||||||
|
|
||||||
|
|
||||||
class WorkerThread():
|
class WorkerThread():
|
||||||
def __init__ (self, id):
|
def __init__ (self, id):
|
||||||
self.id = id
|
self.id = id
|
||||||
self.done = threading.Event()
|
self.done = threading.Event()
|
||||||
self.thread = None
|
self.thread = None
|
||||||
self.workqueue = []
|
self.workqueue = []
|
||||||
self.workdone = []
|
self.workdone = []
|
||||||
self.lock = threading.Lock()
|
self.lock = threading.Lock()
|
||||||
def stop(self):
|
def stop(self):
|
||||||
self.done.set()
|
self.done.set()
|
||||||
def term(self):
|
def term(self):
|
||||||
if self.thread: self.thread.join()
|
if self.thread: self.thread.join()
|
||||||
def add_jobs(self, jobs):
|
def add_jobs(self, jobs):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
self.workqueue.extend(jobs)
|
self.workqueue.extend(jobs)
|
||||||
def return_jobs(self):
|
def return_jobs(self):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
jobs = self.workqueue
|
jobs = self.workqueue
|
||||||
self.workqueue = []
|
self.workqueue = []
|
||||||
return jobs
|
return jobs
|
||||||
def jobcount(self):
|
def jobcount(self):
|
||||||
return len(self.workqueue)
|
return len(self.workqueue)
|
||||||
def collect(self):
|
def collect(self):
|
||||||
wd = copy.copy(self.workdone)
|
wd = copy.copy(self.workdone)
|
||||||
self.workdone = []
|
self.workdone = []
|
||||||
return wd
|
return wd
|
||||||
def start_thread(self):
|
def start_thread(self):
|
||||||
self.thread = threading.Thread(target=self.workloop)
|
self.thread = threading.Thread(target=self.workloop)
|
||||||
self.thread.start()
|
self.thread.start()
|
||||||
def pop_if_possible(self):
|
def pop_if_possible(self):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
if len(self.workqueue):
|
if len(self.workqueue):
|
||||||
job = self.workqueue.pop()
|
job = self.workqueue.pop()
|
||||||
else:
|
else:
|
||||||
job = None
|
job = None
|
||||||
return job
|
return job
|
||||||
def workloop(self):
|
def workloop(self):
|
||||||
success_count = 0
|
success_count = 0
|
||||||
job_count = 0
|
job_count = 0
|
||||||
duration_total = 0
|
duration_total = 0
|
||||||
duration_success_total = 0
|
duration_success_total = 0
|
||||||
while True:
|
while True:
|
||||||
job = self.pop_if_possible()
|
job = self.pop_if_possible()
|
||||||
if job:
|
if job:
|
||||||
nao = time.time()
|
nao = time.time()
|
||||||
job.run()
|
job.run()
|
||||||
spent = time.time() - nao
|
spent = time.time() - nao
|
||||||
if job.failcount == 0:
|
if job.failcount == 0:
|
||||||
duration_success_total += spent
|
duration_success_total += spent
|
||||||
success_count += 1
|
success_count += 1
|
||||||
job_count += 1
|
job_count += 1
|
||||||
duration_total += spent
|
duration_total += spent
|
||||||
self.workdone.append(job)
|
self.workdone.append(job)
|
||||||
elif not self.thread:
|
elif not self.thread:
|
||||||
break
|
break
|
||||||
if self.done.is_set(): break
|
if self.done.is_set(): break
|
||||||
time.sleep(0.01)
|
time.sleep(0.01)
|
||||||
if self.thread:
|
if self.thread:
|
||||||
succ_rate = try_div(success_count, job_count)*100
|
succ_rate = try_div(success_count, job_count)*100
|
||||||
avg_succ_t = try_div(duration_success_total, success_count)
|
avg_succ_t = try_div(duration_success_total, success_count)
|
||||||
avg_fail_t = try_div(duration_total-duration_success_total, job_count-success_count)
|
avg_fail_t = try_div(duration_total-duration_success_total, job_count-success_count)
|
||||||
avg_t = try_div(duration_total, job_count)
|
avg_t = try_div(duration_total, job_count)
|
||||||
_log("terminated, %d/%d (%.2f%%), avg.time S/F/T %.2f, %.2f, %.2f" \
|
_log("terminated, %d/%d (%.2f%%), avg.time S/F/T %.2f, %.2f, %.2f" \
|
||||||
% (success_count, job_count, succ_rate, avg_succ_t, avg_fail_t, avg_t) \
|
% (success_count, job_count, succ_rate, avg_succ_t, avg_fail_t, avg_t) \
|
||||||
, self.id)
|
, self.id)
|
||||||
|
|
||||||
class Proxywatchd():
|
class Proxywatchd():
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
_log('halting... (%d thread(s))' % len([item for item in self.threads if True]), 'watchd')
|
_log('halting... (%d thread(s))' % len([item for item in self.threads if True]), 'watchd')
|
||||||
self.stopping.set()
|
self.stopping.set()
|
||||||
|
|
||||||
def _cleanup(self):
|
def _cleanup(self):
|
||||||
for wt in self.threads:
|
for wt in self.threads:
|
||||||
wt.stop()
|
wt.stop()
|
||||||
for wt in self.threads:
|
for wt in self.threads:
|
||||||
wt.term()
|
wt.term()
|
||||||
self.collect_work()
|
self.collect_work()
|
||||||
self.submit_collected()
|
self.submit_collected()
|
||||||
self.stopped.set()
|
self.stopped.set()
|
||||||
|
|
||||||
def finish(self):
|
def finish(self):
|
||||||
if not self.in_background: self._cleanup()
|
if not self.in_background: self._cleanup()
|
||||||
while not self.stopped.is_set(): time.sleep(0.1)
|
while not self.stopped.is_set(): time.sleep(0.1)
|
||||||
success_rate = try_div(self.totals['success'], self.totals['submitted']) * 100
|
success_rate = try_div(self.totals['success'], self.totals['submitted']) * 100
|
||||||
_log("total results: %d/%d (%.2f%%)"%(self.totals['success'], self.totals['submitted'], success_rate), "watchd")
|
_log("total results: %d/%d (%.2f%%)"%(self.totals['success'], self.totals['submitted'], success_rate), "watchd")
|
||||||
|
|
||||||
def _prep_db(self):
|
def _prep_db(self):
|
||||||
self.mysqlite = mysqlite.mysqlite(config.watchd.database, str)
|
self.mysqlite = mysqlite.mysqlite(config.watchd.database, str)
|
||||||
def _close_db(self):
|
def _close_db(self):
|
||||||
if self.mysqlite:
|
if self.mysqlite:
|
||||||
self.mysqlite.close()
|
self.mysqlite.close()
|
||||||
self.mysqlite = None
|
self.mysqlite = None
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
config.load()
|
config.load()
|
||||||
self.in_background = False
|
self.in_background = False
|
||||||
self.threads = []
|
self.threads = []
|
||||||
self.stopping = threading.Event()
|
self.stopping = threading.Event()
|
||||||
self.stopped = threading.Event()
|
self.stopped = threading.Event()
|
||||||
|
|
||||||
# create table if needed
|
# create table if needed
|
||||||
self._prep_db()
|
self._prep_db()
|
||||||
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, success_count INT, total_duration INT)')
|
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, success_count INT, total_duration INT)')
|
||||||
self.mysqlite.commit()
|
self.mysqlite.commit()
|
||||||
self._close_db()
|
self._close_db()
|
||||||
|
|
||||||
self.submit_after = config.watchd.submit_after # number of collected jobs before writing db
|
self.submit_after = config.watchd.submit_after # number of collected jobs before writing db
|
||||||
self.jobs = []
|
self.jobs = []
|
||||||
self.collected = []
|
self.collected = []
|
||||||
self.totals = {
|
self.totals = {
|
||||||
'submitted':0,
|
'submitted':0,
|
||||||
'success':0,
|
'success':0,
|
||||||
}
|
}
|
||||||
|
|
||||||
def fetch_rows(self):
|
def fetch_rows(self):
|
||||||
q = 'SELECT proxy,proto,failed,success_count,total_duration,country FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()'
|
self.isoldies = False
|
||||||
rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall()
|
q = 'SELECT proxy,proto,failed,success_count,total_duration,country FROM proxylist WHERE failed >= ? and failed < ? and (tested + ? + (failed * ?)) < ? ORDER BY RANDOM()'
|
||||||
# check oldies ?
|
rows = self.mysqlite.execute(q, (0, config.watchd.max_fail, config.watchd.checktime, config.watchd.perfail_checktime, time.time())).fetchall()
|
||||||
if len(rows) < config.watchd.threads and config.watchd.oldies:
|
# check oldies ?
|
||||||
## disable tor safeguard for old proxies
|
if len(rows) < config.watchd.threads and config.watchd.oldies:
|
||||||
if self.tor_safeguard: self.tor_safeguard = False
|
self.isoldies = True
|
||||||
#q += ' LIMIT ?'
|
## disable tor safeguard for old proxies
|
||||||
#rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time(), config.watchd.threads*config.watchd.oldies_multi)).fetchall()
|
if self.tor_safeguard: self.tor_safeguard = False
|
||||||
rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*3, config.watchd.checktime, config.watchd.oldies_checktime, time.time())).fetchall()
|
rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time())).fetchall()
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
def prepare_jobs(self):
|
def prepare_jobs(self):
|
||||||
self._prep_db()
|
self._prep_db()
|
||||||
## enable tor safeguard by default
|
## enable tor safeguard by default
|
||||||
self.tor_safeguard = config.watchd.tor_safeguard
|
self.tor_safeguard = config.watchd.tor_safeguard
|
||||||
rows = self.fetch_rows()
|
rows = self.fetch_rows()
|
||||||
for row in rows:
|
#print('preparing jobbs, oldies: %s' % str(self.isoldies))
|
||||||
job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5])
|
for row in rows:
|
||||||
self.jobs.append(job)
|
job = WorkerJob(row[0], row[1], row[2], row[3], row[4], row[5], self.isoldies)
|
||||||
self._close_db()
|
self.jobs.append(job)
|
||||||
|
self._close_db()
|
||||||
|
|
||||||
def collect_work(self):
|
def collect_work(self):
|
||||||
for wt in self.threads:
|
for wt in self.threads:
|
||||||
self.collected.extend(wt.collect())
|
self.collected.extend(wt.collect())
|
||||||
|
|
||||||
def collect_unfinished(self):
|
def collect_unfinished(self):
|
||||||
for wt in self.threads:
|
for wt in self.threads:
|
||||||
jobs = wt.return_jobs()
|
jobs = wt.return_jobs()
|
||||||
self.jobs.extend(jobs)
|
self.jobs.extend(jobs)
|
||||||
if len(self.jobs):
|
if len(self.jobs):
|
||||||
_log("collected %d unfinished jobs"%len(self.jobs), "watchd")
|
_log("collected %d unfinished jobs"%len(self.jobs), "watchd")
|
||||||
|
|
||||||
def submit_collected(self):
|
def submit_collected(self):
|
||||||
if len(self.collected) == 0: return True
|
if len(self.collected) == 0: return True
|
||||||
sc = 0
|
sc = 0
|
||||||
args = []
|
args = []
|
||||||
for job in self.collected:
|
for job in self.collected:
|
||||||
if job.failcount == 0: sc += 1
|
if job.failcount == 0: sc += 1
|
||||||
args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
|
args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
|
||||||
|
|
||||||
success_rate = (float(sc) / len(self.collected)) * 100
|
success_rate = (float(sc) / len(self.collected)) * 100
|
||||||
ret = True
|
ret = True
|
||||||
if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold and self.tor_safeguard:
|
if len(self.collected) >= 100 and success_rate <= config.watchd.outage_threshold and self.tor_safeguard:
|
||||||
_log("WATCHD %.2f%% SUCCESS RATE - tor circuit blocked? won't submit fails"%success_rate, "ERROR")
|
_log("WATCHD %.2f%% SUCCESS RATE - tor circuit blocked? won't submit fails"%success_rate, "ERROR")
|
||||||
if sc == 0: return False
|
if sc == 0: return False
|
||||||
args = []
|
args = []
|
||||||
for job in self.collected:
|
for job in self.collected:
|
||||||
if job.failcount == 0:
|
if job.failcount == 0:
|
||||||
args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
|
args.append( (job.failcount, job.checktime, 1, job.country, job.proto, job.success_count, job.total_duration, job.proxy) )
|
||||||
ret = False
|
ret = False
|
||||||
|
|
||||||
_log("updating %d DB entries (success rate: %.2f%%)"%(len(self.collected), success_rate), 'watchd')
|
_log("updating %d DB entries (success rate: %.2f%%)"%(len(self.collected), success_rate), 'watchd')
|
||||||
self._prep_db()
|
self._prep_db()
|
||||||
query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=? WHERE proxy=?'
|
query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,success_count=?,total_duration=? WHERE proxy=?'
|
||||||
self.mysqlite.executemany(query, args)
|
self.mysqlite.executemany(query, args)
|
||||||
self.mysqlite.commit()
|
self.mysqlite.commit()
|
||||||
self._close_db()
|
self._close_db()
|
||||||
self.collected = []
|
self.collected = []
|
||||||
self.totals['submitted'] += len(args)
|
self.totals['submitted'] += len(args)
|
||||||
self.totals['success'] += sc
|
self.totals['success'] += sc
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
if config.watchd.threads == 1 and _run_standalone:
|
if config.watchd.threads == 1 and _run_standalone:
|
||||||
return self._run()
|
return self._run()
|
||||||
else:
|
else:
|
||||||
return self._run_background()
|
return self._run_background()
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
if self.in_background:
|
if self.in_background:
|
||||||
while 1: time.sleep(1)
|
while 1: time.sleep(1)
|
||||||
|
|
||||||
def _run_background(self):
|
def _run_background(self):
|
||||||
self.in_background = True
|
self.in_background = True
|
||||||
t = threading.Thread(target=self._run)
|
t = threading.Thread(target=self._run)
|
||||||
t.start()
|
t.start()
|
||||||
|
|
||||||
def _run(self):
|
def _run(self):
|
||||||
_log('starting...', 'watchd')
|
_log('starting...', 'watchd')
|
||||||
|
|
||||||
for i in range(config.watchd.threads):
|
for i in range(config.watchd.threads):
|
||||||
threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
|
threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
|
||||||
wt = WorkerThread(threadid)
|
wt = WorkerThread(threadid)
|
||||||
if self.in_background:
|
if self.in_background:
|
||||||
wt.start_thread()
|
wt.start_thread()
|
||||||
self.threads.append(wt)
|
self.threads.append(wt)
|
||||||
time.sleep( (random.random()/100) )
|
time.sleep( (random.random()/100) )
|
||||||
|
|
||||||
sleeptime = 0
|
sleeptime = 0
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
|
||||||
if self.stopping.is_set():
|
if self.stopping.is_set():
|
||||||
if self.in_background: self._cleanup()
|
if self.in_background: self._cleanup()
|
||||||
break
|
break
|
||||||
|
|
||||||
if sleeptime == 0:
|
if sleeptime == 0:
|
||||||
sleeptime = 1
|
sleeptime = 1
|
||||||
else:
|
else:
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
sleeptime -= 1
|
sleeptime -= 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0:
|
if self.threads[random.choice(xrange(len(self.threads)))].jobcount() == 0:
|
||||||
self.collect_unfinished()
|
self.collect_unfinished()
|
||||||
if not len(self.jobs):
|
if not len(self.jobs):
|
||||||
self.collect_work()
|
self.collect_work()
|
||||||
if not self.submit_collected() and self.tor_safeguard:
|
if not self.submit_collected() and self.tor_safeguard:
|
||||||
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
|
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
|
||||||
self.collect_unfinished()
|
self.collect_unfinished()
|
||||||
sleeptime = 1*60
|
sleeptime = 1*60
|
||||||
else:
|
else:
|
||||||
self.prepare_jobs()
|
self.prepare_jobs()
|
||||||
else:
|
else:
|
||||||
if len(self.jobs) < len(self.threads):
|
if len(self.jobs) < len(self.threads):
|
||||||
# allow threads enough time to consume the jobs
|
# allow threads enough time to consume the jobs
|
||||||
sleeptime = 10
|
sleeptime = 10
|
||||||
if len(self.jobs):
|
if len(self.jobs):
|
||||||
_log("handing out %d jobs to %d thread(s)"% (len(self.jobs), len(self.threads)), 'watchd')
|
_log("handing out %d jobs to %d thread(s)"% (len(self.jobs), len(self.threads)), 'watchd')
|
||||||
jpt = len(self.jobs)/len(self.threads)
|
jpt = len(self.jobs)/len(self.threads)
|
||||||
if len(self.jobs)/float(len(self.threads)) - jpt > 0.0: jpt += 1
|
if len(self.jobs)/float(len(self.threads)) - jpt > 0.0: jpt += 1
|
||||||
for tid in xrange(len(self.threads)):
|
for tid in xrange(len(self.threads)):
|
||||||
self.threads[tid].add_jobs(self.jobs[tid*jpt:tid*jpt+jpt])
|
self.threads[tid].add_jobs(self.jobs[tid*jpt:tid*jpt+jpt])
|
||||||
self.jobs = []
|
self.jobs = []
|
||||||
|
|
||||||
if not self.in_background: # single_thread scenario
|
if not self.in_background: # single_thread scenario
|
||||||
self.threads[0].workloop()
|
self.threads[0].workloop()
|
||||||
|
|
||||||
self.collect_work()
|
self.collect_work()
|
||||||
|
|
||||||
if len(self.collected) > self.submit_after:
|
if len(self.collected) > self.submit_after:
|
||||||
if not self.submit_collected() and self.tor_safeguard:
|
if not self.submit_collected() and self.tor_safeguard:
|
||||||
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
|
_log("zzZzZzzZ sleeping 1 minute(s) due to tor issues - consider decreasing thread number!", "watchd")
|
||||||
self.collect_unfinished()
|
self.collect_unfinished()
|
||||||
sleeptime = 1*60
|
sleeptime = 1*60
|
||||||
|
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
sleeptime -= 1
|
sleeptime -= 1
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
_run_standalone = True
|
_run_standalone = True
|
||||||
|
|
||||||
config.load()
|
config.load()
|
||||||
|
|
||||||
w = Proxywatchd()
|
w = Proxywatchd()
|
||||||
try:
|
try:
|
||||||
w.start()
|
w.start()
|
||||||
w.run()
|
w.run()
|
||||||
except KeyboardInterrupt as e:
|
except KeyboardInterrupt as e:
|
||||||
pass
|
pass
|
||||||
finally:
|
finally:
|
||||||
w.stop()
|
w.stop()
|
||||||
w.finish()
|
w.finish()
|
||||||
|
|||||||
@@ -1,16 +1,14 @@
|
|||||||
https://searx.serneels.xyz
|
|
||||||
https://searx.me
|
|
||||||
https://searx.xyz
|
https://searx.xyz
|
||||||
https://searx.site
|
#https://searx.site
|
||||||
https://searx.win
|
#https://searx.win
|
||||||
https://searx.ru
|
#https://searx.ru
|
||||||
https://stemy.me/searx
|
#https://stemy.me/searx
|
||||||
https://searx.at
|
#https://searx.at
|
||||||
https://listi.me
|
#https://listi.me
|
||||||
https://searx.dk
|
#https://searx.dk
|
||||||
https://searx.laquadrature.net
|
#https://searx.laquadrature.net
|
||||||
## hidden services
|
## hidden services
|
||||||
http://searchb5a7tmimez.onion
|
#http://searchb5a7tmimez.onion
|
||||||
http://nxhhwbbxc4khvvlw.onion
|
#http://nxhhwbbxc4khvvlw.onion
|
||||||
http://searx.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion
|
#http://searx.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion
|
||||||
http://ulrn6sryqaifefld.onion
|
#http://ulrn6sryqaifefld.onion
|
||||||
|
|||||||
26
servers.txt
26
servers.txt
@@ -1,21 +1,10 @@
|
|||||||
eu.ircnet.org
|
|
||||||
eu.undernet.org
|
|
||||||
irc.2600.net
|
irc.2600.net
|
||||||
irc.Undernet.Org
|
irc.Undernet.Org
|
||||||
irc.afterx.net
|
|
||||||
irc.atrum.org
|
|
||||||
irc.atw-inter.net
|
|
||||||
irc.au.dal.net
|
|
||||||
irc.austnet.org
|
|
||||||
irc.azzurra.org
|
|
||||||
irc.blitzed.org
|
|
||||||
irc.bongster.org
|
|
||||||
irc.chat4all.org
|
irc.chat4all.org
|
||||||
irc.chatspike.net
|
irc.chatspike.net
|
||||||
irc.choopa.net
|
irc.choopa.net
|
||||||
irc.coldfront.net
|
irc.coldfront.net
|
||||||
irc.cyberarmy.net
|
irc.cyberarmy.net
|
||||||
irc.d-t-net.de
|
|
||||||
irc.dal.net
|
irc.dal.net
|
||||||
irc.darkmyst.org
|
irc.darkmyst.org
|
||||||
irc.data.lt
|
irc.data.lt
|
||||||
@@ -34,9 +23,6 @@ irc.europnet.org
|
|||||||
irc.eversible.com
|
irc.eversible.com
|
||||||
irc.fdfnet.net
|
irc.fdfnet.net
|
||||||
irc.fef.net
|
irc.fef.net
|
||||||
irc.financialchat.com
|
|
||||||
irc.forestnet.org
|
|
||||||
irc.foreverchat.net
|
|
||||||
irc.freequest.net
|
irc.freequest.net
|
||||||
irc.gamesurge.net
|
irc.gamesurge.net
|
||||||
irc.geekshed.net
|
irc.geekshed.net
|
||||||
@@ -44,18 +30,8 @@ irc.german-freakz.net
|
|||||||
irc.gigairc.net
|
irc.gigairc.net
|
||||||
irc.globalgamers.net
|
irc.globalgamers.net
|
||||||
irc.greekirc.net
|
irc.greekirc.net
|
||||||
irc.icq.com
|
|
||||||
irc.immortal-anime.net
|
|
||||||
irc.inet.tele.dk
|
|
||||||
irc.irc2.hu
|
|
||||||
irc.irc4fun.net
|
|
||||||
irc.irchighway.net
|
|
||||||
irc.ircsource.net
|
|
||||||
irc.irctoo.net
|
irc.irctoo.net
|
||||||
irc.ircube.org
|
irc.ircube.org
|
||||||
irc.ircworld.nl
|
|
||||||
irc.irdsi.net
|
|
||||||
irc.kampungchat.org
|
|
||||||
irc.knightirc.net
|
irc.knightirc.net
|
||||||
irc.krono.net
|
irc.krono.net
|
||||||
irc.langochat.net
|
irc.langochat.net
|
||||||
@@ -91,8 +67,6 @@ irc.swiftirc.net
|
|||||||
irc.teranova.net
|
irc.teranova.net
|
||||||
irc.us.dal.net
|
irc.us.dal.net
|
||||||
irc.us.gamesurge.net
|
irc.us.gamesurge.net
|
||||||
irc.whatnet.org
|
|
||||||
irc.wondernet.nu
|
|
||||||
irc.xevion.net
|
irc.xevion.net
|
||||||
irc.xs4all.nl
|
irc.xs4all.nl
|
||||||
irc.zerofuzion.net
|
irc.zerofuzion.net
|
||||||
|
|||||||
Reference in New Issue
Block a user