username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

introduce success_count and total_duration (proxylist.sqlit

Browse Source 
run those commands to update the database:

sqlite3 proxylist.sqlite "alter table proxylist add success_count int"
sqlite3 proxylist.sqlite "alter table proxylist add total_duration int"
sqlite3 proxylist.sqlite "update proxylist set success_count=0,total_duration=0"
This commit is contained in:
mickael
2019-01-05 22:24:38 +00:00
parent 3348435e21
commit d93f4dcaf2
2 changed files with 13 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ppf.py
View File

@@ -58,10 +58,10 @@ def insert_proxies(proxies, uri, sqlite):
query = [ 'proxy=?' for p in proxies ] query = [ 'proxy=?' for p in proxies ]
known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ] known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ]
new = [ (time_now,i,3,0) for i in proxies if not i in known ] new = [ (time_now,i,3,0,0,0) for i in proxies if not i in known ]
if len(new): if len(new):
sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested) VALUES (?,?,?,?)', new) sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new)
sqlite.commit() sqlite.commit()
_log('+%d item(s) from %s' % (len(new), uri), 'added') _log('+%d item(s) from %s' % (len(new), uri), 'added')
time.sleep(0.1) time.sleep(0.1)
@@ -158,7 +158,7 @@ if __name__ == '__main__':
## create dbs if required ## create dbs if required
sqlite.execute('CREATE TABLE IF NOT EXISTS uris (added INT, url TEXT, check_time INT, error INT, driver INT, hash TEXT)') sqlite.execute('CREATE TABLE IF NOT EXISTS uris (added INT, url TEXT, check_time INT, error INT, driver INT, hash TEXT)')
sqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, dronebl INT, proto TEXT, duration INT)') sqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, dronebl INT, proto TEXT, duration INT, success_count INT, total_duration INT)')
sqlite.commit() sqlite.commit()
import_from_file('import.txt', sqlite) import_from_file('import.txt', sqlite)

16
proxywatchd.py
View File

@@ -14,12 +14,14 @@ import rocksock
_run_standalone = False _run_standalone = False
class WorkerJob(): class WorkerJob():
def __init__(self, proxy, proto, failcount): def __init__(self, proxy, proto, failcount, success_count, total_duration):
self.proxy = proxy self.proxy = proxy
self.proto = proto self.proto = proto
self.failcount = failcount self.failcount = failcount
self.nextcheck = None self.nextcheck = None
self.duration = None self.duration = None
self.success_count = success_count
self.total_duration = total_duration
def is_drone_bl(self, proxy): def is_drone_bl(self, proxy):
p = proxy.split(':')[0] p = proxy.split(':')[0]
@@ -75,6 +77,8 @@ class WorkerJob():
self.proto = proto self.proto = proto
self.duation = duration self.duation = duration
self.failcount = 0 self.failcount = 0
self.success_count = self.success_count + 1
self.total_duration = self.total_duration + duration
_log('%s://%s; c: %s; d: %d sec(s); tor: %s; srv: %s; recv: %s' % (proto, self.proxy, match, duration, tor, srv, recv), 'xxxxx') _log('%s://%s; c: %s; d: %d sec(s); tor: %s; srv: %s; recv: %s' % (proto, self.proxy, match, duration, tor, srv, recv), 'xxxxx')
except KeyboardInterrupt as e: except KeyboardInterrupt as e:
raise e raise e
@@ -153,7 +157,7 @@ class Proxywatchd():
# create table if needed # create table if needed
self._prep_db() self._prep_db()
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)') self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT, success_count INT, total_duration INT)')
self.mysqlite.commit() self.mysqlite.commit()
self._close_db() self._close_db()
@@ -163,10 +167,10 @@ class Proxywatchd():
def prepare_jobs(self): def prepare_jobs(self):
self._prep_db() self._prep_db()
q = 'SELECT proxy,proto,failed FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM()' # ' LIMIT ?' q = 'SELECT proxy,proto,failed,success_count,total_duration FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM()' # ' LIMIT ?'
rows = self.mysqlite.execute(q, (config.maxfail, time.time())).fetchall() rows = self.mysqlite.execute(q, (config.maxfail, time.time())).fetchall()
for row in rows: for row in rows:
job = WorkerJob(row[0], row[1], row[2]) job = WorkerJob(row[0], row[1], row[2], row[3], row[4])
self.jobs.append(job) self.jobs.append(job)
self._close_db() self._close_db()
@@ -176,8 +180,8 @@ class Proxywatchd():
def submit_collected(self): def submit_collected(self):
self._prep_db() self._prep_db()
query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?' query = 'UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=?,success_count=?,total_duration=? WHERE proxy=?'
args = [ (job.failcount, job.nextcheck, 1, 'unknown', job.proto, job.duration, job.proxy) for job in self.collected ] args = [ (job.failcount, job.nextcheck, 1, 'unknown', job.proto, job.success_count, job.total_duration, job.duration, job.proxy) for job in self.collected ]
self.mysqlite.executemany(query, args) self.mysqlite.executemany(query, args)
self.mysqlite.commit() self.mysqlite.commit()
self._close_db() self._close_db()