username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

split databases

Browse Source
This commit is contained in:
mickael
2019-01-10 22:20:09 +00:00
parent b85cb863ba
commit 4c6a83373f
5 changed files with 47 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
ppf.py
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python
import dbs
import random, time
import re
import urllib
@@ -102,7 +103,7 @@ def insert_proxies(proxies, uri, sqlite, timestamp):
_log('+%d item(s) from %s' % (len(new), uri), 'added')
def proxyfind(sqlite = None):
if not sqlite: sqlite = mysqlite.mysqlite(config.common.database,str)
if not sqlite: sqlite = mysqlite.mysqlite(config.ppf.database,str)
uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
if len(uris) > 0 and random.random() < random.random():
@@ -153,7 +154,7 @@ def is_usable_proxy(proxy):
(A == 172 and B >= 16 and B <= 31): return False
return True
def proxyleech(sqlite, rows):
def proxyleech(proxydb, urldb, rows):
for row in rows:
try: content = fetch_contents(row[0])
except KeyboardInterrupt as e: raise e
@@ -181,8 +182,8 @@ def proxyleech(sqlite, rows):
## proxylist was updated: error is zero
else: row[2] = 0
sqlite.execute('UPDATE uris SET error=?,hash=?,check_time=? where url=?', (row[2],hash, int(time.time()),row[0]))
sqlite.commit()
urldb.execute('UPDATE uris SET error=?,hash=?,check_time=? where url=?', (row[2],hash, int(time.time()),row[0]))
urldb.commit()
if not row[1] or row[2] > 0: return
@@ -191,9 +192,9 @@ def proxyleech(sqlite, rows):
for i in uniques:
add.append(i)
if len(add) > 500:
insert_proxies(add, row[0], sqlite, time_now)
insert_proxies(add, row[0], proxydb, time_now)
add = []
if len(add): insert_proxies(add, row[0], sqlite, time_now)
if len(add): insert_proxies(add, row[0], proxydb, time_now)
@@ -201,12 +202,13 @@ if __name__ == '__main__':
config.load()
proxies={'http':'socks4://%s' % random.choice(config.torhosts),'https':'socks4://%s' % random.choice(config.torhosts)}
sqlite = mysqlite.mysqlite(config.common.database, str)
## create dbs if required
sqlite.execute('CREATE TABLE IF NOT EXISTS uris (added INT, url TEXT, check_time INT, error INT, driver INT, hash TEXT)')
sqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, dronebl INT, proto TEXT, success_count INT, total_duration INT)')
sqlite.commit()
import_from_file('import.txt', sqlite)
proxydb = mysqlite.mysqlite(config.watchd.database, str)
dbs.create_table_if_not_exists(proxydb, 'proxylist')
urldb = mysqlite.mysqlite(config.ppf.database, str)
dbs.create_table_if_not_exists(urldb, 'uris')
import_from_file('import.txt', urldb)
if config.ppf.search:
## load search terms
@@ -229,11 +231,11 @@ if __name__ == '__main__':
while True:
try:
## any site that needs to be checked ?
rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time+?+(error*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() ]
rows = [ [i[0],i[1],i[2]] for i in urldb.execute('SELECT url,hash,error FROM uris WHERE (check_time+?+(error*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() ]
if len(rows): proxyleech(sqlite,rows)
if len(rows): proxyleech(proxydb, urldb, rows)
## search for new website during free time
if config.ppf.search: proxyfind(sqlite)
if config.ppf.search: proxyfind(urldb)
## sleep
else: time.sleep(10)