username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move math function inside the sql statement

Browse Source
This commit is contained in:
mickael
2019-01-07 21:11:08 +00:00
parent f47745c8d6
commit aeff09d2b3
3 changed files with 12 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config.ini.sample
View File

@@ -15,3 +15,6 @@ debug = false
search = true search = true
timeout = 30 timeout = 30
threads = 3 threads = 3
checktime = 3600
perfail_checktime = 3600

5
config.py
View File

@@ -4,7 +4,7 @@ _loaded = False
def load(): def load():
if _loaded: return if _loaded: return
global database, maxfail, search, torhosts, watchd_threads, checktime, timeout, read_timeout, submit_after, use_ssl global database, maxfail, search, torhosts, watchd_threads, checktime, timeout, read_timeout, submit_after, use_ssl, checktime, perfail_checktime
## read the config files ## read the config files
parser = SafeConfigParser() parser = SafeConfigParser()
@@ -22,6 +22,9 @@ def load():
use_ssl = parser.getboolean('watcherd', 'use_ssl') use_ssl = parser.getboolean('watcherd', 'use_ssl')
global watchd_debug global watchd_debug
watchd_debug = parser.getboolean('watcherd', 'debug') watchd_debug = parser.getboolean('watcherd', 'debug')
#url_maxfail = parser.get('proxyfind', 'maxfail')
checktime = parser.get('proxyfind', 'checktime')
perfail_checktime = parser.get('proxyfind', 'perfail_checktime')
# allow overriding select items from the commandline # allow overriding select items from the commandline
import argparse import argparse

9
ppf.py
View File

@@ -127,15 +127,15 @@ def proxyleech(sqlite, rows):
hash = hashlib.md5(''.join(uniques)).hexdigest() hash = hashlib.md5(''.join(uniques)).hexdigest()
## empty list of proxies: increment error by two ## empty list of proxies: multiply error by two
if not len(uniques): row[2] = (row[2] * 2) if not len(uniques): row[2] = (row[2] * 2)
## same proxy list: increment error by one ## same proxy list: increment error by one
elif hash == row[1]: row[2] = (row[2] + 1) elif hash == row[1]: row[2] = (row[2] + 1)
## proxylist was updated: error is zero ## proxylist was updated: error is zero
else: row[2] = 0 else: row[2] = 0
check_time = (time.time() + 3600 + (3600 * row[2])) #check_time = (time.time() + 3600 + (3600 * row[2]))
sqlite.execute('UPDATE uris SET error=?,hash=?,check_time=? where url=?', (row[2],hash, check_time,row[0])) sqlite.execute('UPDATE uris SET error=?,hash=?,check_time=? where url=?', (row[2],hash, time.time(),row[0]))
sqlite.commit() sqlite.commit()
if not row[1] or row[2] > 0: return if not row[1] or row[2] > 0: return
@@ -183,7 +183,8 @@ if __name__ == '__main__':
while True: while True:
try: try:
## any site that needs to be checked ? ## any site that needs to be checked ?
rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time<? AND error<?) ORDER BY RANDOM() LIMIT 25', (time.time(), 10)).fetchall() ] rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time+?+(error*?) <?) ORDER BY RANDOM() LIMIT 25', (config.checktime, config.perfail_checktime, time.time())).fetchall() ]
if len(rows): proxyleech(sqlite,rows) if len(rows): proxyleech(sqlite,rows)
## search for new website during free time ## search for new website during free time
elif config.search: proxyfind(sqlite) elif config.search: proxyfind(sqlite)