username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ppf: add new field proxies_added to be able to rate sites

Browse Source 
sqlite3 urls.sqlite "alter table uris add proxies_added INT"
sqlite3 urls.sqlite "update uris set proxies_added=0"
This commit is contained in:
rofl0r
2019-01-18 15:44:03 +00:00
parent 0734635e30
commit 0dad0176f3
2 changed files with 8 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
ppf.py
View File

@@ -42,7 +42,7 @@ def import_from_file(fn, sqlite):
exists = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE url=?',(u,)).fetchall() ]
if exists: continue
print('adding "%s"' % u)
sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count) VALUES (?,?,?,?,?)', (int(time.time()),u,0,0,0))
sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count,proxies_added) VALUES (?,?,?,?,?,?)', (int(time.time()),u,0,0,0,0))
sqlite.commit()
def fetch_contents(url):
@@ -111,9 +111,9 @@ def insert_urls(urls, search, sqlite):
query = [ 'url=?' for u in urls ]
known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ]
time_now = int(time.time())
new = [ (time_now,i,0,0,0,0) for i in urls if not i in known ]
new = [ (time_now,i,0,0,0,0,0) for i in urls if not i in known ]
if not len(new): return
sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals) values(?,?,?,?,?,?)', new)
sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new)
sqlite.commit()
_log('+%d item(s) from %s' % (len(new), search), 'added')
@@ -152,7 +152,7 @@ def insert_proxies(proxydb, proxies, url):
_log('+%d item(s) from %s' % (len(proxies), url), 'added')
def proxyleech(proxydb, urldb, url, stale_count, error, retrievals):
def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added):
try: content = fetch_contents(url)
except KeyboardInterrupt as e: raise e
except: content = ''
@@ -173,7 +173,7 @@ def proxyleech(proxydb, urldb, url, stale_count, error, retrievals):
retrievals += 1
error = 0
urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=? where url=?', (error, stale_count, int(time.time()), retrievals, url))
urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=? where url=?', (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), url))
urldb.commit()
if not len(new): return
@@ -248,10 +248,10 @@ if __name__ == '__main__':
while True:
try:
## any site that needs to be checked ?
rows = urldb.execute('SELECT url,stale_count,error,retrievals FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
rows = urldb.execute('SELECT url,stale_count,error,retrievals,proxies_added FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
for row in rows:
proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3])
proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4])
## search for new website during free time
if config.ppf.search: proxyfind(urldb)