username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ppf: move insert function into dbs.py

Browse Source
This commit is contained in:
rofl0r
2019-01-18 21:43:17 +00:00
parent aba74c8eab
commit 8be5ab1567
2 changed files with 31 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
dbs.py
View File

@@ -1,4 +1,5 @@
import mysqlite import time
from misc import _log
def create_table_if_not_exists(sqlite, dbname): def create_table_if_not_exists(sqlite, dbname):
if dbname == 'proxylist': if dbname == 'proxylist':
@@ -25,3 +26,29 @@ def create_table_if_not_exists(sqlite, dbname):
)""") )""")
sqlite.commit() sqlite.commit()
def insert_proxies(proxydb, proxies, url):
timestamp = int(time.time())
new = []
for p in proxies:
new.append((timestamp,p,3,0,0,0))
while len(new):
proxydb.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new[:500])
new = new[500:]
proxydb.commit()
_log('+%d item(s) from %s' % (len(proxies), url), 'added')
def insert_urls(urls, search, sqlite):
query = [ 'url=?' for u in urls ]
known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ]
time_now = int(time.time())
new = [ (time_now,i,0,0,0,0,0) for i in urls if not i in known ]
if not len(new): return
sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new)
sqlite.commit()
_log('+%d item(s) from %s' % (len(new), search), 'added')

31
ppf.py
View File

@@ -44,32 +44,7 @@ def proxyfind(sqlite = None):
for x in range(1,10): for x in range(1,10):
content = fetch.fetch_contents('%s/?%s&pageno=%d' % (srx,search_arg,x)) content = fetch.fetch_contents('%s/?%s&pageno=%d' % (srx,search_arg,x))
if content: urls = fetch.extract_urls(content, urls, urignore) if content: urls = fetch.extract_urls(content, urls, urignore)
if len(urls): insert_urls(urls, search_arg, sqlite) if len(urls): dbs.insert_urls(urls, search_arg, sqlite)
def insert_urls(urls, search, sqlite):
query = [ 'url=?' for u in urls ]
known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ]
time_now = int(time.time())
new = [ (time_now,i,0,0,0,0,0) for i in urls if not i in known ]
if not len(new): return
sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new)
sqlite.commit()
_log('+%d item(s) from %s' % (len(new), search), 'added')
def insert_proxies(proxydb, proxies, url):
timestamp = int(time.time())
new = []
for p in proxies:
new.append((timestamp,p,3,0,0,0))
while len(new):
proxydb.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new[:500])
new = new[500:]
proxydb.commit()
_log('+%d item(s) from %s' % (len(proxies), url), 'added')
def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added): def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added):
@@ -98,14 +73,14 @@ def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_adde
if not len(new): return if not len(new): return
insert_proxies(proxydb, new, url) dbs.insert_proxies(proxydb, new, url)
def import_proxies_from_file(proxydb, fn): def import_proxies_from_file(proxydb, fn):
content = open(fn, 'r').read() content = open(fn, 'r').read()
unique_count, new = fetch.extract_proxies(content, proxydb) unique_count, new = fetch.extract_proxies(content, proxydb)
if len(new): if len(new):
insert_proxies(proxydb, new, fn) dbs.insert_proxies(proxydb, new, fn)
return 0 return 0
return 1 return 1