username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

import new url as bulk list, misc cleansing

Browse Source
This commit is contained in:
Your Name
2021-02-06 23:25:12 +01:00
parent f321e5a934
commit 60c78be3fb
1 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
ppf.py
View File

@@ -19,7 +19,12 @@ _known_proxies = {}
def import_from_file(fn, sqlite): def import_from_file(fn, sqlite):
with open(fn, 'r') as f: with open(fn, 'r') as f:
urls = [ url for url in f.read().split('\n') if url != '' ] urls = [ url for url in f.read().split('\n') if url != '' ]
dbs.insert_urls(urls, 'import.txt', urldb) cinc = 0
while True:
chunk = urls[cinc:cinc+200]
if len(chunk): dbs.insert_urls(chunk, 'import.txt', urldb)
else: break
cinc = cinc + 200
def get_content_type(url): def get_content_type(url):
@@ -284,8 +289,6 @@ if __name__ == '__main__':
proxydb = mysqlite.mysqlite(config.watchd.database, str) proxydb = mysqlite.mysqlite(config.watchd.database, str)
dbs.create_table_if_not_exists(proxydb, 'proxylist') dbs.create_table_if_not_exists(proxydb, 'proxylist')
#global _known_proxies
#if len(_known_proxies) == 0:
known = proxydb.execute('SELECT proxy FROM proxylist').fetchall() known = proxydb.execute('SELECT proxy FROM proxylist').fetchall()
for k in known: for k in known:
_known_proxies[k[0]] = True _known_proxies[k[0]] = True
@@ -332,8 +335,8 @@ if __name__ == '__main__':
nao = time.time() nao = time.time()
_log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf') _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
args = [ (nao, row[0]) for row in rows ] args = [ (nao, row[0]) for row in rows ]
urldb.executemany('UPDATE uris SET check_time=? where url=?', args) #urldb.executemany('UPDATE uris SET check_time=? where url=?', args)
urldb.commit() #urldb.commit()
for thread in threads: for thread in threads:
@@ -353,6 +356,8 @@ if __name__ == '__main__':
threads = [ thread for thread in threads if thread.is_alive() ] threads = [ thread for thread in threads if thread.is_alive() ]
if len(threads) < config.ppf.threads and len(rows): if len(threads) < config.ppf.threads and len(rows):
row = random.choice(rows) row = random.choice(rows)
urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0]))
urldb.commit()
rows.remove(row) rows.remove(row)
t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5]) t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5])
threads.append(t) threads.append(t)