diff --git a/ppf.py b/ppf.py index b6fedf6..bc84c2a 100755 --- a/ppf.py +++ b/ppf.py @@ -19,7 +19,12 @@ _known_proxies = {} def import_from_file(fn, sqlite): with open(fn, 'r') as f: urls = [ url for url in f.read().split('\n') if url != '' ] - dbs.insert_urls(urls, 'import.txt', urldb) + cinc = 0 + while True: + chunk = urls[cinc:cinc+200] + if len(chunk): dbs.insert_urls(chunk, 'import.txt', urldb) + else: break + cinc = cinc + 200 def get_content_type(url): @@ -284,8 +289,6 @@ if __name__ == '__main__': proxydb = mysqlite.mysqlite(config.watchd.database, str) dbs.create_table_if_not_exists(proxydb, 'proxylist') - #global _known_proxies - #if len(_known_proxies) == 0: known = proxydb.execute('SELECT proxy FROM proxylist').fetchall() for k in known: _known_proxies[k[0]] = True @@ -332,8 +335,8 @@ if __name__ == '__main__': nao = time.time() _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf') args = [ (nao, row[0]) for row in rows ] - urldb.executemany('UPDATE uris SET check_time=? where url=?', args) - urldb.commit() + #urldb.executemany('UPDATE uris SET check_time=? where url=?', args) + #urldb.commit() for thread in threads: @@ -353,6 +356,8 @@ if __name__ == '__main__': threads = [ thread for thread in threads if thread.is_alive() ] if len(threads) < config.ppf.threads and len(rows): row = random.choice(rows) + urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0])) + urldb.commit() rows.remove(row) t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5]) threads.append(t)