import new url as bulk list, misc cleansing

ppf.py

@@ -19,7 +19,12 @@ _known_proxies = {}
 def import_from_file(fn, sqlite):
   with open(fn, 'r') as f:
     urls = [ url for url in f.read().split('\n') if url != '' ]
-    dbs.insert_urls(urls, 'import.txt', urldb)
+    cinc = 0
+    while True:
+      chunk = urls[cinc:cinc+200]
+      if len(chunk): dbs.insert_urls(chunk, 'import.txt', urldb)
+      else: break
+      cinc = cinc + 200
 
 
 def get_content_type(url):
@@ -284,8 +289,6 @@ if __name__ == '__main__':
 
   proxydb = mysqlite.mysqlite(config.watchd.database, str)
   dbs.create_table_if_not_exists(proxydb, 'proxylist')
-  #global _known_proxies
-  #if len(_known_proxies) == 0:
   known = proxydb.execute('SELECT proxy FROM proxylist').fetchall()
   for k in known:
     _known_proxies[k[0]] = True
@@ -332,8 +335,8 @@ if __name__ == '__main__':
           nao = time.time()
           _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
           args = [ (nao,  row[0]) for row in rows ]
-          urldb.executemany('UPDATE uris SET check_time=? where url=?', args)
-          urldb.commit()
+          #urldb.executemany('UPDATE uris SET check_time=? where url=?', args)
+          #urldb.commit()
 
 
       for thread in threads:
@@ -353,6 +356,8 @@ if __name__ == '__main__':
       threads = [ thread for thread in threads if thread.is_alive() ]
       if len(threads) < config.ppf.threads and len(rows):
         row = random.choice(rows)
+        urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0]))
+        urldb.commit()
         rows.remove(row)
         t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5])
         threads.append(t)