misc changes and fixes and

import.txt

@@ -1,3 +1,29 @@
+https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt
+https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt
+https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt
+https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/proxy.txt
+https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt
+https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt
+http://proxy-ip-list.com/
+https://www.proxytm.com/socks-4-5-proxy-server-lists/type-socks4.htm
+https://www.proxytm.com/socks-4-5-proxy-server-lists/type-socks5.htm
+https://www.proxytm.com/fresh-http-proxy-server-lists/type-transparent.htm
+https://www.proxytm.com/fresh-http-proxy-server-lists/type-anonymous.htm
+https://www.proxytm.com/fresh-http-proxy-server-lists/type-elite.htm
+https://www.proxytm.com/public-http-proxy-server-lists/type-transparent.htm
+https://www.proxytm.com/public-http-proxy-server-lists/type-distorting.htm
+https://www.proxytm.com/public-http-proxy-server-lists/type-anonymous.htm
+https://www.proxytm.com/public-http-proxy-server-lists/type-elite.htm
+https://www.proxynova.com/proxy-server-list/elite-proxies/
+https://www.proxynova.com/proxy-server-list/
+https://free-proxy-list.net/
+http://proxydb.net/
+http://proxydb.net/?protocol=http&protocol=https&protocol=socks4&protocol=socks5&anonlvl=1&anonlvl=2&anonlvl=3&anonlvl=4&country=
+https://www.proxylist4all.com/free-proxy-list/free-elite-proxy-list
+http://www.proxytm.com/public-http-proxy-server-lists/port-9999.htm
+https://www.proxytm.com/free-http-proxy-server-lists/type-transparent.htm
+https://www.proxytm.com/free-http-proxy-server-lists/type-anonymous.htm
+https://www.proxytm.com/free-http-proxy-server-lists/type-elite.htm
 http://www.playinator.com/proxylist.txt
 http://neo.kador.free.fr/liste_proxy.txt
 https://portal-css.at.ua/40314proxy.txt

ppf.py

@@ -234,7 +234,7 @@ class Leechered(threading.Thread):
     threading.Thread.__init__(self)
 
   def retrieve(self):
-    return self.url, self.proxylist, self.stale_count, self.error, self.retrievals, self.content_type, self.execute
+    return self.url, self.proxylist, self.stale_count, self.error, self.retrievals, self.content_type, self.proxies_added, self.execute
   def status(self):
     return self.status
 
@@ -258,6 +258,8 @@ class Leechered(threading.Thread):
       if content != '' and proxy_count == 0: # site works but has zero proxy addresses
         #error = 99999
         self.error += 2
+      else:
+        self.error += 1
     else:
       if len(self.proxylist) == 0:
         self.stale_count += 1
@@ -307,25 +309,40 @@ if __name__ == '__main__':
   #start_server(config.httpd.listenip, config.httpd.port)
 
 
-  qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25'
+  #qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25'
+  qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM()'
   threads = []
   rows = []
   rinc = 0
   reqtime = time.time() - 3600
+  statusmsg = time.time()
   while True:
     try:
+      if (time.time() - statusmsg) > 180:
+        _log('running %d thread(s) over %d' % (len(threads), config.ppf.threads), 'ppf')
+        statusmsg = time.time()
+      time.sleep(0.1)
       if not len(rows):
         if (time.time() - reqtime) > 3:
           rows = urldb.execute(qurl, (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
           reqtime = time.time()
         if len(rows) < config.ppf.threads:
           rows = []
+        else:
+          _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
 
       for thread in threads:
         if thread.status == 'ok':
-          url, proxylist, stale_count, error, retrievals, content_type, execute = thread.retrieve()
+          url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve()
+          new = []
+          for p in proxylist:
+            #print(p)
+            if not p in _known_proxies:
+              new.append(p)
+              _known_proxies[p]=1
+          execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url)
           urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute)
-          if len(proxylist): dbs.insert_proxies(proxydb, proxylist, url)
+          if len(new): dbs.insert_proxies(proxydb, new, url)
 
       threads = [ thread for thread in threads if thread.is_alive() ]
       if len(threads) < config.ppf.threads and len(rows):

search_terms.txt

@@ -1,8 +1,16 @@
 elite proxylist
 elite http proxies
 elite socks proxies
-anonymous proxies
-anonymous proxylist
+anonymous proxies updated
+anonymous proxies hourly
+anonymous proxies daily
+anonymous proxylist updated
+anonymous proxylist hourly
+anonymous proxylist daily
+transparent proxylist fresh
+transparent proxylist updated
+transparent proxylist hourly
+transparent proxylist daily
 hourly http proxy
 hourly socks proxy
 daily http proxy