changes

ppf.py

@@ -260,22 +260,35 @@ class Leechered(threading.Thread):
     proxy_count = len(self.proxylist)
 
     if self.retrievals == 0:  # new site
-      if content != '' and proxy_count == 0: # site works but has zero proxy addresses
-        #error = 99999
-        self.error += 2
-      else:
+      if content != '' and len(self.proxylist) == 0: # site works but has zero proxy addresses
         self.error += 1
-    else:
-      if len(self.proxylist) == 0:
         self.stale_count += 1
+      elif proxy_count > 0:
+        self.error = 0
+        self.stale_count = 0
+      else:
+        self.error += 2
+        self.stale_count += 2
+    else:                     # not a new site
+      # proxylist is empty
+      if proxy_count == 0:
+        self.stale_count += 1
+      # proxylist is not empty: site is working
       else:
         self.stale_count = 0
+        self.error = 0
+      # site has no content
       if content == '':
         self.error += 1
-      else:
-        self.retrievals += 1
-        self.error = 0
+        self.stale_count += 1
+      #else:
+      #  self.retrievals += 1
+      #  self.error = 0
+      #  self.stale_count = 0
+      # site has proxies
       if proxy_count:
+        self.error = 0
+        self.stale_count = 0
         extract_urls(content, self.url)
 
     self.execute = (self.error, self.stale_count, int(time.time()), self.retrievals, self.proxies_added+len(self.proxylist), self.content_type, self.url)
@@ -310,10 +323,10 @@ if __name__ == '__main__':
     watcherd = None
 
   #start_server(config.httpd.listenip, config.httpd.port)
-
-
+  
   #qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25'
   qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM()'
+  #qurl = 'SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE stale_count < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM()'
   threads = []
   rows = []
   rinc = 0
@@ -333,9 +346,9 @@ if __name__ == '__main__':
           time.sleep(60)
           rows = []
         else:
-          nao = time.time()
           _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
-          args = [ (nao,  row[0]) for row in rows ]
+          #nao = time.time()
+          #args = [ (nao,  row[0]) for row in rows ]
           #urldb.executemany('UPDATE uris SET check_time=? where url=?', args)
           #urldb.commit()
 
@@ -362,7 +375,7 @@ if __name__ == '__main__':
         t = Leechered(row[0], row[1], row[2], row[3], row[4], row[5])
         threads.append(t)
         t.start()
-        time.sleep(random.random()/100)
+        #time.sleep(random.random()/100)
 
     except KeyboardInterrupt:
       if watcherd: