split to ip/port, "cleanse" ips and ports, bugfixes

fetch.py

@@ -23,7 +23,7 @@ def cleanhtml(raw_html):
 
 def fetch_contents(url, head=False, proxy=None):
 	content = None
-	if len(proxy):
+	if proxy is not None and len(proxy):
 		for p in proxy:
 			content = _fetch_contents(url, head=head, proxy=p)
 			if content is not None: break
@@ -102,6 +102,10 @@ def extract_proxies(content, proxydb):
 
 	uniques_dict = {}
 	for p in matches:
+		ip, port = p.split(':')
+		ip = '.'.join( [ str(int(str(i))) for i in ip.split('.') ] )
+		port = int( port.lstrip('0') )
+		p = '%s:%s' % (ip, port)
 		uniques_dict[p] = True
 
 	uniques = []