diff --git a/config.py b/config.py index ed29ece..d939da4 100644 --- a/config.py +++ b/config.py @@ -29,6 +29,11 @@ class Config(ComboParser): self.add_item(section, 'source_file', str, 'servers.txt', 'server/url list to read from (default: servers.txt)', False) self.add_item(section, 'tor_safeguard', bool, True, 'enable tor safeguard (default: True)', False) + section = 'httpd' + self.add_item(section, 'listenip', str, '127.0.0.1', 'address for the httpd to listen to (default: 127.0.0.1)', True) + self.add_item(section, 'port', int, 8081, 'port for the httpd to listen to (default: 8081)', True) + self.add_item(section, 'enabled', bool, False, 'start httpd (default: False)', True) + section = 'ppf' self.add_item(section, 'debug', bool, False, 'whether to print additional debug info', False) self.add_item(section, 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False) diff --git a/dbs.py b/dbs.py index 9fa2d4f..c3a09ba 100644 --- a/dbs.py +++ b/dbs.py @@ -17,6 +17,7 @@ def create_table_if_not_exists(sqlite, dbname): elif dbname == 'uris': sqlite.execute("""CREATE TABLE IF NOT EXISTS uris ( url TEXT, + content_type TEXT, check_time INT, error INT, stale_count INT, diff --git a/ppf.py b/ppf.py index e1f67c0..bb2cdb5 100755 --- a/ppf.py +++ b/ppf.py @@ -10,6 +10,7 @@ import fetch import sys from bs4 import BeautifulSoup import re +import threading config = Config() @@ -107,6 +108,99 @@ def import_proxies_from_file(proxydb, fn): return 0 return 1 +def serve_loop(hs, done): + client_threads = [] + while not done.is_set(): + c = hs.wait_client() + + evt_done = threading.Event() + cthread = threading.Thread(target=httpsrv_client_thread, args=(c,evt_done)) + cthread.daemon = True + cthread.start() + + ctrm = [] + for ct, ct_done in client_threads: + if ct_done.is_set(): + ctrm.append((ct,ct_done)) + ct.join() + + if len(ctrm): + client_threads = [ x for x in client_threads if not x in ctrm ] + + client_threads.append((cthread, evt_done)) + +def forbidden_page(): + return ( + '\n' + ' \n' + ' \n' + ' Forbidden\n' + ' \n' + ' \n' + '
🖕
\n' + ' \n' + '') + +def httpsrv_client_thread(c, evt_done): + req = c.read_request() + if req is None: pass + elif len(watchlist) == 0: + c.redirect('/config.html') + elif os.path.isdir(req['url'][1:]): + c.send(403,'Forbidden', forbidden_page()) + elif req['url'] == '/': + c.redirect('/index.html') + elif req['url'].startswith('/index.html'): + variables = variables_from_request(req) + r, redir = render_site(variables) + if redir is not "": + c.redirect(redir) + else: + if r == '': r = render_empty(variables=variables) + c.send(200, "OK", r) + elif not '..' in req['url'] and file_exists(os.getcwd() + req['url']): + c.serve_file(os.getcwd() + req['url']) + elif req['url'] == '/robots.txt': + c.send(200, "OK", "User-agent: *\nDisallow: /") + + elif req['url'].startswith('/config.html'): + if args.config > 0: + variables=variables_from_request(req) + r, redir = configpage(req,variables) + else: + redir = '/index.html' + if redir is not "": + c.redirect(redir) + else: + if r == '': r = render_empty(variables=variables) + c.send(200, "OK", r) + + else: + c.send(404, "not exist", "the reqested file not exist!!!1") + c.disconnect() + evt_done.set() + +def start_server(ip, port): + done = threading.Event() + from httpsrv import HttpSrv + hs = HttpSrv(ip, port) + try: + hs.setup() + except socket.error as e: + if e.errno == errno.EADDRINUSE: + sys.stderr.write(( + "ERROR: server socket address in use\n" + "wait a couple seconds and try again.\n" + "in case you're in pdb, you need to quit it\n")) + sys.exit(1) + else: + raise e + + t = threading.Thread(target=serve_loop, args=(hs, done)) + t.daemon = True + t.start() + return t, done + if __name__ == '__main__': config.load() fetch.set_config(config) @@ -130,6 +224,8 @@ if __name__ == '__main__': else: watcherd = None + start_server(config.httpd.listenip, config.httpd.port) + while True: try: ## any site that needs to be checked ? @@ -140,8 +236,6 @@ if __name__ == '__main__': for row in rows: proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4], row[5]) - #time.sleep(10) - except KeyboardInterrupt: if watcherd: watcherd.stop() diff --git a/proxywatchd.py b/proxywatchd.py index 7d5ac04..baf1597 100644 --- a/proxywatchd.py +++ b/proxywatchd.py @@ -2,7 +2,11 @@ import threading import time, random, string, re, copy -from geoip import geolite2 +try: + from geoip import geolite2 + geolite = True +except: + geolite = False from config import Config @@ -24,7 +28,7 @@ def socks4_resolve(srvname, server_port): if srv in cached_dns: srv = cached_dns[srvname] if config.watchd.debug: - _log("using cached ip (%s) for %s (proxy: %s)"%(srv, srvname, self.proxy), "debug") + _log("using cached ip (%s) for %s"%(srv, srvname), "debug") else: dns_fail = False try: @@ -129,7 +133,7 @@ class WorkerJob(): if re.match('^(:|NOTICE)', recv, re.IGNORECASE): duration = (time.time() - duration) - if not self.country or self.country == 'unknown' or self.country == 'N/A': + if geolite and not self.country or self.country == 'unknown' or self.country == 'N/A': match = geolite2.lookup(self.proxy.split(':')[0]) if match is not None: self.country = match.country else: self.country = 'N/A' @@ -270,8 +274,9 @@ class Proxywatchd(): if len(rows) < config.watchd.threads and config.watchd.oldies: ## disable tor safeguard for old proxies if self.tor_safeguard: self.tor_safeguard = False - q += ' LIMIT ?' - rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time(), config.watchd.threads*config.watchd.oldies_multi)).fetchall() + #q += ' LIMIT ?' + #rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time(), config.watchd.threads*config.watchd.oldies_multi)).fetchall() + rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*3, config.watchd.checktime, config.watchd.oldies_checktime, time.time())).fetchall() return rows def prepare_jobs(self): diff --git a/servers.txt b/servers.txt index 2c71e96..0924d39 100644 --- a/servers.txt +++ b/servers.txt @@ -2,7 +2,6 @@ eu.ircnet.org eu.undernet.org irc.2600.net irc.Undernet.Org -irc.accessirc.net irc.afterx.net irc.atrum.org irc.atw-inter.net @@ -15,14 +14,12 @@ irc.chat4all.org irc.chatspike.net irc.choopa.net irc.coldfront.net -irc.cyanide-x.net irc.cyberarmy.net irc.d-t-net.de irc.dal.net irc.darkmyst.org irc.data.lt irc.deepirc.net -irc.deepspace.org irc.drlnet.com irc.dynastynet.net irc.ecnet.org @@ -38,7 +35,6 @@ irc.eversible.com irc.fdfnet.net irc.fef.net irc.financialchat.com -irc.foonetic.net irc.forestnet.org irc.foreverchat.net irc.freequest.net @@ -46,12 +42,10 @@ irc.gamesurge.net irc.geekshed.net irc.german-freakz.net irc.gigairc.net -irc.globalchat.org irc.globalgamers.net irc.greekirc.net irc.icq.com irc.immortal-anime.net -irc.imperium.ca irc.inet.tele.dk irc.irc2.hu irc.irc4fun.net @@ -64,11 +58,9 @@ irc.irdsi.net irc.kampungchat.org irc.knightirc.net irc.krono.net -irc.landoleet.org irc.langochat.net irc.lichtsnel.nl irc.maddshark.net -irc.mozilla.org irc.newnet.net irc.nightstar.net irc.oftc.net @@ -82,7 +74,6 @@ irc.ptnet.org irc.rezosup.org irc.rizon.net irc.scarynet.org -irc.seersirc.net irc.serenia.net irc.serenity-irc.net irc.servercentral.net @@ -93,7 +84,6 @@ irc.snt.utwente.nl irc.sorcery.net irc.spacetronix.net irc.st-city.net -irc.starchat.net irc.starlink-irc.org irc.starlink.org irc.staynet.org @@ -101,8 +91,6 @@ irc.swiftirc.net irc.teranova.net irc.us.dal.net irc.us.gamesurge.net -irc.utonet.org -irc.webchat.org irc.whatnet.org irc.wondernet.nu irc.xevion.net