username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

some changes

Browse Source
This commit is contained in:
Your Name
2021-01-24 03:52:56 +01:00
parent fe2353acb2
commit 78b29a1187
5 changed files with 112 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
config.py
View File

@@ -29,6 +29,11 @@ class Config(ComboParser):
self.add_item(section, 'source_file', str, 'servers.txt', 'server/url list to read from (default: servers.txt)', False) self.add_item(section, 'source_file', str, 'servers.txt', 'server/url list to read from (default: servers.txt)', False)
self.add_item(section, 'tor_safeguard', bool, True, 'enable tor safeguard (default: True)', False) self.add_item(section, 'tor_safeguard', bool, True, 'enable tor safeguard (default: True)', False)
section = 'httpd'
self.add_item(section, 'listenip', str, '127.0.0.1', 'address for the httpd to listen to (default: 127.0.0.1)', True)
self.add_item(section, 'port', int, 8081, 'port for the httpd to listen to (default: 8081)', True)
self.add_item(section, 'enabled', bool, False, 'start httpd (default: False)', True)
section = 'ppf' section = 'ppf'
self.add_item(section, 'debug', bool, False, 'whether to print additional debug info', False) self.add_item(section, 'debug', bool, False, 'whether to print additional debug info', False)
self.add_item(section, 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False) self.add_item(section, 'search', bool, True, 'whether to use searx search engine to find new proxy lists', False)

1
dbs.py
View File

@@ -17,6 +17,7 @@ def create_table_if_not_exists(sqlite, dbname):
elif dbname == 'uris': elif dbname == 'uris':
sqlite.execute("""CREATE TABLE IF NOT EXISTS uris ( sqlite.execute("""CREATE TABLE IF NOT EXISTS uris (
url TEXT, url TEXT,
content_type TEXT,
check_time INT, check_time INT,
error INT, error INT,
stale_count INT, stale_count INT,

98
ppf.py
View File

@@ -10,6 +10,7 @@ import fetch
import sys import sys
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import re import re
import threading
config = Config() config = Config()
@@ -107,6 +108,99 @@ def import_proxies_from_file(proxydb, fn):
return 0 return 0
return 1 return 1
def serve_loop(hs, done):
client_threads = []
while not done.is_set():
c = hs.wait_client()
evt_done = threading.Event()
cthread = threading.Thread(target=httpsrv_client_thread, args=(c,evt_done))
cthread.daemon = True
cthread.start()
ctrm = []
for ct, ct_done in client_threads:
if ct_done.is_set():
ctrm.append((ct,ct_done))
ct.join()
if len(ctrm):
client_threads = [ x for x in client_threads if not x in ctrm ]
client_threads.append((cthread, evt_done))
def forbidden_page():
return (
'<!DOCTYPE html>\n'
' <head>\n'
' <style>div.e{position:fixed;top:25%;bottom:25%;left:25%;right:25%;font-size:150px;text-align:center;}</style>\n'
' <title>Forbidden</title>\n'
' </head>\n'
' <body>\n'
' <div class="e">&#128405;</div>\n'
' </body>\n'
'</html>')
def httpsrv_client_thread(c, evt_done):
req = c.read_request()
if req is None: pass
elif len(watchlist) == 0:
c.redirect('/config.html')
elif os.path.isdir(req['url'][1:]):
c.send(403,'Forbidden', forbidden_page())
elif req['url'] == '/':
c.redirect('/index.html')
elif req['url'].startswith('/index.html'):
variables = variables_from_request(req)
r, redir = render_site(variables)
if redir is not "":
c.redirect(redir)
else:
if r == '': r = render_empty(variables=variables)
c.send(200, "OK", r)
elif not '..' in req['url'] and file_exists(os.getcwd() + req['url']):
c.serve_file(os.getcwd() + req['url'])
elif req['url'] == '/robots.txt':
c.send(200, "OK", "User-agent: *\nDisallow: /")
elif req['url'].startswith('/config.html'):
if args.config > 0:
variables=variables_from_request(req)
r, redir = configpage(req,variables)
else:
redir = '/index.html'
if redir is not "":
c.redirect(redir)
else:
if r == '': r = render_empty(variables=variables)
c.send(200, "OK", r)
else:
c.send(404, "not exist", "the reqested file not exist!!!1")
c.disconnect()
evt_done.set()
def start_server(ip, port):
done = threading.Event()
from httpsrv import HttpSrv
hs = HttpSrv(ip, port)
try:
hs.setup()
except socket.error as e:
if e.errno == errno.EADDRINUSE:
sys.stderr.write((
"ERROR: server socket address in use\n"
"wait a couple seconds and try again.\n"
"in case you're in pdb, you need to quit it\n"))
sys.exit(1)
else:
raise e
t = threading.Thread(target=serve_loop, args=(hs, done))
t.daemon = True
t.start()
return t, done
if __name__ == '__main__': if __name__ == '__main__':
config.load() config.load()
fetch.set_config(config) fetch.set_config(config)
@@ -130,6 +224,8 @@ if __name__ == '__main__':
else: else:
watcherd = None watcherd = None
start_server(config.httpd.listenip, config.httpd.port)
while True: while True:
try: try:
## any site that needs to be checked ? ## any site that needs to be checked ?
@@ -140,8 +236,6 @@ if __name__ == '__main__':
for row in rows: for row in rows:
proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4], row[5]) proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4], row[5])
#time.sleep(10)
except KeyboardInterrupt: except KeyboardInterrupt:
if watcherd: if watcherd:
watcherd.stop() watcherd.stop()

15
proxywatchd.py
View File

@@ -2,7 +2,11 @@
import threading import threading
import time, random, string, re, copy import time, random, string, re, copy
from geoip import geolite2 try:
from geoip import geolite2
geolite = True
except:
geolite = False
from config import Config from config import Config
@@ -24,7 +28,7 @@ def socks4_resolve(srvname, server_port):
if srv in cached_dns: if srv in cached_dns:
srv = cached_dns[srvname] srv = cached_dns[srvname]
if config.watchd.debug: if config.watchd.debug:
_log("using cached ip (%s) for %s (proxy: %s)"%(srv, srvname, self.proxy), "debug") _log("using cached ip (%s) for %s"%(srv, srvname), "debug")
else: else:
dns_fail = False dns_fail = False
try: try:
@@ -129,7 +133,7 @@ class WorkerJob():
if re.match('^(:|NOTICE)', recv, re.IGNORECASE): if re.match('^(:|NOTICE)', recv, re.IGNORECASE):
duration = (time.time() - duration) duration = (time.time() - duration)
if not self.country or self.country == 'unknown' or self.country == 'N/A': if geolite and not self.country or self.country == 'unknown' or self.country == 'N/A':
match = geolite2.lookup(self.proxy.split(':')[0]) match = geolite2.lookup(self.proxy.split(':')[0])
if match is not None: self.country = match.country if match is not None: self.country = match.country
else: self.country = 'N/A' else: self.country = 'N/A'
@@ -270,8 +274,9 @@ class Proxywatchd():
if len(rows) < config.watchd.threads and config.watchd.oldies: if len(rows) < config.watchd.threads and config.watchd.oldies:
## disable tor safeguard for old proxies ## disable tor safeguard for old proxies
if self.tor_safeguard: self.tor_safeguard = False if self.tor_safeguard: self.tor_safeguard = False
q += ' LIMIT ?' #q += ' LIMIT ?'
rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time(), config.watchd.threads*config.watchd.oldies_multi)).fetchall() #rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*2, config.watchd.checktime, config.watchd.oldies_checktime, time.time(), config.watchd.threads*config.watchd.oldies_multi)).fetchall()
rows = self.mysqlite.execute(q, (config.watchd.max_fail, config.watchd.max_fail*3, config.watchd.checktime, config.watchd.oldies_checktime, time.time())).fetchall()
return rows return rows
def prepare_jobs(self): def prepare_jobs(self):

12
servers.txt
View File

@@ -2,7 +2,6 @@ eu.ircnet.org
eu.undernet.org eu.undernet.org
irc.2600.net irc.2600.net
irc.Undernet.Org irc.Undernet.Org
irc.accessirc.net
irc.afterx.net irc.afterx.net
irc.atrum.org irc.atrum.org
irc.atw-inter.net irc.atw-inter.net
@@ -15,14 +14,12 @@ irc.chat4all.org
irc.chatspike.net irc.chatspike.net
irc.choopa.net irc.choopa.net
irc.coldfront.net irc.coldfront.net
irc.cyanide-x.net
irc.cyberarmy.net irc.cyberarmy.net
irc.d-t-net.de irc.d-t-net.de
irc.dal.net irc.dal.net
irc.darkmyst.org irc.darkmyst.org
irc.data.lt irc.data.lt
irc.deepirc.net irc.deepirc.net
irc.deepspace.org
irc.drlnet.com irc.drlnet.com
irc.dynastynet.net irc.dynastynet.net
irc.ecnet.org irc.ecnet.org
@@ -38,7 +35,6 @@ irc.eversible.com
irc.fdfnet.net irc.fdfnet.net
irc.fef.net irc.fef.net
irc.financialchat.com irc.financialchat.com
irc.foonetic.net
irc.forestnet.org irc.forestnet.org
irc.foreverchat.net irc.foreverchat.net
irc.freequest.net irc.freequest.net
@@ -46,12 +42,10 @@ irc.gamesurge.net
irc.geekshed.net irc.geekshed.net
irc.german-freakz.net irc.german-freakz.net
irc.gigairc.net irc.gigairc.net
irc.globalchat.org
irc.globalgamers.net irc.globalgamers.net
irc.greekirc.net irc.greekirc.net
irc.icq.com irc.icq.com
irc.immortal-anime.net irc.immortal-anime.net
irc.imperium.ca
irc.inet.tele.dk irc.inet.tele.dk
irc.irc2.hu irc.irc2.hu
irc.irc4fun.net irc.irc4fun.net
@@ -64,11 +58,9 @@ irc.irdsi.net
irc.kampungchat.org irc.kampungchat.org
irc.knightirc.net irc.knightirc.net
irc.krono.net irc.krono.net
irc.landoleet.org
irc.langochat.net irc.langochat.net
irc.lichtsnel.nl irc.lichtsnel.nl
irc.maddshark.net irc.maddshark.net
irc.mozilla.org
irc.newnet.net irc.newnet.net
irc.nightstar.net irc.nightstar.net
irc.oftc.net irc.oftc.net
@@ -82,7 +74,6 @@ irc.ptnet.org
irc.rezosup.org irc.rezosup.org
irc.rizon.net irc.rizon.net
irc.scarynet.org irc.scarynet.org
irc.seersirc.net
irc.serenia.net irc.serenia.net
irc.serenity-irc.net irc.serenity-irc.net
irc.servercentral.net irc.servercentral.net
@@ -93,7 +84,6 @@ irc.snt.utwente.nl
irc.sorcery.net irc.sorcery.net
irc.spacetronix.net irc.spacetronix.net
irc.st-city.net irc.st-city.net
irc.starchat.net
irc.starlink-irc.org irc.starlink-irc.org
irc.starlink.org irc.starlink.org
irc.staynet.org irc.staynet.org
@@ -101,8 +91,6 @@ irc.swiftirc.net
irc.teranova.net irc.teranova.net
irc.us.dal.net irc.us.dal.net
irc.us.gamesurge.net irc.us.gamesurge.net
irc.utonet.org
irc.webchat.org
irc.whatnet.org irc.whatnet.org
irc.wondernet.nu irc.wondernet.nu
irc.xevion.net irc.xevion.net