initial commit
This commit is contained in:
mickael
23
config.ini
Normal file
23
config.ini
Normal file
@@ -0,0 +1,23 @@
|
||||
[global]
|
||||
tor_host = 127.0.0.1:9050
|
||||
i2p_host = 127.0.0.1:4444
|
||||
common_proxy_ports = 80, 1080, 3124, 3128, 4145, 4444, 8080, 8081, 8118, 8888, 9999
|
||||
proxylist_reload_every = 180
|
||||
database = proxylist.sqlite
|
||||
proxy_max_fail = 5
|
||||
|
||||
[watcherd]
|
||||
enabled = true
|
||||
proxy_file = false
|
||||
checktime = 1800
|
||||
threads = 10
|
||||
timeout = 15
|
||||
read_timeout = 20
|
||||
max_fail = 5
|
||||
|
||||
[proxyfind]
|
||||
enabled = true
|
||||
search = true
|
||||
maxfail = 10
|
||||
timeout = 30
|
||||
threads = 3
|
||||
562
import.txt
Normal file
562
import.txt
Normal file
@@ -0,0 +1,562 @@
|
||||
http://spys.one/en/socks-proxy-list/
|
||||
https://www.my-proxy.com/free-socks-5-proxy.html
|
||||
http://www.proxyserverlist24.top/feeds/posts/default
|
||||
http://spys.me/proxy.txt
|
||||
https://raw.githubusercontent.com/opsxcq/proxy-list/master/list.txt
|
||||
https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt
|
||||
https://raw.githubusercontent.com/flyinwind/proxylist/master/proxies.txt
|
||||
https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list.txt
|
||||
http://multiproxy.org/txt_all/proxy.txt
|
||||
https://github.com/clarketm/proxy-list/blob/master/proxy-list.txt
|
||||
http://www.megaproxylist.net/
|
||||
https://raw.githubusercontent.com/JamesJGoodwin/wreckuests/master/files/proxy.txt
|
||||
https://www.binary-zone.com/files/MyProxyList.txt
|
||||
http://captchasniper.com/new/FreeProxies.txt
|
||||
http://oupfps7we.bkt.clouddn.com/proxy_list.txt
|
||||
http://white55.narod.ru/downloads/proxyold.txt
|
||||
http://www.white55.narod.ru/downloads/proxylist.txt
|
||||
http://xls.su/br/1.txt
|
||||
https://proxyscrape.com/proxies/HTTP_Working_Proxies.txt
|
||||
https://rootjazz.com/proxies/proxies.txt
|
||||
http://www.zarb.org/~saispo/proxylist.txt
|
||||
http://promicom.by/tools/proxy/proxy.txt
|
||||
http://pouuet.free.fr/proxy.txt
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=1
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=2
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=3
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=4
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=5
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=6
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=7
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=8
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=9
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=10
|
||||
http://proxy-ip-list.com/download/free-usa-proxy-ip.txt
|
||||
http://proxy-ip-list.com/download/free-uk-proxy-list.txt
|
||||
http://www.freeproxy.ru/download/lists/goodproxy.txt
|
||||
http://www.proxylists.net/socks4.txt
|
||||
http://multiproxy.org/txt_all/proxy.txt
|
||||
https://raw.githubusercontent.com/Zenaker/IpLogger-Flood/master/proxy.txt
|
||||
https://raw.githubusercontent.com/SEODEMON/EXTREME-PROXY-HARVESTER-AND-CHECKER/master/leeched/leeched_proxies.txt
|
||||
https://proxy.mimvp.com/free.php?proxy=in_hp
|
||||
http://www.coobobo.com/free-http-proxy
|
||||
http://ip.zdaye.com/
|
||||
http://www.mayidaili.com/free/anonymous/%E9%AB%98%E5%8C%BF
|
||||
http://http.taiyangruanjian.com/
|
||||
http://http.zhimaruanjian.com/
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=50
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=100
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=150
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=200
|
||||
http://samair.ru/proxy/list-IP-port/proxy-1.htm
|
||||
http://samair.ru/proxy/list-IP-port/proxy-2.htm
|
||||
http://samair.ru/proxy/list-IP-port/proxy-3.htm
|
||||
http://samair.ru/proxy/list-IP-port/proxy-4.htm
|
||||
http://samair.ru/proxy/list-IP-port/proxy-5.htm
|
||||
http://proxylist.hidemyass.com/search-1304300#listable
|
||||
https://free-proxy-list.net/
|
||||
https://proxy-list.org/english/search.php?search=anonymous-and-elite.ssl-no&country=any&type=anonymous-and-elite&port=any&ssl=no
|
||||
https://proxy-list.org/english/search.php?search=anonymous-and-elite.ssl-no&country=any&type=anonymous-and-elite&port=any&ssl=no&p=2
|
||||
https://hidemy.name/en/proxy-list/?type=h&anon=4#list
|
||||
https://hidemy.name/en/proxy-list/?type=h&anon=4&start=64#list
|
||||
https://hidemy.name/en/proxy-list/?type=h&anon=4&start=128#list
|
||||
https://hidemy.name/en/proxy-list/?type=h&anon=4&start=192#list
|
||||
https://checkerproxy.net/archive/2017-04-01
|
||||
http://aliveproxy.com/fastest-proxies/
|
||||
http://multiproxy.org/anon_proxy.htm
|
||||
http://www.ultraproxies.com/high-anonymous.html
|
||||
http://www.ip-adress.com/proxy_list/
|
||||
http://samair.ru/anonymity-test/
|
||||
http://txt.proxyspy.net/proxy.txt
|
||||
http://nntime.com/
|
||||
http://www.gatherproxy.com/
|
||||
http://proxysearcher.sourceforge.net/Proxy%20List.php?type=http&filtered=true
|
||||
http://proxyserverlist-24.blogspot.com/2016/11/01-11-16-fast-proxy-server-list.html
|
||||
https://cracking.org/threads/190x-anonymous-proxy-list.125920/
|
||||
http://samair.ru/proxy/list-IP-port/ip-address-12.htm
|
||||
http://www.businessadviceforum.com/threads/fastest-securest-proxies-on-the-web-new-proxy-lists-everyday.75326/page-10
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=50
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=100
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=100
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=150
|
||||
http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=200
|
||||
http://www.nntime.com/proxy/
|
||||
http://www.textproxylists.com/proxy.php?codeen
|
||||
http://www.textproxylists.com/proxy.php?anonymous
|
||||
http://www.textproxylists.com/proxy.php?highanonymity
|
||||
http://www.textproxylists.com/proxy.php?transparent
|
||||
http://www.textproxylists.com/proxy.php?nontransparent
|
||||
http://www.textproxylists.com/proxy.php?allproxy
|
||||
http://proxies.my-proxy.com/proxy-list-s1.html
|
||||
http://proxies.my-proxy.com/proxy-list-s2.html
|
||||
http://proxies.my-proxy.com/proxy-list-s3.html
|
||||
http://proxies.my-proxy.com/proxy-list-1.html
|
||||
http://proxies.my-proxy.com/proxy-list-2.html
|
||||
http://proxies.my-proxy.com/proxy-list-3.html
|
||||
http://proxies.my-proxy.com/proxy-list-4.html
|
||||
http://proxies.my-proxy.com/proxy-list-5.html
|
||||
http://proxies.my-proxy.com/proxy-list-6.html
|
||||
http://proxies.my-proxy.com/proxy-list-7.html
|
||||
http://proxies.my-proxy.com/proxy-list-8.html
|
||||
http://proxies.my-proxy.com/proxy-list-9.html
|
||||
http://proxies.my-proxy.com/proxy-list-10.html
|
||||
http://www.proxy-list.net/anonymous-proxy-lists.shtml
|
||||
http://www.proxy-list.net/transparent-proxy-lists.shtml
|
||||
http://www.digitalcybersoft.com/ProxyList/fresh-proxy-list.shtml?
|
||||
http://proxies.my-proxy.com/
|
||||
http://proxies.my-proxy.com/proxy-list-2.html
|
||||
http://proxies.my-proxy.com/proxy-list-3.html
|
||||
http://proxies.my-proxy.com/proxy-list-4.html
|
||||
http://proxies.my-proxy.com/proxy-list-5.html
|
||||
http://proxies.my-proxy.com/proxy-list-6.html
|
||||
http://proxies.my-proxy.com/proxy-list-7.html
|
||||
http://proxies.my-proxy.com/proxy-list-8.html
|
||||
http://proxies.my-proxy.com/proxy-list-9.html
|
||||
http://proxies.my-proxy.com/proxy-list-10.html
|
||||
http://proxies.my-proxy.com/proxy-list-s1.html
|
||||
http://proxies.my-proxy.com/proxy-list-s2.html
|
||||
http://proxies.my-proxy.com/proxy-list-s3.html
|
||||
http://www.cooleasy.com/
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=2
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=3
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=4
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=5
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=6
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=7
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=8
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=9
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=10
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=11
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=12
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=13
|
||||
http://www.cooleasy.com/?act=list&port=&type=&country=&page=14
|
||||
http://www.x-proxy.info/highanon.php
|
||||
http://www.fresh-proxy-list.net/fresh%20proxy%20list.htm
|
||||
http://atomintersoft.com/free_proxy_list
|
||||
http://publicproxylist.org/
|
||||
http://httpproxylist.org/
|
||||
http://www.proxz.com/proxy_list_anonymous_us_0.html
|
||||
http://spys.me/proxy.txt
|
||||
https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt
|
||||
https://free-proxy-list.net/anonymous-proxy.html
|
||||
https://www.aliveproxy.com/proxy-list/proxies.aspx/Russian_Federation-ru
|
||||
http://www.aliveproxy.com/de-proxy-list/
|
||||
http://proxy-ip-list.com/free-usa-proxy-ip.html
|
||||
http://proxy-ip-list.com/8081/transparent-Spain-8081.html
|
||||
https://www.my-proxy.com/blog/tag/proxy-setting
|
||||
http://aliveproxy.com/proxy-list/proxies.aspx/South_Korea-KR
|
||||
https://www.my-proxy.com/free-proxy-list-3.html
|
||||
https://www.my-proxy.com/free-proxy-list-2.html
|
||||
https://www.my-proxy.com/free-proxy-list-4.html
|
||||
https://www.my-proxy.com/free-proxy-list-10.html
|
||||
https://free-proxy-list.net/?ref=binfind.com/web
|
||||
http://www.aliveproxy.com/proxy-list/proxies.aspx/Syria-sy
|
||||
https://www.my-proxy.com/free-proxy-list-9.html
|
||||
https://www.my-proxy.com/free-proxy-list-8.html
|
||||
https://www.my-proxy.com/free-proxy-list-5.html
|
||||
https://www.my-proxy.com/free-proxy-list-6.html
|
||||
https://github.com/a2u/free-proxy-list/blob/master/free-proxy-list.txt
|
||||
https://github.com/fate0/proxylist/blob/master/proxy.list
|
||||
http://proxy-ip-list.com/8080/fresh-servers-list-8080-Indonesia.html
|
||||
https://www.my-proxy.com/free-proxy-list.html
|
||||
https://www.my-proxy.com/free-elite-proxy.html
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=10
|
||||
http://proxy-ip-list.com/high-anonymous/high-anonymous-80-servers-list.html
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=1
|
||||
https://www.my-proxy.com/free-transparent-proxy.html
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=3
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=5
|
||||
https://www.my-proxy.com/free-proxy-list-1.html
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=s2
|
||||
https://www.my-proxy.com/free-anonymous-proxy.html
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=9
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=2
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=s3
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=7
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=4
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=6
|
||||
https://www.my-proxy.com/free-proxy-list.html?list=8
|
||||
http://proxies.my-proxy.com/proxy-list-s1.html
|
||||
http://proxies.my-proxy.com/proxy-list-s2.html
|
||||
http://proxies.my-proxy.com/proxy-list-s3.html
|
||||
http://proxies.my-proxy.com/proxy-list-1.html
|
||||
http://proxies.my-proxy.com/proxy-list-2.html
|
||||
http://proxies.my-proxy.com/proxy-list-3.html
|
||||
http://proxies.my-proxy.com/proxy-list-4.html
|
||||
http://proxies.my-proxy.com/proxy-list-5.html
|
||||
http://proxies.my-proxy.com/proxy-list-6.html
|
||||
http://proxies.my-proxy.com/proxy-list-7.html
|
||||
http://proxies.my-proxy.com/proxy-list-8.html
|
||||
http://proxies.my-proxy.com/proxy-list-9.html
|
||||
http://proxies.my-proxy.com/proxy-list-10.html
|
||||
http://proxies.my-proxy.com/
|
||||
http://www.proxz.com/proxy_list_anonymous_us_0.html
|
||||
https://www.my-proxy.com/free-socks-5-proxy.html
|
||||
http://www.httptunnel.ge/ProxyListForFree.aspx
|
||||
https://www.my-proxy.com/free-socks-4-proxy.html
|
||||
http://proxysearcher.sourceforge.net/Proxy%20List.php?type=socks&filtered=true
|
||||
http://proxysearcher.sourceforge.net/Proxy%20List.php
|
||||
http://www.proxz.com/
|
||||
http://www.proxz.com/proxy_list_port_std_2.html
|
||||
http://www.proxz.com/proxy_list_uk_0_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_11_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_0.html
|
||||
http://www.proxz.com/proxy_list_port_std_28.html
|
||||
http://www.proxz.com/proxy_list_cn_ssl_43.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_61.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_54.html
|
||||
http://www.proxz.com/proxy_list_port_std_38.html
|
||||
http://www.proxz.com/proxy_list_port_std_17.html
|
||||
https://www.proxydocker.com/de/proxylist/country/Germany
|
||||
http://proxysearcher.sourceforge.net/ProxyList.php?type=socks&filtered=true
|
||||
http://proxysearcher.sourceforge.net/ProxyList.php%3Ftype%3Dsocks%26filtered%3Dtrue
|
||||
http://proxysearcher.sourceforge.net/ProxyList.php%3Ftype%3Dsocks%26filtered%3Dtrue%26limit%3D1000
|
||||
http://www.proxz.com/proxy_list_port_nonstd_84_ext.html
|
||||
https://www.proxydocker.com/de/proxylist/country/Austria
|
||||
https://www.proxydocker.com/de/socks-list/
|
||||
https://free-proxy-list.com/
|
||||
http://proxysearcher.sourceforge.net/Proxy%20List.php?type=socks
|
||||
http://www.proxz.com/proxy_list_transparent_17.html
|
||||
http://proxysearcher.sourceforge.net/ProxyList.php?type=http
|
||||
http://proxysearcher.sourceforge.net/ProxyList.php?type=socks
|
||||
http://proxysearcher.sourceforge.net/ProxyList.php?type=http&filtered=true
|
||||
http://proxysearcher.sourceforge.net/ProxyList.php%3Ftype%3Dsocks
|
||||
http://www.proxz.com/proxy_list_port_std_11_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_18_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_5_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_36.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_20_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_15_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_72_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_31.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_8.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_56_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_30_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_58.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_87.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_86.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_151.html
|
||||
http://www.proxz.com/proxy_list_transparent_8_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_42_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_4.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_4.html
|
||||
http://www.proxz.com/proxy_list_transparent_51_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_2.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_117.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_62.html
|
||||
http://www.proxz.com/proxy_list_port_std_9_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_43.html
|
||||
http://www.proxz.com/proxy_list_transparent_5_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_11.html
|
||||
http://www.proxz.com/proxy_list_port_std_41_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_0.html
|
||||
http://www.proxz.com/proxy_list_transparent_3_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_52_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_72.html
|
||||
http://www.proxz.com/proxy_list_fr_1.html
|
||||
http://www.proxz.com/proxy_list_cn_ssl_47_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_27_ext.html
|
||||
http://www.proxz.com/proxy_list_jp_0.html
|
||||
http://www.proxz.com/proxy_list_ca_1.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_127_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_166_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_5_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_24_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_4_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_97.html
|
||||
http://www.proxz.com/proxy_list_anonymous_us_2.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_9.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_91.html
|
||||
http://www.proxz.com/proxy_list_transparent_0.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_24.html
|
||||
http://www.proxz.com/proxy_list_port_std_53_ext.html
|
||||
http://www.proxz.com/proxy_list_cn_ssl_38_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_31_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_12_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_9_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_15_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_59_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_46_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_12_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_17_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_40_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_137_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_132_ext.html
|
||||
https://www.proxydocker.com/?_locale=es
|
||||
https://www.proxydocker.com/en/proxylist/city/Mumbai
|
||||
https://www.proxydocker.com/en/proxylist/city/Dhaka
|
||||
https://www.proxydocker.com/en/proxylist/country/Lebanon
|
||||
https://www.proxydocker.com/de/proxylist/country/Indonesia
|
||||
http://www.proxydocker.com/de/proxylist/port/8118
|
||||
http://www.proxz.com/proxy_list_high_anonymous_14_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_23.html
|
||||
https://www.proxz.com/proxy_list_fr_0.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_36.html
|
||||
http://www.proxz.com/proxy_list_transparent_15_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_66_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_68.html
|
||||
https://www.proxydocker.com/
|
||||
https://www.proxydocker.com/fr/
|
||||
https://www.proxydocker.com/es/
|
||||
https://www.proxydocker.com/en/
|
||||
https://www.proxydocker.com/ms/
|
||||
https://www.proxydocker.com/pt/
|
||||
https://www.proxydocker.com/de/
|
||||
https://www.proxydocker.com/tr/
|
||||
https://www.proxydocker.com/it/
|
||||
http://www.proxz.com/proxy_list_transparent_1_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_38_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_33.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_1_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_40_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_10_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_14_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_64_ext.html
|
||||
https://www.proxydocker.com/en/proxylist/country/Hungary
|
||||
https://www.proxydocker.com/en/proxylist/country/Netherlands
|
||||
https://www.proxydocker.com/en/proxylist/city/Jalalpur
|
||||
https://www.proxydocker.com/en/proxylist/city/Istanbul
|
||||
http://www.proxydocker.com/
|
||||
https://www.proxydocker.com/?_locale=de
|
||||
https://www.proxydocker.com/en/proxylist/city/Singapore
|
||||
https://www.proxydocker.com/en/proxylist/state/Virginia
|
||||
https://www.proxydocker.com/en/proxylist/city/Kiev
|
||||
https://www.proxydocker.com/en/proxylist/state/Florida
|
||||
https://www.proxydocker.com/en/proxylist/country/Moldova
|
||||
https://www.proxydocker.com/en/proxylist/port/53281
|
||||
https://www.proxydocker.com/en/proxylist/country/Indonesia
|
||||
https://www.proxydocker.com/en/proxylist/country/Peru
|
||||
https://www.proxydocker.com/en/proxylist/country/Thailand
|
||||
http://www.proxz.com/proxy_list_port_nonstd_77_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_56.html
|
||||
http://www.proxz.com/proxy_list_cn_ssl_85_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_24_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_7.html
|
||||
https://www.proxydocker.com/en/proxylist/country/United%20States
|
||||
https://www.proxydocker.com/de/proxylist/country/Russia
|
||||
https://www.proxydocker.com/en/proxylist/
|
||||
https://www.proxydocker.com/de/proxylist/country/Italy
|
||||
https://www.proxydocker.com/de/proxylist/port/8080,3128
|
||||
https://www.proxydocker.com/de/proxylist/country/United%20Kingdom
|
||||
https://www.proxydocker.com/en/socks4-list/
|
||||
https://www.proxydocker.com/?_locale=hi
|
||||
https://www.proxydocker.com/?_locale=ms
|
||||
https://www.proxydocker.com/?id=7PeaMYqp7Fk
|
||||
https://www.proxydocker.com/?_locale=fr
|
||||
https://www.proxydocker.com/?_locale=it
|
||||
https://www.proxydocker.com/?_locale=en
|
||||
https://www.proxydocker.com/?_locale=ru
|
||||
https://www.proxydocker.com/en/socks-list/
|
||||
http://www.proxz.com/proxy_list_cn_ssl_88_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_46_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_55.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_29_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_10.html
|
||||
https://www.proxydocker.com/de/proxylist/country/Belgium
|
||||
https://www.proxydocker.com/de/proxylist/country/Cambodia
|
||||
https://www.proxydocker.com/de/proxylist/country/Philippines
|
||||
https://www.proxydocker.com/de/freeproxylist/South%20Africa
|
||||
https://www.proxydocker.com/de/proxylist/city/Paris
|
||||
https://www.proxydocker.com/en/proxylist/type/HTTP
|
||||
https://www.proxydocker.com/de/proxylist/country/Poland
|
||||
http://www.proxydocker.com/de
|
||||
https://www.proxydocker.com/de/proxylist/country/France
|
||||
https://www.proxydocker.com/de/proxylist/country/Vietnam
|
||||
https://www.proxydocker.com/de/proxylist/country/Singapore
|
||||
http://www.proxz.com/proxy_list_transparent_4_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_104_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_45_ext.html
|
||||
https://www.proxydocker.com/de/proxylist/city/Singapore
|
||||
https://www.proxydocker.com/de/proxylist/country/Australia
|
||||
https://www.proxydocker.com/de/socks5-list/
|
||||
https://www.proxydocker.com/en/proxylist/country/France
|
||||
https://www.proxydocker.com/en/proxylist/state/Texas
|
||||
https://www.proxydocker.com/en/proxylist/city/Moscow
|
||||
https://www.proxydocker.com/en/proxylist/country/Iran
|
||||
http://www.proxz.com/proxy_list_cn_ssl_1_ext.html
|
||||
http://www.httptunnel.ge/ProxyListForFree.aspx/Media/httptunnelbanner.swf
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/httptunnelbanner.swf
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/media/Media/ProxyChecker.aspx
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/ProxyListForFree.aspx
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/Default.aspx
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/dpg.aspx
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/media/Media/Media/Default.aspx
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/media/Media/Media/dpg.aspx
|
||||
https://www.proxydocker.com/en/socks5-list/
|
||||
https://www.proxydocker.com/?_locale=pt
|
||||
http://www.httptunnel.ge/ProxyListForFree.aspx/ProxyChecker.aspx
|
||||
http://www.httptunnel.ge/ProxyListForFree.aspx/FAQ.aspx
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/media/Download.aspx
|
||||
http://www.proxz.com/proxy_list_port_nonstd_23_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_129_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_57_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_16_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_54_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_3.html
|
||||
https://proxypedia.org/
|
||||
http://proxysearcher.sourceforge.net/Proxy%20List.php?type=http
|
||||
https://www.proxydocker.com/en/proxylist/state/California
|
||||
https://www.proxydocker.com/en/proxylist/country/South%20korea
|
||||
https://www.proxydocker.com/en/proxylist/port/80
|
||||
https://www.proxydocker.com/en/proxylist/city/New%20York
|
||||
https://www.proxydocker.com/en/proxylist/country/United%20Kingdom
|
||||
https://www.proxydocker.com/ja/proxylist/country/Japan
|
||||
https://www.proxydocker.com/en/proxylist/country/Brazil
|
||||
https://www.proxydocker.com/en/proxylist/port/8080
|
||||
https://www.proxydocker.com/ja/
|
||||
https://www.proxydocker.com/en/proxylist/country/Germany
|
||||
https://www.proxydocker.com/en/proxylist/country/Egypt
|
||||
https://www.proxydocker.com/en/proxylist/country/Kazakhstan
|
||||
https://www.proxydocker.com/en/proxylist/country/Paraguay
|
||||
https://www.proxydocker.com/en/proxylist/country/Malaysia
|
||||
https://www.proxydocker.com/de/proxylist/port/8080,3128,80
|
||||
https://www.proxydocker.com/de/proxylist/
|
||||
https://www.proxydocker.com/de/proxylist/country/Egypt
|
||||
https://www.proxydocker.com/de/proxylist/city/Bangkok
|
||||
http://www.proxz.com/proxy_list_transparent_164_ext.html
|
||||
https://www.proxydocker.com/en/proxylist/city/Lima
|
||||
https://www.proxydocker.com/de/proxylist/country/India
|
||||
http://www.proxz.com/proxy_list_high_anonymous_130_ext.html
|
||||
http://www.proxz.com/proxy_list_cn_ssl_59_ext.html
|
||||
https://www.proxydocker.com/en/proxylist/city/Yekaterinburg
|
||||
http://www.proxz.com/proxy_list_port_std_156_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_155_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_18_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_65_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_99_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_42_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_132.html
|
||||
http://www.proxz.com/proxy_list_transparent_85_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_87_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_128.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_75_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_103_ext.html
|
||||
http://www.proxz.com/proxy_list_transparent_76_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_11_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_90_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_93_ext.html
|
||||
http://www.proxz.com/proxy_list_port_nonstd_146_ext.html
|
||||
http://www.proxz.com/proxy_list_high_anonymous_124_ext.html
|
||||
http://www.proxz.com/proxy_list_fr_4.html
|
||||
https://www.proxydocker.com/en/proxylist/country/China
|
||||
https://www.proxydocker.com/en/proxylist/country/Albania
|
||||
http://www.premiumproxy.net/top-country-proxy-list?country=DE/
|
||||
http://www.premiumproxy.net/top-country-proxy-list?country=HN/
|
||||
http://www.premiumproxy.net/top-country-proxy-list.php?country=HK/
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=-braziljikarkarkA
|
||||
http://www.premiumproxy.net/top-country-proxy-list.php?country=FM
|
||||
http://www.premiumproxy.net/top-country-proxy-list.php?country=KR/
|
||||
http://premiumproxy.net/
|
||||
http://premiumproxy.net/contact-us
|
||||
http://premiumproxy.net/web-proxy-list
|
||||
http://premiumproxy.net/full-proxy-list
|
||||
http://premiumproxy.net/elite-proxy-list
|
||||
http://premiumproxy.net/check-headers-status
|
||||
http://premiumproxy.net/distorting-proxy-list
|
||||
http://premiumproxy.net/http-proxy-list
|
||||
http://premiumproxy.net/transparent-proxy-list.php
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=-bolivia_AND_AARK
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=FR-France_A_or&page=0
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=CA-A&page=3
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=JP-JapanRK
|
||||
http://www.proxz.com/proxy_list_port_nonstd_7_ext.html
|
||||
http://www.proxydocker.com/en/proxylist/country/China
|
||||
http://www.proxz.com/proxy_list_high_anonymous_104_ext.html
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=BG-
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=RU-
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=GR-
|
||||
http://premiumproxy.net/http-proxy-list.php?page=2
|
||||
http://premiumproxy.net/full-proxy-list.php?page=2
|
||||
http://premiumproxy.net/socks-proxy-list.php?page=1
|
||||
http://premiumproxy.net/top-country-proxy-list?country=EG
|
||||
http://premiumproxy.net/top-country-proxy-list?country=BR
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=A-
|
||||
http://premiumproxy.net/top-country-proxy-list?country=PK/
|
||||
https://www.proxydocker.com/de/proxylist/country/United%20States
|
||||
http://www.premiumproxy.net/top-country-proxy-list.php?country=CN&page=1
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=LT-Lithuania&page=3
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=CH-&page=1
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=BR-
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=SE-
|
||||
http://premiumproxy.net/top-country-proxy-list?country=UK
|
||||
http://www.proxz.com/proxy_list_port_std_67_ext.html
|
||||
http://www.proxz.com/proxy_list_port_std_49_ext.html
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=BO-
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=SELECT-
|
||||
https://www.proxydocker.com/en/proxylist/country/Japan
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-vietnam_and__and_skrkskrk
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-singapore
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=IT-Italy
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=AF
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-tanzaniaa
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=SG
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=PA-Panama
|
||||
http://premiumproxy.net/check-headers-status.php?url=nekopoi.lol
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=VN-Vietnam_AND__AND_zOUNzOUNRK
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=MX-Mexico_AND__AND_sUjBsUjBRK
|
||||
https://www.proxydocker.com/de/proxylist/port/8888,3128
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/media/dpg.aspx
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=FR-
|
||||
https://www.proxydocker.com/en/proxylist/city/Surabaya
|
||||
http://www.httptunnel.ge/proxylistforfree.aspx/
|
||||
https://www.proxydocker.com/de/proxylist/city/London
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=%20R-
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-dominicanrk
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-guatemala
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-southwp
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=BR-Brazil
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=NL
|
||||
http://premiumproxy.net/https-ssl-proxy-list.php?page=1
|
||||
https://proxypedia.org/czechia/free-proxy-178-77-206-21-58004
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=COA-
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=MX-
|
||||
http://premiumproxy.net/transparent-proxy-list.php?page=1
|
||||
http://premiumproxy.net/top-country-proxy-list?country=ML
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-namibia
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=IN
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=TH
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=JP-JapanRK
|
||||
http://www.proxydocker.com/en/proxylist/country/Italy
|
||||
http://www.proxz.com/proxy_list_high_anonymous_97_ext.html
|
||||
http://www.proxz.com/proxy_list_cn_ssl_82_ext.html
|
||||
http://www.proxz.com/terms.php
|
||||
http://www.proxz.com/proxy_list_high_anonymous_117_ext.html
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=UZ
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=NA-Namibia
|
||||
http://premiumproxy.net/top-country-proxy-list?country=FM
|
||||
http://www.premiumproxy.net/check-ip-proxy-port?ip-proxy=Anonytun.com&port=443
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=-thailand&page=5
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=-braziljikarkaRKRK&page=5
|
||||
http://www.premiumproxy.net/top-country-proxy-list?country=UK/&page=1
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=-singapore&page=2
|
||||
http://www.proxz.com/proxy_list_port_std_8.html
|
||||
http://premiumproxy.net/http-proxy-list.php?page=1
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=NL-Netherlands_ARKRK_
|
||||
http://premiumproxy.net/anonymous-proxy-list.php
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-bolivia_AND_AARK
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-curacao
|
||||
http://www.proxz.com/proxy_list_high_anonymous_108.html
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-bolivia_AND_A
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=MQ
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-myanmar
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=CN-China_
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=TW
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-malta
|
||||
http://premiumproxy.net/full-proxy-list.php?page=1
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=JP-Japan
|
||||
http://www.premiumproxy.net/top-country-proxy-list?cntr=KR-Southwp
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=VN
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=MX
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-turkey
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=PH-Pilippines
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=%20-_and_rkrs_ajlrnbcdatwcautelegvxdaso
|
||||
http://premiumproxy.net/transparent-proxy-list
|
||||
http://premiumproxy.net/squid-proxy-list
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-bolivia_and_aark
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=KR-Southwp
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-south_koreaaaaa
|
||||
http://premiumproxy.net/top-country-proxy-list.php?country=CL/
|
||||
http://premiumproxy.net/top-country-proxy-list?cntr=-braziljikarkark
|
||||
87
includes/misc.py
Normal file
87
includes/misc.py
Normal file
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import time, random, sys, os, string
|
||||
#import sockschain as socks
|
||||
sys.path.append('./includes')
|
||||
import rocksock
|
||||
|
||||
""" return formatted timestamp """
|
||||
def timestamp():
|
||||
return time.strftime('%H:%M:%S', time.gmtime())
|
||||
|
||||
""" return some random string """
|
||||
def random_string(strlen=20):
|
||||
return ''.join([random.choice(string.letters) for x in xrange(strlen)])
|
||||
|
||||
def _log(strng, level='info'):
|
||||
print '%s/%s\t%s' % (timestamp(), level, strng)
|
||||
|
||||
def option_matches_options(strng, items):
|
||||
try: return [item for item in items if re.match(strng, item)]
|
||||
except: return False
|
||||
|
||||
def prepare_socksocket(self, destination, path, path_item):
|
||||
if path_item in self.paths and self.paths[path_item]['path'] == path:
|
||||
self.paths[path_item]['path'] = False
|
||||
|
||||
#socks.setdefaultproxy()
|
||||
# relay to i2p http proxy if *.i2p domain
|
||||
if destination.endswith('i2p'):
|
||||
proxy = random.choice(self.i2p_host).split(':')
|
||||
path = False
|
||||
# or go with tor
|
||||
else:
|
||||
proxies = [ rocksock.RocksockProxyFromURL('socks5://%s' % random.choice(self.tor_host)) ]
|
||||
#socks.adddefaultproxy(*socks.parseproxy('tor://%s' % random.choice(self.tor_host)))
|
||||
# add 'clearnet' proxies to the chain ?
|
||||
if self.proxify and (not destination.endswith('onion') and not destination.endswith('.exit')):
|
||||
|
||||
# get a proxy path
|
||||
path = build_path(self, path_item, path)
|
||||
|
||||
# if path isn't long enough, break
|
||||
if not len(path): return False, False, False
|
||||
|
||||
# add chain...
|
||||
#for inc in xrange(len(path) - 1): socks.adddefaultproxy(*socks.parseproxy('http://%s' % path[inc]))
|
||||
#for inc in xrange(len(path) - 1): socks.adddefaultproxy(*socks.parseproxy('%s://%s' % (path[inc][1], path[inc][0])))
|
||||
for inc in xrange(len(path)): proxies.append( rocksock.RocksockProxyFromURL('%s://%s' % (path[inc][1], path[inc][0])))
|
||||
|
||||
#return True, socks.socksocket, path
|
||||
return True, proxies, path
|
||||
|
||||
def build_path(self, path_item, path):
|
||||
|
||||
chainlen = random.randint( self.path_len, (self.path_len + self.path_randomlen))
|
||||
# if not enough proxies
|
||||
# FIXME: try to get a proxylist from database
|
||||
if len(self.proxylist) < chainlen: return []
|
||||
|
||||
# valid path already available
|
||||
elif (path_item in self.paths and
|
||||
self.paths[path_item]['path'] and
|
||||
(time.time() - self.paths[path_item]['ticks']) < self.path_duration):
|
||||
|
||||
# take available path if any
|
||||
if path != self.paths[path_item]['path']: path = self.paths[path_item]['path']
|
||||
|
||||
# or nope, none available
|
||||
# build a new one from scratch
|
||||
else:
|
||||
path = []
|
||||
avail = []
|
||||
|
||||
# dec chainlen if we have to select the exit proxy
|
||||
if self.exitcountry is not None: chainlen -= 1
|
||||
#avail = [item[0] for item in self.proxylist if not item[0] in avail and item[1] != str(self.exitcountry).upper()]
|
||||
avail = [[item[0],item[2]] for item in self.proxylist if not item[0] in avail and item[1] != str(self.exitcountry).upper()]
|
||||
path = random.sample(avail, chainlen)
|
||||
|
||||
# choose the exit proxy
|
||||
if self.exitcountry is not None:
|
||||
#avail = [item[0] for item in self.proxylist if not item[0] in path and item[1] == str(self.exitcountry).upper()]
|
||||
avail = [[item[0],item[2]] for item in self.proxylist if not item[0] in path and item[1] == str(self.exitcountry).upper()]
|
||||
if not len(avail): return []
|
||||
path.append(random.choice(avail))
|
||||
|
||||
return path
|
||||
33
includes/mysqlite.py
Normal file
33
includes/mysqlite.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import time, random, sys
|
||||
import sqlite3 as lite
|
||||
|
||||
class mysqlite:
|
||||
|
||||
def execute(self, query, args = None, rmin = 1.5, rmax = 7.0):
|
||||
|
||||
while True:
|
||||
try: return self.cursor.execute(query,args) if args else self.cursor.execute(query)
|
||||
except:
|
||||
print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
|
||||
time.sleep(random.uniform(rmin, rmax))
|
||||
|
||||
def executemany(self, query, args, rmin = 1.5, rmax = 7.0):
|
||||
|
||||
while True:
|
||||
try: return self.cursor.executemany(query,args)
|
||||
except:
|
||||
print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
|
||||
time.sleep(random.uniform(rmin, rmax))
|
||||
|
||||
def commit(self, rmin = 1.5, rmax = 7.0):
|
||||
while True:
|
||||
try: return self.handle.commit()
|
||||
except: time.sleep(random.uniform(rmin, rmax))
|
||||
|
||||
def close(self):
|
||||
self.handle.close()
|
||||
|
||||
def __init__(self, database, factory = None):
|
||||
self.handle = lite.connect(database)
|
||||
if factory: self.handle.text_factory = factory
|
||||
self.cursor = self.handle.cursor()
|
||||
165
includes/proxywatchd.py
Normal file
165
includes/proxywatchd.py
Normal file
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from threading import Thread
|
||||
import threading, commands
|
||||
import socket, time, random, sys, string, re
|
||||
#import sockschain as socks
|
||||
import requests
|
||||
#from geoip import geolite2
|
||||
|
||||
from ConfigParser import SafeConfigParser
|
||||
|
||||
sys.path.append('./includes')
|
||||
import mysqlite
|
||||
from misc import _log
|
||||
import rocksock
|
||||
|
||||
class Proxywatchd(Thread):
|
||||
|
||||
def stop(self):
|
||||
_log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()]))
|
||||
self.running = 0
|
||||
|
||||
def __init__(self, config_file):
|
||||
Thread.__init__(self)
|
||||
|
||||
self.threads = []
|
||||
self.running = 1
|
||||
self.parser = SafeConfigParser()
|
||||
self.parser.read(config_file)
|
||||
|
||||
self.maxfail = self.parser.getint('global', 'proxy_max_fail')
|
||||
self.maxthreads = self.parser.getint('watcherd', 'threads')
|
||||
self.checktime = self.parser.getint('watcherd', 'checktime')
|
||||
self.timeout = self.parser.getint('watcherd', 'timeout')
|
||||
self.database = self.parser.get('global', 'database')
|
||||
self.torhosts = [ str(i).strip() for i in self.parser.get('global', 'tor_host').split(',') ]
|
||||
self.read_timeout = self.parser.getint('watcherd', 'read_timeout')
|
||||
|
||||
# create table if needed
|
||||
self.mysqlite = mysqlite.mysqlite(self.database, str)
|
||||
self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)')
|
||||
self.mysqlite.commit()
|
||||
self.echoise = time.time() - 3600;
|
||||
self.ticks = time.time() - 3600;
|
||||
|
||||
with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n')
|
||||
|
||||
self.start()
|
||||
|
||||
def run(self):
|
||||
_log('Starting proxywatchd..', 'notice')
|
||||
|
||||
threads = []
|
||||
self.mysqlite = mysqlite.mysqlite(self.database, str)
|
||||
|
||||
while self.running:
|
||||
|
||||
if len(threads) < self.maxthreads:
|
||||
t = threading.Thread(target=self.daemon, args=(self.servers,))
|
||||
t.start()
|
||||
threads.append(t)
|
||||
time.sleep( random.choice( xrange(1,3)))
|
||||
|
||||
else: time.sleep(1)
|
||||
|
||||
if (time.time() - self.echoise) >= 180:
|
||||
_log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads))
|
||||
self.echoise = time.time()
|
||||
|
||||
self.mysqlite.close()
|
||||
|
||||
def is_drone_bl(self, proxy):
|
||||
p = proxy.split(':')[0]
|
||||
proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))}
|
||||
resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies)
|
||||
if 'No incidents regarding' in resp.text: return 0
|
||||
else: return 1
|
||||
|
||||
def connect_socket(self, proxy, servers, proto = None):
|
||||
proto = ['http', 'socks5', 'socks4']
|
||||
|
||||
for p in proto:
|
||||
torhost = random.choice(self.torhosts)
|
||||
duration = time.time()
|
||||
proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost),
|
||||
rocksock.RocksockProxyFromURL('%s://%s' % (p, proxy[0])),
|
||||
]
|
||||
|
||||
tst = random.choice(servers).strip()
|
||||
try:
|
||||
sock = rocksock.Rocksock(host=tst, port=6667, ssl=False, proxies=proxies, timeout=self.timeout)
|
||||
sock.connect()
|
||||
sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT']))
|
||||
return sock, p, duration, torhost, tst
|
||||
|
||||
except:
|
||||
#print('socket error, disconnecting')
|
||||
sock.disconnect()
|
||||
|
||||
return False, False, False, False, False
|
||||
|
||||
def daemon(self, servers):
|
||||
sqlite = mysqlite.mysqlite(self.database, str)
|
||||
threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] )
|
||||
|
||||
q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed<? and tested<? ORDER BY RANDOM() LIMIT ?'
|
||||
|
||||
while self.running:
|
||||
|
||||
time.sleep(0.1)
|
||||
sqlite_requests = []
|
||||
rows = sqlite.execute(q, (self.maxfail, time.time(), random.randint(10,20))).fetchall()
|
||||
|
||||
if not len(rows):
|
||||
time.sleep(random.randint(10,20))
|
||||
continue
|
||||
|
||||
abc = ' OR proxy='.join( [ '?' for x in xrange(0, len(rows)) ] )
|
||||
args = [ (time.time() + 180) ]
|
||||
e = [ args.append(i[0]) for i in rows ]
|
||||
sqlite.executemany('UPDATE proxylist SET tested=? WHERE proxy=%s' % abc, (args,))
|
||||
sqlite.commit()
|
||||
|
||||
for proxy in rows:
|
||||
time.sleep(0.1)
|
||||
nextcheck = (time.time() + 1800 + ((1+int(proxy[1])) * 3600))
|
||||
|
||||
sock, proto, duration, tor, tst = self.connect_socket(proxy, servers, proto=proxy[3])
|
||||
if not sock:
|
||||
sqlite_requests.append(((proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
||||
continue
|
||||
|
||||
try:
|
||||
recv = sock.recv(6)
|
||||
|
||||
# good data
|
||||
if re.match('^(:|ERROR|PING|PONG|NOTICE|\*\*\*)', recv, re.IGNORECASE):
|
||||
duration = (time.time() - duration)
|
||||
nextcheck = (time.time() + 1800)
|
||||
|
||||
#match = geolite2.lookup(proxy[0].split(':')[0])
|
||||
match = None
|
||||
if match is not None: match = match.country
|
||||
else: match = 'unknown'
|
||||
|
||||
#dronebl = self.is_drone_bl(proxy[0])
|
||||
sqlite_requests.append( (0, time.time(), 1, match, proto, duration, proxy[0],))
|
||||
_log('%s://%s; c: %s; d: %d sec(s); tor: %s; irc: %s; recv: %s' % (proto, proxy[0], match, duration, tor, tst, recv), threadid)
|
||||
|
||||
# bad data
|
||||
else:
|
||||
sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
||||
|
||||
# also bad
|
||||
except:
|
||||
sqlite_requests.append(( (proxy[1]+1), nextcheck, 1, 'unknown', 'http', 0, proxy[0],))
|
||||
|
||||
finally:
|
||||
sock.disconnect()
|
||||
|
||||
for r in sqlite_requests:
|
||||
sqlite.execute('UPDATE proxylist SET failed=?,tested=?,dronebl=?,country=?,proto=?,duration=? WHERE proxy=?', r)
|
||||
sqlite.commit()
|
||||
|
||||
sqlite.close()
|
||||
437
includes/rocksock.py
Normal file
437
includes/rocksock.py
Normal file
@@ -0,0 +1,437 @@
|
||||
import socket, ssl, select
|
||||
|
||||
# rs_proxyType
|
||||
RS_PT_NONE = 0
|
||||
RS_PT_SOCKS4 = 1
|
||||
RS_PT_SOCKS5 = 2
|
||||
RS_PT_HTTP = 3
|
||||
|
||||
# rs_errorType
|
||||
RS_ET_OWN = 0 # rocksock-specific error
|
||||
RS_ET_SYS = 1 # system error with errno
|
||||
RS_ET_GAI = 2 # dns resolution subsystem error
|
||||
RS_ET_SSL = 3 # ssl subsystem error
|
||||
|
||||
# rs_error
|
||||
RS_E_NO_ERROR = 0
|
||||
RS_E_NULL = 1
|
||||
RS_E_EXCEED_PROXY_LIMIT = 2
|
||||
RS_E_NO_SSL = 3
|
||||
RS_E_NO_SOCKET = 4
|
||||
RS_E_HIT_TIMEOUT = 5
|
||||
RS_E_OUT_OF_BUFFER = 6
|
||||
RS_E_SSL_GENERIC = 7
|
||||
RS_E_SOCKS4_NOAUTH = 8
|
||||
RS_E_SOCKS5_AUTH_EXCEEDSIZE = 9
|
||||
RS_E_SOCKS4_NO_IP6 = 10
|
||||
RS_E_PROXY_UNEXPECTED_RESPONSE = 11
|
||||
RS_E_TARGETPROXY_CONNECT_FAILED = 12
|
||||
RS_E_PROXY_AUTH_FAILED = 13
|
||||
RS_E_HIT_READTIMEOUT = 14
|
||||
RS_E_HIT_WRITETIMEOUT = 15
|
||||
RS_E_HIT_CONNECTTIMEOUT = 16
|
||||
RS_E_PROXY_GENERAL_FAILURE = 17
|
||||
RS_E_TARGETPROXY_NET_UNREACHABLE = 18
|
||||
RS_E_TARGETPROXY_HOST_UNREACHABLE = 19
|
||||
RS_E_TARGETPROXY_CONN_REFUSED = 20
|
||||
RS_E_TARGETPROXY_TTL_EXPIRED = 21
|
||||
RS_E_PROXY_COMMAND_NOT_SUPPORTED = 22
|
||||
RS_E_PROXY_ADDRESSTYPE_NOT_SUPPORTED = 23
|
||||
RS_E_REMOTE_DISCONNECTED = 24
|
||||
RS_E_NO_PROXYSTORAGE = 25
|
||||
RS_E_HOSTNAME_TOO_LONG = 26
|
||||
RS_E_INVALID_PROXY_URL = 27
|
||||
|
||||
|
||||
class RocksockException(Exception):
|
||||
def __init__(self, error, failedproxy=None, errortype=RS_ET_OWN, *args, **kwargs):
|
||||
Exception.__init__(self,*args,**kwargs)
|
||||
self.error = error
|
||||
self.errortype = errortype
|
||||
self.failedproxy = failedproxy
|
||||
|
||||
def get_failedproxy(self):
|
||||
return self.failedproxy
|
||||
|
||||
def get_error(self):
|
||||
return self.error
|
||||
|
||||
def reraise(self):
|
||||
import sys
|
||||
ei = sys.exc_info()
|
||||
raise ei[0], ei[1], ei[2]
|
||||
# import traceback, sys
|
||||
# traceback.print_exc(file=sys.stderr)
|
||||
# raise self
|
||||
|
||||
def get_errormessage(self):
|
||||
errordict = {
|
||||
RS_E_NO_ERROR : "no error",
|
||||
RS_E_NULL: "NULL pointer passed",
|
||||
RS_E_EXCEED_PROXY_LIMIT: "exceeding maximum number of proxies",
|
||||
RS_E_NO_SSL: "can not establish SSL connection, since library was not compiled with USE_SSL define",
|
||||
RS_E_NO_SOCKET: "socket is not set up, maybe you should call connect first",
|
||||
RS_E_HIT_TIMEOUT: "timeout reached on operation",
|
||||
RS_E_OUT_OF_BUFFER: "supplied buffer is too small",
|
||||
RS_E_SSL_GENERIC: "generic SSL error", # the C version uses this error when the SSL library does not report any specific error, otherwise errortype SSL will be set and the SSL errorcode be used
|
||||
RS_E_SOCKS4_NOAUTH:"SOCKS4 authentication not implemented",
|
||||
RS_E_SOCKS5_AUTH_EXCEEDSIZE: "maximum length for SOCKS5 servername/password/username is 255",
|
||||
RS_E_SOCKS4_NO_IP6: "SOCKS4 is not compatible with IPv6",
|
||||
RS_E_PROXY_UNEXPECTED_RESPONSE: "the proxy sent an unexpected response",
|
||||
RS_E_TARGETPROXY_CONNECT_FAILED: "could not connect to target proxy",
|
||||
RS_E_PROXY_AUTH_FAILED: "proxy authentication failed or authd not enabled",
|
||||
RS_E_HIT_READTIMEOUT : "timeout reached on read operation",
|
||||
RS_E_HIT_WRITETIMEOUT : "timeout reached on write operation",
|
||||
RS_E_HIT_CONNECTTIMEOUT : "timeout reached on connect operation",
|
||||
RS_E_PROXY_GENERAL_FAILURE : "proxy general failure",
|
||||
RS_E_TARGETPROXY_NET_UNREACHABLE : "proxy-target: net unreachable",
|
||||
RS_E_TARGETPROXY_HOST_UNREACHABLE : "proxy-target: host unreachable",
|
||||
RS_E_TARGETPROXY_CONN_REFUSED : "proxy-target: connection refused",
|
||||
RS_E_TARGETPROXY_TTL_EXPIRED : "proxy-target: TTL expired",
|
||||
RS_E_PROXY_COMMAND_NOT_SUPPORTED : "proxy: command not supported",
|
||||
RS_E_PROXY_ADDRESSTYPE_NOT_SUPPORTED : "proxy: addresstype not supported",
|
||||
RS_E_REMOTE_DISCONNECTED : "remote socket closed connection",
|
||||
RS_E_NO_PROXYSTORAGE : "no proxy storage assigned",
|
||||
RS_E_HOSTNAME_TOO_LONG : "hostname exceeds 255 chars",
|
||||
RS_E_INVALID_PROXY_URL : "invalid proxy URL string"
|
||||
}
|
||||
if self.errortype == RS_ET_SYS:
|
||||
import errno
|
||||
if self.error in errno.errorcode:
|
||||
msg = "ERRNO: " + errno.errorcode[self.error]
|
||||
else:
|
||||
msg = "ERRNO: invalid errno: " + str(self.error)
|
||||
elif self.errortype == RS_ET_GAI:
|
||||
msg = "GAI: " + self.failedproxy
|
||||
elif self.errortype == RS_ET_SSL:
|
||||
msg = errordict[self.error]
|
||||
if self.error == RS_E_SSL_GENERIC and self.failedproxy != None:
|
||||
msg += ': ' + self.failedproxy #failedproxy is repurposed for SSL exceptions
|
||||
else: #RS_ET_OWN
|
||||
msg = errordict[self.error]
|
||||
return msg
|
||||
|
||||
|
||||
class RocksockHostinfo():
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
|
||||
class RocksockProxy():
|
||||
def __init__(self, host, port, type, username = None, password=None, **kwargs):
|
||||
typemap = { 'none' : RS_PT_NONE,
|
||||
'socks4' : RS_PT_SOCKS4,
|
||||
'socks5' : RS_PT_SOCKS5,
|
||||
'http' : RS_PT_HTTP }
|
||||
self.type = typemap[type] if type in typemap else type
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.hostinfo = RocksockHostinfo(host, port)
|
||||
|
||||
def RocksockProxyFromURL(url):
|
||||
# valid URL: socks5://[user:pass@]hostname:port
|
||||
x = url.find('://')
|
||||
if x == -1: return None
|
||||
t = url[:x]
|
||||
url = url[x+len('://'):]
|
||||
x = url.rfind(':')
|
||||
if x == -1: return None # port is obligatory
|
||||
port = int(url[x+len(':'):]) #TODO: catch exception when port is non-numeric
|
||||
url = url[:x]
|
||||
x = url.find('@')
|
||||
if x != -1:
|
||||
u, p = url[:x].split(':')
|
||||
url = url[x+len('@'):]
|
||||
else:
|
||||
u, p = (None, None)
|
||||
return RocksockProxy(host=url, port=port, type=t, username=u, password=p)
|
||||
|
||||
|
||||
class Rocksock():
|
||||
def __init__(self, host=None, port=0, verifycert=False, timeout=0, proxies=None, **kwargs):
|
||||
if 'ssl' in kwargs and kwargs['ssl'] == True:
|
||||
self.sslcontext = ssl.create_default_context()
|
||||
self.sslcontext.check_hostname = False
|
||||
if not verifycert: self.sslcontext.verify_mode = ssl.CERT_NONE
|
||||
else:
|
||||
self.sslcontext = None
|
||||
self.proxychain = proxies if proxies else []
|
||||
target = RocksockProxy(host, port, RS_PT_NONE)
|
||||
self.proxychain.append(target)
|
||||
self.sock = None
|
||||
self.timeout = timeout
|
||||
|
||||
def connect(self):
|
||||
|
||||
af, sa = self._resolve(self.proxychain[0].hostinfo, True)
|
||||
try:
|
||||
x = af+1
|
||||
except TypeError:
|
||||
raise RocksockException(-3, "unexpected problem resolving DNS, try again", errortype=RS_ET_GAI)
|
||||
# print "GOT A WEIRD AF"
|
||||
# print af
|
||||
# raise RocksockException(-6666, af, errortype=RS_ET_GAI)
|
||||
|
||||
self.sock = socket.socket(af, socket.SOCK_STREAM)
|
||||
self.sock.settimeout(None if self.timeout == 0 else self.timeout)
|
||||
try:
|
||||
self.sock.connect((sa[0], sa[1]))
|
||||
except socket.timeout:
|
||||
raise RocksockException(RS_E_HIT_TIMEOUT)
|
||||
except socket.error as e:
|
||||
raise RocksockException(e.errno, errortype=RS_ET_SYS)
|
||||
|
||||
for pnum in xrange(1, len(self.proxychain)):
|
||||
curr = self.proxychain[pnum]
|
||||
prev = self.proxychain[pnum-1]
|
||||
self._connect_step(pnum)
|
||||
|
||||
if self.sslcontext:
|
||||
try:
|
||||
self.sock = self.sslcontext.wrap_socket(self.sock, server_hostname=self.proxychain[len(self.proxychain)-1].hostinfo.host)
|
||||
except ssl.SSLError as e:
|
||||
reason = self._get_ssl_exception_reason(e)
|
||||
#if hasattr(e, 'library'): subsystem = e.library
|
||||
raise RocksockException(RS_E_SSL_GENERIC, reason, errortype=RS_ET_SSL)
|
||||
except socket.error as e:
|
||||
raise RocksockException(e.errno, errortype=RS_ET_SYS)
|
||||
except Exception as e:
|
||||
raise e
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
self.sock.do_handshake()
|
||||
break
|
||||
except ssl.SSLWantReadError:
|
||||
select.select([self.sock], [], [])
|
||||
except ssl.SSLWantWriteError:
|
||||
select.select([], [self.sock], [])
|
||||
"""
|
||||
|
||||
|
||||
def disconnect(self):
|
||||
if self.sock is None: return
|
||||
try:
|
||||
self.sock.shutdown(socket.SHUT_RDWR)
|
||||
except socket.error:
|
||||
pass
|
||||
self.sock.close()
|
||||
self.sock = None
|
||||
|
||||
def canread(self):
|
||||
return select.select([self.sock], [], [], 0)[0]
|
||||
|
||||
def send(self, buf):
|
||||
if self.sock is None:
|
||||
raise RocksockException(RS_E_NO_SOCKET)
|
||||
return self.sock.sendall(buf)
|
||||
|
||||
def _get_ssl_exception_reason(self, e):
|
||||
s = ''
|
||||
if hasattr(e, 'reason'): s = e.reason
|
||||
elif hasattr(e, 'message'): s = e.message
|
||||
elif hasattr(e, 'args'): s = e.args[0]
|
||||
return s
|
||||
|
||||
def recv(self, count=-1):
|
||||
data = ''
|
||||
while count:
|
||||
try:
|
||||
n = count if count != -1 else 4096
|
||||
if n >= 1024*1024: n = 1024*1024
|
||||
chunk = self.sock.recv(n)
|
||||
except socket.timeout:
|
||||
raise RocksockException(RS_E_HIT_TIMEOUT)
|
||||
except ssl.SSLError as e:
|
||||
s = self._get_ssl_exception_reason(e)
|
||||
if s == 'The read operation timed out':
|
||||
raise RocksockException(RS_E_HIT_READTIMEOUT)
|
||||
else:
|
||||
raise RocksockException(RS_E_SSL_GENERIC, s, errortype=RS_ET_SSL)
|
||||
if len(chunk) == 0:
|
||||
raise RocksockException(RS_E_REMOTE_DISCONNECTED)
|
||||
data += chunk
|
||||
if count == -1: break
|
||||
else: count -= len(chunk)
|
||||
return data
|
||||
|
||||
def recvline(self):
|
||||
s = ''
|
||||
c = '\0'
|
||||
while c != '\n':
|
||||
c = self.recv(1)
|
||||
if c == '': return s
|
||||
s += c
|
||||
return s
|
||||
|
||||
def recvuntil(self, until):
|
||||
s = self.recv(len(until))
|
||||
endc = until[-1:]
|
||||
while not (s[-1:] == endc and s.endswith(until)):
|
||||
s += self.recv(1)
|
||||
return s
|
||||
|
||||
def _isnumericipv4(self, ip):
|
||||
try:
|
||||
a,b,c,d = ip.split('.')
|
||||
if int(a) < 256 and int(b) < 256 and int(c) < 256 and int(d) < 256:
|
||||
return True
|
||||
return False
|
||||
except:
|
||||
return False
|
||||
|
||||
def _resolve(self, hostinfo, want_v4=True):
|
||||
if self._isnumericipv4(hostinfo.host):
|
||||
return socket.AF_INET, (hostinfo.host, hostinfo.port)
|
||||
try:
|
||||
for res in socket.getaddrinfo(hostinfo.host, hostinfo.port, \
|
||||
socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
|
||||
af, socktype, proto, canonname, sa = res
|
||||
if want_v4 and af != socket.AF_INET: continue
|
||||
if af != socket.AF_INET and af != socket.AF_INET6: continue
|
||||
else: return af, sa
|
||||
|
||||
except socket.gaierror as e:
|
||||
eno, str = e.args
|
||||
raise RocksockException(eno, str, errortype=RS_ET_GAI)
|
||||
|
||||
return None, None
|
||||
|
||||
def _ip_to_int(self, ip):
|
||||
a,b,c,d = ip.split('.')
|
||||
h = "0x%.2X%.2X%.2X%.2X"%(int(a),int(b),int(c),int(d))
|
||||
return int(h, 16)
|
||||
|
||||
def _ip_to_bytes(self, ip):
|
||||
ip = self._ip_to_int(ip)
|
||||
a = (ip & 0xff000000) >> 24
|
||||
b = (ip & 0x00ff0000) >> 16
|
||||
c = (ip & 0x0000ff00) >> 8
|
||||
d = (ip & 0x000000ff) >> 0
|
||||
return chr(a) + chr(b) + chr(c) + chr(d)
|
||||
|
||||
def _setup_socks4_header(self, v4a, dest):
|
||||
buf = '\x04\x01'
|
||||
buf += chr(dest.hostinfo.port / 256)
|
||||
buf += chr(dest.hostinfo.port % 256)
|
||||
if v4a:
|
||||
buf += '\0\0\0\x01'
|
||||
else:
|
||||
af, sa = self._resolve(dest, True)
|
||||
if af != socket.AF_INET: raise RocksockException(RS_E_SOCKS4_NO_IP6)
|
||||
buf += self._ip_to_bytes(sa[0])
|
||||
buf += '\0'
|
||||
if v4a: buf += dest.hostinfo.host + '\0'
|
||||
return buf
|
||||
|
||||
def _connect_socks4(self, header, pnum):
|
||||
self.send(header)
|
||||
res = self.recv(8)
|
||||
if len(res) < 8 or ord(res[0]) != 0:
|
||||
raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1)
|
||||
ch = ord(res[1])
|
||||
if ch == 0x5a:
|
||||
pass
|
||||
elif ch == 0x5b:
|
||||
raise RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, pnum-1)
|
||||
elif ch == 0x5c or ch == 0x5d:
|
||||
return RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1)
|
||||
else:
|
||||
raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1)
|
||||
|
||||
def _setup_socks5_header(self, proxy):
|
||||
buf = '\x05'
|
||||
if proxy.username and proxy.password:
|
||||
buf += '\x02\x00\x02'
|
||||
else:
|
||||
buf += '\x01\x00'
|
||||
return buf
|
||||
|
||||
def _connect_socks5(self, header, pnum):
|
||||
self.send(header)
|
||||
res = self.recv(2)
|
||||
if len(res) != 2 or res[0] != '\x05':
|
||||
raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1)
|
||||
if res[1] == '\xff':
|
||||
raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1)
|
||||
|
||||
if ord(res[1]) == 2:
|
||||
px = self.proxychain[pnum-1]
|
||||
if px.username and px.password:
|
||||
pkt = '\x01%c%s%c%s'%(len(px.username),px.username,len(px.password),px.password)
|
||||
self.send(pkt)
|
||||
res = self.recv(2)
|
||||
if len(res) < 2 or res[1] != '\0':
|
||||
raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1)
|
||||
else: raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1)
|
||||
dst = self.proxychain[pnum]
|
||||
numeric = self._isnumericipv4(dst.hostinfo.host)
|
||||
if numeric:
|
||||
dstaddr = self._ip_to_bytes(dst.hostinfo.host)
|
||||
else:
|
||||
dstaddr = chr(len(dst.hostinfo.host)) + dst.hostinfo.host
|
||||
|
||||
pkt = '\x05\x01\x00%c%s%c%c'% (1 if numeric else 3, dstaddr, dst.hostinfo.port / 256, dst.hostinfo.port % 256)
|
||||
self.send(pkt)
|
||||
res = self.recv()
|
||||
if len(res) < 2 or res[0] != '\x05':
|
||||
raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1)
|
||||
ch = ord(res[1])
|
||||
if ch == 0: pass
|
||||
elif ch == 1: raise RocksockException(RS_E_PROXY_GENERAL_FAILURE, pnum-1)
|
||||
elif ch == 2: raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1)
|
||||
elif ch == 3: raise RocksockException(RS_E_TARGETPROXY_NET_UNREACHABLE, pnum-1)
|
||||
elif ch == 4: raise RocksockException(RS_E_TARGETPROXY_HOST_UNREACHABLE, pnum-1)
|
||||
elif ch == 5: raise RocksockException(RS_E_TARGETPROXY_CONN_REFUSED, pnum-1)
|
||||
elif ch == 6: raise RocksockException(RS_E_TARGETPROXY_TTL_EXPIRED, pnum-1)
|
||||
elif ch == 7: raise RocksockException(RS_E_PROXY_COMMAND_NOT_SUPPORTED, pnum-1)
|
||||
elif ch == 8: raise RocksockException(RS_E_PROXY_ADDRESSTYPE_NOT_SUPPORTED, pnum-1)
|
||||
else: raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1)
|
||||
|
||||
|
||||
def _connect_step(self, pnum):
|
||||
prev = self.proxychain[pnum -1]
|
||||
curr = self.proxychain[pnum]
|
||||
if prev.type == RS_PT_SOCKS4:
|
||||
s4a = self._setup_socks4_header(True, curr)
|
||||
try:
|
||||
self._connect_socks4(s4a, pnum)
|
||||
except RocksockException as e:
|
||||
if e.get_error() == RS_E_TARGETPROXY_CONNECT_FAILED:
|
||||
s4 = self._setup_socks4_header(self, False, curr)
|
||||
self._connect_socks4(s4a, pnum)
|
||||
else: raise e
|
||||
elif prev.type == RS_PT_SOCKS5:
|
||||
s5 = self._setup_socks5_header(prev)
|
||||
self._connect_socks5(s5, pnum)
|
||||
elif prev.type == RS_PT_HTTP:
|
||||
dest = self.proxychain[pnum]
|
||||
self.send("CONNECT %s:%d HTTP/1.1\r\n\r\n"%(dest.hostinfo.host, dest.hostinfo.port))
|
||||
resp = self.recv()
|
||||
if len(resp) <12:
|
||||
raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1)
|
||||
if resp[9] != '2':
|
||||
raise RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, pnum-1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
proxies = [
|
||||
# RocksockProxyFromURL("socks5://foo:bar@localhost:1080"),
|
||||
# RocksockProxyFromURL("socks5://10.0.0.3:1080"),
|
||||
RocksockProxyFromURL("socks5://127.0.0.1:31339"),
|
||||
]
|
||||
proxies = None
|
||||
#rs = Rocksock(host='googleff242342423f.com', port=443, ssl=True, proxies=proxies)
|
||||
rs = Rocksock(host='google.com', port=80, ssl=False, proxies=proxies)
|
||||
try:
|
||||
rs.connect()
|
||||
except RocksockException as e:
|
||||
print e.get_errormessage()
|
||||
e.reraise()
|
||||
rs.send('GET / HTTP/1.0\r\n\r\n')
|
||||
print rs.recvline()
|
||||
rs.disconnect()
|
||||
rs.connect()
|
||||
rs.send('GET / HTTP/1.0\r\n\r\n')
|
||||
print rs.recvline()
|
||||
|
||||
245
ppf.py
Executable file
245
ppf.py
Executable file
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import sys
|
||||
import socket
|
||||
import requests
|
||||
import socks
|
||||
import random, time
|
||||
import sqlite3
|
||||
import re
|
||||
import urllib
|
||||
import threading
|
||||
import hashlib
|
||||
import ipcalc
|
||||
from soup_parser import soupify
|
||||
from ConfigParser import SafeConfigParser
|
||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||
from dns import resolver
|
||||
#from selenium import webdriver
|
||||
|
||||
sys.path.append('./includes')
|
||||
import mysqlite
|
||||
from misc import _log
|
||||
|
||||
base_header = {
|
||||
'Accept-Language':'en-US,en;q=0.8',
|
||||
'Cache-Control':'max-age=0',
|
||||
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'DNT': '1',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0'
|
||||
}
|
||||
|
||||
database = 'proxies.sqlite'
|
||||
|
||||
searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' )
|
||||
|
||||
retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded')
|
||||
proxies={ 'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050', 'ftp': 'socks5://127.0.0.1:9050'}
|
||||
|
||||
|
||||
# include own classes
|
||||
sys.path.append('./includes')
|
||||
import proxywatchd, mysqlite
|
||||
from misc import _log
|
||||
|
||||
CONFIG = 'config.ini'
|
||||
|
||||
|
||||
def cleanhtml(raw_html):
|
||||
cleanr = re.compile('<.*?>')
|
||||
cleantext = re.sub(cleanr, ':', raw_html)
|
||||
cleantext = re.sub('::+',':', cleantext)
|
||||
return cleantext
|
||||
|
||||
def import_from_file(fn, sqlite):
|
||||
with open(fn, 'r') as f:
|
||||
for u in f.read().split('\n'):
|
||||
if not len(u): continue
|
||||
exists = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE url=?',(u,)).fetchall() ]
|
||||
if exists: continue
|
||||
print('adding "%s"' % u)
|
||||
sqlite.execute('INSERT INTO uris (added,url,check_time,error) VALUES (?,?,?,?)', (time.time(),u,0,1))
|
||||
sqlite.commit()
|
||||
|
||||
def fetch_contents(uri, driver=None):
|
||||
headers = base_header
|
||||
## use requests (default)
|
||||
if not driver:
|
||||
try: resp = requests.get(uri, timeout=45, headers=headers, verify=False, proxies=proxies)
|
||||
except: return ''
|
||||
data = resp.text
|
||||
|
||||
## phantomjs
|
||||
else:
|
||||
for key, value in enumerate(base_header):
|
||||
capability_key = 'phantomjs.page.customHeaders.{}'.format(key)
|
||||
webdriver.DesiredCapabilities.PHANTOMJS[capability_key] = value
|
||||
|
||||
service_args = ['--proxy=127.0.0.1:9050', '--proxy-type=socks5']
|
||||
driver = webdriver.PhantomJS()
|
||||
try:
|
||||
driver.implicitly_wait(45)
|
||||
driver.set_page_load_timeout(45)
|
||||
driver.get(uri)
|
||||
data = driver.page_source
|
||||
|
||||
except: data = ''
|
||||
finally:driver.quit()
|
||||
|
||||
for retry_message in retry_messages:
|
||||
if retry_message in data: return ''
|
||||
return data
|
||||
|
||||
def update_proxy_sources(sqlite, proxies, uri):
|
||||
for proxy in proxies:
|
||||
md5sum = hashlib.md5(proxy).hexdigest()
|
||||
sqlite.execute('CREATE TABLE IF NOT EXISTS "%s" (uri TEXT)' % md5sum)
|
||||
sqlite.commit()
|
||||
#check = [ i for i in sqlite.execute('SELECT uri FROM "%s" WHERE uri=?' % md5sum, (uri,)).fetchall() ]
|
||||
check = sqlite.execute('SELECT uri FROM "%s" WHERE uri=?' % md5sum, (uri,)).fetchall()
|
||||
if not len(check):
|
||||
sqlite.execute('INSERT INTO "%s" (uri) VALUES(?)' % md5sum, (uri,))
|
||||
sqlite.commit()
|
||||
|
||||
def insert_proxies(proxies, uri, sqlite):
|
||||
time_now = time.time()
|
||||
added = 0
|
||||
|
||||
## very wasteful
|
||||
#update_proxy_sources(sqlite, proxies, uri)
|
||||
|
||||
query = [ 'proxy=?' for p in proxies ]
|
||||
known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ]
|
||||
new = [ (time_now,i,3,0) for i in proxies if not i in known ]
|
||||
if len(new):
|
||||
added = added + len(new)
|
||||
sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested) VALUES (?,?,?,?)', new)
|
||||
sqlite.commit()
|
||||
|
||||
if added > 0: _log('+%d item(s) from %s' % (added, uri), 'added')
|
||||
|
||||
def proxyfind(sqlite = None):
|
||||
#print('entering proxyfind...')
|
||||
|
||||
if not sqlite: sqlite = mysqlite.mysqlite(database,str)
|
||||
|
||||
uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ]
|
||||
|
||||
if len(uris) > 0 and random.random() < random.random():
|
||||
search = urllib.quote_plus('site:%s' % random.choice(uris).split('/')[2])
|
||||
else:
|
||||
search = urllib.quote_plus(random.choice(search_terms))
|
||||
|
||||
choice = random.choice(searx_instances)
|
||||
urls = []
|
||||
|
||||
content = fetch_contents('%s/?q=%s&pageno=%d' % (choice, search, random.randint(0,10)))
|
||||
|
||||
if not content: return
|
||||
soup = soupify(content)
|
||||
|
||||
for a in soup.body.find_all('a'):
|
||||
if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue
|
||||
badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ]
|
||||
if not len(badurl): urls.append(a.attrs['href'])
|
||||
|
||||
if len(urls):
|
||||
query = [ 'url=?' for u in urls ]
|
||||
known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ]
|
||||
time_now = time.time()
|
||||
new = [ (time_now,i,0,5,0) for i in urls if not i in known ]
|
||||
if len(new):
|
||||
sqlite.executemany('INSERT INTO uris (added,url,check_time,error,driver) values(?,?,?,?,?)', new)
|
||||
sqlite.commit()
|
||||
_log('+%d item(s) from %s' % (len(new), search), 'added')
|
||||
|
||||
sqlite.commit()
|
||||
|
||||
def proxyleech(sqlite, rows):
|
||||
#print('entering proxyleech...')
|
||||
|
||||
for row in rows:
|
||||
try: content = fetch_contents(row[0], None)
|
||||
except: content = ''
|
||||
|
||||
uniques = []
|
||||
for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))):
|
||||
if p in uniques: continue
|
||||
try:
|
||||
if str(ipcalc.Network(p.split(':')[0]).subnet) != 'PRIVATE': uniques.append(p)
|
||||
except:
|
||||
pass
|
||||
|
||||
hash = hashlib.md5(''.join(uniques)).hexdigest()
|
||||
#print('unique; hash: %s, len: %d' % (hash, len(uniques)))
|
||||
|
||||
## empty list of proxies: increment error by two
|
||||
if not len(uniques): row[2] = (row[2] * 2)
|
||||
## same proxy list: increment error by one
|
||||
elif hash == row[1]: row[2] = (row[2] + 1)
|
||||
## proxylist was updated: error is zero
|
||||
else: row[2] = 0
|
||||
|
||||
check_time = (time.time() + 3600 + (3600 * row[2]))
|
||||
sqlite.execute('UPDATE uris SET error=?,hash=?,check_time=? where url=?', (row[2],hash, check_time,row[0]))
|
||||
sqlite.commit()
|
||||
|
||||
if not row[1] or row[2] > 0: return
|
||||
|
||||
add = []
|
||||
for i in uniques:
|
||||
add.append(i)
|
||||
if len(add) < 500: continue
|
||||
insert_proxies(add, row[0], sqlite)
|
||||
add = []
|
||||
if len(add): insert_proxies(add, row[0], sqlite)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
## read the config files
|
||||
parser = SafeConfigParser()
|
||||
parser.read(CONFIG)
|
||||
|
||||
database = parser.get('global', 'database')
|
||||
search = parser.getboolean('proxyfind', 'search')
|
||||
|
||||
sqlite = mysqlite.mysqlite(database, str)
|
||||
|
||||
## create dbs if required
|
||||
sqlite.execute('CREATE TABLE IF NOT EXISTS uris (added INT, url TEXT, check_time INT, error INT, driver INT, hash TEXT)')
|
||||
sqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, dronebl INT, proto TEXT, duration INT)')
|
||||
sqlite.commit()
|
||||
|
||||
import_from_file('import.txt', sqlite)
|
||||
|
||||
## load search terms
|
||||
with open('search_terms.txt', 'r') as f:
|
||||
search_terms = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]
|
||||
|
||||
## load bad terms
|
||||
with open('urignore.txt', 'r') as f:
|
||||
urignore = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ]
|
||||
## add searx instances as bad terms (avoid loops)
|
||||
empty = [ urignore.append(i.split('/')[2]) for i in searx_instances ]
|
||||
|
||||
# start proxy watcher
|
||||
watcherd = proxywatchd.Proxywatchd(CONFIG) if parser.getboolean('watcherd', 'enabled') else None
|
||||
|
||||
while True:
|
||||
try:
|
||||
## any site that needs to be checked ?
|
||||
rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time<? AND error<?) ORDER BY RANDOM() LIMIT 25', (time.time(), 10)).fetchall() ]
|
||||
if len(rows): proxyleech(sqlite,rows)
|
||||
## search for new website during free time
|
||||
elif search: proxyfind(sqlite)
|
||||
## sleep
|
||||
else: time.sleep(10)
|
||||
|
||||
except KeyboardInterrupt: break
|
||||
|
||||
print '\r',
|
||||
|
||||
# stop things
|
||||
if watcherd: watcherd.stop()
|
||||
13
search_terms.txt
Normal file
13
search_terms.txt
Normal file
@@ -0,0 +1,13 @@
|
||||
site:github.com :8081 :8888 :8080
|
||||
site:github.com :4444 :1234 :3124
|
||||
site:github.com proxylist
|
||||
hourly http proxy
|
||||
hourly socks proxy
|
||||
daily http proxy
|
||||
daily socks proxy
|
||||
fresh http proxy list
|
||||
fresh socks proxy list
|
||||
updated http proxy list
|
||||
updated socks proxy list
|
||||
download http proxy
|
||||
download socks proxy
|
||||
131
servers.txt
Normal file
131
servers.txt
Normal file
@@ -0,0 +1,131 @@
|
||||
chat.freenode.net
|
||||
irc.dal.net
|
||||
irc.Undernet.Org
|
||||
eu.ircnet.org
|
||||
eu.undernet.org
|
||||
irc.2600.net
|
||||
irc.accessirc.net
|
||||
irc.afterx.net
|
||||
irc.asylo.com
|
||||
irc.atheme.org
|
||||
irc.atrum.org
|
||||
irc.atw-inter.net
|
||||
irc.au.dal.net
|
||||
irc.austnet.org
|
||||
irc.azzurra.org
|
||||
irc.blitzed.org
|
||||
irc.bongster.org
|
||||
irc.chat4all.org
|
||||
irc.chatcafe.net
|
||||
irc.chatnet.org
|
||||
irc.chatspike.net
|
||||
irc.chattingaway.com
|
||||
irc.choopa.net
|
||||
irc.coldfront.net
|
||||
irc.cyanide-x.net
|
||||
irc.cyberarmy.net
|
||||
irc.darkmyst.org
|
||||
irc.data.lt
|
||||
irc.deepirc.net
|
||||
irc.deepspace.org
|
||||
irc.drlnet.com
|
||||
irc.d-t-net.de
|
||||
irc.dynastynet.net
|
||||
irc.ecnet.org
|
||||
irc.efnet.org
|
||||
irc.efnet.pl
|
||||
irc.enterthegame.com
|
||||
irc.esper.net
|
||||
irc.eu.dal.net
|
||||
irc.eu.gamesurge.net
|
||||
irc.euirc.net
|
||||
irc.europnet.org
|
||||
irc.eversible.com
|
||||
irc.fdfnet.net
|
||||
irc.fef.net
|
||||
irc.financialchat.com
|
||||
irc.foonetic.net
|
||||
irc.forestnet.org
|
||||
irc.foreverchat.net
|
||||
irc.freequest.net
|
||||
irc.gamesurge.net
|
||||
irc.geekshed.net
|
||||
irc.german-freakz.net
|
||||
irc.gigairc.net
|
||||
irc.globalchat.org
|
||||
irc.globalgamers.net
|
||||
irc.greekirc.net
|
||||
irc.icq.com
|
||||
irc.immortal-anime.net
|
||||
irc.imperium.ca
|
||||
irc.inet.tele.dk
|
||||
irc.irc2.hu
|
||||
irc.irc4fun.net
|
||||
irc.irchighway.net
|
||||
irc.ircsource.net
|
||||
irc.irctoo.net
|
||||
irc.ircube.org
|
||||
irc.ircworld.nl
|
||||
irc.irdsi.net
|
||||
irc.kampungchat.org
|
||||
irc.kickchat.com
|
||||
irc.knightirc.net
|
||||
irc.krono.net
|
||||
irc.landoleet.org
|
||||
irc.langochat.net
|
||||
irc.lichtsnel.nl
|
||||
irc.maddshark.net
|
||||
irc.matrix-network.org
|
||||
irc.mmoirc.com
|
||||
irc.mozilla.org
|
||||
irc.mugglenet.com
|
||||
irc.netgamers.org
|
||||
irc.newnet.net
|
||||
irc.nightstar.net
|
||||
irc.oftc.net
|
||||
irc.onlinegamesnet.net
|
||||
irc.othernet.org
|
||||
irc.otherworlders.org
|
||||
irc.oz.org
|
||||
irc.p2pchat.net
|
||||
irc.phat-net.de
|
||||
irc.ptnet.org
|
||||
irc.rezosup.org
|
||||
irc.rizon.net
|
||||
irc.scarynet.org
|
||||
irc.seersirc.net
|
||||
irc.serenia.net
|
||||
irc.serenity-irc.net
|
||||
irc.servercentral.net
|
||||
irc.shadowfire.org
|
||||
irc.shadowworld.net
|
||||
irc.simosnap.net
|
||||
irc.slacked.org
|
||||
irc.slashnet.org
|
||||
irc.snt.utwente.nl
|
||||
irc.solidirc.com
|
||||
irc.sorcery.net
|
||||
irc.spacetronix.net
|
||||
irc.starchat.net
|
||||
irc.starlink-irc.org
|
||||
irc.starlink.org
|
||||
irc.staynet.org
|
||||
irc.st-city.net
|
||||
irc.stratics.com
|
||||
irc.swiftirc.net
|
||||
irc.teksavvy.ca
|
||||
irc.teranova.net
|
||||
irc.umich.edu
|
||||
irc.us.dal.net
|
||||
irc.us.gamesurge.net
|
||||
irc.utonet.org
|
||||
irc.webchat.org
|
||||
irc.whatnet.org
|
||||
irc.wikkedwire.com
|
||||
irc.wondernet.nu
|
||||
irc.worldnet.net
|
||||
irc.xevion.net
|
||||
irc.xs4all.nl
|
||||
irc.zerofuzion.net
|
||||
uk.quakenet.org
|
||||
us.quakenet.org
|
||||
36
soup_parser.py
Normal file
36
soup_parser.py
Normal file
@@ -0,0 +1,36 @@
|
||||
from bs4 import BeautifulSoup, SoupStrainer, FeatureNotFound
|
||||
import sys
|
||||
#import gumbo
|
||||
|
||||
parser = 'lxml'
|
||||
def soupify_bs4(html, nohtml=False):
|
||||
global parser
|
||||
parser = 'html.parser'
|
||||
htm = html if nohtml else '<html><body>%s</body></html>'%(html)
|
||||
try:
|
||||
res = BeautifulSoup(htm, parser)
|
||||
except FeatureNotFound as e:
|
||||
parser = 'html.parser'
|
||||
res = BeautifulSoup(htm, parser)
|
||||
return res
|
||||
|
||||
def soupify_gumbo(html, nohtml=False):
|
||||
htm = html if nohtml else '<html><body>%s</body></html>'%(html)
|
||||
try:
|
||||
soup = gumbo.soup_parse(htm)
|
||||
if not soup.body:
|
||||
print "AAAA"
|
||||
print html
|
||||
print "BBBB"
|
||||
print repr(soup)
|
||||
return soup
|
||||
|
||||
except Exception as e:
|
||||
sys.stdout.write(html)
|
||||
raise
|
||||
|
||||
def soupify(html, nohtml=False):
|
||||
# return soupify_gumbo(html, nohtml)
|
||||
return soupify_bs4(html, nohtml)
|
||||
|
||||
|
||||
41
striphtml.py
Normal file
41
striphtml.py
Normal file
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from HTMLParser import HTMLParser
|
||||
import requests
|
||||
import re
|
||||
from selenium.webdriver.common.proxy import *
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.common.by import By
|
||||
phantomjs_path = '/home/mickael/bin/phantomjs'
|
||||
def cleanhtml(raw_html):
|
||||
cleanr = re.compile('<.*?>')
|
||||
cleantext = re.sub(cleanr, ':', raw_html)
|
||||
cleantext = re.sub('::+',':', cleantext)
|
||||
return cleantext
|
||||
|
||||
class MLStripper(HTMLParser):
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
self.fed = []
|
||||
def handle_data(self, d):
|
||||
self.fed.append(d)
|
||||
def get_data(self):
|
||||
return ''.join(self.fed)
|
||||
|
||||
def strip_tags(html):
|
||||
s = MLStripper()
|
||||
s.feed(html)
|
||||
return s.get_data()
|
||||
|
||||
service_args = ['--proxy=127.0.0.1:9050', '--proxy-type=socks5']
|
||||
driver = webdriver.PhantomJS(phantomjs_path,service_args=service_args)
|
||||
try: driver.get('http://www.proxz.com/proxy_list_fr_1_ext.html')
|
||||
except: sys.exit(0)
|
||||
html = driver.page_source
|
||||
driver.quit()
|
||||
|
||||
text = cleanhtml(html)
|
||||
proxies = sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', text))
|
||||
|
||||
print(text)
|
||||
print(proxies)
|
||||
19
urignore.txt
Normal file
19
urignore.txt
Normal file
@@ -0,0 +1,19 @@
|
||||
check-reverse-domain-ip-lookup
|
||||
www.youtube.com
|
||||
youtu.be
|
||||
archive
|
||||
.gov[$/]
|
||||
/blog/
|
||||
/questions/
|
||||
/question/
|
||||
wiki
|
||||
answer
|
||||
thread
|
||||
/robots.txt
|
||||
form
|
||||
webproxy
|
||||
check?i=
|
||||
[0-9]+(?:\.[0-9]+){3}
|
||||
README.md
|
||||
support
|
||||
aspx?p=
|
||||
Reference in New Issue
Block a user