commit 3fb2dc40a60bbe11b0f94e61012ae4efb9e3af28 Author: mickael Date: Thu Jan 3 16:36:31 2019 +0000 initial commit diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..7706dcb --- /dev/null +++ b/config.ini @@ -0,0 +1,23 @@ +[global] +tor_host = 127.0.0.1:9050 +i2p_host = 127.0.0.1:4444 +common_proxy_ports = 80, 1080, 3124, 3128, 4145, 4444, 8080, 8081, 8118, 8888, 9999 +proxylist_reload_every = 180 +database = proxylist.sqlite +proxy_max_fail = 5 + +[watcherd] +enabled = true +proxy_file = false +checktime = 1800 +threads = 10 +timeout = 15 +read_timeout = 20 +max_fail = 5 + +[proxyfind] +enabled = true +search = true +maxfail = 10 +timeout = 30 +threads = 3 diff --git a/import.txt b/import.txt new file mode 100644 index 0000000..5b54e8e --- /dev/null +++ b/import.txt @@ -0,0 +1,562 @@ +http://spys.one/en/socks-proxy-list/ +https://www.my-proxy.com/free-socks-5-proxy.html +http://www.proxyserverlist24.top/feeds/posts/default +http://spys.me/proxy.txt +https://raw.githubusercontent.com/opsxcq/proxy-list/master/list.txt +https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt +https://raw.githubusercontent.com/flyinwind/proxylist/master/proxies.txt +https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list.txt +http://multiproxy.org/txt_all/proxy.txt +https://github.com/clarketm/proxy-list/blob/master/proxy-list.txt +http://www.megaproxylist.net/ +https://raw.githubusercontent.com/JamesJGoodwin/wreckuests/master/files/proxy.txt +https://www.binary-zone.com/files/MyProxyList.txt +http://captchasniper.com/new/FreeProxies.txt +http://oupfps7we.bkt.clouddn.com/proxy_list.txt +http://white55.narod.ru/downloads/proxyold.txt +http://www.white55.narod.ru/downloads/proxylist.txt +http://xls.su/br/1.txt +https://proxyscrape.com/proxies/HTTP_Working_Proxies.txt +https://rootjazz.com/proxies/proxies.txt +http://www.zarb.org/~saispo/proxylist.txt +http://promicom.by/tools/proxy/proxy.txt +http://pouuet.free.fr/proxy.txt +https://www.my-proxy.com/free-proxy-list.html?list=1 +https://www.my-proxy.com/free-proxy-list.html?list=2 +https://www.my-proxy.com/free-proxy-list.html?list=3 +https://www.my-proxy.com/free-proxy-list.html?list=4 +https://www.my-proxy.com/free-proxy-list.html?list=5 +https://www.my-proxy.com/free-proxy-list.html?list=6 +https://www.my-proxy.com/free-proxy-list.html?list=7 +https://www.my-proxy.com/free-proxy-list.html?list=8 +https://www.my-proxy.com/free-proxy-list.html?list=9 +https://www.my-proxy.com/free-proxy-list.html?list=10 +http://proxy-ip-list.com/download/free-usa-proxy-ip.txt +http://proxy-ip-list.com/download/free-uk-proxy-list.txt +http://www.freeproxy.ru/download/lists/goodproxy.txt +http://www.proxylists.net/socks4.txt +http://multiproxy.org/txt_all/proxy.txt +https://raw.githubusercontent.com/Zenaker/IpLogger-Flood/master/proxy.txt +https://raw.githubusercontent.com/SEODEMON/EXTREME-PROXY-HARVESTER-AND-CHECKER/master/leeched/leeched_proxies.txt +https://proxy.mimvp.com/free.php?proxy=in_hp +http://www.coobobo.com/free-http-proxy +http://ip.zdaye.com/ +http://www.mayidaili.com/free/anonymous/%E9%AB%98%E5%8C%BF +http://http.taiyangruanjian.com/ +http://http.zhimaruanjian.com/ +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=50 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=100 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=150 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=3&anonlvl=4&availability=90&offset=200 +http://samair.ru/proxy/list-IP-port/proxy-1.htm +http://samair.ru/proxy/list-IP-port/proxy-2.htm +http://samair.ru/proxy/list-IP-port/proxy-3.htm +http://samair.ru/proxy/list-IP-port/proxy-4.htm +http://samair.ru/proxy/list-IP-port/proxy-5.htm +http://proxylist.hidemyass.com/search-1304300#listable +https://free-proxy-list.net/ +https://proxy-list.org/english/search.php?search=anonymous-and-elite.ssl-no&country=any&type=anonymous-and-elite&port=any&ssl=no +https://proxy-list.org/english/search.php?search=anonymous-and-elite.ssl-no&country=any&type=anonymous-and-elite&port=any&ssl=no&p=2 +https://hidemy.name/en/proxy-list/?type=h&anon=4#list +https://hidemy.name/en/proxy-list/?type=h&anon=4&start=64#list +https://hidemy.name/en/proxy-list/?type=h&anon=4&start=128#list +https://hidemy.name/en/proxy-list/?type=h&anon=4&start=192#list +https://checkerproxy.net/archive/2017-04-01 +http://aliveproxy.com/fastest-proxies/ +http://multiproxy.org/anon_proxy.htm +http://www.ultraproxies.com/high-anonymous.html +http://www.ip-adress.com/proxy_list/ +http://samair.ru/anonymity-test/ +http://txt.proxyspy.net/proxy.txt +http://nntime.com/ +http://www.gatherproxy.com/ +http://proxysearcher.sourceforge.net/Proxy%20List.php?type=http&filtered=true +http://proxyserverlist-24.blogspot.com/2016/11/01-11-16-fast-proxy-server-list.html +https://cracking.org/threads/190x-anonymous-proxy-list.125920/ +http://samair.ru/proxy/list-IP-port/ip-address-12.htm +http://www.businessadviceforum.com/threads/fastest-securest-proxies-on-the-web-new-proxy-lists-everyday.75326/page-10 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=50 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=100 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=100 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=150 +http://proxydb.net/?protocol=http&anonlvl=2&anonlvl=4&availability=90&response_time=10&offset=200 +http://www.nntime.com/proxy/ +http://www.textproxylists.com/proxy.php?codeen +http://www.textproxylists.com/proxy.php?anonymous +http://www.textproxylists.com/proxy.php?highanonymity +http://www.textproxylists.com/proxy.php?transparent +http://www.textproxylists.com/proxy.php?nontransparent +http://www.textproxylists.com/proxy.php?allproxy +http://proxies.my-proxy.com/proxy-list-s1.html +http://proxies.my-proxy.com/proxy-list-s2.html +http://proxies.my-proxy.com/proxy-list-s3.html +http://proxies.my-proxy.com/proxy-list-1.html +http://proxies.my-proxy.com/proxy-list-2.html +http://proxies.my-proxy.com/proxy-list-3.html +http://proxies.my-proxy.com/proxy-list-4.html +http://proxies.my-proxy.com/proxy-list-5.html +http://proxies.my-proxy.com/proxy-list-6.html +http://proxies.my-proxy.com/proxy-list-7.html +http://proxies.my-proxy.com/proxy-list-8.html +http://proxies.my-proxy.com/proxy-list-9.html +http://proxies.my-proxy.com/proxy-list-10.html +http://www.proxy-list.net/anonymous-proxy-lists.shtml +http://www.proxy-list.net/transparent-proxy-lists.shtml +http://www.digitalcybersoft.com/ProxyList/fresh-proxy-list.shtml? +http://proxies.my-proxy.com/ +http://proxies.my-proxy.com/proxy-list-2.html +http://proxies.my-proxy.com/proxy-list-3.html +http://proxies.my-proxy.com/proxy-list-4.html +http://proxies.my-proxy.com/proxy-list-5.html +http://proxies.my-proxy.com/proxy-list-6.html +http://proxies.my-proxy.com/proxy-list-7.html +http://proxies.my-proxy.com/proxy-list-8.html +http://proxies.my-proxy.com/proxy-list-9.html +http://proxies.my-proxy.com/proxy-list-10.html +http://proxies.my-proxy.com/proxy-list-s1.html +http://proxies.my-proxy.com/proxy-list-s2.html +http://proxies.my-proxy.com/proxy-list-s3.html +http://www.cooleasy.com/ +http://www.cooleasy.com/?act=list&port=&type=&country=&page=2 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=3 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=4 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=5 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=6 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=7 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=8 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=9 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=10 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=11 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=12 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=13 +http://www.cooleasy.com/?act=list&port=&type=&country=&page=14 +http://www.x-proxy.info/highanon.php +http://www.fresh-proxy-list.net/fresh%20proxy%20list.htm +http://atomintersoft.com/free_proxy_list +http://publicproxylist.org/ +http://httpproxylist.org/ +http://www.proxz.com/proxy_list_anonymous_us_0.html +http://spys.me/proxy.txt +https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt +https://free-proxy-list.net/anonymous-proxy.html +https://www.aliveproxy.com/proxy-list/proxies.aspx/Russian_Federation-ru +http://www.aliveproxy.com/de-proxy-list/ +http://proxy-ip-list.com/free-usa-proxy-ip.html +http://proxy-ip-list.com/8081/transparent-Spain-8081.html +https://www.my-proxy.com/blog/tag/proxy-setting +http://aliveproxy.com/proxy-list/proxies.aspx/South_Korea-KR +https://www.my-proxy.com/free-proxy-list-3.html +https://www.my-proxy.com/free-proxy-list-2.html +https://www.my-proxy.com/free-proxy-list-4.html +https://www.my-proxy.com/free-proxy-list-10.html +https://free-proxy-list.net/?ref=binfind.com/web +http://www.aliveproxy.com/proxy-list/proxies.aspx/Syria-sy +https://www.my-proxy.com/free-proxy-list-9.html +https://www.my-proxy.com/free-proxy-list-8.html +https://www.my-proxy.com/free-proxy-list-5.html +https://www.my-proxy.com/free-proxy-list-6.html +https://github.com/a2u/free-proxy-list/blob/master/free-proxy-list.txt +https://github.com/fate0/proxylist/blob/master/proxy.list +http://proxy-ip-list.com/8080/fresh-servers-list-8080-Indonesia.html +https://www.my-proxy.com/free-proxy-list.html +https://www.my-proxy.com/free-elite-proxy.html +https://www.my-proxy.com/free-proxy-list.html?list=10 +http://proxy-ip-list.com/high-anonymous/high-anonymous-80-servers-list.html +https://www.my-proxy.com/free-proxy-list.html?list=1 +https://www.my-proxy.com/free-transparent-proxy.html +https://www.my-proxy.com/free-proxy-list.html?list=3 +https://www.my-proxy.com/free-proxy-list.html?list=5 +https://www.my-proxy.com/free-proxy-list-1.html +https://www.my-proxy.com/free-proxy-list.html?list=s2 +https://www.my-proxy.com/free-anonymous-proxy.html +https://www.my-proxy.com/free-proxy-list.html?list=9 +https://www.my-proxy.com/free-proxy-list.html?list=2 +https://www.my-proxy.com/free-proxy-list.html?list=s3 +https://www.my-proxy.com/free-proxy-list.html?list=7 +https://www.my-proxy.com/free-proxy-list.html?list=4 +https://www.my-proxy.com/free-proxy-list.html?list=6 +https://www.my-proxy.com/free-proxy-list.html?list=8 +http://proxies.my-proxy.com/proxy-list-s1.html +http://proxies.my-proxy.com/proxy-list-s2.html +http://proxies.my-proxy.com/proxy-list-s3.html +http://proxies.my-proxy.com/proxy-list-1.html +http://proxies.my-proxy.com/proxy-list-2.html +http://proxies.my-proxy.com/proxy-list-3.html +http://proxies.my-proxy.com/proxy-list-4.html +http://proxies.my-proxy.com/proxy-list-5.html +http://proxies.my-proxy.com/proxy-list-6.html +http://proxies.my-proxy.com/proxy-list-7.html +http://proxies.my-proxy.com/proxy-list-8.html +http://proxies.my-proxy.com/proxy-list-9.html +http://proxies.my-proxy.com/proxy-list-10.html +http://proxies.my-proxy.com/ +http://www.proxz.com/proxy_list_anonymous_us_0.html +https://www.my-proxy.com/free-socks-5-proxy.html +http://www.httptunnel.ge/ProxyListForFree.aspx +https://www.my-proxy.com/free-socks-4-proxy.html +http://proxysearcher.sourceforge.net/Proxy%20List.php?type=socks&filtered=true +http://proxysearcher.sourceforge.net/Proxy%20List.php +http://www.proxz.com/ +http://www.proxz.com/proxy_list_port_std_2.html +http://www.proxz.com/proxy_list_uk_0_ext.html +http://www.proxz.com/proxy_list_transparent_11_ext.html +http://www.proxz.com/proxy_list_port_std_0.html +http://www.proxz.com/proxy_list_port_std_28.html +http://www.proxz.com/proxy_list_cn_ssl_43.html +http://www.proxz.com/proxy_list_high_anonymous_61.html +http://www.proxz.com/proxy_list_port_nonstd_54.html +http://www.proxz.com/proxy_list_port_std_38.html +http://www.proxz.com/proxy_list_port_std_17.html +https://www.proxydocker.com/de/proxylist/country/Germany +http://proxysearcher.sourceforge.net/ProxyList.php?type=socks&filtered=true +http://proxysearcher.sourceforge.net/ProxyList.php%3Ftype%3Dsocks%26filtered%3Dtrue +http://proxysearcher.sourceforge.net/ProxyList.php%3Ftype%3Dsocks%26filtered%3Dtrue%26limit%3D1000 +http://www.proxz.com/proxy_list_port_nonstd_84_ext.html +https://www.proxydocker.com/de/proxylist/country/Austria +https://www.proxydocker.com/de/socks-list/ +https://free-proxy-list.com/ +http://proxysearcher.sourceforge.net/Proxy%20List.php?type=socks +http://www.proxz.com/proxy_list_transparent_17.html +http://proxysearcher.sourceforge.net/ProxyList.php?type=http +http://proxysearcher.sourceforge.net/ProxyList.php?type=socks +http://proxysearcher.sourceforge.net/ProxyList.php?type=http&filtered=true +http://proxysearcher.sourceforge.net/ProxyList.php%3Ftype%3Dsocks +http://www.proxz.com/proxy_list_port_std_11_ext.html +http://www.proxz.com/proxy_list_port_nonstd_18_ext.html +http://www.proxz.com/proxy_list_port_nonstd_5_ext.html +http://www.proxz.com/proxy_list_port_std_36.html +http://www.proxz.com/proxy_list_port_nonstd_20_ext.html +http://www.proxz.com/proxy_list_high_anonymous_15_ext.html +http://www.proxz.com/proxy_list_port_std_72_ext.html +http://www.proxz.com/proxy_list_port_nonstd_31.html +http://www.proxz.com/proxy_list_port_nonstd_8.html +http://www.proxz.com/proxy_list_high_anonymous_56_ext.html +http://www.proxz.com/proxy_list_transparent_30_ext.html +http://www.proxz.com/proxy_list_port_nonstd_58.html +http://www.proxz.com/proxy_list_high_anonymous_87.html +http://www.proxz.com/proxy_list_high_anonymous_86.html +http://www.proxz.com/proxy_list_port_nonstd_151.html +http://www.proxz.com/proxy_list_transparent_8_ext.html +http://www.proxz.com/proxy_list_transparent_42_ext.html +http://www.proxz.com/proxy_list_port_nonstd_4.html +http://www.proxz.com/proxy_list_high_anonymous_4.html +http://www.proxz.com/proxy_list_transparent_51_ext.html +http://www.proxz.com/proxy_list_high_anonymous_2.html +http://www.proxz.com/proxy_list_port_nonstd_117.html +http://www.proxz.com/proxy_list_port_nonstd_62.html +http://www.proxz.com/proxy_list_port_std_9_ext.html +http://www.proxz.com/proxy_list_high_anonymous_43.html +http://www.proxz.com/proxy_list_transparent_5_ext.html +http://www.proxz.com/proxy_list_high_anonymous_11.html +http://www.proxz.com/proxy_list_port_std_41_ext.html +http://www.proxz.com/proxy_list_port_nonstd_0.html +http://www.proxz.com/proxy_list_transparent_3_ext.html +http://www.proxz.com/proxy_list_transparent_52_ext.html +http://www.proxz.com/proxy_list_high_anonymous_72.html +http://www.proxz.com/proxy_list_fr_1.html +http://www.proxz.com/proxy_list_cn_ssl_47_ext.html +http://www.proxz.com/proxy_list_port_std_27_ext.html +http://www.proxz.com/proxy_list_jp_0.html +http://www.proxz.com/proxy_list_ca_1.html +http://www.proxz.com/proxy_list_port_nonstd_127_ext.html +http://www.proxz.com/proxy_list_port_std_166_ext.html +http://www.proxz.com/proxy_list_high_anonymous_5_ext.html +http://www.proxz.com/proxy_list_transparent_24_ext.html +http://www.proxz.com/proxy_list_high_anonymous_4_ext.html +http://www.proxz.com/proxy_list_port_std_97.html +http://www.proxz.com/proxy_list_anonymous_us_2.html +http://www.proxz.com/proxy_list_high_anonymous_9.html +http://www.proxz.com/proxy_list_high_anonymous_91.html +http://www.proxz.com/proxy_list_transparent_0.html +http://www.proxz.com/proxy_list_high_anonymous_24.html +http://www.proxz.com/proxy_list_port_std_53_ext.html +http://www.proxz.com/proxy_list_cn_ssl_38_ext.html +http://www.proxz.com/proxy_list_transparent_31_ext.html +http://www.proxz.com/proxy_list_port_nonstd_12_ext.html +http://www.proxz.com/proxy_list_high_anonymous_9_ext.html +http://www.proxz.com/proxy_list_port_std_15_ext.html +http://www.proxz.com/proxy_list_port_nonstd_59_ext.html +http://www.proxz.com/proxy_list_port_nonstd_46_ext.html +http://www.proxz.com/proxy_list_port_std_12_ext.html +http://www.proxz.com/proxy_list_port_std_17_ext.html +http://www.proxz.com/proxy_list_port_std_40_ext.html +http://www.proxz.com/proxy_list_port_nonstd_137_ext.html +http://www.proxz.com/proxy_list_high_anonymous_132_ext.html +https://www.proxydocker.com/?_locale=es +https://www.proxydocker.com/en/proxylist/city/Mumbai +https://www.proxydocker.com/en/proxylist/city/Dhaka +https://www.proxydocker.com/en/proxylist/country/Lebanon +https://www.proxydocker.com/de/proxylist/country/Indonesia +http://www.proxydocker.com/de/proxylist/port/8118 +http://www.proxz.com/proxy_list_high_anonymous_14_ext.html +http://www.proxz.com/proxy_list_port_nonstd_23.html +https://www.proxz.com/proxy_list_fr_0.html +http://www.proxz.com/proxy_list_high_anonymous_36.html +http://www.proxz.com/proxy_list_transparent_15_ext.html +http://www.proxz.com/proxy_list_port_nonstd_66_ext.html +http://www.proxz.com/proxy_list_transparent_68.html +https://www.proxydocker.com/ +https://www.proxydocker.com/fr/ +https://www.proxydocker.com/es/ +https://www.proxydocker.com/en/ +https://www.proxydocker.com/ms/ +https://www.proxydocker.com/pt/ +https://www.proxydocker.com/de/ +https://www.proxydocker.com/tr/ +https://www.proxydocker.com/it/ +http://www.proxz.com/proxy_list_transparent_1_ext.html +http://www.proxz.com/proxy_list_transparent_38_ext.html +http://www.proxz.com/proxy_list_port_nonstd_33.html +http://www.proxz.com/proxy_list_high_anonymous_1_ext.html +http://www.proxz.com/proxy_list_transparent_40_ext.html +http://www.proxz.com/proxy_list_port_nonstd_10_ext.html +http://www.proxz.com/proxy_list_transparent_14_ext.html +http://www.proxz.com/proxy_list_high_anonymous_64_ext.html +https://www.proxydocker.com/en/proxylist/country/Hungary +https://www.proxydocker.com/en/proxylist/country/Netherlands +https://www.proxydocker.com/en/proxylist/city/Jalalpur +https://www.proxydocker.com/en/proxylist/city/Istanbul +http://www.proxydocker.com/ +https://www.proxydocker.com/?_locale=de +https://www.proxydocker.com/en/proxylist/city/Singapore +https://www.proxydocker.com/en/proxylist/state/Virginia +https://www.proxydocker.com/en/proxylist/city/Kiev +https://www.proxydocker.com/en/proxylist/state/Florida +https://www.proxydocker.com/en/proxylist/country/Moldova +https://www.proxydocker.com/en/proxylist/port/53281 +https://www.proxydocker.com/en/proxylist/country/Indonesia +https://www.proxydocker.com/en/proxylist/country/Peru +https://www.proxydocker.com/en/proxylist/country/Thailand +http://www.proxz.com/proxy_list_port_nonstd_77_ext.html +http://www.proxz.com/proxy_list_transparent_56.html +http://www.proxz.com/proxy_list_cn_ssl_85_ext.html +http://www.proxz.com/proxy_list_port_std_24_ext.html +http://www.proxz.com/proxy_list_high_anonymous_7.html +https://www.proxydocker.com/en/proxylist/country/United%20States +https://www.proxydocker.com/de/proxylist/country/Russia +https://www.proxydocker.com/en/proxylist/ +https://www.proxydocker.com/de/proxylist/country/Italy +https://www.proxydocker.com/de/proxylist/port/8080,3128 +https://www.proxydocker.com/de/proxylist/country/United%20Kingdom +https://www.proxydocker.com/en/socks4-list/ +https://www.proxydocker.com/?_locale=hi +https://www.proxydocker.com/?_locale=ms +https://www.proxydocker.com/?id=7PeaMYqp7Fk +https://www.proxydocker.com/?_locale=fr +https://www.proxydocker.com/?_locale=it +https://www.proxydocker.com/?_locale=en +https://www.proxydocker.com/?_locale=ru +https://www.proxydocker.com/en/socks-list/ +http://www.proxz.com/proxy_list_cn_ssl_88_ext.html +http://www.proxz.com/proxy_list_high_anonymous_46_ext.html +http://www.proxz.com/proxy_list_transparent_55.html +http://www.proxz.com/proxy_list_high_anonymous_29_ext.html +http://www.proxz.com/proxy_list_high_anonymous_10.html +https://www.proxydocker.com/de/proxylist/country/Belgium +https://www.proxydocker.com/de/proxylist/country/Cambodia +https://www.proxydocker.com/de/proxylist/country/Philippines +https://www.proxydocker.com/de/freeproxylist/South%20Africa +https://www.proxydocker.com/de/proxylist/city/Paris +https://www.proxydocker.com/en/proxylist/type/HTTP +https://www.proxydocker.com/de/proxylist/country/Poland +http://www.proxydocker.com/de +https://www.proxydocker.com/de/proxylist/country/France +https://www.proxydocker.com/de/proxylist/country/Vietnam +https://www.proxydocker.com/de/proxylist/country/Singapore +http://www.proxz.com/proxy_list_transparent_4_ext.html +http://www.proxz.com/proxy_list_port_std_104_ext.html +http://www.proxz.com/proxy_list_transparent_45_ext.html +https://www.proxydocker.com/de/proxylist/city/Singapore +https://www.proxydocker.com/de/proxylist/country/Australia +https://www.proxydocker.com/de/socks5-list/ +https://www.proxydocker.com/en/proxylist/country/France +https://www.proxydocker.com/en/proxylist/state/Texas +https://www.proxydocker.com/en/proxylist/city/Moscow +https://www.proxydocker.com/en/proxylist/country/Iran +http://www.proxz.com/proxy_list_cn_ssl_1_ext.html +http://www.httptunnel.ge/ProxyListForFree.aspx/Media/httptunnelbanner.swf +http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/httptunnelbanner.swf +http://www.httptunnel.ge/proxylistforfree.aspx/media/Media/ProxyChecker.aspx +http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/ProxyListForFree.aspx +http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/Default.aspx +http://www.httptunnel.ge/proxylistforfree.aspx/Media/Media/dpg.aspx +http://www.httptunnel.ge/proxylistforfree.aspx/media/Media/Media/Default.aspx +http://www.httptunnel.ge/proxylistforfree.aspx/media/Media/Media/dpg.aspx +https://www.proxydocker.com/en/socks5-list/ +https://www.proxydocker.com/?_locale=pt +http://www.httptunnel.ge/ProxyListForFree.aspx/ProxyChecker.aspx +http://www.httptunnel.ge/ProxyListForFree.aspx/FAQ.aspx +http://www.httptunnel.ge/proxylistforfree.aspx/media/Download.aspx +http://www.proxz.com/proxy_list_port_nonstd_23_ext.html +http://www.proxz.com/proxy_list_transparent_129_ext.html +http://www.proxz.com/proxy_list_high_anonymous_57_ext.html +http://www.proxz.com/proxy_list_transparent_16_ext.html +http://www.proxz.com/proxy_list_port_std_54_ext.html +http://www.proxz.com/proxy_list_high_anonymous_3.html +https://proxypedia.org/ +http://proxysearcher.sourceforge.net/Proxy%20List.php?type=http +https://www.proxydocker.com/en/proxylist/state/California +https://www.proxydocker.com/en/proxylist/country/South%20korea +https://www.proxydocker.com/en/proxylist/port/80 +https://www.proxydocker.com/en/proxylist/city/New%20York +https://www.proxydocker.com/en/proxylist/country/United%20Kingdom +https://www.proxydocker.com/ja/proxylist/country/Japan +https://www.proxydocker.com/en/proxylist/country/Brazil +https://www.proxydocker.com/en/proxylist/port/8080 +https://www.proxydocker.com/ja/ +https://www.proxydocker.com/en/proxylist/country/Germany +https://www.proxydocker.com/en/proxylist/country/Egypt +https://www.proxydocker.com/en/proxylist/country/Kazakhstan +https://www.proxydocker.com/en/proxylist/country/Paraguay +https://www.proxydocker.com/en/proxylist/country/Malaysia +https://www.proxydocker.com/de/proxylist/port/8080,3128,80 +https://www.proxydocker.com/de/proxylist/ +https://www.proxydocker.com/de/proxylist/country/Egypt +https://www.proxydocker.com/de/proxylist/city/Bangkok +http://www.proxz.com/proxy_list_transparent_164_ext.html +https://www.proxydocker.com/en/proxylist/city/Lima +https://www.proxydocker.com/de/proxylist/country/India +http://www.proxz.com/proxy_list_high_anonymous_130_ext.html +http://www.proxz.com/proxy_list_cn_ssl_59_ext.html +https://www.proxydocker.com/en/proxylist/city/Yekaterinburg +http://www.proxz.com/proxy_list_port_std_156_ext.html +http://www.proxz.com/proxy_list_high_anonymous_155_ext.html +http://www.proxz.com/proxy_list_transparent_18_ext.html +http://www.proxz.com/proxy_list_port_std_65_ext.html +http://www.proxz.com/proxy_list_high_anonymous_99_ext.html +http://www.proxz.com/proxy_list_port_std_42_ext.html +http://www.proxz.com/proxy_list_transparent_132.html +http://www.proxz.com/proxy_list_transparent_85_ext.html +http://www.proxz.com/proxy_list_port_nonstd_87_ext.html +http://www.proxz.com/proxy_list_transparent_128.html +http://www.proxz.com/proxy_list_high_anonymous_75_ext.html +http://www.proxz.com/proxy_list_port_nonstd_103_ext.html +http://www.proxz.com/proxy_list_transparent_76_ext.html +http://www.proxz.com/proxy_list_port_nonstd_11_ext.html +http://www.proxz.com/proxy_list_port_nonstd_90_ext.html +http://www.proxz.com/proxy_list_high_anonymous_93_ext.html +http://www.proxz.com/proxy_list_port_nonstd_146_ext.html +http://www.proxz.com/proxy_list_high_anonymous_124_ext.html +http://www.proxz.com/proxy_list_fr_4.html +https://www.proxydocker.com/en/proxylist/country/China +https://www.proxydocker.com/en/proxylist/country/Albania +http://www.premiumproxy.net/top-country-proxy-list?country=DE/ +http://www.premiumproxy.net/top-country-proxy-list?country=HN/ +http://www.premiumproxy.net/top-country-proxy-list.php?country=HK/ +http://www.premiumproxy.net/top-country-proxy-list?cntr=-braziljikarkarkA +http://www.premiumproxy.net/top-country-proxy-list.php?country=FM +http://www.premiumproxy.net/top-country-proxy-list.php?country=KR/ +http://premiumproxy.net/ +http://premiumproxy.net/contact-us +http://premiumproxy.net/web-proxy-list +http://premiumproxy.net/full-proxy-list +http://premiumproxy.net/elite-proxy-list +http://premiumproxy.net/check-headers-status +http://premiumproxy.net/distorting-proxy-list +http://premiumproxy.net/http-proxy-list +http://premiumproxy.net/transparent-proxy-list.php +http://www.premiumproxy.net/top-country-proxy-list?cntr=-bolivia_AND_AARK +http://www.premiumproxy.net/top-country-proxy-list?cntr=FR-France_A_or&page=0 +http://www.premiumproxy.net/top-country-proxy-list?cntr=CA-A&page=3 +http://www.premiumproxy.net/top-country-proxy-list?cntr=JP-JapanRK +http://www.proxz.com/proxy_list_port_nonstd_7_ext.html +http://www.proxydocker.com/en/proxylist/country/China +http://www.proxz.com/proxy_list_high_anonymous_104_ext.html +http://premiumproxy.net/top-country-proxy-list?cntr=BG- +http://premiumproxy.net/top-country-proxy-list?cntr=RU- +http://premiumproxy.net/top-country-proxy-list?cntr=GR- +http://premiumproxy.net/http-proxy-list.php?page=2 +http://premiumproxy.net/full-proxy-list.php?page=2 +http://premiumproxy.net/socks-proxy-list.php?page=1 +http://premiumproxy.net/top-country-proxy-list?country=EG +http://premiumproxy.net/top-country-proxy-list?country=BR +http://premiumproxy.net/top-country-proxy-list?cntr=A- +http://premiumproxy.net/top-country-proxy-list?country=PK/ +https://www.proxydocker.com/de/proxylist/country/United%20States +http://www.premiumproxy.net/top-country-proxy-list.php?country=CN&page=1 +http://www.premiumproxy.net/top-country-proxy-list?cntr=LT-Lithuania&page=3 +http://www.premiumproxy.net/top-country-proxy-list?cntr=CH-&page=1 +http://premiumproxy.net/top-country-proxy-list?cntr=BR- +http://premiumproxy.net/top-country-proxy-list?cntr=SE- +http://premiumproxy.net/top-country-proxy-list?country=UK +http://www.proxz.com/proxy_list_port_std_67_ext.html +http://www.proxz.com/proxy_list_port_std_49_ext.html +http://premiumproxy.net/top-country-proxy-list?cntr=BO- +http://premiumproxy.net/top-country-proxy-list?cntr=SELECT- +https://www.proxydocker.com/en/proxylist/country/Japan +http://premiumproxy.net/top-country-proxy-list?cntr=-vietnam_and__and_skrkskrk +http://premiumproxy.net/top-country-proxy-list?cntr=-singapore +http://premiumproxy.net/top-country-proxy-list?cntr=IT-Italy +http://premiumproxy.net/top-country-proxy-list.php?country=AF +http://premiumproxy.net/top-country-proxy-list?cntr=-tanzaniaa +http://premiumproxy.net/top-country-proxy-list.php?country=SG +http://premiumproxy.net/top-country-proxy-list?cntr=PA-Panama +http://premiumproxy.net/check-headers-status.php?url=nekopoi.lol +http://premiumproxy.net/top-country-proxy-list?cntr=VN-Vietnam_AND__AND_zOUNzOUNRK +http://premiumproxy.net/top-country-proxy-list?cntr=MX-Mexico_AND__AND_sUjBsUjBRK +https://www.proxydocker.com/de/proxylist/port/8888,3128 +http://www.httptunnel.ge/proxylistforfree.aspx/media/dpg.aspx +http://premiumproxy.net/top-country-proxy-list?cntr=FR- +https://www.proxydocker.com/en/proxylist/city/Surabaya +http://www.httptunnel.ge/proxylistforfree.aspx/ +https://www.proxydocker.com/de/proxylist/city/London +http://premiumproxy.net/top-country-proxy-list?cntr=%20R- +http://premiumproxy.net/top-country-proxy-list?cntr=-dominicanrk +http://premiumproxy.net/top-country-proxy-list?cntr=-guatemala +http://premiumproxy.net/top-country-proxy-list?cntr=-southwp +http://premiumproxy.net/top-country-proxy-list?cntr=BR-Brazil +http://premiumproxy.net/top-country-proxy-list.php?country=NL +http://premiumproxy.net/https-ssl-proxy-list.php?page=1 +https://proxypedia.org/czechia/free-proxy-178-77-206-21-58004 +http://premiumproxy.net/top-country-proxy-list?cntr=COA- +http://premiumproxy.net/top-country-proxy-list?cntr=MX- +http://premiumproxy.net/transparent-proxy-list.php?page=1 +http://premiumproxy.net/top-country-proxy-list?country=ML +http://premiumproxy.net/top-country-proxy-list?cntr=-namibia +http://premiumproxy.net/top-country-proxy-list.php?country=IN +http://premiumproxy.net/top-country-proxy-list.php?country=TH +http://premiumproxy.net/top-country-proxy-list?cntr=JP-JapanRK +http://www.proxydocker.com/en/proxylist/country/Italy +http://www.proxz.com/proxy_list_high_anonymous_97_ext.html +http://www.proxz.com/proxy_list_cn_ssl_82_ext.html +http://www.proxz.com/terms.php +http://www.proxz.com/proxy_list_high_anonymous_117_ext.html +http://premiumproxy.net/top-country-proxy-list.php?country=UZ +http://premiumproxy.net/top-country-proxy-list?cntr=NA-Namibia +http://premiumproxy.net/top-country-proxy-list?country=FM +http://www.premiumproxy.net/check-ip-proxy-port?ip-proxy=Anonytun.com&port=443 +http://www.premiumproxy.net/top-country-proxy-list?cntr=-thailand&page=5 +http://www.premiumproxy.net/top-country-proxy-list?cntr=-braziljikarkaRKRK&page=5 +http://www.premiumproxy.net/top-country-proxy-list?country=UK/&page=1 +http://www.premiumproxy.net/top-country-proxy-list?cntr=-singapore&page=2 +http://www.proxz.com/proxy_list_port_std_8.html +http://premiumproxy.net/http-proxy-list.php?page=1 +http://www.premiumproxy.net/top-country-proxy-list?cntr=NL-Netherlands_ARKRK_ +http://premiumproxy.net/anonymous-proxy-list.php +http://premiumproxy.net/top-country-proxy-list?cntr=-bolivia_AND_AARK +http://premiumproxy.net/top-country-proxy-list?cntr=-curacao +http://www.proxz.com/proxy_list_high_anonymous_108.html +http://premiumproxy.net/top-country-proxy-list?cntr=-bolivia_AND_A +http://premiumproxy.net/top-country-proxy-list.php?country=MQ +http://premiumproxy.net/top-country-proxy-list?cntr=-myanmar +http://premiumproxy.net/top-country-proxy-list?cntr=CN-China_ +http://premiumproxy.net/top-country-proxy-list.php?country=TW +http://premiumproxy.net/top-country-proxy-list?cntr=-malta +http://premiumproxy.net/full-proxy-list.php?page=1 +http://premiumproxy.net/top-country-proxy-list?cntr=JP-Japan +http://www.premiumproxy.net/top-country-proxy-list?cntr=KR-Southwp +http://premiumproxy.net/top-country-proxy-list.php?country=VN +http://premiumproxy.net/top-country-proxy-list.php?country=MX +http://premiumproxy.net/top-country-proxy-list?cntr=-turkey +http://premiumproxy.net/top-country-proxy-list?cntr=PH-Pilippines +http://premiumproxy.net/top-country-proxy-list?cntr=%20-_and_rkrs_ajlrnbcdatwcautelegvxdaso +http://premiumproxy.net/transparent-proxy-list +http://premiumproxy.net/squid-proxy-list +http://premiumproxy.net/top-country-proxy-list.php?country= +http://premiumproxy.net/top-country-proxy-list?cntr=-bolivia_and_aark +http://premiumproxy.net/top-country-proxy-list?cntr=KR-Southwp +http://premiumproxy.net/top-country-proxy-list?cntr=-south_koreaaaaa +http://premiumproxy.net/top-country-proxy-list.php?country=CL/ +http://premiumproxy.net/top-country-proxy-list?cntr=-braziljikarkark diff --git a/includes/misc.py b/includes/misc.py new file mode 100644 index 0000000..3c62fda --- /dev/null +++ b/includes/misc.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +import time, random, sys, os, string +#import sockschain as socks +sys.path.append('./includes') +import rocksock + +""" return formatted timestamp """ +def timestamp(): + return time.strftime('%H:%M:%S', time.gmtime()) + +""" return some random string """ +def random_string(strlen=20): + return ''.join([random.choice(string.letters) for x in xrange(strlen)]) + +def _log(strng, level='info'): + print '%s/%s\t%s' % (timestamp(), level, strng) + +def option_matches_options(strng, items): + try: return [item for item in items if re.match(strng, item)] + except: return False + +def prepare_socksocket(self, destination, path, path_item): + if path_item in self.paths and self.paths[path_item]['path'] == path: + self.paths[path_item]['path'] = False + + #socks.setdefaultproxy() + # relay to i2p http proxy if *.i2p domain + if destination.endswith('i2p'): + proxy = random.choice(self.i2p_host).split(':') + path = False + # or go with tor + else: + proxies = [ rocksock.RocksockProxyFromURL('socks5://%s' % random.choice(self.tor_host)) ] + #socks.adddefaultproxy(*socks.parseproxy('tor://%s' % random.choice(self.tor_host))) + # add 'clearnet' proxies to the chain ? + if self.proxify and (not destination.endswith('onion') and not destination.endswith('.exit')): + + # get a proxy path + path = build_path(self, path_item, path) + + # if path isn't long enough, break + if not len(path): return False, False, False + + # add chain... + #for inc in xrange(len(path) - 1): socks.adddefaultproxy(*socks.parseproxy('http://%s' % path[inc])) + #for inc in xrange(len(path) - 1): socks.adddefaultproxy(*socks.parseproxy('%s://%s' % (path[inc][1], path[inc][0]))) + for inc in xrange(len(path)): proxies.append( rocksock.RocksockProxyFromURL('%s://%s' % (path[inc][1], path[inc][0]))) + + #return True, socks.socksocket, path + return True, proxies, path + +def build_path(self, path_item, path): + + chainlen = random.randint( self.path_len, (self.path_len + self.path_randomlen)) + # if not enough proxies + # FIXME: try to get a proxylist from database + if len(self.proxylist) < chainlen: return [] + + # valid path already available + elif (path_item in self.paths and + self.paths[path_item]['path'] and + (time.time() - self.paths[path_item]['ticks']) < self.path_duration): + + # take available path if any + if path != self.paths[path_item]['path']: path = self.paths[path_item]['path'] + + # or nope, none available + # build a new one from scratch + else: + path = [] + avail = [] + + # dec chainlen if we have to select the exit proxy + if self.exitcountry is not None: chainlen -= 1 + #avail = [item[0] for item in self.proxylist if not item[0] in avail and item[1] != str(self.exitcountry).upper()] + avail = [[item[0],item[2]] for item in self.proxylist if not item[0] in avail and item[1] != str(self.exitcountry).upper()] + path = random.sample(avail, chainlen) + + # choose the exit proxy + if self.exitcountry is not None: + #avail = [item[0] for item in self.proxylist if not item[0] in path and item[1] == str(self.exitcountry).upper()] + avail = [[item[0],item[2]] for item in self.proxylist if not item[0] in path and item[1] == str(self.exitcountry).upper()] + if not len(avail): return [] + path.append(random.choice(avail)) + + return path diff --git a/includes/mysqlite.py b/includes/mysqlite.py new file mode 100644 index 0000000..340f94c --- /dev/null +++ b/includes/mysqlite.py @@ -0,0 +1,33 @@ +import time, random, sys +import sqlite3 as lite + +class mysqlite: + + def execute(self, query, args = None, rmin = 1.5, rmax = 7.0): + + while True: + try: return self.cursor.execute(query,args) if args else self.cursor.execute(query) + except: + print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args)) + time.sleep(random.uniform(rmin, rmax)) + + def executemany(self, query, args, rmin = 1.5, rmax = 7.0): + + while True: + try: return self.cursor.executemany(query,args) + except: + print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args)) + time.sleep(random.uniform(rmin, rmax)) + + def commit(self, rmin = 1.5, rmax = 7.0): + while True: + try: return self.handle.commit() + except: time.sleep(random.uniform(rmin, rmax)) + + def close(self): + self.handle.close() + + def __init__(self, database, factory = None): + self.handle = lite.connect(database) + if factory: self.handle.text_factory = factory + self.cursor = self.handle.cursor() diff --git a/includes/proxywatchd.py b/includes/proxywatchd.py new file mode 100644 index 0000000..6e2f73b --- /dev/null +++ b/includes/proxywatchd.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python + +from threading import Thread +import threading, commands +import socket, time, random, sys, string, re +#import sockschain as socks +import requests +#from geoip import geolite2 + +from ConfigParser import SafeConfigParser + +sys.path.append('./includes') +import mysqlite +from misc import _log +import rocksock + +class Proxywatchd(Thread): + + def stop(self): + _log('Requesting proxywatchd to halt (%d thread(s))' % len([item for item in self.threads if item.isAlive()])) + self.running = 0 + + def __init__(self, config_file): + Thread.__init__(self) + + self.threads = [] + self.running = 1 + self.parser = SafeConfigParser() + self.parser.read(config_file) + + self.maxfail = self.parser.getint('global', 'proxy_max_fail') + self.maxthreads = self.parser.getint('watcherd', 'threads') + self.checktime = self.parser.getint('watcherd', 'checktime') + self.timeout = self.parser.getint('watcherd', 'timeout') + self.database = self.parser.get('global', 'database') + self.torhosts = [ str(i).strip() for i in self.parser.get('global', 'tor_host').split(',') ] + self.read_timeout = self.parser.getint('watcherd', 'read_timeout') + + # create table if needed + self.mysqlite = mysqlite.mysqlite(self.database, str) + self.mysqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, source BLOB, dronebl INT, proto TEXT, duration INT)') + self.mysqlite.commit() + self.echoise = time.time() - 3600; + self.ticks = time.time() - 3600; + + with open('servers.txt', 'r') as handle: self.servers = handle.read().split('\n') + + self.start() + + def run(self): + _log('Starting proxywatchd..', 'notice') + + threads = [] + self.mysqlite = mysqlite.mysqlite(self.database, str) + + while self.running: + + if len(threads) < self.maxthreads: + t = threading.Thread(target=self.daemon, args=(self.servers,)) + t.start() + threads.append(t) + time.sleep( random.choice( xrange(1,3))) + + else: time.sleep(1) + + if (time.time() - self.echoise) >= 180: + _log('Proxywatchd threads: %d/%d' % (len(threads), self.maxthreads)) + self.echoise = time.time() + + self.mysqlite.close() + + def is_drone_bl(self, proxy): + p = proxy.split(':')[0] + proxies = {'http':'socks4://%s:%s@%s' % (p,p,random.choice(self.torhosts))} + resp = requests.get('http://dronebl.org/lookup?ip=%s' % p, proxies=proxies) + if 'No incidents regarding' in resp.text: return 0 + else: return 1 + + def connect_socket(self, proxy, servers, proto = None): + proto = ['http', 'socks5', 'socks4'] + + for p in proto: + torhost = random.choice(self.torhosts) + duration = time.time() + proxies = [ rocksock.RocksockProxyFromURL('socks4://%s' % torhost), + rocksock.RocksockProxyFromURL('%s://%s' % (p, proxy[0])), + ] + + tst = random.choice(servers).strip() + try: + sock = rocksock.Rocksock(host=tst, port=6667, ssl=False, proxies=proxies, timeout=self.timeout) + sock.connect() + sock.send('%s\n' % random.choice(['NICK', 'USER', 'JOIN', 'MODE', 'PART', 'INVITE', 'KNOCK', 'WHOIS', 'WHO', 'NOTICE', 'PRIVMSG', 'PING', 'QUIT'])) + return sock, p, duration, torhost, tst + + except: + #print('socket error, disconnecting') + sock.disconnect() + + return False, False, False, False, False + + def daemon(self, servers): + sqlite = mysqlite.mysqlite(self.database, str) + threadid = ''.join( [ random.choice(string.letters) for x in range(5) ] ) + + q = 'SELECT proxy,failed,country,proto FROM proxylist WHERE failed= 1024*1024: n = 1024*1024 + chunk = self.sock.recv(n) + except socket.timeout: + raise RocksockException(RS_E_HIT_TIMEOUT) + except ssl.SSLError as e: + s = self._get_ssl_exception_reason(e) + if s == 'The read operation timed out': + raise RocksockException(RS_E_HIT_READTIMEOUT) + else: + raise RocksockException(RS_E_SSL_GENERIC, s, errortype=RS_ET_SSL) + if len(chunk) == 0: + raise RocksockException(RS_E_REMOTE_DISCONNECTED) + data += chunk + if count == -1: break + else: count -= len(chunk) + return data + + def recvline(self): + s = '' + c = '\0' + while c != '\n': + c = self.recv(1) + if c == '': return s + s += c + return s + + def recvuntil(self, until): + s = self.recv(len(until)) + endc = until[-1:] + while not (s[-1:] == endc and s.endswith(until)): + s += self.recv(1) + return s + + def _isnumericipv4(self, ip): + try: + a,b,c,d = ip.split('.') + if int(a) < 256 and int(b) < 256 and int(c) < 256 and int(d) < 256: + return True + return False + except: + return False + + def _resolve(self, hostinfo, want_v4=True): + if self._isnumericipv4(hostinfo.host): + return socket.AF_INET, (hostinfo.host, hostinfo.port) + try: + for res in socket.getaddrinfo(hostinfo.host, hostinfo.port, \ + socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE): + af, socktype, proto, canonname, sa = res + if want_v4 and af != socket.AF_INET: continue + if af != socket.AF_INET and af != socket.AF_INET6: continue + else: return af, sa + + except socket.gaierror as e: + eno, str = e.args + raise RocksockException(eno, str, errortype=RS_ET_GAI) + + return None, None + + def _ip_to_int(self, ip): + a,b,c,d = ip.split('.') + h = "0x%.2X%.2X%.2X%.2X"%(int(a),int(b),int(c),int(d)) + return int(h, 16) + + def _ip_to_bytes(self, ip): + ip = self._ip_to_int(ip) + a = (ip & 0xff000000) >> 24 + b = (ip & 0x00ff0000) >> 16 + c = (ip & 0x0000ff00) >> 8 + d = (ip & 0x000000ff) >> 0 + return chr(a) + chr(b) + chr(c) + chr(d) + + def _setup_socks4_header(self, v4a, dest): + buf = '\x04\x01' + buf += chr(dest.hostinfo.port / 256) + buf += chr(dest.hostinfo.port % 256) + if v4a: + buf += '\0\0\0\x01' + else: + af, sa = self._resolve(dest, True) + if af != socket.AF_INET: raise RocksockException(RS_E_SOCKS4_NO_IP6) + buf += self._ip_to_bytes(sa[0]) + buf += '\0' + if v4a: buf += dest.hostinfo.host + '\0' + return buf + + def _connect_socks4(self, header, pnum): + self.send(header) + res = self.recv(8) + if len(res) < 8 or ord(res[0]) != 0: + raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1) + ch = ord(res[1]) + if ch == 0x5a: + pass + elif ch == 0x5b: + raise RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, pnum-1) + elif ch == 0x5c or ch == 0x5d: + return RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1) + else: + raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1) + + def _setup_socks5_header(self, proxy): + buf = '\x05' + if proxy.username and proxy.password: + buf += '\x02\x00\x02' + else: + buf += '\x01\x00' + return buf + + def _connect_socks5(self, header, pnum): + self.send(header) + res = self.recv(2) + if len(res) != 2 or res[0] != '\x05': + raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1) + if res[1] == '\xff': + raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1) + + if ord(res[1]) == 2: + px = self.proxychain[pnum-1] + if px.username and px.password: + pkt = '\x01%c%s%c%s'%(len(px.username),px.username,len(px.password),px.password) + self.send(pkt) + res = self.recv(2) + if len(res) < 2 or res[1] != '\0': + raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1) + else: raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1) + dst = self.proxychain[pnum] + numeric = self._isnumericipv4(dst.hostinfo.host) + if numeric: + dstaddr = self._ip_to_bytes(dst.hostinfo.host) + else: + dstaddr = chr(len(dst.hostinfo.host)) + dst.hostinfo.host + + pkt = '\x05\x01\x00%c%s%c%c'% (1 if numeric else 3, dstaddr, dst.hostinfo.port / 256, dst.hostinfo.port % 256) + self.send(pkt) + res = self.recv() + if len(res) < 2 or res[0] != '\x05': + raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1) + ch = ord(res[1]) + if ch == 0: pass + elif ch == 1: raise RocksockException(RS_E_PROXY_GENERAL_FAILURE, pnum-1) + elif ch == 2: raise RocksockException(RS_E_PROXY_AUTH_FAILED, pnum-1) + elif ch == 3: raise RocksockException(RS_E_TARGETPROXY_NET_UNREACHABLE, pnum-1) + elif ch == 4: raise RocksockException(RS_E_TARGETPROXY_HOST_UNREACHABLE, pnum-1) + elif ch == 5: raise RocksockException(RS_E_TARGETPROXY_CONN_REFUSED, pnum-1) + elif ch == 6: raise RocksockException(RS_E_TARGETPROXY_TTL_EXPIRED, pnum-1) + elif ch == 7: raise RocksockException(RS_E_PROXY_COMMAND_NOT_SUPPORTED, pnum-1) + elif ch == 8: raise RocksockException(RS_E_PROXY_ADDRESSTYPE_NOT_SUPPORTED, pnum-1) + else: raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1) + + + def _connect_step(self, pnum): + prev = self.proxychain[pnum -1] + curr = self.proxychain[pnum] + if prev.type == RS_PT_SOCKS4: + s4a = self._setup_socks4_header(True, curr) + try: + self._connect_socks4(s4a, pnum) + except RocksockException as e: + if e.get_error() == RS_E_TARGETPROXY_CONNECT_FAILED: + s4 = self._setup_socks4_header(self, False, curr) + self._connect_socks4(s4a, pnum) + else: raise e + elif prev.type == RS_PT_SOCKS5: + s5 = self._setup_socks5_header(prev) + self._connect_socks5(s5, pnum) + elif prev.type == RS_PT_HTTP: + dest = self.proxychain[pnum] + self.send("CONNECT %s:%d HTTP/1.1\r\n\r\n"%(dest.hostinfo.host, dest.hostinfo.port)) + resp = self.recv() + if len(resp) <12: + raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, pnum-1) + if resp[9] != '2': + raise RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, pnum-1) + + +if __name__ == '__main__': + proxies = [ +# RocksockProxyFromURL("socks5://foo:bar@localhost:1080"), +# RocksockProxyFromURL("socks5://10.0.0.3:1080"), + RocksockProxyFromURL("socks5://127.0.0.1:31339"), + ] + proxies = None + #rs = Rocksock(host='googleff242342423f.com', port=443, ssl=True, proxies=proxies) + rs = Rocksock(host='google.com', port=80, ssl=False, proxies=proxies) + try: + rs.connect() + except RocksockException as e: + print e.get_errormessage() + e.reraise() + rs.send('GET / HTTP/1.0\r\n\r\n') + print rs.recvline() + rs.disconnect() + rs.connect() + rs.send('GET / HTTP/1.0\r\n\r\n') + print rs.recvline() + diff --git a/ppf.py b/ppf.py new file mode 100755 index 0000000..d430ceb --- /dev/null +++ b/ppf.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python + +import os +import sys +import socket +import requests +import socks +import random, time +import sqlite3 +import re +import urllib +import threading +import hashlib +import ipcalc +from soup_parser import soupify +from ConfigParser import SafeConfigParser +from requests.packages.urllib3.exceptions import InsecureRequestWarning +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) +from dns import resolver +#from selenium import webdriver + +sys.path.append('./includes') +import mysqlite +from misc import _log + +base_header = { + 'Accept-Language':'en-US,en;q=0.8', + 'Cache-Control':'max-age=0', + 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'DNT': '1', + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0' +} + +database = 'proxies.sqlite' + +searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' ) + +retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded') +proxies={ 'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050', 'ftp': 'socks5://127.0.0.1:9050'} + + +# include own classes +sys.path.append('./includes') +import proxywatchd, mysqlite +from misc import _log + +CONFIG = 'config.ini' + + +def cleanhtml(raw_html): + cleanr = re.compile('<.*?>') + cleantext = re.sub(cleanr, ':', raw_html) + cleantext = re.sub('::+',':', cleantext) + return cleantext + +def import_from_file(fn, sqlite): + with open(fn, 'r') as f: + for u in f.read().split('\n'): + if not len(u): continue + exists = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE url=?',(u,)).fetchall() ] + if exists: continue + print('adding "%s"' % u) + sqlite.execute('INSERT INTO uris (added,url,check_time,error) VALUES (?,?,?,?)', (time.time(),u,0,1)) + sqlite.commit() + +def fetch_contents(uri, driver=None): + headers = base_header + ## use requests (default) + if not driver: + try: resp = requests.get(uri, timeout=45, headers=headers, verify=False, proxies=proxies) + except: return '' + data = resp.text + + ## phantomjs + else: + for key, value in enumerate(base_header): + capability_key = 'phantomjs.page.customHeaders.{}'.format(key) + webdriver.DesiredCapabilities.PHANTOMJS[capability_key] = value + + service_args = ['--proxy=127.0.0.1:9050', '--proxy-type=socks5'] + driver = webdriver.PhantomJS() + try: + driver.implicitly_wait(45) + driver.set_page_load_timeout(45) + driver.get(uri) + data = driver.page_source + + except: data = '' + finally:driver.quit() + + for retry_message in retry_messages: + if retry_message in data: return '' + return data + +def update_proxy_sources(sqlite, proxies, uri): + for proxy in proxies: + md5sum = hashlib.md5(proxy).hexdigest() + sqlite.execute('CREATE TABLE IF NOT EXISTS "%s" (uri TEXT)' % md5sum) + sqlite.commit() + #check = [ i for i in sqlite.execute('SELECT uri FROM "%s" WHERE uri=?' % md5sum, (uri,)).fetchall() ] + check = sqlite.execute('SELECT uri FROM "%s" WHERE uri=?' % md5sum, (uri,)).fetchall() + if not len(check): + sqlite.execute('INSERT INTO "%s" (uri) VALUES(?)' % md5sum, (uri,)) + sqlite.commit() + +def insert_proxies(proxies, uri, sqlite): + time_now = time.time() + added = 0 + + ## very wasteful + #update_proxy_sources(sqlite, proxies, uri) + + query = [ 'proxy=?' for p in proxies ] + known = [ i[0] for i in sqlite.execute('SELECT proxy FROM proxylist WHERE %s' % ' OR '.join(query), proxies).fetchall() ] + new = [ (time_now,i,3,0) for i in proxies if not i in known ] + if len(new): + added = added + len(new) + sqlite.executemany('INSERT INTO proxylist (added,proxy,failed,tested) VALUES (?,?,?,?)', new) + sqlite.commit() + + if added > 0: _log('+%d item(s) from %s' % (added, uri), 'added') + +def proxyfind(sqlite = None): + #print('entering proxyfind...') + + if not sqlite: sqlite = mysqlite.mysqlite(database,str) + + uris = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE error=0 and url not like "%github%" ORDER BY RANDOM() LIMIT 10').fetchall() ] + + if len(uris) > 0 and random.random() < random.random(): + search = urllib.quote_plus('site:%s' % random.choice(uris).split('/')[2]) + else: + search = urllib.quote_plus(random.choice(search_terms)) + + choice = random.choice(searx_instances) + urls = [] + + content = fetch_contents('%s/?q=%s&pageno=%d' % (choice, search, random.randint(0,10))) + + if not content: return + soup = soupify(content) + + for a in soup.body.find_all('a'): + if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue + badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ] + if not len(badurl): urls.append(a.attrs['href']) + + if len(urls): + query = [ 'url=?' for u in urls ] + known = [ i[0] for i in sqlite.execute('SELECT url FROM uris WHERE %s' % ' OR '.join(query),urls).fetchall() ] + time_now = time.time() + new = [ (time_now,i,0,5,0) for i in urls if not i in known ] + if len(new): + sqlite.executemany('INSERT INTO uris (added,url,check_time,error,driver) values(?,?,?,?,?)', new) + sqlite.commit() + _log('+%d item(s) from %s' % (len(new), search), 'added') + + sqlite.commit() + +def proxyleech(sqlite, rows): + #print('entering proxyleech...') + + for row in rows: + try: content = fetch_contents(row[0], None) + except: content = '' + + uniques = [] + for p in sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', cleanhtml(content))): + if p in uniques: continue + try: + if str(ipcalc.Network(p.split(':')[0]).subnet) != 'PRIVATE': uniques.append(p) + except: + pass + + hash = hashlib.md5(''.join(uniques)).hexdigest() + #print('unique; hash: %s, len: %d' % (hash, len(uniques))) + + ## empty list of proxies: increment error by two + if not len(uniques): row[2] = (row[2] * 2) + ## same proxy list: increment error by one + elif hash == row[1]: row[2] = (row[2] + 1) + ## proxylist was updated: error is zero + else: row[2] = 0 + + check_time = (time.time() + 3600 + (3600 * row[2])) + sqlite.execute('UPDATE uris SET error=?,hash=?,check_time=? where url=?', (row[2],hash, check_time,row[0])) + sqlite.commit() + + if not row[1] or row[2] > 0: return + + add = [] + for i in uniques: + add.append(i) + if len(add) < 500: continue + insert_proxies(add, row[0], sqlite) + add = [] + if len(add): insert_proxies(add, row[0], sqlite) + + +if __name__ == '__main__': + ## read the config files + parser = SafeConfigParser() + parser.read(CONFIG) + + database = parser.get('global', 'database') + search = parser.getboolean('proxyfind', 'search') + + sqlite = mysqlite.mysqlite(database, str) + + ## create dbs if required + sqlite.execute('CREATE TABLE IF NOT EXISTS uris (added INT, url TEXT, check_time INT, error INT, driver INT, hash TEXT)') + sqlite.execute('CREATE TABLE IF NOT EXISTS proxylist (proxy BLOB, country BLOB, added INT, failed INT, tested INT, dronebl INT, proto TEXT, duration INT)') + sqlite.commit() + + import_from_file('import.txt', sqlite) + + ## load search terms + with open('search_terms.txt', 'r') as f: + search_terms = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ] + + ## load bad terms + with open('urignore.txt', 'r') as f: + urignore = [ i.strip() for i in f.read().split('\n') if len(i.strip()) ] + ## add searx instances as bad terms (avoid loops) + empty = [ urignore.append(i.split('/')[2]) for i in searx_instances ] + + # start proxy watcher + watcherd = proxywatchd.Proxywatchd(CONFIG) if parser.getboolean('watcherd', 'enabled') else None + + while True: + try: + ## any site that needs to be checked ? + rows = [ [i[0],i[1],i[2]] for i in sqlite.execute('SELECT url,hash,error FROM uris WHERE (check_time%s'%(html) + try: + res = BeautifulSoup(htm, parser) + except FeatureNotFound as e: + parser = 'html.parser' + res = BeautifulSoup(htm, parser) + return res + +def soupify_gumbo(html, nohtml=False): + htm = html if nohtml else '%s'%(html) + try: + soup = gumbo.soup_parse(htm) + if not soup.body: + print "AAAA" + print html + print "BBBB" + print repr(soup) + return soup + + except Exception as e: + sys.stdout.write(html) + raise + +def soupify(html, nohtml=False): +# return soupify_gumbo(html, nohtml) + return soupify_bs4(html, nohtml) + + diff --git a/striphtml.py b/striphtml.py new file mode 100644 index 0000000..ef195c4 --- /dev/null +++ b/striphtml.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +from HTMLParser import HTMLParser +import requests +import re +from selenium.webdriver.common.proxy import * +from selenium import webdriver +from selenium.webdriver.common.by import By +phantomjs_path = '/home/mickael/bin/phantomjs' +def cleanhtml(raw_html): + cleanr = re.compile('<.*?>') + cleantext = re.sub(cleanr, ':', raw_html) + cleantext = re.sub('::+',':', cleantext) + return cleantext + +class MLStripper(HTMLParser): + def __init__(self): + self.reset() + self.fed = [] + def handle_data(self, d): + self.fed.append(d) + def get_data(self): + return ''.join(self.fed) + +def strip_tags(html): + s = MLStripper() + s.feed(html) + return s.get_data() + +service_args = ['--proxy=127.0.0.1:9050', '--proxy-type=socks5'] +driver = webdriver.PhantomJS(phantomjs_path,service_args=service_args) +try: driver.get('http://www.proxz.com/proxy_list_fr_1_ext.html') +except: sys.exit(0) +html = driver.page_source +driver.quit() + +text = cleanhtml(html) +proxies = sorted(re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', text)) + +print(text) +print(proxies) diff --git a/urignore.txt b/urignore.txt new file mode 100644 index 0000000..69be6c6 --- /dev/null +++ b/urignore.txt @@ -0,0 +1,19 @@ +check-reverse-domain-ip-lookup +www.youtube.com +youtu.be +archive +.gov[$/] +/blog/ +/questions/ +/question/ +wiki +answer +thread +/robots.txt +form +webproxy +check?i= +[0-9]+(?:\.[0-9]+){3} +README.md +support +aspx?p=