From 7e91ae52373ddab9f61b1771853a9fd979903a07 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 6 Feb 2021 21:50:08 +0100 Subject: [PATCH] changes --- import.txt | 267 +++++++++++++++++++++++++++++++++++++++++++++++++++++ ppf.py | 10 +- 2 files changed, 275 insertions(+), 2 deletions(-) diff --git a/import.txt b/import.txt index 138f0e0..67c98f0 100644 --- a/import.txt +++ b/import.txt @@ -1,3 +1,270 @@ +https://www.coderduck.com/free-proxy-list +https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt +https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt +https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt +https://www.proxyscrape.com/free-proxy-list +https://proxy-daily.com/ +https://us-proxy.org/ +https://freeproxylists.net/ +https://www.proxy-list.download/ +https://cuteseotools.net/free-proxy-list +http://proxydb.net/?country=BR +https://www.proxy-list.download/ +https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt +https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list.txt +https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt +http://hideme.ru/proxy-list/http +http://www.multiproxy.org/anon_proxy.htm +http://www.proxylist.net/list/0/0/2/0/0 +http://www.proxylist.net/list/0/0/2/0/1 +http://www.proxylist.net/list/0/0/2/0/2 +http://hideme.ru/proxy-list/socks5 +http://cool-proxy.ru/spisok-elitnyx-proksi-20-08-11.html#more-29751 +http://cool-proxy.ru/spisok-soksov-20-08-11.html#more-29750 +http://cool-proxy.ru/spisok-xttps-proksi-20-08-11.html#more-29754 +http://cool-proxy.ru/spisok-anonimnyx-proksi-20-08-11.html#more-29752 +http://x-proxy.info/socks.php +http://fineproxy.ru/ +http://fineproxy.ru/page/2 +http://fineproxy.ru/page/3 +http://veliki-ustyug.info/curl-socks5-2011-07-31.html +http://proxy-free-list.ru/spisok-soksov-01-08-11.html#more-20115 +http://px.at.ua/index/0-7 +http://proxy.vvs777.org.ua/exp.php?67ce1418 +http://www.multiproxy.org/anon_proxy.htm +http://www.proxylist.net/list/0/0/2/0/0 +http://www.proxylist.net/list/0/0/2/0/1 +http://www.proxylist.net/list/0/0/2/0/2 +http://www.proxylist.net/list/0/0/2/0/3 +http://www.proxylist.net/list/0/0/2/0/4 +http://www.proxylist.net/list/0/0/2/0/5 +http://veliki-ustyug.info/online-proxy-surf-2011-08-03.html +http://abc.planet.ee/free-socks-list/ +http://aliveproxy.com/socks4-list/ +http://freetao.org/socks/ +http://proxy.cnfei.net/socks/ +http://www.proxybox.ru/socks_list.php +http://www.proxycn.net/socks4.php +http://www.proxycn.net/socks5.php +http://www.shram.kiev.ua/hacker/proxy/ +http://www.xroxy.com/proxy-type-Socks4.htm +http://www.xroxy.com/proxy-type-Socks5.htm +http://x-proxy.info/socks.php +http://aliveproxy.com/socks5-list/ +http://atomintersoft.com/products/alive-proxy/socks5-list/ +http://best-proxy.ru/category/socks-proksi +http://best-proxylist.blogspot.com/ +http://eiloo.us/ +http://spys.ru/en/socks-proxy-list/ +http://proxydb.ru/ +http://www.avivi.info/ +http://www.vipsocks.cz.cc/ +http://www.livesocks.cz.cc/ +http://www.socks24.org/ +http://socksus.blogspot.com/ +http://megasocks.blogspot.com/ +http://www.lb-ru.ru/tag/socks-proxy/ +http://hideme.ru/proxy-list/socks4/ +http://hideme.ru/proxy-list/socks5/ +http://www.proxylists.net/proxylist.php?SOCKS +http://www.proxybox.ru/socks_list.php +http://www.proxybox.ru/socks_list.htm +http://www.socks5list.com/ +http://persianwz.blogspot.com/ +http://cool-proxy.ru/category/socks-proxy +http://anon-proxy.ru/category/socks-proksi +http://3d2f.com/tags/socks5/list/ +http://3d2f.com/tags/socks5/list/ +http://4proxy.org.ua/free-socks-list-18-07-11.html +http://4proxy.org.ua/tag/free-socks +http://abc.planet.ee/free-socks-list/ +http://aliveproxy.com/ +http://aliveproxy.com/socks-list/socks5.aspx/France-fr +http://aliveproxy.com/socks-list/socks5.aspx/Russian_Federation-ru +http://aliveproxy.com/socks4-list/ +http://aliveproxy.com/socks5-list/ +http://all-proxy.ru/free-socks-list-03-08-11.html +http://all-proxy.ru/free-socks-list-08-07-11.html +http://anon-proxy.ru/free-socks-list-17-06-11.html +http://anon-proxy.ru/free-socks-list-24-07-11.html +http://appfinder.lisisoft.com/ipad-iphone-apps/free-socks5-proxy-list.html +http://ardoino.com/16-c-functions-socks-list-validation/ +http://athesironhy.narod.ru/skachat/proxy_socks4_i_socks5.html +http://atomintersoft.com/products/alive-proxy/socks5-list/ +http://best-proxy.ru/ +http://best-proxy.ru/free-socks-list-01-08-11.html +http://best-proxy.ru/free-socks-list-11-06-11.html +http://best-proxy.ru/free-socks-list-18-07-11.html +http://best-proxylist.blogspot.com/2011/04/free-proxy-list-socks-5-on-monday-18.html +http://best-proxylist.blogspot.com/2011/04/socks-4-anonymous-proxy-server-list-on.html +http://best-proxylist.blogspot.com/2011/04/socks-4-fresh-proxy-list-on-thursday-07.html +http://best-proxylist.blogspot.com/2011/06/socks-4-free-proxy-server-list-on.html +http://biohazard.org.ru/forum/11 +http://biohazard.org.ru/forum/11-3-1 +http://blog.kingsocks.net/ +http://buzzcode.us/ +http://buzzmodule.us/ +http://cgi.ebay.com/MENS-GOLDTOE-FLUFFIES-SOCKS-4-PACK-NAVY-/150506034626 +http://composhop.info/socks-proxy-server-2011-07-21.html +http://cool-proxy.ru/ +http://cool-proxy.ru/free-socks-list-14-07-11.html +http://crackingforum.com/proxy-lists-wordlists/27495-list-fresh-socks-4-5-live-checked-daily-%7C-april-17-2011-update-1-a.html +http://css-templates-free.info/ +http://css-templates-free.info/proxy-list-socks-4-2011-06-28.html +http://css-templates-free.info/socks-5-proxies-2011-05-14.html +http://dailysocksbycountry.blogspot.com/ +http://dawhois.com/site/socksproxylists.com.html +http://depolariseinconseque.hotbox.ru/8.html +http://dumpz.ru/archive/index.php/t-12468.html +http://eiloo.us/ +http://ekaterinburg-ru.info/socks4-vs-socks5-2011-05-11.html +http://ekaterinburg-ru.info/socks5-proxy-download-2011-08-01.html +http://elite-proxy.ru/free-socks-list-05-08-11.html +http://en.softonic.com/s/socks-proxy-list +http://exchangemoneyforum.com/showthread.php/37754-SOCKS24.US-Free-Socks-5-vip-Proxy +http://fastproxyservers.org/ +http://forum.lug.am/index.php?topic=1849.0 +http://forums.digitalpoint.com/showthread.php?t=2107853 +http://free-socks-proxy-list.downloadsoftware4free.com/ +http://free-socks-proxy-list.suggestsoft.com/ +http://free_socks_proxy_list.blueprograms.com/ +http://freeproxilist.info/ +http://freeproxy.org.ua/free-socks-list-01-08-11.html +http://freeproxy.org.ua/free-socks-list-06-08-11.html +http://freeproxy.org.ua/free-socks-list-20-07-11.html +http://freeproxylist.free-webmaster-resources.org/2011/04/16/16-apr-2011-free-socks-proxy-list/ +http://freeproxylist.free-webmaster-resources.org/tag/free-socks4-list/ +http://freetao.org/socks/ +http://get-proxy.ru/free-socks-list-06-08-11.html +http://gfreeproxylist.com/search/google-chrome-socks5 +http://gigaline.us/ +http://good-proxy.ru/free-socks-list-03-08-11.html +http://good-proxy.ru/free-socks-list-09-07-11.html +http://gotovicomp.info/socks4-2011-05-07.html +http://hacker-attack.ru/showthread.php?p=6680 +http://heatkeys.com/socks+proxy+list +http://help.vnn.ru/Free-Socks-5-Proxy-Lists +http://hide-me.ru/free-socks-list-06-07-11.html +http://hide-me.ru/free-socks-list-08-07-11.html +http://hide-me.ru/free-socks-list-24-06-11.html +http://hideme.ru/proxy-list/socks4/ +http://hideme.ru/proxy-list/socks5/ +http://hidemefirst.com/ +http://hidemefirst.com/proxy-list/socks-proxy-1-1 +http://hidemefirst.com/proxy-list/socks5-proxy-1 +http://informe.com/sockslist.net/ +http://joomla-templates-free.info/ +http://joomla-templates-free.info/socks-5-2011-04-17.html +http://joomla-templates-free.info/socks-5-lists-2011-07-20.html +http://kamia.us/ +http://memori.ru/bookmarks/513/?page=&perpage=50 +http://mikhed.newmail.ru/ru/programs/index.htm +http://mogilev.in/proxy-socks-list-2011-07-30.html +http://myproxylists.com/ +http://myproxylists.com/socks-proxy-lists +http://novosibirsksity.info/ +http://novosibirsksity.info/free-socks-2011-08-01.html +http://omoney.info/socks5-liste-2011-07-14.html +http://omoney.info/uk-socks5-proxy-2011-07-27.html +http://www.zarb.org/~saispo/proxylist.txt +https://spys.one/en/socks-proxy-list/ +https://proxies.su/?page=1 +https://proxies.su/?page=2 +https://proxies.su/?page=3 +https://proxies.su/?page=4 +https://proxies.su/?page=5 +https://proxies.su/?page=6 +https://proxies.su/?page=7 +https://proxies.su/?page=8 +https://proxies.su/?page=9 +https://proxies.su/?page=10 +http://onlinechecker.freeproxy.ru/free_proxy_lists.php +http://paolosoftware.com/download/software/Fresh_Socks4_Proxy_List.htm +http://pastebin.com/dxWphRy3 +http://phrase-history.ru/phrases/free_proxy_list_socks5/ +http://phrase-history.ru/phrases/socks_list/?all_pos=1 +http://premium-proxies.com/knowledge/help/netherlands%20free%20socks%20list +http://proxies.my-proxy.com/proxy-list-socks4.html +http://proxies.my-proxy.com/proxy-list-socks5.html +http://proxy-free-list.ru/ +http://proxy-free-list.ru/free-socks-list-28-07-11.html +http://proxy-ip-list.com/3128/Switzerland-high-anonymous-3128.html +http://proxy-ip-list.com/s/Germany+Socks+5+List/ +http://proxy.atomintersoft.com/products/alive-proxy/socks4-list/ +http://proxy.cnfei.net/socks/ +http://proxy.wmzond.ru/category/russian-socks/ +http://proxydb.ru/ +http://proxylist.vbank.us/free_socks_proxylist.html +http://proxylist.vbank.us/free_socks_proxylist/socks_4,5/port_all/country_all/pages_1.html +http://proxynext.com/ +http://ptf.com/socks5/socks5+proxy+list+skype/ +http://rosinstrument.com/pb/m/13641.htm +http://russianproxy.ru/socks5_proxy_list_fastest +http://samair.ru/proxy/socks.htm +http://scriptsforwordpress.info/ +http://shareme.com/showall/free-socks-proxy-list.html +http://shutkeys.com/socks+5+proxy+list +http://socks-4-proxy.fyxm.net/ +http://socks-proxy-list.softplatz.net/ +http://socks.softrecipe.com/ +http://socks4list.com/ +http://socks4proxylist.com/ +http://socks5proxylist.blogspot.com/2011/05/free-proxy-list-free-socks-5-list-free_29.html +http://sockslist.net/ +http://sockslist.net/list/proxy-socks-4-list/ +http://sockslist.net/list/proxy-socks-5-list/ +http://soft.mydiv.net/win/download-Socks-Proxy-Search.html +http://softwaretopic.informer.com/socks-proxy-list-4-yahoo/ +http://softwaretopic.informer.com/socks-proxy-list-for-yahoo/ +http://softwaretopic.informer.com/socks-proxy-list-with-password/ +http://softwaretopic.informer.com/socks-v5-list/ +http://sovetwebmasters.info/ +http://spys.ru/en/ +http://spys.ru/en/socks-proxy-list/ +http://spys.ru/proxies/ +http://stealthproxy.org/buy-proxy-list/ +http://telestate.us/ +http://temawordpress.info/socks-proxy-server-2011-05-04.html +http://theend2010.info/ +http://thoughtlinks.us/ +http://tinycast.us/free-socks-proxy-2011-08-04.html +http://tinysource.us/ +http://tools.rosinstrument.com/proxy/ +http://free-proxy.cz/ +http://free-proxy.cz/ru/proxylist/country/ru/all/ping/all +https://hidemy.name/ru/proxy-list/ +https://proxyscrape.com/free-proxy-list +https://free.proxy-sale.com/ +https://openproxy.space/list +https://free-proxy-list.net/ +https://www.vpnside.com/proxy/list/ +https://freeproxyip.net/ +https://www.proxyscan.io/ +https://sockslist.net/ +https://www.proxydocker.com/en/socks-list/ +https://premproxy.com/socks-list/ +https://www.proxy-list.download/SOCKS5 +http://www.aliveproxy.com/socks5-list/ +https://proxyscrape.com/free-proxy-list +https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt +https://www.proxyfish.com/proxylist/all_socks_proxies.php +https://sur.ly/o/sockslist.net/proxy-list +https://advanced.name/freeproxy?type=socks4 +http://free-proxy.cz/en/proxylist/country/all/socks/ping/all +https://50na50.net/ru/proxy/socks4list +https://advanced.name/freeproxy?type=socks4&ddexp4attempt=1 +https://scrapingant.com/free-proxies/ +https://www.my-proxy.com/free-elite-proxy.html +https://www.proxynova.com/proxy-server-list/elite-proxies/ +https://advanced.name/freeproxy?type=elite +https://www.us-proxy.org/ +https://eliteproxy.net/ +http://free-proxy.cz/en/proxylist/country/all/socks4/ping/all +https://www.proxy-list.download/SOCKS4 +http://proxydb.net/?protocol=socks4&protocol=socks5 +https://advanced.name/freeproxy?type=elite +https://www.proxyscan.io/ +http://free-proxy.cz/ru/proxylist/country/all/socks4/ping/all https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt diff --git a/ppf.py b/ppf.py index e0a9c76..b6fedf6 100755 --- a/ppf.py +++ b/ppf.py @@ -98,9 +98,9 @@ def extract_urls(html, url): if not item.startswith('/'): item = '/%s' % item item = '%s://%s%s' % (proto,domain,item) - if is_bad_url(item, domain=domain, samedomain=config.ppf.extract_samedomain): + elif is_bad_url(item, domain=domain, samedomain=config.ppf.extract_samedomain): continue - elif not item in urls: urls.append(item) + if not item in urls: urls.append(item) if len(urls): dbs.insert_urls(urls, url, urldb) #insert_if_not_exists(urls) @@ -329,7 +329,12 @@ if __name__ == '__main__': if len(rows) < config.ppf.threads: rows = [] else: + nao = time.time() _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf') + args = [ (nao, row[0]) for row in rows ] + urldb.executemany('UPDATE uris SET check_time=? where url=?', args) + urldb.commit() + for thread in threads: if thread.status == 'ok': @@ -342,6 +347,7 @@ if __name__ == '__main__': _known_proxies[p]=1 execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url) urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', execute) + urldb.commit() if len(new): dbs.insert_proxies(proxydb, new, url) threads = [ thread for thread in threads if thread.is_alive() ]