username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

check content type

Browse Source
This commit is contained in:
Mickaël Serneels
2019-04-28 01:04:32 +02:00
parent 0962019386
commit 9a624819d3
1 changed files with 14 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
ppf.py
View File

@@ -21,6 +21,7 @@ def import_from_file(fn, sqlite):
sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count,proxies_added,retrievals) VALUES (?,?,?,?,?,?,?)', (int(time.time()),u,0,0,0,0,0)) sqlite.execute('INSERT INTO uris (added,url,check_time,error,stale_count,proxies_added,retrievals) VALUES (?,?,?,?,?,?,?)', (int(time.time()),u,0,0,0,0,0))
sqlite.commit() sqlite.commit()
def get_content_type(url): def get_content_type(url):
hdr = fetch.fetch_contents(url, head=True) hdr = fetch.fetch_contents(url, head=True)
@@ -29,9 +30,17 @@ def get_content_type(url):
return '' return ''
def is_good_content_type(string):
allowed_ct = [ 'text/html', 'text/plain', 'atom+xml' ]
for ct in allowed_ct:
if ct.lower() in string.lower(): return True
return False
def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added, content_type): def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_added, content_type):
if not content_type: content_type = get_content_type(url) if not content_type: content_type = get_content_type(url)
if content_type.lower().startswith('text/') or 'atom+xml' in content_type.lower():
#if content_type.lower().startswith('text/') or 'atom+xml' in content_type.lower():
if is_good_content_type(content_type):
try: content = fetch.fetch_contents(url) try: content = fetch.fetch_contents(url)
except KeyboardInterrupt as e: raise e except KeyboardInterrupt as e: raise e
except: content = '' except: content = ''
@@ -55,7 +64,7 @@ def proxyleech(proxydb, urldb, url, stale_count, error, retrievals, proxies_adde
retrievals += 1 retrievals += 1
error = 0 error = 0
urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=? where url=?', (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), url)) urldb.execute('UPDATE uris SET error=?,stale_count=?,check_time=?,retrievals=?,proxies_added=?,content_type=? where url=?', (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, url))
urldb.commit() urldb.commit()
if not len(new): return if not len(new): return
@@ -96,10 +105,12 @@ if __name__ == '__main__':
## any site that needs to be checked ? ## any site that needs to be checked ?
rows = urldb.execute('SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall() rows = urldb.execute('SELECT url,stale_count,error,retrievals,proxies_added,content_type FROM uris WHERE error < ? and (check_time+?+((error+stale_count)*?) <?) ORDER BY RANDOM() LIMIT 25', (config.ppf.max_fail, config.ppf.checktime, config.ppf.perfail_checktime, int(time.time()))).fetchall()
if not len(rows): time.sleep(10)
for row in rows: for row in rows:
proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4], row[5]) proxyleech(proxydb, urldb, row[0], row[1], row[2], row[3], row[4], row[5])
time.sleep(10) #time.sleep(10)
except KeyboardInterrupt: except KeyboardInterrupt:
if watcherd: if watcherd: