username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fetch.py: improve readability of extract_urls

Browse Source
This commit is contained in:
rofl0r
2019-01-18 19:32:37 +00:00
parent 4a41796b19
commit b99f83a991
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fetch.py
View File

@@ -107,7 +107,11 @@ def extract_urls(content, urls = None, urignore=None):
soup = soupify(content) soup = soupify(content)
for a in soup.body.find_all('a'): for a in soup.body.find_all('a'):
if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue
badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ] bad = False
if not len(badurl): urls.append(a.attrs['href']) for i in urignore:
if re.findall(i,a.attrs['href'], re.IGNORECASE):
bad = True
break
if not bad: urls.append(a.attrs['href'])
return urls return urls