fetch.py: improve readability of extract_urls

This commit is contained in:
rofl0r
2019-01-18 19:32:37 +00:00
parent 4a41796b19
commit b99f83a991

View File

@@ -107,7 +107,11 @@ def extract_urls(content, urls = None, urignore=None):
soup = soupify(content)
for a in soup.body.find_all('a'):
if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue
badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ]
if not len(badurl): urls.append(a.attrs['href'])
bad = False
for i in urignore:
if re.findall(i,a.attrs['href'], re.IGNORECASE):
bad = True
break
if not bad: urls.append(a.attrs['href'])
return urls