fetch.py: improve readability of extract_urls
This commit is contained in:
8
fetch.py
8
fetch.py
@@ -107,7 +107,11 @@ def extract_urls(content, urls = None, urignore=None):
|
||||
soup = soupify(content)
|
||||
for a in soup.body.find_all('a'):
|
||||
if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue
|
||||
badurl = [ i for i in urignore if re.findall(i,a.attrs['href'], re.IGNORECASE) ]
|
||||
if not len(badurl): urls.append(a.attrs['href'])
|
||||
bad = False
|
||||
for i in urignore:
|
||||
if re.findall(i,a.attrs['href'], re.IGNORECASE):
|
||||
bad = True
|
||||
break
|
||||
if not bad: urls.append(a.attrs['href'])
|
||||
return urls
|
||||
|
||||
|
||||
Reference in New Issue
Block a user