fetch.py: factor out twice used var

This commit is contained in:
rofl0r
2019-03-29 23:07:44 +00:00
committed by Mickaël Serneels
parent 096ee21286
commit bf7ec03fbf

View File

@@ -108,10 +108,11 @@ def extract_urls(content, urls = None, urignore=None):
for a in soup.body.find_all('a'):
if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue
bad = False
href = a.attrs['href']
for i in urignore:
if re.findall(i,a.attrs['href'], re.IGNORECASE):
if re.findall(i, href):
bad = True
break
if not bad: urls.append(a.attrs['href'])
if not bad: urls.append(href)
return urls