username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fetch.py: factor out twice used var

Browse Source
This commit is contained in:
rofl0r
2019-03-29 23:07:44 +00:00
committed by Mickaël Serneels
parent 096ee21286
commit bf7ec03fbf
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fetch.py
View File

@@ -108,10 +108,11 @@ def extract_urls(content, urls = None, urignore=None):
for a in soup.body.find_all('a'): for a in soup.body.find_all('a'):
if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue if not 'rel' in a.attrs or not 'noreferrer' in a.attrs['rel'] or a.attrs['href'] in urls: continue
bad = False bad = False
href = a.attrs['href']
for i in urignore: for i in urignore:
if re.findall(i,a.attrs['href'], re.IGNORECASE): if re.findall(i, href):
bad = True bad = True
break break
if not bad: urls.append(a.attrs['href']) if not bad: urls.append(href)
return urls return urls