username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ppf: improve cleanhtml() and cache compiled re's

Browse Source 
now it transforms e.g. '<td>118.114.116.36</td>\n<td>1080</td>'
correctly.
(the newline was formerly preventing success)
This commit is contained in:
rofl0r
2019-01-09 22:48:03 +00:00
parent befb346941
commit 087559637e
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
ppf.py
View File

@@ -21,11 +21,18 @@ base_header = {
searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' ) searx_instances = ('https://searx.me', 'https://searx.xyz', 'https://searx.site', 'https://searx.win', 'https://searx.ru', 'https://stemy.me/searx', 'https://searx.at', 'https://listi.me', 'https://searx.dk', 'https://searx.laquadrature.net' )
retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded') retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded')
cleanhtml_re = [
re.compile('<.*?>'),
re.compile('\s+'),
re.compile('::+'),
]
def cleanhtml(raw_html): def cleanhtml(raw_html):
cleanr = re.compile('<.*?>') html = raw_html.replace('&nbsp;', ' ')
cleantext = re.sub(cleanr, ':', raw_html) html = re.sub(cleanhtml_re[0], ':', html)
cleantext = re.sub('::+',':', cleantext) html = re.sub(cleanhtml_re[1], ':', html)
return cleantext html = re.sub(cleanhtml_re[2], ':', html)
return html
def import_from_file(fn, sqlite): def import_from_file(fn, sqlite):
with open(fn, 'r') as f: with open(fn, 'r') as f: