Mikaela/Limnoria
1
0
Fork 0
mirror of https://github.com/Mikaela/Limnoria.git synced 2024-11-27 13:19:24 +01:00
Code Issues Projects Releases Wiki Activity

utils/web.py: Add some 'try' statements to make decoding as error-permissive as possible.

Browse Source
This commit is contained in:
Valentin Lorentz 2013-06-13 17:22:33 +00:00
parent 2eec980c8e
commit cca1e6dba5
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/utils/web.py
View File

@ -191,14 +191,16 @@ def htmlToText(s, tagReplace=' '):
""" """
try: try:
import chardet.universaldetector import chardet.universaldetector
except ImportError:
if sys.version_info[0] < 3 or isinstance(s, bytes):
s = s.decode('utf8')
else:
u = chardet.universaldetector.UniversalDetector() u = chardet.universaldetector.UniversalDetector()
u.feed(s) u.feed(s)
u.close() u.close()
s = s.decode(u.result['encoding']) s = s.decode(u.result['encoding'])
except:
try:
if sys.version_info[0] < 3 or isinstance(s, bytes):
s = s.decode('utf8')
except:
pass
x = HtmlToText(tagReplace) x = HtmlToText(tagReplace)
x.feed(s) x.feed(s)
return x.getText() return x.getText()