From ff96b898f9262f86d03115b4ea072b60c066d8f8 Mon Sep 17 00:00:00 2001 From: James McCoy Date: Sat, 22 Oct 2011 15:23:56 -0400 Subject: [PATCH] RSS._getConverter: Encode strings before handing them off to other functions When the feed has a specified encoding, we'll be dealing with unicode objects in the response from feedparser.parse(). To avoid possible UnicodeErrors, we need to encode() before handing the string off to other functions, so the other functions are always dealing with bytestrings instead of bytestrings and unicode objects. Mixing unicode and bytestrings will cause implicit conversions of the unicode objects, which will most likely use the wrong encoding. Signed-off-by: James McCoy Conflicts: plugins/RSS/plugin.py --- plugins/RSS/plugin.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/RSS/plugin.py b/plugins/RSS/plugin.py index fecc57ef3..5945145a8 100644 --- a/plugins/RSS/plugin.py +++ b/plugins/RSS/plugin.py @@ -291,10 +291,12 @@ class RSS(callbacks.Plugin): toText = utils.web.htmlToText if 'encoding' in feed: def conv(s): - try: - return toText(s).strip().encode(feed['encoding'],'replace') - except UnicodeEncodeError: - return toText(s.encode('utf-8', 'ignore')).strip() + # encode() first so there implicit encoding doesn't happen in + # other functions when unicode and bytestring objects are used + # together + s = s.encode(feed['encoding'], 'replace') + s = toText(s).strip() + return s return conv else: return lambda s: toText(s).strip()