From 2ace534bdb326d18654f968e92b23750733dbee8 Mon Sep 17 00:00:00 2001
From: Valentin Lorentz <progval@progval.net>
Date: Wed, 23 Jan 2013 15:48:24 +0100
Subject: [PATCH] utils.str: Prevent issue with tokens like '\x80' in Python 3.

---
 src/callbacks.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/callbacks.py b/src/callbacks.py
index 48a42cae6..a8366ead6 100644
--- a/src/callbacks.py
+++ b/src/callbacks.py
@@ -307,7 +307,10 @@ class Tokenizer(object):
             else:
                 token = codecs.getencoder('utf8')(token)[0]
                 token = codecs.getdecoder('unicode_escape')(token)[0]
-                token = token.encode('iso-8859-1').decode()
+                try:
+                    token = token.encode('iso-8859-1').decode()
+                except: # Prevent issue with tokens like '"\\x80"'.
+                    pass
         return token
 
     def _insideBrackets(self, lexer):