Fixed bug #832590.

src/callbacks.py

@@ -226,7 +226,7 @@ class Tokenizer:
     #
     # These are the characters valid in a token.  Everything printable except
     # double-quote, left-bracket, and right-bracket.
-    validChars = string.ascii[33:].translate(string.ascii, '"[]')
+    validChars = string.ascii.translate(string.ascii, '\x00\r\n \t"[]')
     def __init__(self, tokens=''):
         # Add a '|' to tokens to have the pipe syntax.
         self.validChars = self.validChars.translate(string.ascii, tokens)

test/test_callbacks.py

@@ -98,6 +98,19 @@ class TokenizerTestCase(unittest.TestCase):
                              ['baz', 'quux', ['foo', 'bar']])
         finally:
             conf.enablePipeSyntax = False
+
+    def testBold(self):
+        s = '\x02foo\x02'
+        self.assertEqual(tokenize(s), [s])
+        s = s[:-1] + '\x0f'
+        self.assertEqual(tokenize(s), [s])
+        
+
+    def testColor(self):
+        s = '\x032,3foo\x03'
+        self.assertEqual(tokenize(s), [s])
+        s = s[:-1] + '\x0f'
+        self.assertEqual(tokenize(s), [s])
         
 
 class FunctionsTestCase(unittest.TestCase):