From 895ecf718f8654ab3c8f703f38e26484a3854f18 Mon Sep 17 00:00:00 2001
From: Jeremy Fincher <jemfinch@users.sourceforge.net>
Date: Thu, 30 Oct 2003 05:27:25 +0000
Subject: [PATCH] Fixed bug #832590.

---
 src/callbacks.py       |  2 +-
 test/test_callbacks.py | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/callbacks.py b/src/callbacks.py
index 1419d20c6..3ad7d4aa9 100644
--- a/src/callbacks.py
+++ b/src/callbacks.py
@@ -226,7 +226,7 @@ class Tokenizer:
     #
     # These are the characters valid in a token.  Everything printable except
     # double-quote, left-bracket, and right-bracket.
-    validChars = string.ascii[33:].translate(string.ascii, '"[]')
+    validChars = string.ascii.translate(string.ascii, '\x00\r\n \t"[]')
     def __init__(self, tokens=''):
         # Add a '|' to tokens to have the pipe syntax.
         self.validChars = self.validChars.translate(string.ascii, tokens)
diff --git a/test/test_callbacks.py b/test/test_callbacks.py
index d8a308a4c..b832a0213 100644
--- a/test/test_callbacks.py
+++ b/test/test_callbacks.py
@@ -98,6 +98,19 @@ class TokenizerTestCase(unittest.TestCase):
                              ['baz', 'quux', ['foo', 'bar']])
         finally:
             conf.enablePipeSyntax = False
+
+    def testBold(self):
+        s = '\x02foo\x02'
+        self.assertEqual(tokenize(s), [s])
+        s = s[:-1] + '\x0f'
+        self.assertEqual(tokenize(s), [s])
+        
+
+    def testColor(self):
+        s = '\x032,3foo\x03'
+        self.assertEqual(tokenize(s), [s])
+        s = s[:-1] + '\x0f'
+        self.assertEqual(tokenize(s), [s])
         
 
 class FunctionsTestCase(unittest.TestCase):