From eb31db6277d987c9f3c1b7007f37d0a088ac5ee9 Mon Sep 17 00:00:00 2001
From: Jeremy Fincher <jemfinch@users.sourceforge.net>
Date: Mon, 20 Oct 2003 04:26:37 +0000
Subject: [PATCH] Added memoization optimization to tokenize function.

---
 src/callbacks.py | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/src/callbacks.py b/src/callbacks.py
index d29b7a413..3cfeadc6f 100644
--- a/src/callbacks.py
+++ b/src/callbacks.py
@@ -41,6 +41,7 @@ how to use them.
 import fix
 
 import re
+import copy
 import sets
 import time
 import shlex
@@ -281,22 +282,8 @@ def tokenize(s):
         _lastTokenized = None
         _lastTokenizedResult = None
         raise SyntaxError, str(e)
-    debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
-    return _lastTokenizeResult
-
-## def tokenize(s):
-##     """A utility function to create a Tokenizer and tokenize a string."""
-##     start = time.time()
-##     try:
-##         if conf.enablePipeSyntax:
-##             tokens = '|'
-##         else:
-##             tokens = ''
-##         args = Tokenizer(tokens).tokenize(s)
-##     except ValueError, e:
-##         raise SyntaxError, str(e)
-##     #debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
-##     return args
+    #debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
+    return copy.deepcopy(_lastTokenizeResult)
 
 def getCommands(tokens):
     """Given tokens as output by tokenize, returns the command names."""