Mikaela/Limnoria
1
0
Fork 0
mirror of https://github.com/Mikaela/Limnoria.git synced 2024-12-24 03:33:11 +01:00
Code Issues Projects Releases Wiki Activity

Added memoization optimization to tokenize function.

Browse Source
This commit is contained in:
Jeremy Fincher 2003-10-20 04:17:14 +00:00
parent 06cd9ae285
commit 68fd496516
1 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/callbacks.py
View File

@ -284,19 +284,19 @@ def tokenize(s):
debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
return _lastTokenizeResult
def tokenize(s):
"""A utility function to create a Tokenizer and tokenize a string."""
start = time.time()
try:
if conf.enablePipeSyntax:
tokens = '|'
else:
tokens = ''
args = Tokenizer(tokens).tokenize(s)
except ValueError, e:
raise SyntaxError, str(e)
#debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
return args
## def tokenize(s):
## """A utility function to create a Tokenizer and tokenize a string."""
## start = time.time()
## try:
## if conf.enablePipeSyntax:
## tokens = '|'
## else:
## tokens = ''
## args = Tokenizer(tokens).tokenize(s)
## except ValueError, e:
## raise SyntaxError, str(e)
## #debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
## return args
def getCommands(tokens):
"""Given tokens as output by tokenize, returns the command names."""