Mikaela/Limnoria
1
0
Fork 0
mirror of https://github.com/Mikaela/Limnoria.git synced 2024-12-24 11:42:52 +01:00
Code Issues Projects Releases Wiki Activity

Added memoization optimization to tokenize function.

Browse Source
This commit is contained in:
Jeremy Fincher 2003-10-20 04:17:14 +00:00
parent 06cd9ae285
commit 68fd496516
1 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/callbacks.py
View File

@ -284,19 +284,19 @@ def tokenize(s):
debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose') debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
return _lastTokenizeResult return _lastTokenizeResult
def tokenize(s): ## def tokenize(s):
"""A utility function to create a Tokenizer and tokenize a string.""" ## """A utility function to create a Tokenizer and tokenize a string."""
start = time.time() ## start = time.time()
try: ## try:
if conf.enablePipeSyntax: ## if conf.enablePipeSyntax:
tokens = '|' ## tokens = '|'
else: ## else:
tokens = '' ## tokens = ''
args = Tokenizer(tokens).tokenize(s) ## args = Tokenizer(tokens).tokenize(s)
except ValueError, e: ## except ValueError, e:
raise SyntaxError, str(e) ## raise SyntaxError, str(e)
#debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose') ## #debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
return args ## return args
def getCommands(tokens): def getCommands(tokens):
"""Given tokens as output by tokenize, returns the command names.""" """Given tokens as output by tokenize, returns the command names."""