Completely restructured our utils modules.

Tons of changes.  Here's the summary of things that matter most:

* There is no more supybot.fix.
* There is no more supybot.webutils; now there is supybot.utils.web.
  * It's no longer webutils.WebError, but just utils.web.Error.
* You shouldn't import itertools, ideally, but instead import utils.iter.
* No more using imap/ifilter in commands unless absolutely necessary.  It's
  premature optimization and annoying.
* utils.str.format isn't quite ready yet, but will be soon.  That'll be the
  next big thing to fix in our code.

plugins/Admin/plugin.py

@@ -27,16 +27,9 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot
-
-__author__ = supybot.authors.jemfinch
-
-import supybot.fix as fix
-
 import sys
 import time
 import pprint
-from itertools import imap
 
 import supybot.log as log
 import supybot.conf as conf
@@ -164,7 +157,7 @@ class Admin(callbacks.Privmsg):
         L = irc.state.channels.keys()
         if L:
             utils.sortBy(ircutils.toLower, L)
-            irc.reply(utils.commaAndify(L))
+            irc.reply(utils.str.commaAndify(L))
         else:
             irc.reply('I\'m not currently in any channels.')
     channels = wrap(channels, ['private'])
@@ -194,7 +187,7 @@ class Admin(callbacks.Privmsg):
         irc = self.pendingNickChanges.get(irc, None)
         if irc is not None:
             irc.error('I can\'t change nicks, the server said %s.' %
-                      utils.quoted(msg.args[2]), private=True)
+                      utils.str.quoted(msg.args[2]), private=True)
         else:
             self.log.debug('Got 438 without Admin.nick being called.')
 
@@ -332,7 +325,7 @@ class Admin(callbacks.Privmsg):
         """
         # XXX Add the expirations.
         if ircdb.ignores.hostmasks:
-            irc.reply(utils.commaAndify(imap(repr, ircdb.ignores.hostmasks)))
+            irc.reply(utils.str.commaAndify(map(repr,ircdb.ignores.hostmasks)))
         else:
             irc.reply('I\'m not currently globally ignoring anyone.')
     ignores = wrap(ignores)

plugins/Babelfish/plugin.py

@@ -27,11 +27,7 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot
-
-import sets
 import random
-from itertools import imap
 
 import babelfish
 
@@ -69,7 +65,7 @@ class Babelfish(callbacks.Privmsg):
 
         Returns the languages that Babelfish can translate to/from.
         """
-        irc.reply(utils.commaAndify(babelfish.available_languages))
+        irc.reply(utils.str.commaAndify(babelfish.available_languages))
 
     def translate(self, irc, msg, args, fromLang, toLang, text):
         """<from-language> [to] <to-language> <text>
@@ -87,15 +83,15 @@ class Babelfish(callbacks.Privmsg):
                     irc.error('I do not speak any other languages.')
                     return
                 else:
-                    irc.error('I only speak %s.' % utils.commaAndify(langs))
+                    irc.error('I only speak %s.' % utils.str.commaAndify(langs))
                     return
             translation = babelfish.translate(text, fromLang, toLang)
-            irc.reply(utils.htmlToText(translation))
+            irc.reply(utils.web.htmlToText(translation))
         except (KeyError, babelfish.LanguageNotAvailableError), e:
             languages = self.registryValue('languages', chan)
             if languages:
                 languages = 'Valid languages include %s' % \
-                        utils.commaAndify(sorted(languages))
+                        utils.str.commaAndify(sorted(languages))
             else:
                 languages = 'I do not speak any other languages.'
             irc.errorInvalid('language', str(e), languages)
@@ -125,16 +121,16 @@ class Babelfish(callbacks.Privmsg):
                     irc.error('I do not speak any other languages.')
                     return
                 else:
-                    irc.error('I only speak %s.' % utils.commaAndify(langs,
+                    irc.error('I only speak %s.' % utils.str.commaAndify(langs,
                                                                      And='or'))
                     return
             translations = babelfish.babelize(text, fromLang, toLang)
-            irc.reply(utils.htmlToText(translations[-1]))
+            irc.reply(utils.web.htmlToText(translations[-1]))
         except (KeyError, babelfish.LanguageNotAvailableError), e:
             languages = self.registryValue('languages', chan)
             if languages:
                 languages = 'Valid languages include %s' % \
-                        utils.commaAndify(sorted(languages))
+                        utils.str.commaAndify(sorted(languages))
             else:
                 languages = 'I do not speak any other languages.'
             irc.errorInvalid('language', str(e), languages)

plugins/Channel/config.py

@@ -47,12 +47,12 @@ class BanmaskStyle(registry.SpaceSeparatedSetOfStrings):
                                   'This is a bug.'
         registry.SpaceSeparatedSetOfStrings.__init__(self, *args, **kwargs)
         self.__doc__ = 'Valid values include %s.' % \
-                        utils.commaAndify(map(repr, self.validStrings))
+                        utils.str.commaAndify(map(repr, self.validStrings))
 
     def help(self):
         strings = [s for s in self.validStrings if s]
         return '%s  Valid strings: %s.' % \
-                (self._help, utils.commaAndify(strings))
+                (self._help, utils.str.commaAndify(strings))
 
     def normalize(self, s):
         lowered = s.lower()

plugins/Channel/plugin.py

@@ -30,8 +30,6 @@
 import sys
 import time
 
-from itertools import imap
-
 import supybot.conf as conf
 import supybot.ircdb as ircdb
 import supybot.utils as utils
@@ -287,12 +285,12 @@ class Channel(callbacks.Privmsg):
         self.log.debug('In kban')
         if not irc.isNick(bannedNick):
             self.log.warning('%s tried to kban a non nick: %s',
-                             utils.quoted(msg.prefix),
+                             utils.str.quoted(msg.prefix),
-                             utils.quoted(bannedNick))
+                             utils.str.quoted(bannedNick))
             raise callbacks.ArgumentError
         elif bannedNick == irc.nick:
             self.log.warning('%s tried to make me kban myself.',
-                             utils.quoted(msg.prefix))
+                             utils.str.quoted(msg.prefix))
             irc.error('I cowardly refuse to kickban myself.')
             return
         if not reason:
@@ -330,7 +328,7 @@ class Channel(callbacks.Privmsg):
         if ircutils.hostmaskPatternEqual(banmask, irc.prefix):
             if ircutils.hostmaskPatternEqual(banmask, irc.prefix):
                 self.log.warning('%s tried to make me kban myself.',
-                                 utils.quoted(msg.prefix))
+                                 utils.str.quoted(msg.prefix))
                 irc.error('I cowardly refuse to ban myself.')
                 return
             else:
@@ -354,7 +352,7 @@ class Channel(callbacks.Privmsg):
         elif ircdb.checkCapability(msg.prefix, capability):
             if ircdb.checkCapability(bannedHostmask, capability):
                 self.log.warning('%s tried to ban %s, but both have %s',
-                                 msg.prefix, utils.quoted(bannedHostmask),
+                                 msg.prefix, utils.str.quoted(bannedHostmask),
                                  capability)
                 irc.error('%s has %s too, you can\'t ban him/her/it.' %
                           (bannedNick, capability))
@@ -362,7 +360,7 @@ class Channel(callbacks.Privmsg):
                 doBan()
         else:
             self.log.warning('%s attempted kban without %s',
-                             utils.quoted(msg.prefix), capability)
+                             utils.str.quoted(msg.prefix), capability)
             irc.errorNoCapability(capability)
             exact,nick,user,host
     kban = wrap(kban,
@@ -515,7 +513,7 @@ class Channel(callbacks.Privmsg):
         # XXX Add the expirations.
         c = ircdb.channels.getChannel(channel)
         if c.bans:
-            irc.reply(utils.commaAndify(map(utils.dqrepr, c.bans)))
+            irc.reply(utils.str.commaAndify(map(utils.str.dqrepr, c.bans)))
         else:
             irc.reply('There are currently no permanent bans on %s' % channel)
     permbans = wrap(permbans, [('checkChannelCapability', 'op')])
@@ -561,11 +559,11 @@ class Channel(callbacks.Privmsg):
         c = ircdb.channels.getChannel(channel)
         if len(c.ignores) == 0:
             s = 'I\'m not currently ignoring any hostmasks in %s' % \
-                utils.quoted(channel)
+                utils.str.quoted(channel)
             irc.reply(s)
         else:
             L = sorted(c.ignores)
-            irc.reply(utils.commaAndify(imap(repr, L)))
+            irc.reply(utils.str.commaAndify(map(repr, L)))
     ignores = wrap(ignores, [('checkChannelCapability', 'op')])
 
     def addcapability(self, irc, msg, args, channel, user, capabilities):
@@ -602,8 +600,9 @@ class Channel(callbacks.Privmsg):
         ircdb.users.setUser(user)
         if fail:
             irc.error('That user didn\'t have the %s %s.' %
-                      (utils.commaAndify(fail),
+                      (utils.str.commaAndify(fail),
-                       utils.pluralize('capability', len(fail))), Raise=True)
+                       utils.str.pluralize('capability', len(fail))),
+                      Raise=True)
         irc.replySuccess()
     removecapability = wrap(removecapability,
                             [('checkChannelCapability', 'op'),
@@ -662,8 +661,9 @@ class Channel(callbacks.Privmsg):
         ircdb.channels.setChannel(channel, chan)
         if fail:
             irc.error('I do not know about the %s %s.' %
-                      (utils.commaAndify(fail),
+                      (utils.str.commaAndify(fail),
-                       utils.pluralize('capability', len(fail))), Raise=True)
+                       utils.str.pluralize('capability', len(fail))),
+                      Raise=True)
         irc.replySuccess()
     unsetcapability = wrap(unsetcapability,
                            [('checkChannelCapability', 'op'),
@@ -774,7 +774,7 @@ class Channel(callbacks.Privmsg):
                 L.append(channel)
         if L:
             L.sort()
-            s = 'I\'m currently lobotomized in %s.' % utils.commaAndify(L)
+            s = 'I\'m currently lobotomized in %s.' % utils.str.commaAndify(L)
             irc.reply(s)
         else:
             irc.reply('I\'m not currently lobotomized in any channels.')
@@ -787,7 +787,7 @@ class Channel(callbacks.Privmsg):
         """
         L = list(irc.state.channels[channel].users)
         utils.sortBy(str.lower, L)
-        irc.reply(utils.commaAndify(L))
+        irc.reply(utils.str.commaAndify(L))
     nicks = wrap(nicks, ['inChannel']) # XXX Check that the caller is in chan.
 
     def alertOps(self, irc, channel, s, frm=None):

plugins/Config/plugin.py

@@ -28,7 +28,6 @@
 ###
 
 import os
-import getopt
 import signal
 
 import supybot.log as log
@@ -131,7 +130,7 @@ class Config(callbacks.Privmsg):
         """
         L = self._list(group)
         if L:
-            irc.reply(utils.commaAndify(L))
+            irc.reply(utils.str.commaAndify(L))
         else:
             irc.error('There don\'t seem to be any values in %s.' % group._name)
     list = wrap(list, ['configVar'])
@@ -148,7 +147,7 @@ class Config(callbacks.Privmsg):
                 if not ircutils.isChannel(possibleChannel):
                     L.append(name)
         if L:
-            irc.reply(utils.commaAndify(L))
+            irc.reply(utils.str.commaAndify(L))
         else:
             irc.reply('There were no matching configuration variables.')
     search = wrap(search, ['lowered']) # XXX compose with withoutSpaces?

plugins/Dict/plugin.py

@@ -53,7 +53,7 @@ class Dict(callbacks.Privmsg):
             conn = dictclient.Connection(server)
             dbs = conn.getdbdescs().keys()
             dbs.sort()
-            irc.reply(utils.commaAndify(dbs))
+            irc.reply(utils.str.commaAndify(dbs))
         except socket.error, e:
             irc.error(webutils.strError(e))
     dictionaries = wrap(dictionaries)
@@ -102,21 +102,21 @@ class Dict(callbacks.Privmsg):
         if not definitions:
             if dictionary == '*':
                 irc.reply('No definition for %s could be found.' %
-                          utils.quoted(word))
+                          utils.str.quoted(word))
             else:
                 irc.reply('No definition for %s could be found in %s' %
-                          (utils.quoted(word), ircutils.bold(dictionary)))
+                          (utils.str.quoted(word), ircutils.bold(dictionary)))
             return
         L = []
         for d in definitions:
             dbs.add(ircutils.bold(d.getdb().getname()))
             (db, s) = (d.getdb().getname(), d.getdefstr())
             db = ircutils.bold(db)
-            s = utils.normalizeWhitespace(s).rstrip(';.,')
+            s = utils.str.normalizeWhitespace(s).rstrip(';.,')
             L.append('%s: %s' % (db, s))
         utils.sortBy(len, L)
         if dictionary == '*' and len(dbs) > 1:
-            s = '%s responded: %s' % (utils.commaAndify(dbs), '; '.join(L))
+            s = '%s responded: %s' % (utils.str.commaAndify(dbs), '; '.join(L))
         else:
             s = '; '.join(L)
         irc.reply(s)

plugins/Format/plugin.py

@@ -144,7 +144,7 @@ class Format(callbacks.Privmsg):
 
         Returns the text surrounded by double quotes.
         """
-        irc.reply(utils.dqrepr(text))
+        irc.reply(utils.str.dqrepr(text))
     repr = wrap(repr, ['text'])
 
     def concat(self, irc, msg, args, first, second):

plugins/Math/plugin.py

@@ -29,14 +29,11 @@
 
 from __future__ import division
 
-import supybot.plugins as plugins
-
 import re
 import math
 import cmath
 import types
 import string
-from itertools import imap
 
 import supybot.utils as utils
 from supybot.commands import *
@@ -160,12 +157,12 @@ class Math(callbacks.Privmsg):
         crash to the bot with something like 10**10**10**10.  One consequence
         is that large values such as 10**24 might not be exact.
         """
-        if text != text.translate(string.ascii, '_[]'):
+        if text != text.translate(utils.str.chars, '_[]'):
             irc.error('There\'s really no reason why you should have '
                            'underscores or brackets in your mathematical '
                            'expression.  Please remove them.')
             return
-        #text = text.translate(string.ascii, '_[] \t')
+        #text = text.translate(utils.str.chars, '_[] \t')
         if 'lambda' in text:
             irc.error('You can\'t use lambda in this command.')
             return
@@ -188,7 +185,7 @@ class Math(callbacks.Privmsg):
         text = self._mathRe.sub(handleMatch, text)
         try:
             self.log.info('evaluating %s from %s' %
-                          (utils.quoted(text), msg.prefix))
+                          (utils.str.quoted(text), msg.prefix))
             x = complex(eval(text, self._mathEnv, self._mathEnv))
             irc.reply(self._complexToString(x))
         except OverflowError:
@@ -209,21 +206,21 @@ class Math(callbacks.Privmsg):
         math, and can thus cause the bot to suck up CPU.  Hence it requires
         the 'trusted' capability to use.
         """
-        if text != text.translate(string.ascii, '_[]'):
+        if text != text.translate(utils.str.chars, '_[]'):
             irc.error('There\'s really no reason why you should have '
                            'underscores or brackets in your mathematical '
                            'expression.  Please remove them.')
             return
         # This removes spaces, too, but we'll leave the removal of _[] for
         # safety's sake.
-        text = text.translate(string.ascii, '_[] \t')
+        text = text.translate(utils.str.chars, '_[] \t')
         if 'lambda' in text:
             irc.error('You can\'t use lambda in this command.')
             return
         text = text.replace('lambda', '')
         try:
             self.log.info('evaluating %s from %s' %
-                          (utils.quoted(text), msg.prefix))
+                          (utils.str.quoted(text), msg.prefix))
             irc.reply(str(eval(text, self._mathEnv, self._mathEnv)))
         except OverflowError:
             maxFloat = math.ldexp(0.9999999999999999, 1024)
@@ -280,12 +277,12 @@ class Math(callbacks.Privmsg):
                         stack.append(eval(s, self._mathEnv, self._mathEnv))
                     except SyntaxError:
                         irc.error('%s is not a defined function.' %
-                                  utils.quoted(arg))
+                                  utils.str.quoted(arg))
                         return
         if len(stack) == 1:
             irc.reply(str(self._complexToString(complex(stack[0]))))
         else:
-            s = ', '.join(imap(self._complexToString, imap(complex, stack)))
+            s = ', '.join(map(self._complexToString, map(complex, stack)))
             irc.reply('Stack: [%s]' % s)
 
     def convert(self, irc, msg, args, number, unit1, unit2):

plugins/Misc/plugin.py

@@ -27,16 +27,10 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot
-
-import supybot.fix as fix
-
 import os
 import sys
 import time
 
-from itertools import imap, ifilter
-
 import supybot.log as log
 import supybot.conf as conf
 import supybot.utils as utils
@@ -46,9 +40,10 @@ import supybot.ircdb as ircdb
 import supybot.irclib as irclib
 import supybot.ircmsgs as ircmsgs
 import supybot.ircutils as ircutils
-import supybot.webutils as webutils
 import supybot.callbacks as callbacks
 
+from supybot.utils.iter import ifilter
+
 class Misc(callbacks.Privmsg):
     def __init__(self):
         super(Misc, self).__init__()
@@ -123,7 +118,7 @@ class Misc(callbacks.Privmsg):
                         (not private and isPublic(cb))]
             names.sort()
             if names:
-                irc.reply(utils.commaAndify(names))
+                irc.reply(utils.str.commaAndify(names))
             else:
                 if private:
                     irc.reply('There are no private plugins.')
@@ -148,7 +143,7 @@ class Misc(callbacks.Privmsg):
                             commands.append(s)
                 if commands:
                     commands.sort()
-                    irc.reply(utils.commaAndify(commands))
+                    irc.reply(utils.str.commaAndify(commands))
                 else:
                     irc.error('That plugin exists, but it has no '
                               'commands with help.')
@@ -177,7 +172,7 @@ class Misc(callbacks.Privmsg):
                     L.append('%s %s' % (name, key))
         if L:
             L.sort()
-            irc.reply(utils.commaAndify(L))
+            irc.reply(utils.str.commaAndify(L))
         else:
             irc.reply('No appropriate commands were found.')
     apropos = wrap(apropos, ['lowered'])
@@ -211,7 +206,7 @@ class Misc(callbacks.Privmsg):
                 names = sorted([cb.name() for cb in cbs])
                 irc.error('That command exists in the %s plugins.  '
                           'Please specify exactly which plugin command '
-                          'you want help with.'% utils.commaAndify(names))
+                          'you want help with.'% utils.str.commaAndify(names))
             else:
                 getHelp(cbs[0])
         else:
@@ -235,9 +230,9 @@ class Misc(callbacks.Privmsg):
         Returns the version of the current bot.
         """
         try:
-            newest = webutils.getUrl('http://supybot.sf.net/version.txt')
+            newest = utils.web.getUrl('http://supybot.sf.net/version.txt')
             newest ='The newest version available online is %s.'%newest.strip()
-        except webutils.WebError, e:
+        except utils.web.Error, e:
             self.log.warning('Couldn\'t get website version: %r', e)
             newest = 'I couldn\'t fetch the newest version ' \
                      'from the Supybot website.'
@@ -265,13 +260,13 @@ class Misc(callbacks.Privmsg):
         if cbs:
             names = [cb.name() for cb in cbs]
             names.sort()
-            plugin = utils.commaAndify(names)
+            plugin = utils.str.commaAndify(names)
             if irc.nested:
-                irc.reply(utils.commaAndify(names))
+                irc.reply(utils.str.commaAndify(names))
             else:
                 irc.reply('The %s command is available in the %s %s.' %
-                          (utils.quoted(command), plugin,
+                          (utils.str.quoted(command), plugin,
-                           utils.pluralize('plugin', len(names))))
+                           utils.str.pluralize('plugin', len(names))))
         else:
             irc.error('There is no such command %s.' % command)
     plugin = wrap(plugin, ['commandName'])
@@ -287,7 +282,7 @@ class Misc(callbacks.Privmsg):
             return
         module = sys.modules[cb.__class__.__module__]
         if hasattr(module, '__author__') and module.__author__:
-            irc.reply(utils.mungeEmailForWeb(str(module.__author__)))
+            irc.reply(utils.web.mungeEmail(str(module.__author__)))
         else:
             irc.reply('That plugin doesn\'t have an author that claims it.')
     author = wrap(author, [('plugin')])
@@ -317,7 +312,7 @@ class Misc(callbacks.Privmsg):
             chunk = L.pop()
             if L:
                 chunk += ' \x02(%s)\x0F' % \
-                         utils.nItems('message', len(L), 'more')
+                         utils.str.nItems('message', len(L), 'more')
             irc.reply(chunk, True)
         except KeyError:
             irc.error('You haven\'t asked me a command; perhaps you want '
@@ -380,7 +375,7 @@ class Misc(callbacks.Privmsg):
                 nolimit = True
         iterable = ifilter(self._validLastMsg, reversed(irc.state.history))
         iterable.next() # Drop the first message.
-        predicates = list(utils.flatten(predicates.itervalues()))
+        predicates = list(utils.iter.flatten(predicates.itervalues()))
         resp = []
         if irc.nested and not \
           self.registryValue('last.nested.includeTimestamp'):
@@ -409,7 +404,7 @@ class Misc(callbacks.Privmsg):
             irc.error('I couldn\'t find a message matching that criteria in '
                       'my history of %s messages.' % len(irc.state.history))
         else:
-            irc.reply(utils.commaAndify(resp))
+            irc.reply(utils.str.commaAndify(resp))
     last = wrap(last, [getopts({'nolimit': '',
                                 'on': 'something',
                                 'with': 'something',
@@ -496,7 +491,7 @@ class Misc(callbacks.Privmsg):
             shortname[, shortname and shortname].
             """
             L = [getShortName(n) for n in longList]
-            return utils.commaAndify(L)
+            return utils.str.commaAndify(L)
         def sortAuthors():
             """
             Sort the list of 'long names' based on the number of contributions
@@ -520,7 +515,7 @@ class Misc(callbacks.Privmsg):
             hasContribs = False
             if getattr(module, '__author__', None):
                 author = 'was written by %s' % \
-                    utils.mungeEmailForWeb(str(module.__author__))
+                    utils.web.mungeEmail(str(module.__author__))
                 hasAuthor = True
             if getattr(module, '__contributors__', None):
                 contribs = sortAuthors()
@@ -532,7 +527,7 @@ class Misc(callbacks.Privmsg):
                 if contribs:
                     contrib = '%s %s contributed to it.' % \
                         (buildContributorsString(contribs),
-                        utils.has(len(contribs)))
+                        utils.str.has(len(contribs)))
                     hasContribs = True
                 elif hasAuthor:
                     contrib = 'has no additional contributors listed'
@@ -549,7 +544,7 @@ class Misc(callbacks.Privmsg):
             if not authorInfo:
                 return 'The nick specified (%s) is not a registered ' \
                        'contributor' % nick
-            fullName = utils.mungeEmailForWeb(str(authorInfo))
+            fullName = utils.web.mungeEmail(str(authorInfo))
             contributions = []
             if hasattr(module, '__contributors__'):
                 if authorInfo not in module.__contributors__:
@@ -558,22 +553,21 @@ class Misc(callbacks.Privmsg):
                 contributions = module.__contributors__[authorInfo]
             if getattr(module, '__author__', False) == authorInfo:
                 isAuthor = True
-            # XXX Partition needs moved to utils.
+            (nonCommands, commands) = utils.iter.partition(lambda s: ' ' in s,
-            (nonCommands, commands) = fix.partition(lambda s: ' ' in s,
+                                                           contributions)
-                                                    contributions)
             results = []
             if commands:
                 results.append(
-                    'the %s %s' %(utils.commaAndify(commands),
+                    'the %s %s' %(utils.str.commaAndify(commands),
-                                  utils.pluralize('command',len(commands))))
+                                  utils.str.pluralize('command',len(commands))))
             if nonCommands:
-                results.append('the %s' % utils.commaAndify(nonCommands))
+                results.append('the %s' % utils.str.commaAndify(nonCommands))
             if results and isAuthor:
                 return '%s wrote the %s plugin and also contributed %s' % \
-                    (fullName, cb.name(), utils.commaAndify(results))
+                    (fullName, cb.name(), utils.str.commaAndify(results))
             elif results and not isAuthor:
                 return '%s contributed %s to the %s plugin' % \
-                    (fullName, utils.commaAndify(results), cb.name())
+                    (fullName, utils.str.commaAndify(results), cb.name())
             elif isAuthor and not results:
                 return '%s wrote the %s plugin' % (fullName, cb.name())
             else:

plugins/Network/plugin.py

@@ -27,10 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot
-
-import supybot.plugins as plugins
-
 import time
 
 import supybot.conf as conf
@@ -172,19 +168,19 @@ class Network(callbacks.Privmsg):
                     normal.append(channel)
             L = []
             if ops:
-                L.append('is an op on %s' % utils.commaAndify(ops))
+                L.append('is an op on %s' % utils.str.commaAndify(ops))
             if halfops:
-                L.append('is a halfop on %s' % utils.commaAndify(halfops))
+                L.append('is a halfop on %s' % utils.str.commaAndify(halfops))
             if voices:
-                L.append('is voiced on %s' % utils.commaAndify(voices))
+                L.append('is voiced on %s' % utils.str.commaAndify(voices))
             if normal:
                 if L:
-                    L.append('is also on %s' % utils.commaAndify(normal))
+                    L.append('is also on %s' % utils.str.commaAndify(normal))
                 else:
-                    L.append('is on %s' % utils.commaAndify(normal))
+                    L.append('is on %s' % utils.str.commaAndify(normal))
         else:
             L = ['isn\'t on any non-secret channels']
-        channels = utils.commaAndify(L)
+        channels = utils.str.commaAndify(L)
         if '317' in d:
             idle = utils.timeElapsed(d['317'].args[2])
             signon = time.strftime(conf.supybot.reply.format.time(),
@@ -246,7 +242,7 @@ class Network(callbacks.Privmsg):
         """
         L = ['%s: %s' % (ircd.network, ircd.server) for ircd in world.ircs]
         utils.sortBy(str.lower, L)
-        irc.reply(utils.commaAndify(L))
+        irc.reply(utils.str.commaAndify(L))
     networks = wrap(networks)
 
     def doPong(self, irc, msg):

plugins/Owner/plugin.py

@@ -27,16 +27,12 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot.fix as fix
-
 import gc
 import os
 import imp
 import sre
 import sys
-import getopt
 import socket
-import logging
 import linecache
 
 import supybot.log as log
@@ -424,7 +420,7 @@ class Owner(callbacks.Privmsg):
             return
         except ImportError, e:
             if name in str(e):
-                irc.error('No plugin named %s exists.' % utils.dqrepr(name))
+                irc.error('No plugin named %s exists.' % utils.str.dqrepr(name))
             else:
                 irc.error(str(e))
             return

plugins/Status/plugin.py

@@ -76,9 +76,9 @@ class Status(callbacks.Privmsg):
             networks.setdefault(Irc.network, []).append(Irc.nick)
         networks = networks.items()
         networks.sort()
-        networks = ['%s as %s' % (net, utils.commaAndify(nicks))
+        networks = ['%s as %s' % (net, utils.str.commaAndify(nicks))
                     for (net, nicks) in networks]
-        L = ['I am connected to %s.' % utils.commaAndify(networks)]
+        L = ['I am connected to %s.' % utils.str.commaAndify(networks)]
         if world.profiling:
             L.append('I am currently in code profiling mode.')
         irc.reply('  '.join(L))
@@ -92,9 +92,10 @@ class Status(callbacks.Privmsg):
         threads = [t.getName() for t in threading.enumerate()]
         threads.sort()
         s = 'I have spawned %s; %s %s still currently active: %s.' % \
-            (utils.nItems('thread', world.threadsSpawned),
+            (utils.str.nItems('thread', world.threadsSpawned),
-             utils.nItems('thread', len(threads)), utils.be(len(threads)),
+             utils.str.nItems('thread', len(threads)),
-             utils.commaAndify(threads))
+             utils.str.be(len(threads)),
+             utils.str.commaAndify(threads))
         irc.reply(s)
     threads = wrap(threads)
 
@@ -137,7 +138,7 @@ class Status(callbacks.Privmsg):
                    'of system time, for a total of %.2f seconds of CPU ' \
                    'time.  %s' % (user, system, user + system, children)
         if self.registryValue('cpu.threads', target):
-            spawned = utils.nItems('thread', world.threadsSpawned)
+            spawned = utils.str.nItems('thread', world.threadsSpawned)
             response += 'I have spawned %s; I currently have %s still ' \
                         'running.' % (spawned, activeThreads)
         if self.registryValue('cpu.memory', target):
@@ -159,7 +160,7 @@ class Status(callbacks.Privmsg):
                 response += '  I\'m taking up %s kB of memory.' % mem
             except Exception:
                 self.log.exception('Uncaught exception in cpu.memory:')
-        irc.reply(utils.normalizeWhitespace(response))
+        irc.reply(utils.str.normalizeWhitespace(response))
     cpu = wrap(cpu)
 
     def cmd(self, irc, msg, args):
@@ -178,9 +179,9 @@ class Status(callbacks.Privmsg):
                        attr == callbacks.canonicalName(attr):
                         commands += 1
         s = 'I offer a total of %s in %s.  I have processed %s.' % \
-            (utils.nItems('command', commands),
+            (utils.str.nItems('command', commands),
-             utils.nItems('plugin', callbacksPrivmsg, 'command-based'),
+             utils.str.nItems('plugin', callbacksPrivmsg, 'command-based'),
-             utils.nItems('command', world.commandsProcessed))
+             utils.str.nItems('command', world.commandsProcessed))
         irc.reply(s)
     cmd = wrap(cmd)
 
@@ -199,7 +200,7 @@ class Status(callbacks.Privmsg):
                         commands.add(attr)
         commands = list(commands)
         commands.sort()
-        irc.reply(utils.commaAndify(commands))
+        irc.reply(utils.str.commaAndify(commands))
     commands = wrap(commands)
 
     def uptime(self, irc, msg, args):

plugins/User/plugin.py

@@ -27,12 +27,8 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot.fix as fix
-
 import re
-import getopt
 import fnmatch
-from itertools import imap, ifilter
 
 import supybot.conf as conf
 import supybot.utils as utils
@@ -75,7 +71,7 @@ class User(callbacks.Privmsg):
                 users.append(u.name)
         if users:
             utils.sortBy(str.lower, users)
-            irc.reply(utils.commaAndify(users))
+            irc.reply(utils.str.commaAndify(users))
         else:
             if predicates:
                 irc.reply('There are no matching registered users.')
@@ -158,7 +154,7 @@ class User(callbacks.Privmsg):
         """
         try:
             id = ircdb.users.getUserId(newname)
-            irc.error('%s is already registered.' % utils.quoted(newname))
+            irc.error('%s is already registered.' % utils.str.quoted(newname))
             return
         except KeyError:
             pass
@@ -295,7 +291,7 @@ class User(callbacks.Privmsg):
         def getHostmasks(user):
             hostmasks = map(repr, user.hostmasks)
             hostmasks.sort()
-            return utils.commaAndify(hostmasks)
+            return utils.str.commaAndify(hostmasks)
         try:
             user = ircdb.users.getUser(msg.prefix)
             if name:
@@ -418,8 +414,8 @@ class User(callbacks.Privmsg):
         irc.reply('I have %s registered users '
                   'with %s registered hostmasks; '
                   '%s and %s.' % (users, hostmasks,
-                                  utils.nItems('owner', owners),
+                                  utils.str.nItems('owner', owners),
-                                  utils.nItems('admin', admins)))
+                                  utils.str.nItems('admin', admins)))
     stats = wrap(stats)
 
 

plugins/__init__.py

@@ -27,10 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-__revision__ = "$Id: __init__.py,v 1.44 2005/01/08 07:22:46 jamessan Exp $"
-
-import supybot.fix as fix
-
 import gc
 import os
 import re
@@ -106,7 +102,7 @@ class NoSuitableDatabase(Exception):
         return 'No suitable databases were found.  Suitable databases ' \
                'include %s.  If you have one of these databases installed, ' \
                'make sure it is listed in the supybot.databases ' \
-               'configuration variable.' % utils.commaAndify(self.suitable)
+               'configuration variable.' % utils.str.commaAndify(self.suitable)
 
 def DB(filename, types):
     filename = conf.supybot.directories.data.dirize(filename)
@@ -325,7 +321,7 @@ class ChannelUserDB(ChannelUserDictionary):
             log.debug('Exception: %s', utils.exnToString(e))
 
     def flush(self):
-        fd = utils.transactionalFile(self.filename, makeBackupIfSmaller=False)
+        fd = utils.file.AtomicFile(self.filename, makeBackupIfSmaller=False)
         writer = csv.writer(fd)
         items = self.items()
         if not items:
@@ -389,9 +385,9 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
 
     def getCommandHelp(self, name):
         help = self.__parent.getCommandHelp(name)
-        help = help.replace('$Types', utils.pluralize(self.name()))
+        help = help.replace('$Types', utils.str.pluralize(self.name()))
         help = help.replace('$Type', self.name())
-        help = help.replace('$types', utils.pluralize(self.name().lower()))
+        help = help.replace('$types', utils.str.pluralize(self.name().lower()))
         help = help.replace('$type', self.name().lower())
         return help
 
@@ -442,7 +438,7 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
     remove = wrap(remove, ['user', 'channeldb', 'id'])
 
     def searchSerializeRecord(self, record):
-        text = utils.quoted(utils.ellipsisify(record.text, 50))
+        text = utils.str.quoted(utils.str.ellipsisify(record.text, 50))
         return '#%s: %s' % (record.id, text)
 
     def search(self, irc, msg, args, channel, optlist, glob):
@@ -471,10 +467,10 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
             L.append(self.searchSerializeRecord(record))
         if L:
             L.sort()
-            irc.reply('%s found: %s' % (len(L), utils.commaAndify(L)))
+            irc.reply('%s found: %s' % (len(L), utils.str.commaAndify(L)))
         else:
             irc.reply('No matching %s were found.' %
-                      utils.pluralize(self.name().lower()))
+                      utils.str.pluralize(self.name().lower()))
     search = wrap(search, ['channeldb',
                            getopts({'by': 'otherUser',
                                     'regexp': 'regexpMatcher'}),
@@ -485,7 +481,8 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
         at = time.localtime(record.at)
         timeS = time.strftime(conf.supybot.reply.format.time(), at)
         return '%s #%s: %s (added by %s at %s)' % \
-               (self.name(), record.id, utils.quoted(record.text), name, timeS)
+               (self.name(), record.id,
+                utils.str.quoted(record.text), name, timeS)
 
     def get(self, irc, msg, args, channel, id):
         """[<channel>] <id>
@@ -527,7 +524,7 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
         """
         n = self.db.size(channel)
         irc.reply('There %s %s in my database.' %
-                  (utils.be(n), utils.nItems(self.name().lower(), n)))
+                  (utils.str.be(n), utils.str.nItems(self.name().lower(), n)))
     stats = wrap(stats, ['channeldb'])
 
 
@@ -592,7 +589,7 @@ class PeriodicFileDownloader(object):
                 self.log.warning('Error downloading %s: %s', url, e)
                 return
             confDir = conf.supybot.directories.data()
-            newFilename = os.path.join(confDir, utils.mktemp())
+            newFilename = os.path.join(confDir, utils.file.mktemp())
             outfd = file(newFilename, 'wb')
             start = time.time()
             s = infd.read(4096)

scripts/supybot

@@ -33,7 +33,7 @@
 This is the main program to run Supybot.
 """
 
-
+import supybot
 
 import re
 import os

setup.py

@@ -103,11 +103,13 @@ if clean:
             sys.exit(-1)
 
 packages = ['supybot',
+            'supybot.utils',
             'supybot.drivers',
             'supybot.plugins',] + \
             ['supybot.plugins.'+s for s in plugins]
 
 package_dir = {'supybot': 'src',
+               'supybot.utils': 'src/utils',
                'supybot.plugins': 'plugins',
                'supybot.drivers': 'src/drivers',}
 

src/callbacks.py

@@ -38,10 +38,6 @@ how to use them.
 
 import supybot
 
-
-
-import supybot.fix as fix
-
 import re
 import copy
 import sets
@@ -166,7 +162,7 @@ def canonicalName(command):
     while command and command[-1] in special:
         reAppend = command[-1] + reAppend
         command = command[:-1]
-    return command.translate(string.ascii, special).lower() + reAppend
+    return command.translate(utils.str.chars, special).lower() + reAppend
 
 def reply(msg, s, prefixName=None, private=None,
           notice=None, to=None, action=None, error=False):
@@ -233,7 +229,7 @@ def getHelp(method, name=None):
     if doclines:
         help = ' '.join(doclines)
         s = '(%s) -- %s' % (ircutils.bold(s), help)
-    return utils.normalizeWhitespace(s)
+    return utils.str.normalizeWhitespace(s)
 
 def getSyntax(method, name=None):
     if name is None:
@@ -256,10 +252,10 @@ class Tokenizer(object):
     #
     # These are the characters valid in a token.  Everything printable except
     # double-quote, left-bracket, and right-bracket.
-    validChars = string.ascii.translate(string.ascii, '\x00\r\n \t')
+    validChars = utils.str.chars.translate(utils.str.chars, '\x00\r\n \t')
     def __init__(self, brackets='', pipe=False, quotes='"'):
         if brackets:
-            self.validChars = self.validChars.translate(string.ascii, brackets)
+            self.validChars=self.validChars.translate(utils.str.chars, brackets)
             self.left = brackets[0]
             self.right = brackets[1]
         else:
@@ -267,9 +263,9 @@ class Tokenizer(object):
             self.right = ''
         self.pipe = pipe
         if self.pipe:
-            self.validChars = self.validChars.translate(string.ascii, '|')
+            self.validChars = self.validChars.translate(utils.str.chars, '|')
         self.quotes = quotes
-        self.validChars = self.validChars.translate(string.ascii, quotes)
+        self.validChars = self.validChars.translate(utils.str.chars, quotes)
 
 
     def _handleToken(self, token):
@@ -459,7 +455,7 @@ class RichReplyMethods(object):
         if prefixer is None:
             prefixer = ''
         if joiner is None:
-            joiner = utils.commaAndify
+            joiner = utils.str.commaAndify
         if isinstance(prefixer, basestring):
             prefixer = prefixer.__add__
         if isinstance(joiner, basestring):
@@ -494,7 +490,7 @@ class RichReplyMethods(object):
             kwargs['Raise'] = True
         if isinstance(capability, basestring): # checkCommandCapability!
             log.warning('Denying %s for lacking %s capability.',
-                        self.msg.prefix, utils.quoted(capability))
+                        self.msg.prefix, utils.str.quoted(capability))
             if not self._getConfig(conf.supybot.reply.error.noCapability):
                 v = self._getConfig(conf.supybot.replies.noCapability)
                 s = self.__makeReply(v % capability, s)
@@ -750,7 +746,7 @@ class IrcObjectProxy(RichReplyMethods):
                               'Please specify the plugin whose command you '
                               'wish to call by using its name as a command '
                               'before %s.' %
-                              (command, utils.commaAndify(names), command))
+                              (command, utils.str.commaAndify(names), command))
         else:
             cb = cbs[0]
             del self.args[0] # Remove the command.
@@ -870,7 +866,7 @@ class IrcObjectProxy(RichReplyMethods):
                     response = msgs.pop()
                     if msgs:
                         n = ircutils.bold('(%s)')
-                        n %= utils.nItems('message', len(msgs), 'more')
+                        n %= utils.str.nItems('message', len(msgs), 'more')
                         response = '%s %s' % (response, n)
                     prefix = msg.prefix
                     if self.to and ircutils.isNick(self.to):
@@ -1123,7 +1119,7 @@ class Privmsg(irclib.IrcCallback):
         """Gets the given command from this plugin."""
         name = canonicalName(name)
         assert self.isCommand(name), '%s is not a command.' % \
-                                     utils.quoted(name)
+                                     utils.str.quoted(name)
         return getattr(self, name)
 
     def callCommand(self, name, irc, msg, *L, **kwargs):
@@ -1155,11 +1151,11 @@ class Privmsg(irclib.IrcCallback):
         command = self.getCommand(name)
         if hasattr(command, 'isDispatcher') and \
            command.isDispatcher and self.__doc__:
-            return utils.normalizeWhitespace(self.__doc__)
+            return utils.str.normalizeWhitespace(self.__doc__)
         elif hasattr(command, '__doc__'):
             return getHelp(command)
         else:
-            return 'The %s command has no help.' % utils.quoted(name)
+            return 'The %s command has no help.' % utils.str.quoted(name)
 
     def registryValue(self, name, channel=None, value=True):
         plugin = self.name()
@@ -1291,7 +1287,7 @@ class PrivmsgRegexp(Privmsg):
                     self.res.append((r, name))
                 except re.error, e:
                     self.log.warning('Invalid regexp: %s (%s)',
-                                     utils.quoted(value.__doc__), e)
+                                     utils.str.quoted(value.__doc__), e)
         utils.sortBy(operator.itemgetter(1), self.res)
 
     def callCommand(self, name, irc, msg, *L, **kwargs):

src/cdb.py

@@ -32,10 +32,6 @@ Database module, similar to dbhash.  Uses a format similar to (if not entirely
 the same as) DJB's CDB <http://cr.yp.to/cdb.html>.
 """
 
-
-
-import supybot.fix as fix
-
 import os
 import sys
 import sets
@@ -134,7 +130,7 @@ def make(dbFilename, readFilename=None):
 class Maker(object):
     """Class for making CDB databases."""
     def __init__(self, filename):
-        self.fd = utils.transactionalFile(filename)
+        self.fd = utils.file.AtomicFile(filename)
         self.filename = filename
         self.fd.seek(2048)
         self.hashPointers = [(0, 0)] * 256

src/commands.py

@@ -31,10 +31,6 @@
 Includes wrappers for commands.
 """
 
-
-
-import supybot.fix as fix
-
 import time
 import types
 import getopt
@@ -233,7 +229,7 @@ def getExpiry(irc, msg, args, state):
 
 def getBoolean(irc, msg, args, state):
     try:
-        state.args.append(utils.toBool(args[0]))
+        state.args.append(utils.str.toBool(args[0]))
         del args[0]
     except ValueError:
         irc.errorInvalid('boolean', args[0])
@@ -324,8 +320,8 @@ def _getRe(f):
             irc.errorInvalid('regular expression', s)
     return get
 
-getMatcher = _getRe(utils.perlReToPythonRe)
+getMatcher = _getRe(utils.str.perlReToPythonRe)
-getReplacer = _getRe(utils.perlReToReplacer)
+getReplacer = _getRe(utils.str.perlReToReplacer)
 
 def getNick(irc, msg, args, state):
     if ircutils.isNick(args[0]):
@@ -494,7 +490,7 @@ def getCommandName(irc, msg, args, state):
         state.args.append(callbacks.canonicalName(args.pop(0)))
 
 def getIp(irc, msg, args, state):
-    if utils.isIP(args[0]):
+    if utils.net.isIP(args[0]):
         state.args.append(args.pop(0))
     else:
         irc.errorInvalid('ip', args[0])
@@ -845,7 +841,7 @@ class Spec(object):
     def __init__(self, types, allowExtra=False):
         self.types = types
         self.allowExtra = allowExtra
-        utils.mapinto(contextify, self.types)
+        utils.seq.mapinto(contextify, self.types)
 
     def __call__(self, irc, msg, args, stateAttrs={}):
         state = self._state(self.types[:], stateAttrs)

src/conf.py

@@ -27,8 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot.fix as fix
-
 import os
 import sys
 import time
@@ -70,6 +68,12 @@ daemonized = False
 ###
 allowDefaultOwner = False
 
+###
+# Here we replace values in other modules as appropriate.
+###
+utils.web.defaultHeaders['User-agent'] = \
+                         'Mozilla/5.0 (Compatible; Supybot %s)' % version
+
 ###
 # The standard registry.
 ###
@@ -405,7 +409,7 @@ registerChannelValue(supybot.reply, 'showSimpleSyntax',
 class ValidPrefixChars(registry.String):
     """Value must contain only ~!@#$%^&*()_-+=[{}]\\|'\";:,<.>/?"""
     def setValue(self, v):
-        if v.translate(string.ascii, '`~!@#$%^&*()_-+=[{}]\\|\'";:,<.>/?'):
+        if v.translate(utils.str.chars, '`~!@#$%^&*()_-+=[{}]\\|\'";:,<.>/?'):
             self.error()
         registry.String.setValue(self, v)
 
@@ -579,9 +583,9 @@ registerChannelValue(supybot.commands.nested, 'pipeSyntax',
     example: 'bot: foo | bar'."""))
 
 registerGroup(supybot.commands, 'defaultPlugins',
-    orderAlphabetically=True, help=utils.normalizeWhitespace("""Determines
+    orderAlphabetically=True, help="""Determines what commands have default
-    what commands have default plugins set, and which plugins are set to
+    plugins set, and which plugins are set to be the default for each of those
-    be the default for each of those commands."""))
+    commands.""")
 registerGlobalValue(supybot.commands.defaultPlugins, 'importantPlugins',
     registry.SpaceSeparatedSetOfStrings(
         ['Admin', 'Channel', 'Config', 'Misc', 'Owner', 'User'],
@@ -704,12 +708,8 @@ registerGlobalValue(supybot.directories.data, 'tmp',
     DataFilenameDirectory('tmp', """Determines what directory temporary files
     are put into."""))
 
-# Remember, we're *meant* to replace this nice little wrapper.
+utils.file.AtomicFile.default.tmpDir = supybot.directories.data.tmp
-def transactionalFile(*args, **kwargs):
+utils.file.AtomicFile.default.backupDir = supybot.directories.backup
-    kwargs['tmpDir'] = supybot.directories.data.tmp()
-    kwargs['backupDir'] = supybot.directories.backup()
-    return utils.AtomicFile(*args, **kwargs)
-utils.transactionalFile = transactionalFile
 
 class PluginDirectories(registry.CommaSeparatedListOfStrings):
     def __call__(self):
@@ -931,6 +931,7 @@ registerGlobalValue(supybot.protocols.http, 'peekSize',
 registerGlobalValue(supybot.protocols.http, 'proxy',
     registry.String('', """Determines what proxy all HTTP requests should go
     through.  The value should be of the form 'host:port'."""))
+utils.web.proxy = supybot.protocols.http.proxy
 
 
 ###
@@ -945,7 +946,7 @@ registerGlobalValue(supybot, 'defaultIgnore',
 class IP(registry.String):
     """Value must be a valid IP."""
     def setValue(self, v):
-        if v and not (utils.isIP(v) or utils.isIPV6(v)):
+        if v and not (utils.net.isIP(v) or utils.net.isIPV6(v)):
             self.error()
         else:
             registry.String.setValue(self, v)

src/dbi.py

@@ -31,10 +31,6 @@
 Module for some slight database-independence for simple databases.
 """
 
-
-
-import supybot.fix as fix
-
 import csv
 import math
 import sets
@@ -262,8 +258,7 @@ class FlatfileMapping(MappingInterface):
 
     def vacuum(self):
         infd = file(self.filename)
-        outfd = utils.transactionalFile(self.filename,
+        outfd = utils.file.AstomicFile(self.filename,makeBackupIfSmaller=False)
-                                        makeBackupIfSmaller=False)
         outfd.write(infd.readline()) # First line, nextId.
         for line in infd:
             if not line.startswith('-'):

src/fix.py

@@ -1,239 +0,0 @@
-###
-# Copyright (c) 2002-2005, Jeremiah Fincher
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-#   * Redistributions of source code must retain the above copyright notice,
-#     this list of conditions, and the following disclaimer.
-#   * Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions, and the following disclaimer in the
-#     documentation and/or other materials provided with the distribution.
-#   * Neither the name of the author of this software nor the name of
-#     contributors to this software may be used to endorse or promote products
-#     derived from this software without specific prior written consent.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-###
-
-"""
-Fixes stuff that Python should have but doesn't.
-"""
-
-from __future__ import division
-
-__all__ = []
-
-exported = ['ignore', 'window', 'group', 'partition', 'set', 'frozenset',
-            'any', 'all', 'rsplit', 'dynamic']
-
-import sys
-import new
-import atexit
-import string
-string.ascii = string.maketrans('', '')
-
-import random
-_choice = random.choice
-def choice(iterable):
-    if isinstance(iterable, (list, tuple)):
-        return _choice(iterable)
-    else:
-        n = 1
-        m = new.module('') # Guaranteed unique value.
-        ret = m
-        for x in iterable:
-            if random.random() < 1/n:
-                ret = x
-            n += 1
-        if ret is m:
-            raise IndexError
-        return ret
-random.choice = choice
-
-def ignore(*args, **kwargs):
-    """Simply ignore the arguments sent to it."""
-    pass
-
-class DynamicScope(object):
-    def _getLocals(self, name):
-        f = sys._getframe().f_back.f_back # _getLocals <- __[gs]etattr__ <- ...
-        while f:
-            if name in f.f_locals:
-                return f.f_locals
-            f = f.f_back
-        raise NameError, name
-    
-    def __getattr__(self, name):
-        try:
-            return self._getLocals(name)[name]
-        except (NameError, KeyError):
-            return None
-
-    def __setattr__(self, name, value):
-        self._getLocals(name)[name] = value
-dynamic = DynamicScope()
-        
-
-if sys.version_info < (2, 4, 0):
-    def reversed(L):
-        """Iterates through a sequence in reverse."""
-        for i in xrange(len(L) - 1, -1, -1):
-            yield L[i]
-    exported.append('reversed')
-
-def window(L, size):
-    """Returns a sliding 'window' through the list L of size size."""
-    assert not isinstance(L, int), 'Argument order swapped: window(L, size)'
-    if size < 1:
-        raise ValueError, 'size <= 0 disallowed.'
-    for i in xrange(len(L) - (size-1)):
-        yield L[i:i+size]
-
-import itertools
-def ilen(iterable):
-    """Returns the length of an iterator."""
-    i = 0
-    for _ in iterable:
-        i += 1
-    return i
-
-def trueCycle(iterable):
-    while 1:
-        yielded = False
-        for x in iterable:
-            yield x
-            yielded = True
-        if not yielded:
-            raise StopIteration
-
-itertools.trueCycle = trueCycle
-itertools.ilen = ilen
-
-def groupby(key, iterable):
-    if key is None:
-        key = lambda x: x
-    it = iter(iterable)
-    value = it.next() # If there are no items, this takes an early exit
-    oldkey = key(value)
-    group = [value]
-    for value in it:
-        newkey = key(value)
-        if newkey != oldkey:
-            yield group
-            group = []
-            oldkey = newkey
-        group.append(value)
-    yield group
-itertools.groupby = groupby
-
-def group(seq, groupSize, noneFill=True):
-    """Groups a given sequence into sublists of length groupSize."""
-    ret = []
-    L = []
-    i = groupSize
-    for elt in seq:
-        if i > 0:
-            L.append(elt)
-        else:
-            ret.append(L)
-            i = groupSize
-            L = []
-            L.append(elt)
-        i -= 1
-    if L:
-        if noneFill:
-            while len(L) < groupSize:
-                L.append(None)
-        ret.append(L)
-    return ret
-
-def partition(p, L):
-    """Partitions a list L based on a predicate p.  Returns a (yes,no) tuple"""
-    no = []
-    yes = []
-    for elt in L:
-        if p(elt):
-            yes.append(elt)
-        else:
-            no.append(elt)
-    return (yes, no)
-
-def any(p, seq):
-    """Returns true if any element in seq satisfies predicate p."""
-    for elt in itertools.ifilter(p, seq):
-        return True
-    else:
-        return False
-
-def all(p, seq):
-    """Returns true if all elements in seq satisfy predicate p."""
-    for elt in itertools.ifilterfalse(p, seq):
-        return False
-    else:
-        return True
-
-def rsplit(s, sep=None, maxsplit=-1):
-    """Equivalent to str.split, except splitting from the right."""
-    if sys.version_info < (2, 4, 0):
-        if sep is not None:
-            sep = sep[::-1]
-        L = s[::-1].split(sep, maxsplit)
-        L.reverse()
-        return [s[::-1] for s in L]
-    else:
-        return s.rsplit(sep, maxsplit)
-
-if sys.version_info < (2, 4, 0):
-    import operator
-    def itemgetter(i):
-        return lambda x: x[i]
-
-    def attrgetter(attr):
-        return lambda x: getattr(x, attr)
-    operator.itemgetter = itemgetter
-    operator.attrgetter = attrgetter
-
-import csv
-import cStringIO as StringIO
-def join(L):
-    fd = StringIO.StringIO()
-    writer = csv.writer(fd)
-    writer.writerow(L)
-    return fd.getvalue().rstrip('\r\n')
-
-def split(s):
-    fd = StringIO.StringIO(s)
-    reader = csv.reader(fd)
-    return reader.next()
-csv.join = join
-csv.split = split
-
-import sets
-set = sets.Set
-frozenset = sets.ImmutableSet
-
-import socket
-# Some socket modules don't have sslerror, so we'll just make it an error.
-if not hasattr(socket, 'sslerror'):
-    socket.sslerror = socket.error
-
-g = globals()
-for name in exported:
-    __builtins__[name] = g[name]
-
-
-
-# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
-

src/ircdb.py

@@ -27,18 +27,13 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-
-
 from __future__ import division
 
-import supybot.fix as fix
-
 import os
 import sets
 import time
 import string
 import operator
-from itertools import imap, ilen, ifilter
 
 import supybot.log as log
 import supybot.conf as conf
@@ -47,6 +42,7 @@ import supybot.world as world
 import supybot.ircutils as ircutils
 import supybot.registry as registry
 import supybot.unpreserve as unpreserve
+from utils.iter import imap, ilen, ifilter
 
 def isCapability(capability):
     return len(capability.split(None, 1)) == 1
@@ -114,7 +110,7 @@ def canonicalCapability(capability):
     return capability.lower()
 
 def unWildcardHostmask(hostmask):
-    return hostmask.translate(string.ascii, '!@*?')
+    return hostmask.translate(utils.str.chars, '!@*?')
 
 _invert = invertCapability
 class CapabilitySet(sets.Set):
@@ -228,7 +224,7 @@ class IrcUser(object):
         return '%s(id=%s, ignore=%s, password="", name=%s, hashed=%r, ' \
                'capabilities=%r, hostmasks=[], secure=%r)\n' % \
                (self.__class__.__name__, self.id, self.ignore,
-                utils.quoted(self.name), self.hashed, self.capabilities,
+                utils.str.quoted(self.name), self.hashed, self.capabilities,
                 self.secure)
 
     def __hash__(self):
@@ -611,7 +607,7 @@ class UsersDictionary(utils.IterableMap):
             if self.filename is not None:
                 L = self.users.items()
                 L.sort()
-                fd = utils.transactionalFile(self.filename)
+                fd = utils.file.AtomicFile(self.filename)
                 for (id, u) in L:
                     fd.write('user %s' % id)
                     fd.write(os.linesep)
@@ -657,7 +653,7 @@ class UsersDictionary(utils.IterableMap):
                               'Removing the offending hostmasks.')
                     for (id, hostmask) in ids.iteritems():
                         log.error('Removing %s from user %s.',
-                                  utils.quoted(hostmask), id)
+                                  utils.str.quoted(hostmask), id)
                         self.users[id].removeHostmask(hostmask)
                     raise DuplicateHostmask, 'Ids %r matched.' % ids
         else: # Not a hostmask, must be a name.
@@ -789,7 +785,7 @@ class ChannelsDictionary(utils.IterableMap):
         """Flushes the channel database to its file."""
         if not self.noFlush:
             if self.filename is not None:
-                fd = utils.transactionalFile(self.filename)
+                fd = utils.file.AtomicFile(self.filename)
                 for (channel, c) in self.channels.iteritems():
                     fd.write('channel %s' % channel)
                     fd.write(os.linesep)
@@ -845,7 +841,7 @@ class IgnoresDB(object):
     def open(self, filename):
         self.filename = filename
         fd = file(self.filename)
-        for line in utils.nonCommentNonEmptyLines(fd):
+        for line in utils.file.nonCommentNonEmptyLines(fd):
             try:
                 line = line.rstrip('\r\n')
                 L = line.split()
@@ -857,12 +853,12 @@ class IgnoresDB(object):
                 self.add(hostmask, expiration)
             except Exception, e:
                 log.error('Invalid line in ignores database: %s',
-                          utils.quoted(line))
+                          utils.str.quoted(line))
         fd.close()
 
     def flush(self):
         if self.filename is not None:
-            fd = utils.transactionalFile(self.filename)
+            fd = utils.file.AtomicFile(self.filename)
             now = time.time()
             for (hostmask, expiration) in self.hostmasks.items():
                 if now < expiration or not expiration:

src/irclib.py

@@ -27,16 +27,11 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-
-
-import supybot.fix as fix
-
 import copy
 import sets
 import time
 import random
 import operator
-from itertools import imap, chain, cycle
 
 import supybot.log as log
 import supybot.conf as conf
@@ -45,6 +40,7 @@ import supybot.world as world
 import supybot.ircdb as ircdb
 import supybot.ircmsgs as ircmsgs
 import supybot.ircutils as ircutils
+from utils.iter import imap, chain, cycle
 from supybot.structures import queue, smallqueue, RingBuffer
 
 ###
@@ -173,7 +169,7 @@ class IrcMsgQueue(object):
            not conf.supybot.protocols.irc.queueDuplicateMessages():
             s = str(msg).strip()
             log.info('Not adding message %s to queue, already added.',
-                     utils.quoted(s))
+                     utils.str.quoted(s))
             return False
         else:
             self.msgs.add(msg)
@@ -642,7 +638,7 @@ class Irc(IrcCommandDispatcher):
         name = name.lower()
         def nameMatches(cb):
             return cb.name().lower() == name
-        (bad, good) = partition(nameMatches, self.callbacks)
+        (bad, good) = utils.iter.partition(nameMatches, self.callbacks)
         self.callbacks[:] = good
         return bad
 

src/ircmsgs.py

@@ -34,10 +34,6 @@ construct such messages in an easier way than the constructor for the IrcMsg
 object (which, as you'll read later, is quite...full-featured :))
 """
 
-
-
-import supybot.fix as fix
-
 import re
 import time
 import string
@@ -192,8 +188,8 @@ class IrcMsg(object):
         if self._repr is not None:
             return self._repr
         self._repr = 'IrcMsg(prefix=%s, command=%s, args=%r)' % \
-                     (utils.quoted(self.prefix), utils.quoted(self.command),
+                     (utils.str.quoted(self.prefix),
-                      self.args)
+                      utils.str.quoted(self.command), self.args)
         return self._repr
 
     def __reduce__(self):
@@ -586,7 +582,8 @@ def join(channel, key=None, prefix='', msg=None):
         return IrcMsg(prefix=prefix, command='JOIN', args=(channel,), msg=msg)
     else:
         if conf.supybot.protocols.irc.strictRfc():
-            assert key.translate(string.ascii, string.ascii[128:]) == key and \
+            assert key.translate(utils.str.chars,
+                                 utils.str.chars[128:]) == key and \
                    '\x00' not in key and \
                    '\r' not in key and \
                    '\n' not in key and \
@@ -613,7 +610,8 @@ def joins(channels, keys=None, prefix='', msg=None):
     else:
         for key in keys:
             if conf.supybot.protocols.irc.strictRfc():
-                assert key.translate(string.ascii,string.ascii[128:])==key and\
+                assert key.translate(utils.str.chars,
+                                     utils.str.chars[128:])==key and \
                        '\x00' not in key and \
                        '\r' not in key and \
                        '\n' not in key and \

src/ircutils.py

@@ -34,16 +34,11 @@ nick class to handle nicks (so comparisons and hashing and whatnot work in an
 IRC-case-insensitive fashion), and numerous other things.
 """
 
-
-
-import supybot.fix as fix
-
 import re
 import time
 import random
 import string
 import textwrap
-from itertools import imap, ilen
 from cStringIO import StringIO as sio
 
 import supybot.utils as utils
@@ -109,7 +104,7 @@ def toLower(s, casemapping=None):
     elif casemapping == 'ascii': # freenode
         return s.lower()
     else:
-        raise ValueError, 'Invalid casemapping: %s' % utils.quoted(casemapping)
+        raise ValueError, 'Invalid casemapping: %r' % casemapping
 
 def strEqual(nick1, nick2):
     """s1, s2 => bool
@@ -197,11 +192,11 @@ def banmask(hostmask):
     """
     assert isUserHostmask(hostmask)
     host = hostFromHostmask(hostmask)
-    if utils.isIP(host):
+    if utils.net.isIP(host):
         L = host.split('.')
         L[-1] = '*'
         return '*!*@' + '.'.join(L)
-    elif utils.isIPV6(host):
+    elif utils.net.isIPV6(host):
         L = host.split(':')
         L[-1] = '*'
         return '*!*@' + ':'.join(L)
@@ -450,7 +445,7 @@ def safeArgument(s):
     if isinstance(s, unicode):
         s = s.encode('utf-8')
     elif not isinstance(s, basestring):
-        debug('Got a non-string in safeArgument: %s', utils.quoted(s))
+        debug('Got a non-string in safeArgument: %r', s)
         s = str(s)
     if isValidArgument(s):
         return s
@@ -466,7 +461,7 @@ def replyTo(msg):
 
 def dccIP(ip):
     """Returns in IP in the proper for DCC."""
-    assert utils.isIP(ip), \
+    assert utils.net.isIP(ip), \
            'argument must be a string ip in xxx.yyy.zzz.www format.'
     i = 0
     x = 256**3
@@ -483,7 +478,7 @@ def unDccIP(i):
         L.append(i % 256)
         i /= 256
     L.reverse()
-    return '.'.join(imap(str, L))
+    return '.'.join(utils.iter.imap(str, L))
 
 class IrcString(str):
     """This class does case-insensitive comparison and hashing of nicks."""
@@ -662,7 +657,7 @@ def standardSubstitute(irc, msg, text, env=None):
         })
     if env is not None:
         vars.update(env)
-    return utils.perlVariableSubstitute(vars, text)
+    return utils.str.perlVariableSubstitute(vars, text)
 
 
 if __name__ == '__main__':

src/log.py

@@ -27,8 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot.fix as fix
-
 import os
 import sys
 import time

src/plugin.py

@@ -49,7 +49,7 @@ def loadPluginModule(name, ignoreDeprecation=False):
             files.extend(os.listdir(dir))
         except EnvironmentError: # OSError, IOError superclass.
             log.warning('Invalid plugin directory: %s; removing.',
-                        utils.quoted(dir))
+                        utils.str.quoted(dir))
             conf.supybot.directories.plugins().remove(dir)
     loweredFiles = map(str.lower, files)
     try:
@@ -72,7 +72,7 @@ def loadPluginModule(name, ignoreDeprecation=False):
             log.warning('Deprecated plugin loaded: %s', name)
         else:
             raise Deprecated, 'Attempted to load deprecated plugin %s' % \
-                              utils.quoted(name)
+                              utils.str.quoted(name)
     if module.__name__ in sys.modules:
         sys.modules[module.__name__] = module
     linecache.checkcache()

src/registry.py

@@ -34,12 +34,11 @@ import time
 import string
 import textwrap
 
-import supybot.fix as fix
 import supybot.utils as utils
 
 def error(s):
-    """Replace me with something better from another module!"""
+   """Replace me with something better from another module!"""
-    print '***', s
+   print '***', s
 
 def exception(s):
     """Ditto!"""
@@ -68,7 +67,7 @@ def open(filename, clear=False):
     if clear:
         _cache.clear()
     _fd = file(filename)
-    fd = utils.nonCommentNonEmptyLines(_fd)
+    fd = utils.file.nonCommentNonEmptyLines(_fd)
     acc = ''
     for line in fd:
         line = line.rstrip('\r\n')
@@ -94,7 +93,7 @@ def open(filename, clear=False):
 
 def close(registry, filename, private=True):
     first = True
-    fd = utils.transactionalFile(filename)
+    fd = utils.file.AtomicFile(filename)
     for (name, value) in registry.getValues(getChildren=True):
         help = value.help()
         if help:
@@ -162,7 +161,7 @@ class Group(object):
     """A group; it doesn't hold a value unless handled by a subclass."""
     def __init__(self, help='', supplyDefault=False,
                  orderAlphabetically=False, private=False):
-        self._help = help
+        self._help = utils.str.normalizeWhitespace(help)
         self._name = 'unset'
         self._added = []
         self._children = utils.InsensitivePreservingDict()
@@ -299,7 +298,7 @@ class Value(Group):
         self.__parent.__init__(help, **kwargs)
         self._default = default
         self._showDefault = showDefault
-        self._help = utils.normalizeWhitespace(help.strip())
+        self._help = utils.str.normalizeWhitespace(help.strip())
         if setDefault:
             self.setValue(default)
 
@@ -309,7 +308,7 @@ class Value(Group):
         else:
             s = """%s has no docstring.  If you're getting this message,
             report it, because we forgot to put a proper help string here."""
-        e = InvalidRegistryValue(utils.normalizeWhitespace(s % self._name))
+        e = InvalidRegistryValue(utils.str.normalizeWhitespace(s % self._name))
         e.value = self
         raise e
 
@@ -356,7 +355,7 @@ class Boolean(Value):
     """Value must be either True or False (or On or Off)."""
     def set(self, s):
         try:
-            v = utils.toBool(s)
+            v = utils.str.toBool(s)
         except ValueError:
             if s.strip().lower() == 'toggle':
                 v = not self.value
@@ -440,7 +439,7 @@ class String(Value):
 
     _printable = string.printable[:-4]
     def _needsQuoting(self, s):
-        return s.translate(string.ascii, self._printable) and s.strip() != s
+        return s.translate(utils.str.chars, self._printable) and s.strip() != s
 
     def __str__(self):
         s = self.value
@@ -456,12 +455,12 @@ class OnlySomeStrings(String):
         self.__parent = super(OnlySomeStrings, self)
         self.__parent.__init__(*args, **kwargs)
         self.__doc__ = 'Valid values include %s.' % \
-                        utils.commaAndify(map(repr, self.validStrings))
+                        utils.str.commaAndify(map(repr, self.validStrings))
 
     def help(self):
         strings = [s for s in self.validStrings if s]
         return '%s  Valid strings: %s.' % \
-                (self._help, utils.commaAndify(strings))
+                (self._help, utils.str.commaAndify(strings))
 
     def normalize(self, s):
         lowered = s.lower()
@@ -487,7 +486,7 @@ class NormalizedString(String):
         self._showDefault = False
 
     def normalize(self, s):
-        return utils.normalizeWhitespace(s.strip())
+        return utils.str.normalizeWhitespace(s.strip())
 
     def set(self, s):
         s = self.normalize(s)
@@ -540,7 +539,7 @@ class Regexp(Value):
     def set(self, s):
         try:
             if s:
-                self.setValue(utils.perlReToPythonRe(s), sr=s)
+                self.setValue(utils.str.perlReToPythonRe(s), sr=s)
             else:
                 self.setValue(None)
         except ValueError, e:

src/schedule.py

@@ -32,10 +32,6 @@ Schedule plugin with a subclass of drivers.IrcDriver in order to be run as a
 Supybot driver.
 """
 
-
-
-import supybot.fix as fix
-
 import time
 import heapq
 

src/structures.py

@@ -31,13 +31,9 @@
 Data structures for Python.
 """
 
-
-
-import supybot.fix as fix
-
 import time
 import types
-from itertools import imap, ilen
+from itertools import imap
 
 class RingBuffer(object):
     """Class to represent a fixed-size ring buffer."""
@@ -345,8 +341,12 @@ class TimeoutQueue(object):
                 yield elt
 
     def __len__(self):
-        return ilen(self)
+        # No dependency on utils.
-
+        # return ilen(self)
+        i = 0
+        for _ in self:
+            i += 1
+        return i
 
 class MaxLengthQueue(queue):
     __slots__ = ('length',)

src/test.py

@@ -27,8 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-import supybot.fix as fix
-
 import gc
 import os
 import re
@@ -96,7 +94,7 @@ class SupyTestCase(unittest.TestCase):
     def setUp(self):
         log.critical('Beginning test case %s', self.id())
         threads = [t.getName() for t in threading.enumerate()]
-        log.critical('Threads: %s' % utils.commaAndify(threads))
+        log.critical('Threads: %s' % utils.str.commaAndify(threads))
         unittest.TestCase.setUp(self)
 
     def tearDown(self):

src/unpreserve.py

@@ -27,8 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-
-
 class Reader(object):
     def __init__(self, Creator, *args, **kwargs):
         self.Creator = Creator

src/utils.py

@@ -1,864 +0,0 @@
-###
-# Copyright (c) 2002-2005, Jeremiah Fincher
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-#   * Redistributions of source code must retain the above copyright notice,
-#     this list of conditions, and the following disclaimer.
-#   * Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions, and the following disclaimer in the
-#     documentation and/or other materials provided with the distribution.
-#   * Neither the name of the author of this software nor the name of
-#     contributors to this software may be used to endorse or promote products
-#     derived from this software without specific prior written consent.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-###
-
-"""
-Simple utility functions.
-"""
-
-import supybot.fix as fix
-
-import os
-import re
-import sys
-import md5
-import new
-import sha
-import sets
-import time
-import types
-import random
-import shutil
-import socket
-import string
-import sgmllib
-import compiler
-import textwrap
-import UserDict
-import itertools
-import traceback
-import htmlentitydefs
-from itertools import imap, ifilter
-
-from supybot.structures import TwoWayDictionary
-
-curry = new.instancemethod
-
-def normalizeWhitespace(s):
-    """Normalizes the whitespace in a string; \s+ becomes one space."""
-    return ' '.join(s.split())
-
-class HtmlToText(sgmllib.SGMLParser):
-    """Taken from some eff-bot code on c.l.p."""
-    entitydefs = htmlentitydefs.entitydefs.copy()
-    entitydefs['nbsp'] = ' '
-    def __init__(self, tagReplace=' '):
-        self.data = []
-        self.tagReplace = tagReplace
-        sgmllib.SGMLParser.__init__(self)
-
-    def unknown_starttag(self, tag, attr):
-        self.data.append(self.tagReplace)
-
-    def unknown_endtag(self, tag):
-        self.data.append(self.tagReplace)
-
-    def handle_data(self, data):
-        self.data.append(data)
-
-    def getText(self):
-        text = ''.join(self.data).strip()
-        return normalizeWhitespace(text)
-
-def htmlToText(s, tagReplace=' '):
-    """Turns HTML into text.  tagReplace is a string to replace HTML tags with.
-    """
-    x = HtmlToText(tagReplace)
-    x.feed(s)
-    return x.getText()
-
-def abbrev(strings, d=None):
-    """Returns a dictionary mapping unambiguous abbreviations to full forms."""
-    def eachSubstring(s):
-        for i in xrange(1, len(s)+1):
-            yield s[:i]
-    if len(strings) != len(set(strings)):
-        raise ValueError, \
-              'strings given to utils.abbrev have duplicates: %r' % strings
-    if d is None:
-        d = {}
-    for s in strings:
-        for abbreviation in eachSubstring(s):
-            if abbreviation not in d:
-                d[abbreviation] = s
-            else:
-                if abbreviation not in strings:
-                    d[abbreviation] = None
-    removals = []
-    for key in d:
-        if d[key] is None:
-            removals.append(key)
-    for key in removals:
-        del d[key]
-    return d
-
-def timeElapsed(elapsed, short=False, leadingZeroes=False, years=True,
-                weeks=True, days=True, hours=True, minutes=True, seconds=True):
-    """Given <elapsed> seconds, returns a string with an English description of
-    how much time as passed.  leadingZeroes determines whether 0 days, 0 hours,
-    etc. will be printed; the others determine what larger time periods should
-    be used.
-    """
-    ret = []
-    def format(s, i):
-        if i or leadingZeroes or ret:
-            if short:
-                ret.append('%s%s' % (i, s[0]))
-            else:
-                ret.append(nItems(s, i))
-    elapsed = int(elapsed)
-    assert years or weeks or days or \
-           hours or minutes or seconds, 'One flag must be True'
-    if years:
-        (yrs, elapsed) = (elapsed // 31536000, elapsed % 31536000)
-        format('year', yrs)
-    if weeks:
-        (wks, elapsed) = (elapsed // 604800, elapsed % 604800)
-        format('week', wks)
-    if days:
-        (ds, elapsed) = (elapsed // 86400, elapsed % 86400)
-        format('day', ds)
-    if hours:
-        (hrs, elapsed) = (elapsed // 3600, elapsed % 3600)
-        format('hour', hrs)
-    if minutes or seconds:
-        (mins, secs) = (elapsed // 60, elapsed % 60)
-        if leadingZeroes or mins:
-            format('minute', mins)
-        if seconds:
-            leadingZeroes = True
-            format('second', secs)
-    if not ret:
-        raise ValueError, 'Time difference not great enough to be noted.'
-    if short:
-        return ' '.join(ret)
-    else:
-        return commaAndify(ret)
-
-def distance(s, t):
-    """Returns the levenshtein edit distance between two strings."""
-    n = len(s)
-    m = len(t)
-    if n == 0:
-        return m
-    elif m == 0:
-        return n
-    d = []
-    for i in range(n+1):
-        d.append([])
-        for j in range(m+1):
-            d[i].append(0)
-            d[0][j] = j
-        d[i][0] = i
-    for i in range(1, n+1):
-        cs = s[i-1]
-        for j in range(1, m+1):
-            ct = t[j-1]
-            cost = int(cs != ct)
-            d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cost)
-    return d[n][m]
-
-_soundextrans = string.maketrans(string.ascii_uppercase,
-                                 '01230120022455012623010202')
-_notUpper = string.ascii.translate(string.ascii, string.ascii_uppercase)
-def soundex(s, length=4):
-    """Returns the soundex hash of a given string."""
-    s = s.upper() # Make everything uppercase.
-    s = s.translate(string.ascii, _notUpper) # Delete non-letters.
-    if not s:
-        raise ValueError, 'Invalid string for soundex: %s'
-    firstChar = s[0] # Save the first character.
-    s = s.translate(_soundextrans) # Convert to soundex numbers.
-    s = s.lstrip(s[0]) # Remove all repeated first characters.
-    L = [firstChar]
-    for c in s:
-        if c != L[-1]:
-            L.append(c)
-    L = [c for c in L if c != '0'] + (['0']*(length-1))
-    s = ''.join(L)
-    return length and s[:length] or s.rstrip('0')
-
-def dqrepr(s):
-    """Returns a repr() of s guaranteed to be in double quotes."""
-    # The wankers-that-be decided not to use double-quotes anymore in 2.3.
-    # return '"' + repr("'\x00" + s)[6:]
-    return '"%s"' % s.encode('string_escape').replace('"', '\\"')
-
-def quoted(s):
-    """Returns a quoted s."""
-    return '"%s"' % s
-
-def _getSep(s):
-    if len(s) < 2:
-        raise ValueError, 'string given to _getSep is too short: %r' % s
-    if s.startswith('m') or s.startswith('s'):
-        separator = s[1]
-    else:
-        separator = s[0]
-    if separator.isalnum() or separator in '{}[]()<>':
-        raise ValueError, \
-              'Invalid separator: separator must not be alphanumeric or in ' \
-              '"{}[]()<>"'
-    return separator
-
-def _getSplitterRe(s):
-    separator = _getSep(s)
-    return re.compile(r'(?<!\\)%s' % re.escape(separator))
-        
-def perlReToPythonRe(s):
-    """Converts a string representation of a Perl regular expression (i.e.,
-    m/^foo$/i or /foo|bar/) to a Python regular expression.
-    """
-    sep = _getSep(s)
-    splitter = _getSplitterRe(s)
-    try:
-        (kind, regexp, flags) = splitter.split(s)
-    except ValueError: # Unpack list of wrong size.
-        raise ValueError, 'Must be of the form m/.../ or /.../'
-    regexp = regexp.replace('\\'+sep, sep)
-    if kind not in ('', 'm'):
-        raise ValueError, 'Invalid kind: must be in ("", "m")'
-    flag = 0
-    try:
-        for c in flags.upper():
-            flag |= getattr(re, c)
-    except AttributeError:
-        raise ValueError, 'Invalid flag: %s' % c
-    try:
-        return re.compile(regexp, flag)
-    except re.error, e:
-        raise ValueError, str(e)
-
-def perlReToReplacer(s):
-    """Converts a string representation of a Perl regular expression (i.e.,
-    s/foo/bar/g or s/foo/bar/i) to a Python function doing the equivalent
-    replacement.
-    """
-    sep = _getSep(s)
-    splitter = _getSplitterRe(s)
-    try:
-        (kind, regexp, replace, flags) = splitter.split(s)
-    except ValueError: # Unpack list of wrong size.
-        raise ValueError, 'Must be of the form s/.../.../'
-    regexp = regexp.replace('\x08', r'\b')
-    replace = replace.replace('\\'+sep, sep)
-    for i in xrange(10):
-        replace = replace.replace(chr(i), r'\%s' % i)
-    if kind != 's':
-        raise ValueError, 'Invalid kind: must be "s"'
-    g = False
-    if 'g' in flags:
-        g = True
-        flags = filter('g'.__ne__, flags)
-    r = perlReToPythonRe('/'.join(('', regexp, flags)))
-    if g:
-        return curry(r.sub, replace)
-    else:
-        return lambda s: r.sub(replace, s, 1)
-
-_perlVarSubstituteRe = re.compile(r'\$\{([^}]+)\}|\$([a-zA-Z][a-zA-Z0-9]*)')
-def perlVariableSubstitute(vars, text):
-    def replacer(m):
-        (braced, unbraced) = m.groups()
-        var = braced or unbraced
-        try:
-            x = vars[var]
-            if callable(x):
-                return x()
-            else:
-                return str(x)
-        except KeyError:
-            if braced:
-                return '${%s}' % braced
-            else:
-                return '$' + unbraced
-    return _perlVarSubstituteRe.sub(replacer, text)
-
-def findBinaryInPath(s):
-    """Return full path of a binary if it's in PATH, otherwise return None."""
-    cmdLine = None
-    for dir in os.getenv('PATH').split(':'):
-        filename = os.path.join(dir, s)
-        if os.path.exists(filename):
-            cmdLine = filename
-            break
-    return cmdLine
-
-def commaAndify(seq, comma=',', And='and'):
-    """Given a a sequence, returns an English clause for that sequence.
-
-    I.e., given [1, 2, 3], returns '1, 2, and 3'
-    """
-    L = list(seq)
-    if len(L) == 0:
-        return ''
-    elif len(L) == 1:
-        return ''.join(L) # We need this because it raises TypeError.
-    elif len(L) == 2:
-        L.insert(1, And)
-        return ' '.join(L)
-    else:
-        L[-1] = '%s %s' % (And, L[-1])
-        sep = '%s ' % comma
-        return sep.join(L)
-
-_unCommaTheRe = re.compile(r'(.*),\s*(the)$', re.I)
-def unCommaThe(s):
-    """Takes a string of the form 'foo, the' and turns it into 'the foo'."""
-    m = _unCommaTheRe.match(s)
-    if m is not None:
-        return '%s %s' % (m.group(2), m.group(1))
-    else:
-        return s
-
-def ellipsisify(s, n):
-    """Returns a shortened version of s.  Produces up to the first n chars at
-    the nearest word boundary.
-    """
-    if len(s) <= n:
-        return s
-    else:
-        return (textwrap.wrap(s, n-3)[0] + '...')
-
-plurals = TwoWayDictionary({})
-def matchCase(s1, s2):
-    """Matches the case of s1 in s2"""
-    if s1.isupper():
-        return s2.upper()
-    else:
-        L = list(s2)
-        for (i, char) in enumerate(s1[:len(s2)]):
-            if char.isupper():
-                L[i] = L[i].upper()
-        return ''.join(L)
-
-consonants = 'bcdfghjklmnpqrstvwxz'
-_pluralizeRegex = re.compile('[%s]y$' % consonants)
-def pluralize(s, i=2):
-    """Returns the plural of s based on its number i.  Put any exceptions to
-    the general English rule of appending 's' in the plurals dictionary.
-    """
-    if i == 1:
-        return s
-    else:
-        lowered = s.lower()
-        # Exception dictionary
-        if lowered in plurals:
-            return matchCase(s, plurals[lowered])
-        # Words ending with 'ch', 'sh' or 'ss' such as 'punch(es)', 'fish(es)
-        # and miss(es)
-        elif any(lowered.endswith, ['x', 'ch', 'sh', 'ss']):
-            return matchCase(s, s+'es')
-        # Words ending with a consonant followed by a 'y' such as
-        # 'try (tries)' or 'spy (spies)'
-        elif _pluralizeRegex.search(lowered):
-            return matchCase(s, s[:-1] + 'ies')
-        # In all other cases, we simply add an 's' to the base word
-        else:
-            return matchCase(s, s+'s')
-
-_depluralizeRegex = re.compile('[%s]ies' % consonants)
-def depluralize(s):
-    """Returns the singular of s."""
-    lowered = s.lower()
-    if lowered in plurals:
-        return matchCase(s, plurals[lowered])
-    elif any(lowered.endswith, ['ches', 'shes', 'sses']):
-        return s[:-2]
-    elif re.search(_depluralizeRegex, lowered):
-        return s[:-3] + 'y'
-    else:
-        if lowered.endswith('s'):
-            return s[:-1] # Chop off 's'.
-        else:
-            return s # Don't know what to do.
-
-def nItems(item, n, between=None):
-    """Works like this:
-
-    >>> nItems('clock', 1)
-    '1 clock'
-
-    >>> nItems('clock', 10)
-    '10 clocks'
-
-    >>> nItems('clock', 10, between='grandfather')
-    '10 grandfather clocks'
-    """
-    if between is None:
-        return '%s %s' % (n, pluralize(item, n))
-    else:
-        return '%s %s %s' % (n, between, pluralize(item, n))
-
-def be(i):
-    """Returns the form of the verb 'to be' based on the number i."""
-    if i == 1:
-        return 'is'
-    else:
-        return 'are'
-
-def has(i):
-    """Returns the form of the verb 'to have' based on the number i."""
-    if i == 1:
-        return 'has'
-    else:
-        return 'have'
-
-def sortBy(f, L):
-    """Uses the decorate-sort-undecorate pattern to sort L by function f."""
-    for (i, elt) in enumerate(L):
-        L[i] = (f(elt), i, elt)
-    L.sort()
-    for (i, elt) in enumerate(L):
-        L[i] = L[i][2]
-
-if sys.version_info < (2, 4, 0):
-    def sorted(iterable, cmp=None, key=None, reversed=False):
-        L = list(iterable)
-        if key is not None:
-            assert cmp is None, 'Can\'t use both cmp and key.'
-            sortBy(key, L)
-        else:
-            L.sort(cmp)
-        if reversed:
-            L.reverse()
-        return L
-
-    __builtins__['sorted'] = sorted
-
-def mktemp(suffix=''):
-    """Gives a decent random string, suitable for a filename."""
-    r = random.Random()
-    m = md5.md5(suffix)
-    r.seed(time.time())
-    s = str(r.getstate())
-    for x in xrange(0, random.randrange(400), random.randrange(1, 5)):
-        m.update(str(x))
-        m.update(s)
-        m.update(str(time.time()))
-        s = m.hexdigest()
-    return sha.sha(s + str(time.time())).hexdigest() + suffix
-
-def itersplit(isSeparator, iterable, maxsplit=-1, yieldEmpty=False):
-    """itersplit(isSeparator, iterable, maxsplit=-1, yieldEmpty=False)
-
-    Splits an iterator based on a predicate isSeparator."""
-    if isinstance(isSeparator, basestring):
-        f = lambda s: s == isSeparator
-    else:
-        f = isSeparator
-    acc = []
-    for element in iterable:
-        if maxsplit == 0 or not f(element):
-            acc.append(element)
-        else:
-            maxsplit -= 1
-            if acc or yieldEmpty:
-                yield acc
-            acc = []
-    if acc or yieldEmpty:
-        yield acc
-
-def flatten(seq, strings=False):
-    """Flattens a list of lists into a single list.  See the test for examples.
-    """
-    for elt in seq:
-        if not strings and type(elt) == str or type(elt) == unicode:
-            yield elt
-        else:
-            try:
-                for x in flatten(elt):
-                    yield x
-            except TypeError:
-                yield elt
-
-def saltHash(password, salt=None, hash='sha'):
-    if salt is None:
-        salt = mktemp()[:8]
-    if hash == 'sha':
-        hasher = sha.sha
-    elif hash == 'md5':
-        hasher = md5.md5
-    return '|'.join([salt, hasher(salt + password).hexdigest()])
-
-def safeEval(s, namespace={'True': True, 'False': False, 'None': None}):
-    """Evaluates s, safely.  Useful for turning strings into tuples/lists/etc.
-    without unsafely using eval()."""
-    try:
-        node = compiler.parse(s)
-    except SyntaxError, e:
-        raise ValueError, 'Invalid string: %s.' % e
-    nodes = compiler.parse(s).node.nodes
-    if not nodes:
-        if node.__class__ is compiler.ast.Module:
-            return node.doc
-        else:
-            raise ValueError, 'Unsafe string: %s' % quoted(s)
-    node = nodes[0]
-    if node.__class__ is not compiler.ast.Discard:
-        raise ValueError, 'Invalid expression: %s' % quoted(s)
-    node = node.getChildNodes()[0]
-    def checkNode(node):
-        if node.__class__ is compiler.ast.Const:
-            return True
-        if node.__class__ in (compiler.ast.List,
-                              compiler.ast.Tuple,
-                              compiler.ast.Dict):
-            return all(checkNode, node.getChildNodes())
-        if node.__class__ is compiler.ast.Name:
-            if node.name in namespace:
-                return True
-            else:
-                return False
-        else:
-            return False
-    if checkNode(node):
-        return eval(s, namespace, namespace)
-    else:
-        raise ValueError, 'Unsafe string: %s' % quoted(s)
-
-def exnToString(e):
-    """Turns a simple exception instance into a string (better than str(e))"""
-    strE = str(e)
-    if strE:
-        return '%s: %s' % (e.__class__.__name__, strE)
-    else:
-        return e.__class__.__name__
-
-class IterableMap(object):
-    """Define .iteritems() in a class and subclass this to get the other iters.
-    """
-    def iteritems(self):
-        raise NotImplementedError
-
-    def iterkeys(self):
-        for (key, _) in self.iteritems():
-            yield key
-    __iter__ = iterkeys
-
-    def itervalues(self):
-        for (_, value) in self.iteritems():
-            yield value
-
-    def items(self):
-        return list(self.iteritems())
-
-    def keys(self):
-        return list(self.iterkeys())
-
-    def values(self):
-        return list(self.itervalues())
-
-    def __len__(self):
-        ret = 0
-        for _ in self.iteritems():
-            ret += 1
-        return ret
-
-    def __nonzero__(self):
-        for _ in self.iteritems():
-            return True
-        return False
-
-
-def nonCommentLines(fd):
-    for line in fd:
-        if not line.startswith('#'):
-            yield line
-
-def nonEmptyLines(fd):
-##     for line in fd:
-##         if line.strip():
-##             yield line
-    return ifilter(str.strip, fd)
-
-def nonCommentNonEmptyLines(fd):
-    return nonEmptyLines(nonCommentLines(fd))
-
-def changeFunctionName(f, name, doc=None):
-    if doc is None:
-        doc = f.__doc__
-    newf = types.FunctionType(f.func_code, f.func_globals, name,
-                              f.func_defaults, f.func_closure)
-    newf.__doc__ = doc
-    return newf
-
-def getSocket(host):
-    """Returns a socket of the correct AF_INET type (v4 or v6) in order to
-    communicate with host.
-    """
-    host = socket.gethostbyname(host)
-    if isIP(host):
-        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    elif isIPV6(host):
-        return socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-    else:
-        raise socket.error, 'Something wonky happened.'
-
-def isIP(s):
-    """Returns whether or not a given string is an IPV4 address.
-
-    >>> isIP('255.255.255.255')
-    1
-
-    >>> isIP('abc.abc.abc.abc')
-    0
-    """
-    try:
-        return bool(socket.inet_aton(s))
-    except socket.error:
-        return False
-
-def bruteIsIPV6(s):
-    if s.count('::') <= 1:
-        L = s.split(':')
-        if len(L) <= 8:
-            for x in L:
-                if x:
-                    try:
-                        int(x, 16)
-                    except ValueError:
-                        return False
-            return True
-    return False
-
-def isIPV6(s):
-    """Returns whether or not a given string is an IPV6 address."""
-    try:
-        if hasattr(socket, 'inet_pton'):
-            return bool(socket.inet_pton(socket.AF_INET6, s))
-        else:
-            return bruteIsIPV6(s)
-    except socket.error:
-        try:
-            socket.inet_pton(socket.AF_INET6, '::')
-        except socket.error:
-            # We gotta fake it.
-            return bruteIsIPV6(s)
-        return False
-
-class InsensitivePreservingDict(UserDict.DictMixin, object):
-    def key(self, s):
-        """Override this if you wish."""
-        if s is not None:
-            s = s.lower()
-        return s
-
-    def __init__(self, dict=None, key=None):
-        if key is not None:
-            self.key = key
-        self.data = {}
-        if dict is not None:
-            self.update(dict)
-
-    def __repr__(self):
-        return '%s(%s)' % (self.__class__.__name__,
-                           super(InsensitivePreservingDict, self).__repr__())
-
-    def fromkeys(cls, keys, s=None, dict=None, key=None):
-        d = cls(dict=dict, key=key)
-        for key in keys:
-            d[key] = s
-        return d
-    fromkeys = classmethod(fromkeys)
-
-    def __getitem__(self, k):
-        return self.data[self.key(k)][1]
-
-    def __setitem__(self, k, v):
-        self.data[self.key(k)] = (k, v)
-
-    def __delitem__(self, k):
-        del self.data[self.key(k)]
-
-    def iteritems(self):
-        return self.data.itervalues()
-
-    def keys(self):
-        L = []
-        for (k, _) in self.iteritems():
-            L.append(k)
-        return L
-
-    def __reduce__(self):
-        return (self.__class__, (dict(self.data.values()),))
-
-
-class NormalizingSet(sets.Set):
-    def __init__(self, iterable=()):
-        iterable = itertools.imap(self.normalize, iterable)
-        super(NormalizingSet, self).__init__(iterable)
-
-    def normalize(self, x):
-        return x
-
-    def add(self, x):
-        return super(NormalizingSet, self).add(self.normalize(x))
-
-    def remove(self, x):
-        return super(NormalizingSet, self).remove(self.normalize(x))
-
-    def discard(self, x):
-        return super(NormalizingSet, self).discard(self.normalize(x))
-
-    def __contains__(self, x):
-        return super(NormalizingSet, self).__contains__(self.normalize(x))
-    has_key = __contains__
-
-def mungeEmailForWeb(s):
-    s = s.replace('@', ' AT ')
-    s = s.replace('.', ' DOT ')
-    return s
-
-class AtomicFile(file):
-    """Used for files that need to be atomically written -- i.e., if there's a
-    failure, the original file remains, unmodified.  mode must be 'w' or 'wb'"""
-    def __init__(self, filename, mode='w', allowEmptyOverwrite=True,
-                 makeBackupIfSmaller=True, tmpDir=None, backupDir=None):
-        if mode not in ('w', 'wb'):
-            raise ValueError, 'Invalid mode: %s' % quoted(mode)
-        self.rolledback = False
-        self.allowEmptyOverwrite = allowEmptyOverwrite
-        self.makeBackupIfSmaller = makeBackupIfSmaller
-        self.filename = filename
-        self.backupDir = backupDir
-        if tmpDir is None:
-            # If not given a tmpDir, we'll just put a random token on the end
-            # of our filename and put it in the same directory.
-            self.tempFilename = '%s.%s' % (self.filename, mktemp())
-        else:
-            # If given a tmpDir, we'll get the basename (just the filename, no
-            # directory), put our random token on the end, and put it in tmpDir
-            tempFilename = '%s.%s' % (os.path.basename(self.filename), mktemp())
-            self.tempFilename = os.path.join(tmpDir, tempFilename)
-        # This doesn't work because of the uncollectable garbage effect.
-        # self.__parent = super(AtomicFile, self)
-        super(AtomicFile, self).__init__(self.tempFilename, mode)
-
-    def rollback(self):
-        if not self.closed:
-            super(AtomicFile, self).close()
-            if os.path.exists(self.tempFilename):
-                os.remove(self.tempFilename)
-            self.rolledback = True
-
-    def close(self):
-        if not self.rolledback:
-            super(AtomicFile, self).close()
-            # We don't mind writing an empty file if the file we're overwriting
-            # doesn't exist.
-            newSize = os.path.getsize(self.tempFilename)
-            originalExists = os.path.exists(self.filename)
-            if newSize or self.allowEmptyOverwrite or not originalExists:
-                if originalExists:
-                    oldSize = os.path.getsize(self.filename)
-                    if self.makeBackupIfSmaller and newSize < oldSize:
-                        now = int(time.time())
-                        backupFilename = '%s.backup.%s' % (self.filename, now)
-                        if self.backupDir is not None:
-                            backupFilename = os.path.basename(backupFilename)
-                            backupFilename = os.path.join(self.backupDir,
-                                                          backupFilename)
-                        shutil.copy(self.filename, backupFilename)
-                # We use shutil.move here instead of os.rename because
-                # the latter doesn't work on Windows when self.filename
-                # (the target) already exists.  shutil.move handles those
-                # intricacies for us.
-
-                # This raises IOError if we can't write to the file.  Since
-                # in *nix, it only takes write perms to the *directory* to
-                # rename a file (and shutil.move will use os.rename if
-                # possible), we first check if we have the write permission
-                # and only then do we write.
-                fd = file(self.filename, 'a')
-                fd.close()
-                shutil.move(self.tempFilename, self.filename)
-                
-        else:
-            raise ValueError, 'AtomicFile.close called after rollback.'
-
-    def __del__(self):
-        # We rollback because if we're deleted without being explicitly closed,
-        # that's bad.  We really should log this here, but as of yet we've got
-        # no logging facility in utils.  I've got some ideas for this, though.
-        self.rollback()
-
-def transactionalFile(*args, **kwargs):
-    # This exists so it can be replaced by a function that provides the tmpDir.
-    # We do that replacement in conf.py.
-    return AtomicFile(*args, **kwargs)
-
-def stackTrace(frame=None, compact=True):
-    if frame is None:
-        frame = sys._getframe()
-    if compact:
-        L = []
-        while frame:
-            lineno = frame.f_lineno
-            funcname = frame.f_code.co_name
-            filename = os.path.basename(frame.f_code.co_filename)
-            L.append('[%s|%s|%s]' % (filename, funcname, lineno))
-            frame = frame.f_back
-        return textwrap.fill(' '.join(L))
-    else:
-        return traceback.format_stack(frame)
-
-def callTracer(fd=None, basename=True):
-    if fd is None:
-        fd = sys.stdout
-    def tracer(frame, event, _):
-        if event == 'call':
-            code = frame.f_code
-            lineno = frame.f_lineno
-            funcname = code.co_name
-            filename = code.co_filename
-            if basename:
-                filename = os.path.basename(filename)
-            print >>fd, '%s: %s(%s)' % (filename, funcname, lineno)
-    return tracer
-
-
-def toBool(s):
-    s = s.strip().lower()
-    if s in ('true', 'on', 'enable', 'enabled', '1'):
-        return True
-    elif s in ('false', 'off', 'disable', 'disabled', '0'):
-        return False
-    else:
-        raise ValueError, 'Invalid string for toBool: %s' % quoted(s)
-
-def mapinto(f, L):
-    for (i, x) in enumerate(L):
-        L[i] = f(x)
-
-if __name__ == '__main__':
-    import doctest
-    doctest.testmod(sys.modules['__main__'])
-
-
-# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

src/utils/__init__.py

@@ -0,0 +1,106 @@
+###
+# Copyright (c) 2002-2005, Jeremiah Fincher
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice,
+#     this list of conditions, and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions, and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#   * Neither the name of the author of this software nor the name of
+#     contributors to this software may be used to endorse or promote products
+#     derived from this software without specific prior written consent.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###
+
+import sys
+
+###
+# csv.{join,split} -- useful functions that should exist.
+###
+import csv
+import cStringIO as StringIO
+def join(L):
+    fd = StringIO.StringIO()
+    writer = csv.writer(fd)
+    writer.writerow(L)
+    return fd.getvalue().rstrip('\r\n')
+
+def split(s):
+    fd = StringIO.StringIO(s)
+    reader = csv.reader(fd)
+    return reader.next()
+csv.join = join
+csv.split = split
+
+# We use this often enough that we're going to stick it in builtins.
+def force(x):
+    if callable(x):
+        return x()
+    else:
+        return x
+__builtins__['force'] = force
+        
+if sys.version_info < (2, 4, 0):
+    def reversed(L):
+        """Iterates through a sequence in reverse."""
+        for i in xrange(len(L) - 1, -1, -1):
+            yield L[i]
+    __builtins__['reversed'] = reversed
+
+    def sorted(iterable, cmp=None, key=None, reversed=False):
+        L = list(iterable)
+        if key is not None:
+            assert cmp is None, 'Can\'t use both cmp and key.'
+            sortBy(key, L)
+        else:
+            L.sort(cmp)
+        if reversed:
+            L.reverse()
+        return L
+
+    __builtins__['sorted'] = sorted
+
+    import operator
+    def itemgetter(i):
+        return lambda x: x[i]
+
+    def attrgetter(attr):
+        return lambda x: getattr(x, attr)
+    operator.itemgetter = itemgetter
+    operator.attrgetter = attrgetter
+
+    import sets
+    __builtins__['set'] = sets.Set
+    __builtins__['frozenset'] = sets.ImmutableSet
+
+    import socket
+    # Some socket modules don't have sslerror, so we'll just make it an error.
+    if not hasattr(socket, 'sslerror'):
+        socket.sslerror = socket.error
+
+# These imports need to happen below the block above, so things get put into
+# __builtins__ appropriately.
+from gen import *
+import net
+import web
+import seq
+import str
+import file
+import iter
+
+# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

src/utils/file.py

@@ -0,0 +1,150 @@
+###
+# Copyright (c) 2002-2005, Jeremiah Fincher
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice,
+#     this list of conditions, and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions, and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#   * Neither the name of the author of this software nor the name of
+#     contributors to this software may be used to endorse or promote products
+#     derived from this software without specific prior written consent.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###
+
+import os
+import md5
+import sha
+import time
+import random
+import shutil
+import os.path
+from iter import ifilter
+
+def mktemp(suffix=''):
+    """Gives a decent random string, suitable for a filename."""
+    r = random.Random()
+    m = md5.md5(suffix)
+    r.seed(time.time())
+    s = str(r.getstate())
+    for x in xrange(0, random.randrange(400), random.randrange(1, 5)):
+        m.update(str(x))
+        m.update(s)
+        m.update(str(time.time()))
+        s = m.hexdigest()
+    return sha.sha(s + str(time.time())).hexdigest() + suffix
+
+def nonCommentLines(fd):
+    for line in fd:
+        if not line.startswith('#'):
+            yield line
+
+def nonEmptyLines(fd):
+    return ifilter(str.strip, fd)
+
+def nonCommentNonEmptyLines(fd):
+    return nonEmptyLines(nonCommentLines(fd))
+
+class AtomicFile(file):
+    """Used for files that need to be atomically written -- i.e., if there's a
+    failure, the original file remains, unmodified.  mode must be 'w' or 'wb'"""
+    class default(object): # Holder for values.
+        # Callables?
+        tmpDir = None
+        backupDir = None
+        makeBackupIfSmaller = True
+        allowEmptyOverwrite = True
+    def __init__(self, filename, mode='w', allowEmptyOverwrite=None,
+                 makeBackupIfSmaller=None, tmpDir=None, backupDir=None):
+        if tmpDir is None:
+            tmpDir = force(self.default.tmpDir)
+        if backupDir is None:
+            backupDir = force(self.default.backupDir)
+        if makeBackupIfSmaller is None:
+            makeBackupIfSmaller = force(self.default.makeBackupIfSmaller)
+        if allowEmptyOverwrite is None:
+            allowEmptyOverwrite = force(self.default.allowEmptyOverwrite)
+        if mode not in ('w', 'wb'):
+            raise ValueError, 'Invalid mode: %s' % quoted(mode)
+        self.rolledback = False
+        self.allowEmptyOverwrite = allowEmptyOverwrite
+        self.makeBackupIfSmaller = makeBackupIfSmaller
+        self.filename = filename
+        self.backupDir = backupDir
+        if tmpDir is None:
+            # If not given a tmpDir, we'll just put a random token on the end
+            # of our filename and put it in the same directory.
+            self.tempFilename = '%s.%s' % (self.filename, mktemp())
+        else:
+            # If given a tmpDir, we'll get the basename (just the filename, no
+            # directory), put our random token on the end, and put it in tmpDir
+            tempFilename = '%s.%s' % (os.path.basename(self.filename), mktemp())
+            self.tempFilename = os.path.join(tmpDir, tempFilename)
+        # This doesn't work because of the uncollectable garbage effect.
+        # self.__parent = super(AtomicFile, self)
+        super(AtomicFile, self).__init__(self.tempFilename, mode)
+
+    def rollback(self):
+        if not self.closed:
+            super(AtomicFile, self).close()
+            if os.path.exists(self.tempFilename):
+                os.remove(self.tempFilename)
+            self.rolledback = True
+
+    def close(self):
+        if not self.rolledback:
+            super(AtomicFile, self).close()
+            # We don't mind writing an empty file if the file we're overwriting
+            # doesn't exist.
+            newSize = os.path.getsize(self.tempFilename)
+            originalExists = os.path.exists(self.filename)
+            if newSize or self.allowEmptyOverwrite or not originalExists:
+                if originalExists:
+                    oldSize = os.path.getsize(self.filename)
+                    if self.makeBackupIfSmaller and newSize < oldSize:
+                        now = int(time.time())
+                        backupFilename = '%s.backup.%s' % (self.filename, now)
+                        if self.backupDir is not None:
+                            backupFilename = os.path.basename(backupFilename)
+                            backupFilename = os.path.join(self.backupDir,
+                                                          backupFilename)
+                        shutil.copy(self.filename, backupFilename)
+                # We use shutil.move here instead of os.rename because
+                # the latter doesn't work on Windows when self.filename
+                # (the target) already exists.  shutil.move handles those
+                # intricacies for us.
+
+                # This raises IOError if we can't write to the file.  Since
+                # in *nix, it only takes write perms to the *directory* to
+                # rename a file (and shutil.move will use os.rename if
+                # possible), we first check if we have the write permission
+                # and only then do we write.
+                fd = file(self.filename, 'a')
+                fd.close()
+                shutil.move(self.tempFilename, self.filename)
+                
+        else:
+            raise ValueError, 'AtomicFile.close called after rollback.'
+
+    def __del__(self):
+        # We rollback because if we're deleted without being explicitly closed,
+        # that's bad.  We really should log this here, but as of yet we've got
+        # no logging facility in utils.  I've got some ideas for this, though.
+        self.rollback()
+
+# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

src/utils/gen.py

@@ -0,0 +1,326 @@
+###
+# Copyright (c) 2002-2005, Jeremiah Fincher
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice,
+#     this list of conditions, and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions, and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#   * Neither the name of the author of this software nor the name of
+#     contributors to this software may be used to endorse or promote products
+#     derived from this software without specific prior written consent.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###
+
+import os
+import sys
+import md5
+import new
+import sha
+import time
+import types
+import compiler
+import textwrap
+import UserDict
+import traceback
+
+from iter import imap
+from file import mktemp
+from str import quoted, nItems, commaAndify
+
+def abbrev(strings, d=None):
+    """Returns a dictionary mapping unambiguous abbreviations to full forms."""
+    def eachSubstring(s):
+        for i in xrange(1, len(s)+1):
+            yield s[:i]
+    if len(strings) != len(set(strings)):
+        raise ValueError, \
+              'strings given to utils.abbrev have duplicates: %r' % strings
+    if d is None:
+        d = {}
+    for s in strings:
+        for abbreviation in eachSubstring(s):
+            if abbreviation not in d:
+                d[abbreviation] = s
+            else:
+                if abbreviation not in strings:
+                    d[abbreviation] = None
+    removals = []
+    for key in d:
+        if d[key] is None:
+            removals.append(key)
+    for key in removals:
+        del d[key]
+    return d
+
+def timeElapsed(elapsed, short=False, leadingZeroes=False, years=True,
+                weeks=True, days=True, hours=True, minutes=True, seconds=True):
+    """Given <elapsed> seconds, returns a string with an English description of
+    how much time as passed.  leadingZeroes determines whether 0 days, 0 hours,
+    etc. will be printed; the others determine what larger time periods should
+    be used.
+    """
+    ret = []
+    def format(s, i):
+        if i or leadingZeroes or ret:
+            if short:
+                ret.append('%s%s' % (i, s[0]))
+            else:
+                ret.append(nItems(s, i))
+    elapsed = int(elapsed)
+    assert years or weeks or days or \
+           hours or minutes or seconds, 'One flag must be True'
+    if years:
+        (yrs, elapsed) = (elapsed // 31536000, elapsed % 31536000)
+        format('year', yrs)
+    if weeks:
+        (wks, elapsed) = (elapsed // 604800, elapsed % 604800)
+        format('week', wks)
+    if days:
+        (ds, elapsed) = (elapsed // 86400, elapsed % 86400)
+        format('day', ds)
+    if hours:
+        (hrs, elapsed) = (elapsed // 3600, elapsed % 3600)
+        format('hour', hrs)
+    if minutes or seconds:
+        (mins, secs) = (elapsed // 60, elapsed % 60)
+        if leadingZeroes or mins:
+            format('minute', mins)
+        if seconds:
+            leadingZeroes = True
+            format('second', secs)
+    if not ret:
+        raise ValueError, 'Time difference not great enough to be noted.'
+    if short:
+        return ' '.join(ret)
+    else:
+        return commaAndify(ret)
+
+def findBinaryInPath(s):
+    """Return full path of a binary if it's in PATH, otherwise return None."""
+    cmdLine = None
+    for dir in os.getenv('PATH').split(':'):
+        filename = os.path.join(dir, s)
+        if os.path.exists(filename):
+            cmdLine = filename
+            break
+    return cmdLine
+
+def sortBy(f, L):
+    """Uses the decorate-sort-undecorate pattern to sort L by function f."""
+    for (i, elt) in enumerate(L):
+        L[i] = (f(elt), i, elt)
+    L.sort()
+    for (i, elt) in enumerate(L):
+        L[i] = L[i][2]
+
+def saltHash(password, salt=None, hash='sha'):
+    if salt is None:
+        salt = mktemp()[:8]
+    if hash == 'sha':
+        hasher = sha.sha
+    elif hash == 'md5':
+        hasher = md5.md5
+    return '|'.join([salt, hasher(salt + password).hexdigest()])
+
+def safeEval(s, namespace={'True': True, 'False': False, 'None': None}):
+    """Evaluates s, safely.  Useful for turning strings into tuples/lists/etc.
+    without unsafely using eval()."""
+    try:
+        node = compiler.parse(s)
+    except SyntaxError, e:
+        raise ValueError, 'Invalid string: %s.' % e
+    nodes = compiler.parse(s).node.nodes
+    if not nodes:
+        if node.__class__ is compiler.ast.Module:
+            return node.doc
+        else:
+            raise ValueError, 'Unsafe string: %s' % quoted(s)
+    node = nodes[0]
+    if node.__class__ is not compiler.ast.Discard:
+        raise ValueError, 'Invalid expression: %s' % quoted(s)
+    node = node.getChildNodes()[0]
+    def checkNode(node):
+        if node.__class__ is compiler.ast.Const:
+            return True
+        if node.__class__ in (compiler.ast.List,
+                              compiler.ast.Tuple,
+                              compiler.ast.Dict):
+            return all(checkNode, node.getChildNodes())
+        if node.__class__ is compiler.ast.Name:
+            if node.name in namespace:
+                return True
+            else:
+                return False
+        else:
+            return False
+    if checkNode(node):
+        return eval(s, namespace, namespace)
+    else:
+        raise ValueError, 'Unsafe string: %s' % quoted(s)
+
+def exnToString(e):
+    """Turns a simple exception instance into a string (better than str(e))"""
+    strE = str(e)
+    if strE:
+        return '%s: %s' % (e.__class__.__name__, strE)
+    else:
+        return e.__class__.__name__
+
+class IterableMap(object):
+    """Define .iteritems() in a class and subclass this to get the other iters.
+    """
+    def iteritems(self):
+        raise NotImplementedError
+
+    def iterkeys(self):
+        for (key, _) in self.iteritems():
+            yield key
+    __iter__ = iterkeys
+
+    def itervalues(self):
+        for (_, value) in self.iteritems():
+            yield value
+
+    def items(self):
+        return list(self.iteritems())
+
+    def keys(self):
+        return list(self.iterkeys())
+
+    def values(self):
+        return list(self.itervalues())
+
+    def __len__(self):
+        ret = 0
+        for _ in self.iteritems():
+            ret += 1
+        return ret
+
+    def __nonzero__(self):
+        for _ in self.iteritems():
+            return True
+        return False
+
+
+def changeFunctionName(f, name, doc=None):
+    if doc is None:
+        doc = f.__doc__
+    newf = types.FunctionType(f.func_code, f.func_globals, name,
+                              f.func_defaults, f.func_closure)
+    newf.__doc__ = doc
+    return newf
+
+class InsensitivePreservingDict(UserDict.DictMixin, object):
+    def key(self, s):
+        """Override this if you wish."""
+        if s is not None:
+            s = s.lower()
+        return s
+
+    def __init__(self, dict=None, key=None):
+        if key is not None:
+            self.key = key
+        self.data = {}
+        if dict is not None:
+            self.update(dict)
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__,
+                           super(InsensitivePreservingDict, self).__repr__())
+
+    def fromkeys(cls, keys, s=None, dict=None, key=None):
+        d = cls(dict=dict, key=key)
+        for key in keys:
+            d[key] = s
+        return d
+    fromkeys = classmethod(fromkeys)
+
+    def __getitem__(self, k):
+        return self.data[self.key(k)][1]
+
+    def __setitem__(self, k, v):
+        self.data[self.key(k)] = (k, v)
+
+    def __delitem__(self, k):
+        del self.data[self.key(k)]
+
+    def iteritems(self):
+        return self.data.itervalues()
+
+    def keys(self):
+        L = []
+        for (k, _) in self.iteritems():
+            L.append(k)
+        return L
+
+    def __reduce__(self):
+        return (self.__class__, (dict(self.data.values()),))
+
+
+class NormalizingSet(set):
+    def __init__(self, iterable=()):
+        iterable = imap(self.normalize, iterable)
+        super(NormalizingSet, self).__init__(iterable)
+
+    def normalize(self, x):
+        return x
+
+    def add(self, x):
+        return super(NormalizingSet, self).add(self.normalize(x))
+
+    def remove(self, x):
+        return super(NormalizingSet, self).remove(self.normalize(x))
+
+    def discard(self, x):
+        return super(NormalizingSet, self).discard(self.normalize(x))
+
+    def __contains__(self, x):
+        return super(NormalizingSet, self).__contains__(self.normalize(x))
+    has_key = __contains__
+
+def stackTrace(frame=None, compact=True):
+    if frame is None:
+        frame = sys._getframe()
+    if compact:
+        L = []
+        while frame:
+            lineno = frame.f_lineno
+            funcname = frame.f_code.co_name
+            filename = os.path.basename(frame.f_code.co_filename)
+            L.append('[%s|%s|%s]' % (filename, funcname, lineno))
+            frame = frame.f_back
+        return textwrap.fill(' '.join(L))
+    else:
+        return traceback.format_stack(frame)
+
+def callTracer(fd=None, basename=True):
+    if fd is None:
+        fd = sys.stdout
+    def tracer(frame, event, _):
+        if event == 'call':
+            code = frame.f_code
+            lineno = frame.f_lineno
+            funcname = code.co_name
+            filename = code.co_filename
+            if basename:
+                filename = os.path.basename(filename)
+            print >>fd, '%s: %s(%s)' % (filename, funcname, lineno)
+    return tracer
+
+# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

src/utils/iter.py

@@ -0,0 +1,147 @@
+###
+# Copyright (c) 2002-2005, Jeremiah Fincher
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice,
+#     this list of conditions, and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions, and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#   * Neither the name of the author of this software nor the name of
+#     contributors to this software may be used to endorse or promote products
+#     derived from this software without specific prior written consent.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###
+
+import new
+import random
+
+from itertools import *
+
+def len(iterable):
+    """Returns the length of an iterator."""
+    i = 0
+    for _ in iterable:
+        i += 1
+    return i
+
+def trueCycle(iterable):
+    while 1:
+        yielded = False
+        for x in iterable:
+            yield x
+            yielded = True
+        if not yielded:
+            raise StopIteration
+
+def groupby(key, iterable):
+    if key is None:
+        key = lambda x: x
+    it = iter(iterable)
+    value = it.next() # If there are no items, this takes an early exit
+    oldkey = key(value)
+    group = [value]
+    for value in it:
+        newkey = key(value)
+        if newkey != oldkey:
+            yield group
+            group = []
+            oldkey = newkey
+        group.append(value)
+    yield group
+
+def partition(p, iterable):
+    """Partitions an iterable based on a predicate p.
+    Returns a (yes,no) tuple"""
+    no = []
+    yes = []
+    for elt in iterable:
+        if p(elt):
+            yes.append(elt)
+        else:
+            no.append(elt)
+    return (yes, no)
+
+def any(p, iterable):
+    """Returns true if any element in iterable satisfies predicate p."""
+    for elt in ifilter(p, iterable):
+        return True
+    else:
+        return False
+
+def all(p, iterable):
+    """Returns true if all elements in iterable satisfy predicate p."""
+    for elt in ifilterfalse(p, iterable):
+        return False
+    else:
+        return True
+
+def choice(iterable):
+    if isinstance(iterable, (list, tuple)):
+        return random.choice(iterable)
+    else:
+        n = 1
+        m = new.module('') # Guaranteed unique value.
+        ret = m
+        for x in iterable:
+            if random.random() < 1/n:
+                ret = x
+            n += 1
+        if ret is m:
+            raise IndexError
+        return ret
+
+def flatten(iterable, strings=False):
+    """Flattens a list of lists into a single list.  See the test for examples.
+    """
+    for elt in iterable:
+        if not strings and isinstance(elt, basestring):
+            yield elt
+        else:
+            try:
+                for x in flatten(elt):
+                    yield x
+            except TypeError:
+                yield elt
+
+def split(isSeparator, iterable, maxsplit=-1, yieldEmpty=False):
+    """split(isSeparator, iterable, maxsplit=-1, yieldEmpty=False)
+
+    Splits an iterator based on a predicate isSeparator."""
+    if isinstance(isSeparator, basestring):
+        f = lambda s: s == isSeparator
+    else:
+        f = isSeparator
+    acc = []
+    for element in iterable:
+        if maxsplit == 0 or not f(element):
+            acc.append(element)
+        else:
+            maxsplit -= 1
+            if acc or yieldEmpty:
+                yield acc
+            acc = []
+    if acc or yieldEmpty:
+        yield acc
+
+def ilen(iterable):
+    i = 0
+    for _ in iterable:
+        i += 1
+    return i
+
+# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

src/utils/net.py

@@ -0,0 +1,90 @@
+###
+# Copyright (c) 2002-2005, Jeremiah Fincher
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice,
+#     this list of conditions, and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions, and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#   * Neither the name of the author of this software nor the name of
+#     contributors to this software may be used to endorse or promote products
+#     derived from this software without specific prior written consent.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###
+
+"""
+Simple utility modules.
+"""
+
+import socket
+
+def getSocket(host):
+    """Returns a socket of the correct AF_INET type (v4 or v6) in order to
+    communicate with host.
+    """
+    host = socket.gethostbyname(host)
+    if isIP(host):
+        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    elif isIPV6(host):
+        return socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    else:
+        raise socket.error, 'Something wonky happened.'
+
+def isIP(s):
+    """Returns whether or not a given string is an IPV4 address.
+
+    >>> isIP('255.255.255.255')
+    1
+
+    >>> isIP('abc.abc.abc.abc')
+    0
+    """
+    try:
+        return bool(socket.inet_aton(s))
+    except socket.error:
+        return False
+
+def bruteIsIPV6(s):
+    if s.count('::') <= 1:
+        L = s.split(':')
+        if len(L) <= 8:
+            for x in L:
+                if x:
+                    try:
+                        int(x, 16)
+                    except ValueError:
+                        return False
+            return True
+    return False
+
+def isIPV6(s):
+    """Returns whether or not a given string is an IPV6 address."""
+    try:
+        if hasattr(socket, 'inet_pton'):
+            return bool(socket.inet_pton(socket.AF_INET6, s))
+        else:
+            return bruteIsIPV6(s)
+    except socket.error:
+        try:
+            socket.inet_pton(socket.AF_INET6, '::')
+        except socket.error:
+            # We gotta fake it.
+            return bruteIsIPV6(s)
+        return False
+
+# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

src/utils/seq.py

@@ -1,5 +1,5 @@
 ###
-# Copyright (c) 2004-2005, Jeremiah Fincher
+# Copyright (c) 2002-2005, Jeremiah Fincher
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,21 +27,19 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-from supybot.test import *
+def window(L, size):
+    """Returns a sliding 'window' through the list L of size size."""
+    assert not isinstance(L, int), 'Argument order swapped: window(L, size)'
+    if size < 1:
+        raise ValueError, 'size <= 0 disallowed.'
+    for i in xrange(len(L) - (size-1)):
+        yield L[i:i+size]
 
-import supybot.webutils as webutils
+def mapinto(f, L):
+    for (i, x) in enumerate(L):
+        L[i] = f(x)
 
 
-class WebutilsTestCase(SupyTestCase):
-    def testGetDomain(self):
-        self.assertEqual(webutils.getDomain('http://slashdot.org/foo/bar.exe'),
-                         'slashdot.org')
-
-    if network:
-        def testGetUrlWithSize(self):
-            url = 'http://slashdot.org/'
-            self.failUnless(len(webutils.getUrl(url, 1024)) == 1024)
-
 
 # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
 

src/utils/str.py

@@ -0,0 +1,354 @@
+###
+# Copyright (c) 2002-2005, Jeremiah Fincher
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice,
+#     this list of conditions, and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions, and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#   * Neither the name of the author of this software nor the name of
+#     contributors to this software may be used to endorse or promote products
+#     derived from this software without specific prior written consent.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###
+
+"""
+Simple utility functions related to strings.
+"""
+
+import re
+import new
+import string
+import textwrap
+
+import supybot.structures as structures
+
+curry = new.instancemethod
+
+chars = string.maketrans('', '')
+
+def rsplit(s, sep=None, maxsplit=-1):
+    """Equivalent to str.split, except splitting from the right."""
+    if sys.version_info < (2, 4, 0):
+        if sep is not None:
+            sep = sep[::-1]
+        L = s[::-1].split(sep, maxsplit)
+        L.reverse()
+        return [s[::-1] for s in L]
+    else:
+        return s.rsplit(sep, maxsplit)
+
+def normalizeWhitespace(s):
+    """Normalizes the whitespace in a string; \s+ becomes one space."""
+    return ' '.join(s.split())
+
+def distance(s, t):
+    """Returns the levenshtein edit distance between two strings."""
+    n = len(s)
+    m = len(t)
+    if n == 0:
+        return m
+    elif m == 0:
+        return n
+    d = []
+    for i in xrange(n+1):
+        d.append([])
+        for j in xrange(m+1):
+            d[i].append(0)
+            d[0][j] = j
+        d[i][0] = i
+    for i in xrange(1, n+1):
+        cs = s[i-1]
+        for j in xrange(1, m+1):
+            ct = t[j-1]
+            cost = int(cs != ct)
+            d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cost)
+    return d[n][m]
+
+_soundextrans = string.maketrans(string.ascii_uppercase,
+                                 '01230120022455012623010202')
+_notUpper = chars.translate(chars, string.ascii_uppercase)
+def soundex(s, length=4):
+    """Returns the soundex hash of a given string."""
+    s = s.upper() # Make everything uppercase.
+    s = s.translate(chars, _notUpper) # Delete non-letters.
+    if not s:
+        raise ValueError, 'Invalid string for soundex: %s'
+    firstChar = s[0] # Save the first character.
+    s = s.translate(_soundextrans) # Convert to soundex numbers.
+    s = s.lstrip(s[0]) # Remove all repeated first characters.
+    L = [firstChar]
+    for c in s:
+        if c != L[-1]:
+            L.append(c)
+    L = [c for c in L if c != '0'] + (['0']*(length-1))
+    s = ''.join(L)
+    return length and s[:length] or s.rstrip('0')
+
+def dqrepr(s):
+    """Returns a repr() of s guaranteed to be in double quotes."""
+    # The wankers-that-be decided not to use double-quotes anymore in 2.3.
+    # return '"' + repr("'\x00" + s)[6:]
+    return '"%s"' % s.encode('string_escape').replace('"', '\\"')
+
+def quoted(s):
+    """Returns a quoted s."""
+    return '"%s"' % s
+
+def _getSep(s):
+    if len(s) < 2:
+        raise ValueError, 'string given to _getSep is too short: %r' % s
+    if s.startswith('m') or s.startswith('s'):
+        separator = s[1]
+    else:
+        separator = s[0]
+    if separator.isalnum() or separator in '{}[]()<>':
+        raise ValueError, \
+              'Invalid separator: separator must not be alphanumeric or in ' \
+              '"{}[]()<>"'
+    return separator
+
+def _getSplitterRe(s):
+    separator = _getSep(s)
+    return re.compile(r'(?<!\\)%s' % re.escape(separator))
+        
+def perlReToPythonRe(s):
+    """Converts a string representation of a Perl regular expression (i.e.,
+    m/^foo$/i or /foo|bar/) to a Python regular expression.
+    """
+    sep = _getSep(s)
+    splitter = _getSplitterRe(s)
+    try:
+        (kind, regexp, flags) = splitter.split(s)
+    except ValueError: # Unpack list of wrong size.
+        raise ValueError, 'Must be of the form m/.../ or /.../'
+    regexp = regexp.replace('\\'+sep, sep)
+    if kind not in ('', 'm'):
+        raise ValueError, 'Invalid kind: must be in ("", "m")'
+    flag = 0
+    try:
+        for c in flags.upper():
+            flag |= getattr(re, c)
+    except AttributeError:
+        raise ValueError, 'Invalid flag: %s' % c
+    try:
+        return re.compile(regexp, flag)
+    except re.error, e:
+        raise ValueError, str(e)
+
+def perlReToReplacer(s):
+    """Converts a string representation of a Perl regular expression (i.e.,
+    s/foo/bar/g or s/foo/bar/i) to a Python function doing the equivalent
+    replacement.
+    """
+    sep = _getSep(s)
+    splitter = _getSplitterRe(s)
+    try:
+        (kind, regexp, replace, flags) = splitter.split(s)
+    except ValueError: # Unpack list of wrong size.
+        raise ValueError, 'Must be of the form s/.../.../'
+    regexp = regexp.replace('\x08', r'\b')
+    replace = replace.replace('\\'+sep, sep)
+    for i in xrange(10):
+        replace = replace.replace(chr(i), r'\%s' % i)
+    if kind != 's':
+        raise ValueError, 'Invalid kind: must be "s"'
+    g = False
+    if 'g' in flags:
+        g = True
+        flags = filter('g'.__ne__, flags)
+    r = perlReToPythonRe('/'.join(('', regexp, flags)))
+    if g:
+        return curry(r.sub, replace)
+    else:
+        return lambda s: r.sub(replace, s, 1)
+
+_perlVarSubstituteRe = re.compile(r'\$\{([^}]+)\}|\$([a-zA-Z][a-zA-Z0-9]*)')
+def perlVariableSubstitute(vars, text):
+    def replacer(m):
+        (braced, unbraced) = m.groups()
+        var = braced or unbraced
+        try:
+            x = vars[var]
+            if callable(x):
+                return x()
+            else:
+                return str(x)
+        except KeyError:
+            if braced:
+                return '${%s}' % braced
+            else:
+                return '$' + unbraced
+    return _perlVarSubstituteRe.sub(replacer, text)
+
+def commaAndify(seq, comma=',', And='and'):
+    """Given a a sequence, returns an English clause for that sequence.
+
+    I.e., given [1, 2, 3], returns '1, 2, and 3'
+    """
+    L = list(seq)
+    if len(L) == 0:
+        return ''
+    elif len(L) == 1:
+        return ''.join(L) # We need this because it raises TypeError.
+    elif len(L) == 2:
+        L.insert(1, And)
+        return ' '.join(L)
+    else:
+        L[-1] = '%s %s' % (And, L[-1])
+        sep = '%s ' % comma
+        return sep.join(L)
+
+_unCommaTheRe = re.compile(r'(.*),\s*(the)$', re.I)
+def unCommaThe(s):
+    """Takes a string of the form 'foo, the' and turns it into 'the foo'."""
+    m = _unCommaTheRe.match(s)
+    if m is not None:
+        return '%s %s' % (m.group(2), m.group(1))
+    else:
+        return s
+
+def ellipsisify(s, n):
+    """Returns a shortened version of s.  Produces up to the first n chars at
+    the nearest word boundary.
+    """
+    if len(s) <= n:
+        return s
+    else:
+        return (textwrap.wrap(s, n-3)[0] + '...')
+
+plurals = structures.TwoWayDictionary({})
+def matchCase(s1, s2):
+    """Matches the case of s1 in s2"""
+    if s1.isupper():
+        return s2.upper()
+    else:
+        L = list(s2)
+        for (i, char) in enumerate(s1[:len(s2)]):
+            if char.isupper():
+                L[i] = L[i].upper()
+        return ''.join(L)
+
+consonants = 'bcdfghjklmnpqrstvwxz'
+_pluralizeRegex = re.compile('[%s]y$' % consonants)
+def pluralize(s, i=2):
+    """Returns the plural of s based on its number i.  Put any exceptions to
+    the general English rule of appending 's' in the plurals dictionary.
+    """
+    if i == 1:
+        return s
+    else:
+        lowered = s.lower()
+        # Exception dictionary
+        if lowered in plurals:
+            return matchCase(s, plurals[lowered])
+        # Words ending with 'ch', 'sh' or 'ss' such as 'punch(es)', 'fish(es)
+        # and miss(es)
+        elif any(lowered.endswith, ['x', 'ch', 'sh', 'ss']):
+            return matchCase(s, s+'es')
+        # Words ending with a consonant followed by a 'y' such as
+        # 'try (tries)' or 'spy (spies)'
+        elif _pluralizeRegex.search(lowered):
+            return matchCase(s, s[:-1] + 'ies')
+        # In all other cases, we simply add an 's' to the base word
+        else:
+            return matchCase(s, s+'s')
+
+_depluralizeRegex = re.compile('[%s]ies' % consonants)
+def depluralize(s):
+    """Returns the singular of s."""
+    lowered = s.lower()
+    if lowered in plurals:
+        return matchCase(s, plurals[lowered])
+    elif any(lowered.endswith, ['ches', 'shes', 'sses']):
+        return s[:-2]
+    elif re.search(_depluralizeRegex, lowered):
+        return s[:-3] + 'y'
+    else:
+        if lowered.endswith('s'):
+            return s[:-1] # Chop off 's'.
+        else:
+            return s # Don't know what to do.
+
+def nItems(item, n, between=None):
+    """Works like this:
+
+    >>> nItems('clock', 1)
+    '1 clock'
+
+    >>> nItems('clock', 10)
+    '10 clocks'
+
+    >>> nItems('clock', 10, between='grandfather')
+    '10 grandfather clocks'
+    """
+    if between is None:
+        return '%s %s' % (n, pluralize(item, n))
+    else:
+        return '%s %s %s' % (n, between, pluralize(item, n))
+
+def be(i):
+    """Returns the form of the verb 'to be' based on the number i."""
+    if i == 1:
+        return 'is'
+    else:
+        return 'are'
+
+def has(i):
+    """Returns the form of the verb 'to have' based on the number i."""
+    if i == 1:
+        return 'has'
+    else:
+        return 'have'
+
+_formatRe = re.compile('%([isfbhL])')
+def format(s, *args, **kwargs):
+    kwargs.setdefault('decimalSeparator', decimalSeparator)
+    kwargs.setdefault('thousandsSeparator', thousandsSeparator)
+    args = list(args)
+    args.reverse() # For more efficiency popping.
+    def sub(match):
+        char = match.group(1)
+        if char == 's': # Plain string.
+            return str(args.pop())
+        elif char == 'i': # Integer
+            # XXX Improve me!
+            return str(args.pop())
+        elif char == 'f': # Float
+            # XXX Improve me!
+            return str(args.pop())
+        elif char == 'b': # form of the verb 'to be'
+            return be(args.pop())
+        elif char == 'h': # form of the verb 'to have'
+            return has(args.pop())
+        elif char == 'L': # commaAndify the list.
+            return commaAndify(args.pop())
+        else:
+            assert False, 'Invalid char in sub (in format).'
+    return _formatRe.sub(sub, s)
+            
+def toBool(s):
+    s = s.strip().lower()
+    if s in ('true', 'on', 'enable', 'enabled', '1'):
+        return True
+    elif s in ('false', 'off', 'disable', 'disabled', '0'):
+        return False
+    else:
+        raise ValueError, 'Invalid string for toBool: %s' % quoted(s)
+
+# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

src/utils/web.py

@@ -27,24 +27,22 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
 
-
-
-import supybot.fix as fix
-
 import re
 import socket
 import urllib
 import urllib2
 import httplib
+import sgmllib
 import urlparse
+import htmlentitydefs
 
-import supybot.conf as conf
+from str import normalizeWhitespace
 
 Request = urllib2.Request
 urlquote = urllib.quote
 urlunquote = urllib.unquote
 
-class WebError(Exception):
+class Error(Exception):
     pass
 
 # XXX We should tighten this up a bit.
@@ -75,14 +73,18 @@ def strError(e):
     else:
         return str(e)
 
-_headers = {
+defaultHeaders = {
-    'User-agent': 'Mozilla/4.0 (compatible; Supybot %s)' % conf.version,
+    'User-agent': 'Mozilla/5.0 (compatible; utils.web python module)'
     }
 
+# Other modules should feel free to replace this with an appropriate
+# application-specific function.  Feel free to use a callable here.
+proxy = None
+
 def getUrlFd(url, headers=None):
     """Gets a file-like object for a url."""
     if headers is None:
-        headers = _headers
+        headers = defaultHeaders
     try:
         if not isinstance(url, urllib2.Request):
             if '#' in url:
@@ -90,7 +92,7 @@ def getUrlFd(url, headers=None):
             request = urllib2.Request(url, headers=headers)
         else:
             request = url
-        httpProxy = conf.supybot.protocols.http.proxy()
+        httpProxy = force(proxy)
         if httpProxy:
             request.set_proxy(httpProxy, 'http')
         fd = urllib2.urlopen(request)
@@ -125,5 +127,39 @@ def getUrl(url, size=None, headers=None):
 def getDomain(url):
     return urlparse.urlparse(url)[1]
 
+class HtmlToText(sgmllib.SGMLParser):
+    """Taken from some eff-bot code on c.l.p."""
+    entitydefs = htmlentitydefs.entitydefs.copy()
+    entitydefs['nbsp'] = ' '
+    def __init__(self, tagReplace=' '):
+        self.data = []
+        self.tagReplace = tagReplace
+        sgmllib.SGMLParser.__init__(self)
+
+    def unknown_starttag(self, tag, attr):
+        self.data.append(self.tagReplace)
+
+    def unknown_endtag(self, tag):
+        self.data.append(self.tagReplace)
+
+    def handle_data(self, data):
+        self.data.append(data)
+
+    def getText(self):
+        text = ''.join(self.data).strip()
+        return normalizeWhitespace(text)
+
+def htmlToText(s, tagReplace=' '):
+    """Turns HTML into text.  tagReplace is a string to replace HTML tags with.
+    """
+    x = HtmlToText(tagReplace)
+    x.feed(s)
+    return x.getText()
+
+def mungeEmail(s):
+    s = s.replace('@', ' AT ')
+    s = s.replace('.', ' DOT ')
+    return s
+
 # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
 

src/world.py

@@ -31,10 +31,6 @@
 Module for general worldly stuff, like global variables and whatnot.
 """
 
-
-
-import supybot.fix as fix
-
 import gc
 import os
 import sys
@@ -54,8 +50,6 @@ startedAt = time.time() # Just in case it doesn't get set later.
 starting = False
 
 mainThread = threading.currentThread()
-# ??? Should we do this?  What do we gain?
-# assert 'MainThread' in repr(mainThread)
 
 def isMainThread():
     return mainThread is threading.currentThread()

test/__init__.py

@@ -27,6 +27,7 @@
 # POSSIBILITY OF SUCH DAMAGE.
 ###
     
+# We're just masquerading as a plugin :)
 import test
 
 # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

test/test_callbacks.py

@@ -42,7 +42,7 @@ class TokenizerTestCase(SupyTestCase):
         self.assertEqual(tokenize(''), [])
 
     def testNullCharacter(self):
-        self.assertEqual(tokenize(utils.dqrepr('\0')), ['\0'])
+        self.assertEqual(tokenize(utils.str.dqrepr('\0')), ['\0'])
 
     def testSingleDQInDQString(self):
         self.assertEqual(tokenize('"\\""'), ['"'])

test/test_utils.py

@@ -28,11 +28,24 @@
 ###
 
 from supybot.test import *
-import sets
 
+import time
 import supybot.utils as utils
 
-class UtilsTest(SupyTestCase):
+class GenTest(SupyTestCase):
+    def testInsensitivePreservingDict(self):
+        ipd = utils.InsensitivePreservingDict
+        d = ipd(dict(Foo=10))
+        self.failUnless(d['foo'] == 10)
+        self.assertEqual(d.keys(), ['Foo'])
+        self.assertEqual(d.get('foo'), 10)
+        self.assertEqual(d.get('Foo'), 10)
+
+    def testFindBinaryInPath(self):
+        if os.name == 'posix':
+            self.assertEqual(None, utils.findBinaryInPath('asdfhjklasdfhjkl'))
+            self.failUnless(utils.findBinaryInPath('sh').endswith('/bin/sh'))
+
     def testExnToString(self):
         try:
             raise KeyError, 1
@@ -43,270 +56,21 @@ class UtilsTest(SupyTestCase):
         except Exception, e:
             self.assertEqual(utils.exnToString(e), 'EOFError')
 
-    def testMatchCase(self):
+    def testSaltHash(self):
-        f = utils.matchCase
+        s = utils.saltHash('jemfinch')
-        self.assertEqual('bar', f('foo', 'bar'))
+        (salt, hash) = s.split('|')
-        self.assertEqual('Bar', f('Foo', 'bar'))
+        self.assertEqual(utils.saltHash('jemfinch', salt=salt), s)
-        self.assertEqual('BAr', f('FOo', 'bar'))
-        self.assertEqual('BAR', f('FOO', 'bar'))
-        self.assertEqual('bAR', f('fOO', 'bar'))
-        self.assertEqual('baR', f('foO', 'bar'))
-        self.assertEqual('BaR', f('FoO', 'bar'))
 
-    def testPluralize(self):
+    def testSafeEval(self):
-        f = utils.pluralize
+        for s in ['1', '()', '(1,)', '[]', '{}', '{1:2}', '{1:(2,3)}',
-        self.assertEqual('bike', f('bike', 1))
+                  '1.0', '[1,2,3]', 'True', 'False', 'None',
-        self.assertEqual('bikes', f('bike', 2))
+                  '(True,False,None)', '"foo"', '{"foo": "bar"}']:
-        self.assertEqual('BIKE', f('BIKE', 1))
+            self.assertEqual(eval(s), utils.safeEval(s))
-        self.assertEqual('BIKES', f('BIKE', 2))
+        for s in ['lambda: 2', 'import foo', 'foo.bar']:
-        self.assertEqual('match', f('match', 1))
+            self.assertRaises(ValueError, utils.safeEval, s)
-        self.assertEqual('matches', f('match', 2))
-        self.assertEqual('Patch', f('Patch', 1))
-        self.assertEqual('Patches', f('Patch', 2))
-        self.assertEqual('fish', f('fish', 1))
-        self.assertEqual('fishes', f('fish', 2))
-        self.assertEqual('try', f('try', 1))
-        self.assertEqual('tries', f('try', 2))
-        self.assertEqual('day', f('day', 1))
-        self.assertEqual('days', f('day', 2))
 
-    def testDepluralize(self):
+    def testSafeEvalTurnsSyntaxErrorIntoValueError(self):
-        f = utils.depluralize
+        self.assertRaises(ValueError, utils.safeEval, '/usr/local/')
-        self.assertEqual('bike', f('bikes'))
-        self.assertEqual('Bike', f('Bikes'))
-        self.assertEqual('BIKE', f('BIKES'))
-        self.assertEqual('match', f('matches'))
-        self.assertEqual('Match', f('Matches'))
-        self.assertEqual('fish', f('fishes'))
-        self.assertEqual('try', f('tries'))
-
-    def testTimeElapsed(self):
-        self.assertRaises(ValueError, utils.timeElapsed, 0,
-                          leadingZeroes=False, seconds=False)
-        then = 0
-        now = 0
-        for now, expected in [(0, '0 seconds'),
-                              (1, '1 second'),
-                              (60, '1 minute and 0 seconds'),
-                              (61, '1 minute and 1 second'),
-                              (62, '1 minute and 2 seconds'),
-                              (122, '2 minutes and 2 seconds'),
-                              (3722, '1 hour, 2 minutes, and 2 seconds'),
-                              (7322, '2 hours, 2 minutes, and 2 seconds'),
-                              (90061,'1 day, 1 hour, 1 minute, and 1 second'),
-                              (180122, '2 days, 2 hours, 2 minutes, '
-                                       'and 2 seconds')]:
-            self.assertEqual(utils.timeElapsed(now - then), expected)
-
-    def timeElapsedShort(self):
-        self.assertEqual(utils.timeElapsed(123, short=True), '2m 3s')
-
-    def testDistance(self):
-        self.assertEqual(utils.distance('', ''), 0)
-        self.assertEqual(utils.distance('a', 'b'), 1)
-        self.assertEqual(utils.distance('a', 'a'), 0)
-        self.assertEqual(utils.distance('foobar', 'jemfinch'), 8)
-        self.assertEqual(utils.distance('a', 'ab'), 1)
-        self.assertEqual(utils.distance('foo', ''), 3)
-        self.assertEqual(utils.distance('', 'foo'), 3)
-        self.assertEqual(utils.distance('appel', 'nappe'), 2)
-        self.assertEqual(utils.distance('nappe', 'appel'), 2)
-
-    def testAbbrev(self):
-        L = ['abc', 'bcd', 'bbe', 'foo', 'fool']
-        d = utils.abbrev(L)
-        def getItem(s):
-            return d[s]
-        self.assertRaises(KeyError, getItem, 'f')
-        self.assertRaises(KeyError, getItem, 'fo')
-        self.assertRaises(KeyError, getItem, 'b')
-        self.assertEqual(d['bb'], 'bbe')
-        self.assertEqual(d['bc'], 'bcd')
-        self.assertEqual(d['a'], 'abc')
-        self.assertEqual(d['ab'], 'abc')
-        self.assertEqual(d['fool'], 'fool')
-        self.assertEqual(d['foo'], 'foo')
-
-    def testAbbrevFailsWithDups(self):
-        L = ['english', 'english']
-        self.assertRaises(ValueError, utils.abbrev, L)
-
-    def testSoundex(self):
-        L = [('Euler', 'E460'),
-             ('Ellery', 'E460'),
-             ('Gauss', 'G200'),
-             ('Ghosh', 'G200'),
-             ('Hilbert', 'H416'),
-             ('Heilbronn', 'H416'),
-             ('Knuth', 'K530'),
-             ('Kant', 'K530'),
-             ('Lloyd', 'L300'),
-             ('Ladd', 'L300'),
-             ('Lukasiewicz', 'L222'),
-             ('Lissajous', 'L222')]
-        for (name, key) in L:
-            soundex = utils.soundex(name)
-            self.assertEqual(soundex, key,
-                             '%s was %s, not %s' % (name, soundex, key))
-        self.assertRaises(ValueError, utils.soundex, '3')
-        self.assertRaises(ValueError, utils.soundex, "'")
-
-
-    def testDQRepr(self):
-        L = ['foo', 'foo\'bar', 'foo"bar', '"', '\\', '', '\x00']
-        for s in L:
-            r = utils.dqrepr(s)
-            self.assertEqual(s, eval(r), s)
-            self.failUnless(r[0] == '"' and r[-1] == '"', s)
-
-##     def testQuoted(self):
-##         s = 'foo'
-##         t = 'let\'s'
-##         self.assertEqual("'%s'" % s, utils.quoted(s), s)
-##         self.assertEqual('"%s"' % t, utils.quoted(t), t)
-
-    def testPerlReToPythonRe(self):
-        r = utils.perlReToPythonRe('m/foo/')
-        self.failUnless(r.search('foo'))
-        r = utils.perlReToPythonRe('/foo/')
-        self.failUnless(r.search('foo'))
-        r = utils.perlReToPythonRe('m/\\//')
-        self.failUnless(r.search('/'))
-        r = utils.perlReToPythonRe('m/cat/i')
-        self.failUnless(r.search('CAT'))
-        self.assertRaises(ValueError, utils.perlReToPythonRe, 'm/?/')
-
-    def testP2PReDifferentSeparator(self):
-        r = utils.perlReToPythonRe('m!foo!')
-        self.failUnless(r.search('foo'))
-
-    def testPerlReToReplacer(self):
-        f = utils.perlReToReplacer('s/foo/bar/')
-        self.assertEqual(f('foobarbaz'), 'barbarbaz')
-        f = utils.perlReToReplacer('s/fool/bar/')
-        self.assertEqual(f('foobarbaz'), 'foobarbaz')
-        f = utils.perlReToReplacer('s/foo//')
-        self.assertEqual(f('foobarbaz'), 'barbaz')
-        f = utils.perlReToReplacer('s/ba//')
-        self.assertEqual(f('foobarbaz'), 'foorbaz')
-        f = utils.perlReToReplacer('s/ba//g')
-        self.assertEqual(f('foobarbaz'), 'foorz')
-        f = utils.perlReToReplacer('s/ba\\///g')
-        self.assertEqual(f('fooba/rba/z'), 'foorz')
-        f = utils.perlReToReplacer('s/cat/dog/i')
-        self.assertEqual(f('CATFISH'), 'dogFISH')
-        f = utils.perlReToReplacer('s/foo/foo\/bar/')
-        self.assertEqual(f('foo'), 'foo/bar')
-        f = utils.perlReToReplacer('s/^/foo/')
-        self.assertEqual(f('bar'), 'foobar')
-
-    def testPReToReplacerDifferentSeparator(self):
-        f = utils.perlReToReplacer('s#foo#bar#')
-        self.assertEqual(f('foobarbaz'), 'barbarbaz')
-
-    def testPerlReToReplacerBug850931(self):
-        f = utils.perlReToReplacer('s/\b(\w+)\b/\1./g')
-        self.assertEqual(f('foo bar baz'), 'foo. bar. baz.')
-
-    def testPerlVariableSubstitute(self):
-        f = utils.perlVariableSubstitute
-        vars = {'foo': 'bar', 'b a z': 'baz', 'b': 'c', 'i': 100,
-                'f': lambda: 'called'}
-        self.assertEqual(f(vars, '$foo'), 'bar')
-        self.assertEqual(f(vars, '${foo}'), 'bar')
-        self.assertEqual(f(vars, '$b'), 'c')
-        self.assertEqual(f(vars, '${b}'), 'c')
-        self.assertEqual(f(vars, '$i'), '100')
-        self.assertEqual(f(vars, '${i}'), '100')
-        self.assertEqual(f(vars, '$f'), 'called')
-        self.assertEqual(f(vars, '${f}'), 'called')
-        self.assertEqual(f(vars, '${b a z}'), 'baz')
-        self.assertEqual(f(vars, '$b:$i'), 'c:100')
-        
-                         
-    def testFindBinaryInPath(self):
-        if os.name == 'posix':
-            self.assertEqual(None, utils.findBinaryInPath('asdfhjklasdfhjkl'))
-            self.failUnless(utils.findBinaryInPath('sh').endswith('/bin/sh'))
-
-    def testCommaAndify(self):
-        L = ['foo']
-        original = L[:]
-        self.assertEqual(utils.commaAndify(L), 'foo')
-        self.assertEqual(utils.commaAndify(L, And='or'), 'foo')
-        self.assertEqual(L, original)
-        L.append('bar')
-        original = L[:]
-        self.assertEqual(utils.commaAndify(L), 'foo and bar')
-        self.assertEqual(utils.commaAndify(L, And='or'), 'foo or bar')
-        self.assertEqual(L, original)
-        L.append('baz')
-        original = L[:]
-        self.assertEqual(utils.commaAndify(L), 'foo, bar, and baz')
-        self.assertEqual(utils.commaAndify(L, And='or'), 'foo, bar, or baz')
-        self.assertEqual(utils.commaAndify(L, comma=';'), 'foo; bar; and baz')
-        self.assertEqual(utils.commaAndify(L, comma=';', And='or'),
-                         'foo; bar; or baz')
-        self.assertEqual(L, original)
-        self.failUnless(utils.commaAndify(sets.Set(L)))
-
-    def testCommaAndifyRaisesTypeError(self):
-        L = [(2,)]
-        self.assertRaises(TypeError, utils.commaAndify, L)
-        L.append((3,))
-        self.assertRaises(TypeError, utils.commaAndify, L)
-
-    def testUnCommaThe(self):
-        self.assertEqual(utils.unCommaThe('foo bar'), 'foo bar')
-        self.assertEqual(utils.unCommaThe('foo bar, the'), 'the foo bar')
-        self.assertEqual(utils.unCommaThe('foo bar, The'), 'The foo bar')
-        self.assertEqual(utils.unCommaThe('foo bar,the'), 'the foo bar')
-
-    def testNormalizeWhitespace(self):
-        self.assertEqual(utils.normalizeWhitespace('foo   bar'), 'foo bar')
-        self.assertEqual(utils.normalizeWhitespace('foo\nbar'), 'foo bar')
-        self.assertEqual(utils.normalizeWhitespace('foo\tbar'), 'foo bar')
-
-    def testSortBy(self):
-        L = ['abc', 'z', 'AD']
-        utils.sortBy(len, L)
-        self.assertEqual(L, ['z', 'AD', 'abc'])
-        utils.sortBy(str.lower, L)
-        self.assertEqual(L, ['abc', 'AD', 'z'])
-        L = ['supybot', 'Supybot']
-        utils.sortBy(str.lower, L)
-        self.assertEqual(L, ['supybot', 'Supybot'])
-
-    def testSorted(self):
-        L = ['a', 'c', 'b']
-        self.assertEqual(utils.sorted(L), ['a', 'b', 'c'])
-        self.assertEqual(L, ['a', 'c', 'b'])
-        def mycmp(x, y):
-            return -cmp(x, y)
-        self.assertEqual(utils.sorted(L, mycmp), ['c', 'b', 'a'])
-
-    def testNItems(self):
-        self.assertEqual(utils.nItems('tool', 1, 'crazy'), '1 crazy tool')
-        self.assertEqual(utils.nItems('tool', 1), '1 tool')
-        self.assertEqual(utils.nItems('tool', 2, 'crazy'), '2 crazy tools')
-        self.assertEqual(utils.nItems('tool', 2), '2 tools')
-
-    def testItersplit(self):
-        itersplit = utils.itersplit
-        L = [1, 2, 3] * 3
-        s = 'foo bar baz'
-        self.assertEqual(list(itersplit(lambda x: x == 3, L)),
-                         [[1, 2], [1, 2], [1, 2]])
-        self.assertEqual(list(itersplit(lambda x: x == 3, L, yieldEmpty=True)),
-                         [[1, 2], [1, 2], [1, 2], []])
-        self.assertEqual(list(itersplit(lambda x: x, [])), [])
-        self.assertEqual(list(itersplit(lambda c: c.isspace(), s)),
-                         map(list, s.split()))
-        self.assertEqual(list(itersplit('for'.__eq__, ['foo', 'for', 'bar'])),
-                         [['foo'], ['bar']])
-        self.assertEqual(list(itersplit('for'.__eq__,
-                                        ['foo','for','bar','for', 'baz'], 1)),
-                         [['foo'], ['bar', 'for', 'baz']])
 
     def testIterableMap(self):
         class alist(utils.IterableMap):
@@ -333,9 +97,277 @@ class UtilsTest(SupyTestCase):
         self.assertEqual(list(AL.itervalues()), [2, 3, 4])
         self.assertEqual(len(AL), 3)
 
+    def testSortBy(self):
+        L = ['abc', 'z', 'AD']
+        utils.sortBy(len, L)
+        self.assertEqual(L, ['z', 'AD', 'abc'])
+        utils.sortBy(str.lower, L)
+        self.assertEqual(L, ['abc', 'AD', 'z'])
+        L = ['supybot', 'Supybot']
+        utils.sortBy(str.lower, L)
+        self.assertEqual(L, ['supybot', 'Supybot'])
+
+    def testSorted(self):
+        L = ['a', 'c', 'b']
+        self.assertEqual(utils.sorted(L), ['a', 'b', 'c'])
+        self.assertEqual(L, ['a', 'c', 'b'])
+        def mycmp(x, y):
+            return -cmp(x, y)
+        self.assertEqual(utils.sorted(L, mycmp), ['c', 'b', 'a'])
+
+    def testTimeElapsed(self):
+        self.assertRaises(ValueError, utils.timeElapsed, 0,
+                          leadingZeroes=False, seconds=False)
+        then = 0
+        now = 0
+        for now, expected in [(0, '0 seconds'),
+                              (1, '1 second'),
+                              (60, '1 minute and 0 seconds'),
+                              (61, '1 minute and 1 second'),
+                              (62, '1 minute and 2 seconds'),
+                              (122, '2 minutes and 2 seconds'),
+                              (3722, '1 hour, 2 minutes, and 2 seconds'),
+                              (7322, '2 hours, 2 minutes, and 2 seconds'),
+                              (90061,'1 day, 1 hour, 1 minute, and 1 second'),
+                              (180122, '2 days, 2 hours, 2 minutes, '
+                                       'and 2 seconds')]:
+            self.assertEqual(utils.timeElapsed(now - then), expected)
+
+    def timeElapsedShort(self):
+        self.assertEqual(utils.timeElapsed(123, short=True), '2m 3s')
+
+    def testAbbrev(self):
+        L = ['abc', 'bcd', 'bbe', 'foo', 'fool']
+        d = utils.abbrev(L)
+        def getItem(s):
+            return d[s]
+        self.assertRaises(KeyError, getItem, 'f')
+        self.assertRaises(KeyError, getItem, 'fo')
+        self.assertRaises(KeyError, getItem, 'b')
+        self.assertEqual(d['bb'], 'bbe')
+        self.assertEqual(d['bc'], 'bcd')
+        self.assertEqual(d['a'], 'abc')
+        self.assertEqual(d['ab'], 'abc')
+        self.assertEqual(d['fool'], 'fool')
+        self.assertEqual(d['foo'], 'foo')
+
+    def testAbbrevFailsWithDups(self):
+        L = ['english', 'english']
+        self.assertRaises(ValueError, utils.abbrev, L)
+
+
+class StrTest(SupyTestCase):
+    def testMatchCase(self):
+        f = utils.str.matchCase
+        self.assertEqual('bar', f('foo', 'bar'))
+        self.assertEqual('Bar', f('Foo', 'bar'))
+        self.assertEqual('BAr', f('FOo', 'bar'))
+        self.assertEqual('BAR', f('FOO', 'bar'))
+        self.assertEqual('bAR', f('fOO', 'bar'))
+        self.assertEqual('baR', f('foO', 'bar'))
+        self.assertEqual('BaR', f('FoO', 'bar'))
+
+    def testPluralize(self):
+        f = utils.str.pluralize
+        self.assertEqual('bike', f('bike', 1))
+        self.assertEqual('bikes', f('bike', 2))
+        self.assertEqual('BIKE', f('BIKE', 1))
+        self.assertEqual('BIKES', f('BIKE', 2))
+        self.assertEqual('match', f('match', 1))
+        self.assertEqual('matches', f('match', 2))
+        self.assertEqual('Patch', f('Patch', 1))
+        self.assertEqual('Patches', f('Patch', 2))
+        self.assertEqual('fish', f('fish', 1))
+        self.assertEqual('fishes', f('fish', 2))
+        self.assertEqual('try', f('try', 1))
+        self.assertEqual('tries', f('try', 2))
+        self.assertEqual('day', f('day', 1))
+        self.assertEqual('days', f('day', 2))
+
+    def testDepluralize(self):
+        f = utils.str.depluralize
+        self.assertEqual('bike', f('bikes'))
+        self.assertEqual('Bike', f('Bikes'))
+        self.assertEqual('BIKE', f('BIKES'))
+        self.assertEqual('match', f('matches'))
+        self.assertEqual('Match', f('Matches'))
+        self.assertEqual('fish', f('fishes'))
+        self.assertEqual('try', f('tries'))
+
+    def testDistance(self):
+        self.assertEqual(utils.str.distance('', ''), 0)
+        self.assertEqual(utils.str.distance('a', 'b'), 1)
+        self.assertEqual(utils.str.distance('a', 'a'), 0)
+        self.assertEqual(utils.str.distance('foobar', 'jemfinch'), 8)
+        self.assertEqual(utils.str.distance('a', 'ab'), 1)
+        self.assertEqual(utils.str.distance('foo', ''), 3)
+        self.assertEqual(utils.str.distance('', 'foo'), 3)
+        self.assertEqual(utils.str.distance('appel', 'nappe'), 2)
+        self.assertEqual(utils.str.distance('nappe', 'appel'), 2)
+
+    def testSoundex(self):
+        L = [('Euler', 'E460'),
+             ('Ellery', 'E460'),
+             ('Gauss', 'G200'),
+             ('Ghosh', 'G200'),
+             ('Hilbert', 'H416'),
+             ('Heilbronn', 'H416'),
+             ('Knuth', 'K530'),
+             ('Kant', 'K530'),
+             ('Lloyd', 'L300'),
+             ('Ladd', 'L300'),
+             ('Lukasiewicz', 'L222'),
+             ('Lissajous', 'L222')]
+        for (name, key) in L:
+            soundex = utils.str.soundex(name)
+            self.assertEqual(soundex, key,
+                             '%s was %s, not %s' % (name, soundex, key))
+        self.assertRaises(ValueError, utils.str.soundex, '3')
+        self.assertRaises(ValueError, utils.str.soundex, "'")
+
+    def testDQRepr(self):
+        L = ['foo', 'foo\'bar', 'foo"bar', '"', '\\', '', '\x00']
+        for s in L:
+            r = utils.str.dqrepr(s)
+            self.assertEqual(s, eval(r), s)
+            self.failUnless(r[0] == '"' and r[-1] == '"', s)
+
+    def testPerlReToPythonRe(self):
+        f = utils.str.perlReToPythonRe
+        r = f('m/foo/')
+        self.failUnless(r.search('foo'))
+        r = f('/foo/')
+        self.failUnless(r.search('foo'))
+        r = f('m/\\//')
+        self.failUnless(r.search('/'))
+        r = f('m/cat/i')
+        self.failUnless(r.search('CAT'))
+        self.assertRaises(ValueError, f, 'm/?/')
+
+    def testP2PReDifferentSeparator(self):
+        r = utils.str.perlReToPythonRe('m!foo!')
+        self.failUnless(r.search('foo'))
+
+    def testPerlReToReplacer(self):
+        PRTR = utils.str.perlReToReplacer
+        f = PRTR('s/foo/bar/')
+        self.assertEqual(f('foobarbaz'), 'barbarbaz')
+        f = PRTR('s/fool/bar/')
+        self.assertEqual(f('foobarbaz'), 'foobarbaz')
+        f = PRTR('s/foo//')
+        self.assertEqual(f('foobarbaz'), 'barbaz')
+        f = PRTR('s/ba//')
+        self.assertEqual(f('foobarbaz'), 'foorbaz')
+        f = PRTR('s/ba//g')
+        self.assertEqual(f('foobarbaz'), 'foorz')
+        f = PRTR('s/ba\\///g')
+        self.assertEqual(f('fooba/rba/z'), 'foorz')
+        f = PRTR('s/cat/dog/i')
+        self.assertEqual(f('CATFISH'), 'dogFISH')
+        f = PRTR('s/foo/foo\/bar/')
+        self.assertEqual(f('foo'), 'foo/bar')
+        f = PRTR('s/^/foo/')
+        self.assertEqual(f('bar'), 'foobar')
+
+    def testPReToReplacerDifferentSeparator(self):
+        f = utils.str.perlReToReplacer('s#foo#bar#')
+        self.assertEqual(f('foobarbaz'), 'barbarbaz')
+
+    def testPerlReToReplacerBug850931(self):
+        f = utils.str.perlReToReplacer('s/\b(\w+)\b/\1./g')
+        self.assertEqual(f('foo bar baz'), 'foo. bar. baz.')
+
+    def testPerlVariableSubstitute(self):
+        f = utils.str.perlVariableSubstitute
+        vars = {'foo': 'bar', 'b a z': 'baz', 'b': 'c', 'i': 100,
+                'f': lambda: 'called'}
+        self.assertEqual(f(vars, '$foo'), 'bar')
+        self.assertEqual(f(vars, '${foo}'), 'bar')
+        self.assertEqual(f(vars, '$b'), 'c')
+        self.assertEqual(f(vars, '${b}'), 'c')
+        self.assertEqual(f(vars, '$i'), '100')
+        self.assertEqual(f(vars, '${i}'), '100')
+        self.assertEqual(f(vars, '$f'), 'called')
+        self.assertEqual(f(vars, '${f}'), 'called')
+        self.assertEqual(f(vars, '${b a z}'), 'baz')
+        self.assertEqual(f(vars, '$b:$i'), 'c:100')
+        
+    def testCommaAndify(self):
+        f = utils.str.commaAndify
+        L = ['foo']
+        original = L[:]
+        self.assertEqual(f(L), 'foo')
+        self.assertEqual(f(L, And='or'), 'foo')
+        self.assertEqual(L, original)
+        L.append('bar')
+        original = L[:]
+        self.assertEqual(f(L), 'foo and bar')
+        self.assertEqual(f(L, And='or'), 'foo or bar')
+        self.assertEqual(L, original)
+        L.append('baz')
+        original = L[:]
+        self.assertEqual(f(L), 'foo, bar, and baz')
+        self.assertEqual(f(L, And='or'), 'foo, bar, or baz')
+        self.assertEqual(f(L, comma=';'), 'foo; bar; and baz')
+        self.assertEqual(f(L, comma=';', And='or'),
+                         'foo; bar; or baz')
+        self.assertEqual(L, original)
+        self.failUnless(f(set(L)))
+
+    def testCommaAndifyRaisesTypeError(self):
+        L = [(2,)]
+        self.assertRaises(TypeError, utils.str.commaAndify, L)
+        L.append((3,))
+        self.assertRaises(TypeError, utils.str.commaAndify, L)
+
+    def testUnCommaThe(self):
+        f = utils.str.unCommaThe
+        self.assertEqual(f('foo bar'), 'foo bar')
+        self.assertEqual(f('foo bar, the'), 'the foo bar')
+        self.assertEqual(f('foo bar, The'), 'The foo bar')
+        self.assertEqual(f('foo bar,the'), 'the foo bar')
+
+    def testNormalizeWhitespace(self):
+        f = utils.str.normalizeWhitespace
+        self.assertEqual(f('foo   bar'), 'foo bar')
+        self.assertEqual(f('foo\nbar'), 'foo bar')
+        self.assertEqual(f('foo\tbar'), 'foo bar')
+
+    def testNItems(self):
+        nItems = utils.str.nItems
+        self.assertEqual(nItems('tool', 1, 'crazy'), '1 crazy tool')
+        self.assertEqual(nItems('tool', 1), '1 tool')
+        self.assertEqual(nItems('tool', 2, 'crazy'), '2 crazy tools')
+        self.assertEqual(nItems('tool', 2), '2 tools')
+
+    def testEllipsisify(self):
+        f = utils.str.ellipsisify
+        self.assertEqual(f('x'*30, 30), 'x'*30)
+        self.failUnless(len(f('x'*35, 30)) <= 30)
+        self.failUnless(f(' '.join(['xxxx']*10), 30)[:-3].endswith('xxxx'))
+
+
+class IterTest(SupyTestCase):
+    def testSplit(self):
+        itersplit = utils.iter.split
+        L = [1, 2, 3] * 3
+        s = 'foo bar baz'
+        self.assertEqual(list(itersplit(lambda x: x == 3, L)),
+                         [[1, 2], [1, 2], [1, 2]])
+        self.assertEqual(list(itersplit(lambda x: x == 3, L, yieldEmpty=True)),
+                         [[1, 2], [1, 2], [1, 2], []])
+        self.assertEqual(list(itersplit(lambda x: x, [])), [])
+        self.assertEqual(list(itersplit(lambda c: c.isspace(), s)),
+                         map(list, s.split()))
+        self.assertEqual(list(itersplit('for'.__eq__, ['foo', 'for', 'bar'])),
+                         [['foo'], ['bar']])
+        self.assertEqual(list(itersplit('for'.__eq__,
+                                        ['foo','for','bar','for', 'baz'], 1)),
+                         [['foo'], ['bar', 'for', 'baz']])
+
     def testFlatten(self):
         def lflatten(seq):
-            return list(utils.flatten(seq))
+            return list(utils.iter.flatten(seq))
         self.assertEqual(lflatten([]), [])
         self.assertEqual(lflatten([1]), [1])
         self.assertEqual(lflatten(range(10)), range(10))
@@ -347,60 +379,43 @@ class UtilsTest(SupyTestCase):
         self.assertEqual(lflatten([1, [2, [3, 4], 5], 6]), [1, 2, 3, 4, 5, 6])
         self.assertRaises(TypeError, lflatten, 1)
 
-    def testEllipsisify(self):
-        f = utils.ellipsisify
-        self.assertEqual(f('x'*30, 30), 'x'*30)
-        self.failUnless(len(f('x'*35, 30)) <= 30)
-        self.failUnless(f(' '.join(['xxxx']*10), 30)[:-3].endswith('xxxx'))
-
-    def testSaltHash(self):
-        s = utils.saltHash('jemfinch')
-        (salt, hash) = s.split('|')
-        self.assertEqual(utils.saltHash('jemfinch', salt=salt), s)
-
-    def testSafeEval(self):
-        for s in ['1', '()', '(1,)', '[]', '{}', '{1:2}', '{1:(2,3)}',
-                  '1.0', '[1,2,3]', 'True', 'False', 'None',
-                  '(True,False,None)', '"foo"', '{"foo": "bar"}']:
-            self.assertEqual(eval(s), utils.safeEval(s))
-        for s in ['lambda: 2', 'import foo', 'foo.bar']:
-            self.assertRaises(ValueError, utils.safeEval, s)
-
-
-    def testSafeEvalTurnsSyntaxErrorIntoValueError(self):
-        self.assertRaises(ValueError, utils.safeEval, '/usr/local/')
 
+class FileTest(SupyTestCase):
     def testLines(self):
         L = ['foo', 'bar', '#baz', '  ', 'biff']
-        self.assertEqual(list(utils.nonEmptyLines(L)),
+        self.assertEqual(list(utils.file.nonEmptyLines(L)),
                          ['foo', 'bar', '#baz', 'biff'])
-        self.assertEqual(list(utils.nonCommentLines(L)),
+        self.assertEqual(list(utils.file.nonCommentLines(L)),
                          ['foo', 'bar', '  ', 'biff'])
-        self.assertEqual(list(utils.nonCommentNonEmptyLines(L)),
+        self.assertEqual(list(utils.file.nonCommentNonEmptyLines(L)),
                          ['foo', 'bar', 'biff'])
 
+
+class NetTest(SupyTestCase):
     def testIsIP(self):
-        self.failIf(utils.isIP('a.b.c'))
+        isIP = utils.net.isIP
-        self.failIf(utils.isIP('256.0.0.0'))
+        self.failIf(isIP('a.b.c'))
-        self.failUnless(utils.isIP('127.1'))
+        self.failIf(isIP('256.0.0.0'))
-        self.failUnless(utils.isIP('0.0.0.0'))
+        self.failUnless(isIP('127.1'))
-        self.failUnless(utils.isIP('100.100.100.100'))
+        self.failUnless(isIP('0.0.0.0'))
+        self.failUnless(isIP('100.100.100.100'))
         # This test is too flaky to bother with.
         # self.failUnless(utils.isIP('255.255.255.255'))
 
     def testIsIPV6(self):
-        f = utils.isIPV6
+        f = utils.net.isIPV6
         self.failUnless(f('2001::'))
         self.failUnless(f('2001:888:0:1::666'))
 
-    def testInsensitivePreservingDict(self):
+class WebTest(SupyTestCase):
-        ipd = utils.InsensitivePreservingDict
+    def testGetDomain(self):
-        d = ipd(dict(Foo=10))
+        url = 'http://slashdot.org/foo/bar.exe'
-        self.failUnless(d['foo'] == 10)
+        self.assertEqual(utils.web.getDomain(url), 'slashdot.org')
-        self.assertEqual(d.keys(), ['Foo'])
-        self.assertEqual(d.get('foo'), 10)
-        self.assertEqual(d.get('Foo'), 10)
 
+    if network:
+        def testGetUrlWithSize(self):
+            url = 'http://slashdot.org/'
+            self.failUnless(len(utils.web.getUrl(url, 1024)) == 1024)
 
 # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: