Mikaela/Limnoria
1
0
Fork 0
mirror of https://github.com/Mikaela/Limnoria.git synced 2024-11-19 08:59:27 +01:00
Code Issues Projects Releases Wiki Activity

Completely restructured our utils modules.

Browse Source 
Tons of changes.  Here's the summary of things that matter most:

* There is no more supybot.fix.
* There is no more supybot.webutils; now there is supybot.utils.web.
  * It's no longer webutils.WebError, but just utils.web.Error.
* You shouldn't import itertools, ideally, but instead import utils.iter.
* No more using imap/ifilter in commands unless absolutely necessary.  It's
  premature optimization and annoying.
* utils.str.format isn't quite ready yet, but will be soon.  That'll be the
  next big thing to fix in our code.
This commit is contained in:
Jeremy Fincher 2005-01-27 06:59:08 +00:00
parent e62585092d
commit 5fd6bbb52d
46 changed files with 1744 additions and 1705 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
plugins/Admin/plugin.py
View File

@ -27,16 +27,9 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot
__author__ = supybot.authors.jemfinch
import supybot.fix as fix
import sys
import time
import pprint
from itertools import imap
import supybot.log as log
import supybot.conf as conf
@ -164,7 +157,7 @@ class Admin(callbacks.Privmsg):
L = irc.state.channels.keys()
if L:
utils.sortBy(ircutils.toLower, L)
irc.reply(utils.commaAndify(L))
irc.reply(utils.str.commaAndify(L))
else:
irc.reply('I\'m not currently in any channels.')
channels = wrap(channels, ['private'])
@ -194,7 +187,7 @@ class Admin(callbacks.Privmsg):
irc = self.pendingNickChanges.get(irc, None)
if irc is not None:
irc.error('I can\'t change nicks, the server said %s.' %
utils.quoted(msg.args[2]), private=True)
utils.str.quoted(msg.args[2]), private=True)
else:
self.log.debug('Got 438 without Admin.nick being called.')
@ -332,7 +325,7 @@ class Admin(callbacks.Privmsg):
"""
# XXX Add the expirations.
if ircdb.ignores.hostmasks:
irc.reply(utils.commaAndify(imap(repr, ircdb.ignores.hostmasks)))
irc.reply(utils.str.commaAndify(map(repr,ircdb.ignores.hostmasks)))
else:
irc.reply('I\'m not currently globally ignoring anyone.')
ignores = wrap(ignores)

18
plugins/Babelfish/plugin.py
View File

@ -27,11 +27,7 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot
import sets
import random
from itertools import imap
import babelfish
@ -69,7 +65,7 @@ class Babelfish(callbacks.Privmsg):
Returns the languages that Babelfish can translate to/from.
"""
irc.reply(utils.commaAndify(babelfish.available_languages))
irc.reply(utils.str.commaAndify(babelfish.available_languages))
def translate(self, irc, msg, args, fromLang, toLang, text):
"""<from-language> [to] <to-language> <text>
@ -87,15 +83,15 @@ class Babelfish(callbacks.Privmsg):
irc.error('I do not speak any other languages.')
return
else:
irc.error('I only speak %s.' % utils.commaAndify(langs))
irc.error('I only speak %s.' % utils.str.commaAndify(langs))
return
translation = babelfish.translate(text, fromLang, toLang)
irc.reply(utils.htmlToText(translation))
irc.reply(utils.web.htmlToText(translation))
except (KeyError, babelfish.LanguageNotAvailableError), e:
languages = self.registryValue('languages', chan)
if languages:
languages = 'Valid languages include %s' % \
utils.commaAndify(sorted(languages))
utils.str.commaAndify(sorted(languages))
else:
languages = 'I do not speak any other languages.'
irc.errorInvalid('language', str(e), languages)
@ -125,16 +121,16 @@ class Babelfish(callbacks.Privmsg):
irc.error('I do not speak any other languages.')
return
else:
irc.error('I only speak %s.' % utils.commaAndify(langs,
irc.error('I only speak %s.' % utils.str.commaAndify(langs,
And='or'))
return
translations = babelfish.babelize(text, fromLang, toLang)
irc.reply(utils.htmlToText(translations[-1]))
irc.reply(utils.web.htmlToText(translations[-1]))
except (KeyError, babelfish.LanguageNotAvailableError), e:
languages = self.registryValue('languages', chan)
if languages:
languages = 'Valid languages include %s' % \
utils.commaAndify(sorted(languages))
utils.str.commaAndify(sorted(languages))
else:
languages = 'I do not speak any other languages.'
irc.errorInvalid('language', str(e), languages)

4
plugins/Channel/config.py
View File

@ -47,12 +47,12 @@ class BanmaskStyle(registry.SpaceSeparatedSetOfStrings):
'This is a bug.'
registry.SpaceSeparatedSetOfStrings.__init__(self, *args, **kwargs)
self.__doc__ = 'Valid values include %s.' % \
utils.commaAndify(map(repr, self.validStrings))
utils.str.commaAndify(map(repr, self.validStrings))
def help(self):
strings = [s for s in self.validStrings if s]
return '%s Valid strings: %s.' % \
(self._help, utils.commaAndify(strings))
(self._help, utils.str.commaAndify(strings))
def normalize(self, s):
lowered = s.lower()

34
plugins/Channel/plugin.py
View File

@ -30,8 +30,6 @@
import sys
import time
from itertools import imap
import supybot.conf as conf
import supybot.ircdb as ircdb
import supybot.utils as utils
@ -287,12 +285,12 @@ class Channel(callbacks.Privmsg):
self.log.debug('In kban')
if not irc.isNick(bannedNick):
self.log.warning('%s tried to kban a non nick: %s',
utils.quoted(msg.prefix),
utils.quoted(bannedNick))
utils.str.quoted(msg.prefix),
utils.str.quoted(bannedNick))
raise callbacks.ArgumentError
elif bannedNick == irc.nick:
self.log.warning('%s tried to make me kban myself.',
utils.quoted(msg.prefix))
utils.str.quoted(msg.prefix))
irc.error('I cowardly refuse to kickban myself.')
return
if not reason:
@ -330,7 +328,7 @@ class Channel(callbacks.Privmsg):
if ircutils.hostmaskPatternEqual(banmask, irc.prefix):
if ircutils.hostmaskPatternEqual(banmask, irc.prefix):
self.log.warning('%s tried to make me kban myself.',
utils.quoted(msg.prefix))
utils.str.quoted(msg.prefix))
irc.error('I cowardly refuse to ban myself.')
return
else:
@ -354,7 +352,7 @@ class Channel(callbacks.Privmsg):
elif ircdb.checkCapability(msg.prefix, capability):
if ircdb.checkCapability(bannedHostmask, capability):
self.log.warning('%s tried to ban %s, but both have %s',
msg.prefix, utils.quoted(bannedHostmask),
msg.prefix, utils.str.quoted(bannedHostmask),
capability)
irc.error('%s has %s too, you can\'t ban him/her/it.' %
(bannedNick, capability))
@ -362,7 +360,7 @@ class Channel(callbacks.Privmsg):
doBan()
else:
self.log.warning('%s attempted kban without %s',
utils.quoted(msg.prefix), capability)
utils.str.quoted(msg.prefix), capability)
irc.errorNoCapability(capability)
exact,nick,user,host
kban = wrap(kban,
@ -515,7 +513,7 @@ class Channel(callbacks.Privmsg):
# XXX Add the expirations.
c = ircdb.channels.getChannel(channel)
if c.bans:
irc.reply(utils.commaAndify(map(utils.dqrepr, c.bans)))
irc.reply(utils.str.commaAndify(map(utils.str.dqrepr, c.bans)))
else:
irc.reply('There are currently no permanent bans on %s' % channel)
permbans = wrap(permbans, [('checkChannelCapability', 'op')])
@ -561,11 +559,11 @@ class Channel(callbacks.Privmsg):
c = ircdb.channels.getChannel(channel)
if len(c.ignores) == 0:
s = 'I\'m not currently ignoring any hostmasks in %s' % \
utils.quoted(channel)
utils.str.quoted(channel)
irc.reply(s)
else:
L = sorted(c.ignores)
irc.reply(utils.commaAndify(imap(repr, L)))
irc.reply(utils.str.commaAndify(map(repr, L)))
ignores = wrap(ignores, [('checkChannelCapability', 'op')])
def addcapability(self, irc, msg, args, channel, user, capabilities):
@ -602,8 +600,9 @@ class Channel(callbacks.Privmsg):
ircdb.users.setUser(user)
if fail:
irc.error('That user didn\'t have the %s %s.' %
(utils.commaAndify(fail),
utils.pluralize('capability', len(fail))), Raise=True)
(utils.str.commaAndify(fail),
utils.str.pluralize('capability', len(fail))),
Raise=True)
irc.replySuccess()
removecapability = wrap(removecapability,
[('checkChannelCapability', 'op'),
@ -662,8 +661,9 @@ class Channel(callbacks.Privmsg):
ircdb.channels.setChannel(channel, chan)
if fail:
irc.error('I do not know about the %s %s.' %
(utils.commaAndify(fail),
utils.pluralize('capability', len(fail))), Raise=True)
(utils.str.commaAndify(fail),
utils.str.pluralize('capability', len(fail))),
Raise=True)
irc.replySuccess()
unsetcapability = wrap(unsetcapability,
[('checkChannelCapability', 'op'),
@ -774,7 +774,7 @@ class Channel(callbacks.Privmsg):
L.append(channel)
if L:
L.sort()
s = 'I\'m currently lobotomized in %s.' % utils.commaAndify(L)
s = 'I\'m currently lobotomized in %s.' % utils.str.commaAndify(L)
irc.reply(s)
else:
irc.reply('I\'m not currently lobotomized in any channels.')
@ -787,7 +787,7 @@ class Channel(callbacks.Privmsg):
"""
L = list(irc.state.channels[channel].users)
utils.sortBy(str.lower, L)
irc.reply(utils.commaAndify(L))
irc.reply(utils.str.commaAndify(L))
nicks = wrap(nicks, ['inChannel']) # XXX Check that the caller is in chan.
def alertOps(self, irc, channel, s, frm=None):

5
plugins/Config/plugin.py
View File

@ -28,7 +28,6 @@
###
import os
import getopt
import signal
import supybot.log as log
@ -131,7 +130,7 @@ class Config(callbacks.Privmsg):
"""
L = self._list(group)
if L:
irc.reply(utils.commaAndify(L))
irc.reply(utils.str.commaAndify(L))
else:
irc.error('There don\'t seem to be any values in %s.' % group._name)
list = wrap(list, ['configVar'])
@ -148,7 +147,7 @@ class Config(callbacks.Privmsg):
if not ircutils.isChannel(possibleChannel):
L.append(name)
if L:
irc.reply(utils.commaAndify(L))
irc.reply(utils.str.commaAndify(L))
else:
irc.reply('There were no matching configuration variables.')
search = wrap(search, ['lowered']) # XXX compose with withoutSpaces?

10
plugins/Dict/plugin.py
View File

@ -53,7 +53,7 @@ class Dict(callbacks.Privmsg):
conn = dictclient.Connection(server)
dbs = conn.getdbdescs().keys()
dbs.sort()
irc.reply(utils.commaAndify(dbs))
irc.reply(utils.str.commaAndify(dbs))
except socket.error, e:
irc.error(webutils.strError(e))
dictionaries = wrap(dictionaries)
@ -102,21 +102,21 @@ class Dict(callbacks.Privmsg):
if not definitions:
if dictionary == '*':
irc.reply('No definition for %s could be found.' %
utils.quoted(word))
utils.str.quoted(word))
else:
irc.reply('No definition for %s could be found in %s' %
(utils.quoted(word), ircutils.bold(dictionary)))
(utils.str.quoted(word), ircutils.bold(dictionary)))
return
L = []
for d in definitions:
dbs.add(ircutils.bold(d.getdb().getname()))
(db, s) = (d.getdb().getname(), d.getdefstr())
db = ircutils.bold(db)
s = utils.normalizeWhitespace(s).rstrip(';.,')
s = utils.str.normalizeWhitespace(s).rstrip(';.,')
L.append('%s: %s' % (db, s))
utils.sortBy(len, L)
if dictionary == '*' and len(dbs) > 1:
s = '%s responded: %s' % (utils.commaAndify(dbs), '; '.join(L))
s = '%s responded: %s' % (utils.str.commaAndify(dbs), '; '.join(L))
else:
s = '; '.join(L)
irc.reply(s)

2
plugins/Format/plugin.py
View File

@ -144,7 +144,7 @@ class Format(callbacks.Privmsg):
Returns the text surrounded by double quotes.
"""
irc.reply(utils.dqrepr(text))
irc.reply(utils.str.dqrepr(text))
repr = wrap(repr, ['text'])
def concat(self, irc, msg, args, first, second):

19
plugins/Math/plugin.py
View File

@ -29,14 +29,11 @@
from __future__ import division
import supybot.plugins as plugins
import re
import math
import cmath
import types
import string
from itertools import imap
import supybot.utils as utils
from supybot.commands import *
@ -160,12 +157,12 @@ class Math(callbacks.Privmsg):
crash to the bot with something like 10**10**10**10. One consequence
is that large values such as 10**24 might not be exact.
"""
if text != text.translate(string.ascii, '_[]'):
if text != text.translate(utils.str.chars, '_[]'):
irc.error('There\'s really no reason why you should have '
'underscores or brackets in your mathematical '
'expression. Please remove them.')
return
#text = text.translate(string.ascii, '_[] \t')
#text = text.translate(utils.str.chars, '_[] \t')
if 'lambda' in text:
irc.error('You can\'t use lambda in this command.')
return
@ -188,7 +185,7 @@ class Math(callbacks.Privmsg):
text = self._mathRe.sub(handleMatch, text)
try:
self.log.info('evaluating %s from %s' %
(utils.quoted(text), msg.prefix))
(utils.str.quoted(text), msg.prefix))
x = complex(eval(text, self._mathEnv, self._mathEnv))
irc.reply(self._complexToString(x))
except OverflowError:
@ -209,21 +206,21 @@ class Math(callbacks.Privmsg):
math, and can thus cause the bot to suck up CPU. Hence it requires
the 'trusted' capability to use.
"""
if text != text.translate(string.ascii, '_[]'):
if text != text.translate(utils.str.chars, '_[]'):
irc.error('There\'s really no reason why you should have '
'underscores or brackets in your mathematical '
'expression. Please remove them.')
return
# This removes spaces, too, but we'll leave the removal of _[] for
# safety's sake.
text = text.translate(string.ascii, '_[] \t')
text = text.translate(utils.str.chars, '_[] \t')
if 'lambda' in text:
irc.error('You can\'t use lambda in this command.')
return
text = text.replace('lambda', '')
try:
self.log.info('evaluating %s from %s' %
(utils.quoted(text), msg.prefix))
(utils.str.quoted(text), msg.prefix))
irc.reply(str(eval(text, self._mathEnv, self._mathEnv)))
except OverflowError:
maxFloat = math.ldexp(0.9999999999999999, 1024)
@ -280,12 +277,12 @@ class Math(callbacks.Privmsg):
stack.append(eval(s, self._mathEnv, self._mathEnv))
except SyntaxError:
irc.error('%s is not a defined function.' %
utils.quoted(arg))
utils.str.quoted(arg))
return
if len(stack) == 1:
irc.reply(str(self._complexToString(complex(stack[0]))))
else:
s = ', '.join(imap(self._complexToString, imap(complex, stack)))
s = ', '.join(map(self._complexToString, map(complex, stack)))
irc.reply('Stack: [%s]' % s)
def convert(self, irc, msg, args, number, unit1, unit2):

58
plugins/Misc/plugin.py
View File

@ -27,16 +27,10 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot
import supybot.fix as fix
import os
import sys
import time
from itertools import imap, ifilter
import supybot.log as log
import supybot.conf as conf
import supybot.utils as utils
@ -46,9 +40,10 @@ import supybot.ircdb as ircdb
import supybot.irclib as irclib
import supybot.ircmsgs as ircmsgs
import supybot.ircutils as ircutils
import supybot.webutils as webutils
import supybot.callbacks as callbacks
from supybot.utils.iter import ifilter
class Misc(callbacks.Privmsg):
def __init__(self):
super(Misc, self).__init__()
@ -123,7 +118,7 @@ class Misc(callbacks.Privmsg):
(not private and isPublic(cb))]
names.sort()
if names:
irc.reply(utils.commaAndify(names))
irc.reply(utils.str.commaAndify(names))
else:
if private:
irc.reply('There are no private plugins.')
@ -148,7 +143,7 @@ class Misc(callbacks.Privmsg):
commands.append(s)
if commands:
commands.sort()
irc.reply(utils.commaAndify(commands))
irc.reply(utils.str.commaAndify(commands))
else:
irc.error('That plugin exists, but it has no '
'commands with help.')
@ -177,7 +172,7 @@ class Misc(callbacks.Privmsg):
L.append('%s %s' % (name, key))
if L:
L.sort()
irc.reply(utils.commaAndify(L))
irc.reply(utils.str.commaAndify(L))
else:
irc.reply('No appropriate commands were found.')
apropos = wrap(apropos, ['lowered'])
@ -211,7 +206,7 @@ class Misc(callbacks.Privmsg):
names = sorted([cb.name() for cb in cbs])
irc.error('That command exists in the %s plugins. '
'Please specify exactly which plugin command '
'you want help with.'% utils.commaAndify(names))
'you want help with.'% utils.str.commaAndify(names))
else:
getHelp(cbs[0])
else:
@ -235,9 +230,9 @@ class Misc(callbacks.Privmsg):
Returns the version of the current bot.
"""
try:
newest = webutils.getUrl('http://supybot.sf.net/version.txt')
newest = utils.web.getUrl('http://supybot.sf.net/version.txt')
newest ='The newest version available online is %s.'%newest.strip()
except webutils.WebError, e:
except utils.web.Error, e:
self.log.warning('Couldn\'t get website version: %r', e)
newest = 'I couldn\'t fetch the newest version ' \
'from the Supybot website.'
@ -265,13 +260,13 @@ class Misc(callbacks.Privmsg):
if cbs:
names = [cb.name() for cb in cbs]
names.sort()
plugin = utils.commaAndify(names)
plugin = utils.str.commaAndify(names)
if irc.nested:
irc.reply(utils.commaAndify(names))
irc.reply(utils.str.commaAndify(names))
else:
irc.reply('The %s command is available in the %s %s.' %
(utils.quoted(command), plugin,
utils.pluralize('plugin', len(names))))
(utils.str.quoted(command), plugin,
utils.str.pluralize('plugin', len(names))))
else:
irc.error('There is no such command %s.' % command)
plugin = wrap(plugin, ['commandName'])
@ -287,7 +282,7 @@ class Misc(callbacks.Privmsg):
return
module = sys.modules[cb.__class__.__module__]
if hasattr(module, '__author__') and module.__author__:
irc.reply(utils.mungeEmailForWeb(str(module.__author__)))
irc.reply(utils.web.mungeEmail(str(module.__author__)))
else:
irc.reply('That plugin doesn\'t have an author that claims it.')
author = wrap(author, [('plugin')])
@ -317,7 +312,7 @@ class Misc(callbacks.Privmsg):
chunk = L.pop()
if L:
chunk += ' \x02(%s)\x0F' % \
utils.nItems('message', len(L), 'more')
utils.str.nItems('message', len(L), 'more')
irc.reply(chunk, True)
except KeyError:
irc.error('You haven\'t asked me a command; perhaps you want '
@ -380,7 +375,7 @@ class Misc(callbacks.Privmsg):
nolimit = True
iterable = ifilter(self._validLastMsg, reversed(irc.state.history))
iterable.next() # Drop the first message.
predicates = list(utils.flatten(predicates.itervalues()))
predicates = list(utils.iter.flatten(predicates.itervalues()))
resp = []
if irc.nested and not \
self.registryValue('last.nested.includeTimestamp'):
@ -409,7 +404,7 @@ class Misc(callbacks.Privmsg):
irc.error('I couldn\'t find a message matching that criteria in '
'my history of %s messages.' % len(irc.state.history))
else:
irc.reply(utils.commaAndify(resp))
irc.reply(utils.str.commaAndify(resp))
last = wrap(last, [getopts({'nolimit': '',
'on': 'something',
'with': 'something',
@ -496,7 +491,7 @@ class Misc(callbacks.Privmsg):
shortname[, shortname and shortname].
"""
L = [getShortName(n) for n in longList]
return utils.commaAndify(L)
return utils.str.commaAndify(L)
def sortAuthors():
"""
Sort the list of 'long names' based on the number of contributions
@ -520,7 +515,7 @@ class Misc(callbacks.Privmsg):
hasContribs = False
if getattr(module, '__author__', None):
author = 'was written by %s' % \
utils.mungeEmailForWeb(str(module.__author__))
utils.web.mungeEmail(str(module.__author__))
hasAuthor = True
if getattr(module, '__contributors__', None):
contribs = sortAuthors()
@ -532,7 +527,7 @@ class Misc(callbacks.Privmsg):
if contribs:
contrib = '%s %s contributed to it.' % \
(buildContributorsString(contribs),
utils.has(len(contribs)))
utils.str.has(len(contribs)))
hasContribs = True
elif hasAuthor:
contrib = 'has no additional contributors listed'
@ -549,7 +544,7 @@ class Misc(callbacks.Privmsg):
if not authorInfo:
return 'The nick specified (%s) is not a registered ' \
'contributor' % nick
fullName = utils.mungeEmailForWeb(str(authorInfo))
fullName = utils.web.mungeEmail(str(authorInfo))
contributions = []
if hasattr(module, '__contributors__'):
if authorInfo not in module.__contributors__:
@ -558,22 +553,21 @@ class Misc(callbacks.Privmsg):
contributions = module.__contributors__[authorInfo]
if getattr(module, '__author__', False) == authorInfo:
isAuthor = True
# XXX Partition needs moved to utils.
(nonCommands, commands) = fix.partition(lambda s: ' ' in s,
(nonCommands, commands) = utils.iter.partition(lambda s: ' ' in s,
contributions)
results = []
if commands:
results.append(
'the %s %s' %(utils.commaAndify(commands),
utils.pluralize('command',len(commands))))
'the %s %s' %(utils.str.commaAndify(commands),
utils.str.pluralize('command',len(commands))))
if nonCommands:
results.append('the %s' % utils.commaAndify(nonCommands))
results.append('the %s' % utils.str.commaAndify(nonCommands))
if results and isAuthor:
return '%s wrote the %s plugin and also contributed %s' % \
(fullName, cb.name(), utils.commaAndify(results))
(fullName, cb.name(), utils.str.commaAndify(results))
elif results and not isAuthor:
return '%s contributed %s to the %s plugin' % \
(fullName, utils.commaAndify(results), cb.name())
(fullName, utils.str.commaAndify(results), cb.name())
elif isAuthor and not results:
return '%s wrote the %s plugin' % (fullName, cb.name())
else:

18
plugins/Network/plugin.py
View File

@ -27,10 +27,6 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot
import supybot.plugins as plugins
import time
import supybot.conf as conf
@ -172,19 +168,19 @@ class Network(callbacks.Privmsg):
normal.append(channel)
L = []
if ops:
L.append('is an op on %s' % utils.commaAndify(ops))
L.append('is an op on %s' % utils.str.commaAndify(ops))
if halfops:
L.append('is a halfop on %s' % utils.commaAndify(halfops))
L.append('is a halfop on %s' % utils.str.commaAndify(halfops))
if voices:
L.append('is voiced on %s' % utils.commaAndify(voices))
L.append('is voiced on %s' % utils.str.commaAndify(voices))
if normal:
if L:
L.append('is also on %s' % utils.commaAndify(normal))
L.append('is also on %s' % utils.str.commaAndify(normal))
else:
L.append('is on %s' % utils.commaAndify(normal))
L.append('is on %s' % utils.str.commaAndify(normal))
else:
L = ['isn\'t on any non-secret channels']
channels = utils.commaAndify(L)
channels = utils.str.commaAndify(L)
if '317' in d:
idle = utils.timeElapsed(d['317'].args[2])
signon = time.strftime(conf.supybot.reply.format.time(),
@ -246,7 +242,7 @@ class Network(callbacks.Privmsg):
"""
L = ['%s: %s' % (ircd.network, ircd.server) for ircd in world.ircs]
utils.sortBy(str.lower, L)
irc.reply(utils.commaAndify(L))
irc.reply(utils.str.commaAndify(L))
networks = wrap(networks)
def doPong(self, irc, msg):

6
plugins/Owner/plugin.py
View File

@ -27,16 +27,12 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.fix as fix
import gc
import os
import imp
import sre
import sys
import getopt
import socket
import logging
import linecache
import supybot.log as log
@ -424,7 +420,7 @@ class Owner(callbacks.Privmsg):
return
except ImportError, e:
if name in str(e):
irc.error('No plugin named %s exists.' % utils.dqrepr(name))
irc.error('No plugin named %s exists.' % utils.str.dqrepr(name))
else:
irc.error(str(e))
return

23
plugins/Status/plugin.py
View File

@ -76,9 +76,9 @@ class Status(callbacks.Privmsg):
networks.setdefault(Irc.network, []).append(Irc.nick)
networks = networks.items()
networks.sort()
networks = ['%s as %s' % (net, utils.commaAndify(nicks))
networks = ['%s as %s' % (net, utils.str.commaAndify(nicks))
for (net, nicks) in networks]
L = ['I am connected to %s.' % utils.commaAndify(networks)]
L = ['I am connected to %s.' % utils.str.commaAndify(networks)]
if world.profiling:
L.append('I am currently in code profiling mode.')
irc.reply(' '.join(L))
@ -92,9 +92,10 @@ class Status(callbacks.Privmsg):
threads = [t.getName() for t in threading.enumerate()]
threads.sort()
s = 'I have spawned %s; %s %s still currently active: %s.' % \
(utils.nItems('thread', world.threadsSpawned),
utils.nItems('thread', len(threads)), utils.be(len(threads)),
utils.commaAndify(threads))
(utils.str.nItems('thread', world.threadsSpawned),
utils.str.nItems('thread', len(threads)),
utils.str.be(len(threads)),
utils.str.commaAndify(threads))
irc.reply(s)
threads = wrap(threads)
@ -137,7 +138,7 @@ class Status(callbacks.Privmsg):
'of system time, for a total of %.2f seconds of CPU ' \
'time. %s' % (user, system, user + system, children)
if self.registryValue('cpu.threads', target):
spawned = utils.nItems('thread', world.threadsSpawned)
spawned = utils.str.nItems('thread', world.threadsSpawned)
response += 'I have spawned %s; I currently have %s still ' \
'running.' % (spawned, activeThreads)
if self.registryValue('cpu.memory', target):
@ -159,7 +160,7 @@ class Status(callbacks.Privmsg):
response += ' I\'m taking up %s kB of memory.' % mem
except Exception:
self.log.exception('Uncaught exception in cpu.memory:')
irc.reply(utils.normalizeWhitespace(response))
irc.reply(utils.str.normalizeWhitespace(response))
cpu = wrap(cpu)
def cmd(self, irc, msg, args):
@ -178,9 +179,9 @@ class Status(callbacks.Privmsg):
attr == callbacks.canonicalName(attr):
commands += 1
s = 'I offer a total of %s in %s. I have processed %s.' % \
(utils.nItems('command', commands),
utils.nItems('plugin', callbacksPrivmsg, 'command-based'),
utils.nItems('command', world.commandsProcessed))
(utils.str.nItems('command', commands),
utils.str.nItems('plugin', callbacksPrivmsg, 'command-based'),
utils.str.nItems('command', world.commandsProcessed))
irc.reply(s)
cmd = wrap(cmd)
@ -199,7 +200,7 @@ class Status(callbacks.Privmsg):
commands.add(attr)
commands = list(commands)
commands.sort()
irc.reply(utils.commaAndify(commands))
irc.reply(utils.str.commaAndify(commands))
commands = wrap(commands)
def uptime(self, irc, msg, args):

14
plugins/User/plugin.py
View File

@ -27,12 +27,8 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.fix as fix
import re
import getopt
import fnmatch
from itertools import imap, ifilter
import supybot.conf as conf
import supybot.utils as utils
@ -75,7 +71,7 @@ class User(callbacks.Privmsg):
users.append(u.name)
if users:
utils.sortBy(str.lower, users)
irc.reply(utils.commaAndify(users))
irc.reply(utils.str.commaAndify(users))
else:
if predicates:
irc.reply('There are no matching registered users.')
@ -158,7 +154,7 @@ class User(callbacks.Privmsg):
"""
try:
id = ircdb.users.getUserId(newname)
irc.error('%s is already registered.' % utils.quoted(newname))
irc.error('%s is already registered.' % utils.str.quoted(newname))
return
except KeyError:
pass
@ -295,7 +291,7 @@ class User(callbacks.Privmsg):
def getHostmasks(user):
hostmasks = map(repr, user.hostmasks)
hostmasks.sort()
return utils.commaAndify(hostmasks)
return utils.str.commaAndify(hostmasks)
try:
user = ircdb.users.getUser(msg.prefix)
if name:
@ -418,8 +414,8 @@ class User(callbacks.Privmsg):
irc.reply('I have %s registered users '
'with %s registered hostmasks; '
'%s and %s.' % (users, hostmasks,
utils.nItems('owner', owners),
utils.nItems('admin', admins)))
utils.str.nItems('owner', owners),
utils.str.nItems('admin', admins)))
stats = wrap(stats)

25
plugins/__init__.py
View File

@ -27,10 +27,6 @@
# POSSIBILITY OF SUCH DAMAGE.
###
__revision__ = "$Id: __init__.py,v 1.44 2005/01/08 07:22:46 jamessan Exp $"
import supybot.fix as fix
import gc
import os
import re
@ -106,7 +102,7 @@ class NoSuitableDatabase(Exception):
return 'No suitable databases were found. Suitable databases ' \
'include %s. If you have one of these databases installed, ' \
'make sure it is listed in the supybot.databases ' \
'configuration variable.' % utils.commaAndify(self.suitable)
'configuration variable.' % utils.str.commaAndify(self.suitable)
def DB(filename, types):
filename = conf.supybot.directories.data.dirize(filename)
@ -325,7 +321,7 @@ class ChannelUserDB(ChannelUserDictionary):
log.debug('Exception: %s', utils.exnToString(e))
def flush(self):
fd = utils.transactionalFile(self.filename, makeBackupIfSmaller=False)
fd = utils.file.AtomicFile(self.filename, makeBackupIfSmaller=False)
writer = csv.writer(fd)
items = self.items()
if not items:
@ -389,9 +385,9 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
def getCommandHelp(self, name):
help = self.__parent.getCommandHelp(name)
help = help.replace('$Types', utils.pluralize(self.name()))
help = help.replace('$Types', utils.str.pluralize(self.name()))
help = help.replace('$Type', self.name())
help = help.replace('$types', utils.pluralize(self.name().lower()))
help = help.replace('$types', utils.str.pluralize(self.name().lower()))
help = help.replace('$type', self.name().lower())
return help
@ -442,7 +438,7 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
remove = wrap(remove, ['user', 'channeldb', 'id'])
def searchSerializeRecord(self, record):
text = utils.quoted(utils.ellipsisify(record.text, 50))
text = utils.str.quoted(utils.str.ellipsisify(record.text, 50))
return '#%s: %s' % (record.id, text)
def search(self, irc, msg, args, channel, optlist, glob):
@ -471,10 +467,10 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
L.append(self.searchSerializeRecord(record))
if L:
L.sort()
irc.reply('%s found: %s' % (len(L), utils.commaAndify(L)))
irc.reply('%s found: %s' % (len(L), utils.str.commaAndify(L)))
else:
irc.reply('No matching %s were found.' %
utils.pluralize(self.name().lower()))
utils.str.pluralize(self.name().lower()))
search = wrap(search, ['channeldb',
getopts({'by': 'otherUser',
'regexp': 'regexpMatcher'}),
@ -485,7 +481,8 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
at = time.localtime(record.at)
timeS = time.strftime(conf.supybot.reply.format.time(), at)
return '%s #%s: %s (added by %s at %s)' % \
(self.name(), record.id, utils.quoted(record.text), name, timeS)
(self.name(), record.id,
utils.str.quoted(record.text), name, timeS)
def get(self, irc, msg, args, channel, id):
"""[<channel>] <id>
@ -527,7 +524,7 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg):
"""
n = self.db.size(channel)
irc.reply('There %s %s in my database.' %
(utils.be(n), utils.nItems(self.name().lower(), n)))
(utils.str.be(n), utils.str.nItems(self.name().lower(), n)))
stats = wrap(stats, ['channeldb'])
@ -592,7 +589,7 @@ class PeriodicFileDownloader(object):
self.log.warning('Error downloading %s: %s', url, e)
return
confDir = conf.supybot.directories.data()
newFilename = os.path.join(confDir, utils.mktemp())
newFilename = os.path.join(confDir, utils.file.mktemp())
outfd = file(newFilename, 'wb')
start = time.time()
s = infd.read(4096)

2
scripts/supybot
View File

@ -33,7 +33,7 @@
This is the main program to run Supybot.
"""
import supybot
import re
import os

2
setup.py
View File

@ -103,11 +103,13 @@ if clean:
sys.exit(-1)
packages = ['supybot',
'supybot.utils',
'supybot.drivers',
'supybot.plugins',] + \
['supybot.plugins.'+s for s in plugins]
package_dir = {'supybot': 'src',
'supybot.utils': 'src/utils',
'supybot.plugins': 'plugins',
'supybot.drivers': 'src/drivers',}

32
src/callbacks.py
View File

@ -38,10 +38,6 @@ how to use them.
import supybot
import supybot.fix as fix
import re
import copy
import sets
@ -166,7 +162,7 @@ def canonicalName(command):
while command and command[-1] in special:
reAppend = command[-1] + reAppend
command = command[:-1]
return command.translate(string.ascii, special).lower() + reAppend
return command.translate(utils.str.chars, special).lower() + reAppend
def reply(msg, s, prefixName=None, private=None,
notice=None, to=None, action=None, error=False):
@ -233,7 +229,7 @@ def getHelp(method, name=None):
if doclines:
help = ' '.join(doclines)
s = '(%s) -- %s' % (ircutils.bold(s), help)
return utils.normalizeWhitespace(s)
return utils.str.normalizeWhitespace(s)
def getSyntax(method, name=None):
if name is None:
@ -256,10 +252,10 @@ class Tokenizer(object):
#
# These are the characters valid in a token. Everything printable except
# double-quote, left-bracket, and right-bracket.
validChars = string.ascii.translate(string.ascii, '\x00\r\n \t')
validChars = utils.str.chars.translate(utils.str.chars, '\x00\r\n \t')
def __init__(self, brackets='', pipe=False, quotes='"'):
if brackets:
self.validChars = self.validChars.translate(string.ascii, brackets)
self.validChars=self.validChars.translate(utils.str.chars, brackets)
self.left = brackets[0]
self.right = brackets[1]
else:
@ -267,9 +263,9 @@ class Tokenizer(object):
self.right = ''
self.pipe = pipe
if self.pipe:
self.validChars = self.validChars.translate(string.ascii, '|')
self.validChars = self.validChars.translate(utils.str.chars, '|')
self.quotes = quotes
self.validChars = self.validChars.translate(string.ascii, quotes)
self.validChars = self.validChars.translate(utils.str.chars, quotes)
def _handleToken(self, token):
@ -459,7 +455,7 @@ class RichReplyMethods(object):
if prefixer is None:
prefixer = ''
if joiner is None:
joiner = utils.commaAndify
joiner = utils.str.commaAndify
if isinstance(prefixer, basestring):
prefixer = prefixer.__add__
if isinstance(joiner, basestring):
@ -494,7 +490,7 @@ class RichReplyMethods(object):
kwargs['Raise'] = True
if isinstance(capability, basestring): # checkCommandCapability!
log.warning('Denying %s for lacking %s capability.',
self.msg.prefix, utils.quoted(capability))
self.msg.prefix, utils.str.quoted(capability))
if not self._getConfig(conf.supybot.reply.error.noCapability):
v = self._getConfig(conf.supybot.replies.noCapability)
s = self.__makeReply(v % capability, s)
@ -750,7 +746,7 @@ class IrcObjectProxy(RichReplyMethods):
'Please specify the plugin whose command you '
'wish to call by using its name as a command '
'before %s.' %
(command, utils.commaAndify(names), command))
(command, utils.str.commaAndify(names), command))
else:
cb = cbs[0]
del self.args[0] # Remove the command.
@ -870,7 +866,7 @@ class IrcObjectProxy(RichReplyMethods):
response = msgs.pop()
if msgs:
n = ircutils.bold('(%s)')
n %= utils.nItems('message', len(msgs), 'more')
n %= utils.str.nItems('message', len(msgs), 'more')
response = '%s %s' % (response, n)
prefix = msg.prefix
if self.to and ircutils.isNick(self.to):
@ -1123,7 +1119,7 @@ class Privmsg(irclib.IrcCallback):
"""Gets the given command from this plugin."""
name = canonicalName(name)
assert self.isCommand(name), '%s is not a command.' % \
utils.quoted(name)
utils.str.quoted(name)
return getattr(self, name)
def callCommand(self, name, irc, msg, *L, **kwargs):
@ -1155,11 +1151,11 @@ class Privmsg(irclib.IrcCallback):
command = self.getCommand(name)
if hasattr(command, 'isDispatcher') and \
command.isDispatcher and self.__doc__:
return utils.normalizeWhitespace(self.__doc__)
return utils.str.normalizeWhitespace(self.__doc__)
elif hasattr(command, '__doc__'):
return getHelp(command)
else:
return 'The %s command has no help.' % utils.quoted(name)
return 'The %s command has no help.' % utils.str.quoted(name)
def registryValue(self, name, channel=None, value=True):
plugin = self.name()
@ -1291,7 +1287,7 @@ class PrivmsgRegexp(Privmsg):
self.res.append((r, name))
except re.error, e:
self.log.warning('Invalid regexp: %s (%s)',
utils.quoted(value.__doc__), e)
utils.str.quoted(value.__doc__), e)
utils.sortBy(operator.itemgetter(1), self.res)
def callCommand(self, name, irc, msg, *L, **kwargs):

6
src/cdb.py
View File

@ -32,10 +32,6 @@ Database module, similar to dbhash. Uses a format similar to (if not entirely
the same as) DJB's CDB <http://cr.yp.to/cdb.html>.
"""
import supybot.fix as fix
import os
import sys
import sets
@ -134,7 +130,7 @@ def make(dbFilename, readFilename=None):
class Maker(object):
"""Class for making CDB databases."""
def __init__(self, filename):
self.fd = utils.transactionalFile(filename)
self.fd = utils.file.AtomicFile(filename)
self.filename = filename
self.fd.seek(2048)
self.hashPointers = [(0, 0)] * 256

14
src/commands.py
View File

@ -31,10 +31,6 @@
Includes wrappers for commands.
"""
import supybot.fix as fix
import time
import types
import getopt
@ -233,7 +229,7 @@ def getExpiry(irc, msg, args, state):
def getBoolean(irc, msg, args, state):
try:
state.args.append(utils.toBool(args[0]))
state.args.append(utils.str.toBool(args[0]))
del args[0]
except ValueError:
irc.errorInvalid('boolean', args[0])
@ -324,8 +320,8 @@ def _getRe(f):
irc.errorInvalid('regular expression', s)
return get
getMatcher = _getRe(utils.perlReToPythonRe)
getReplacer = _getRe(utils.perlReToReplacer)
getMatcher = _getRe(utils.str.perlReToPythonRe)
getReplacer = _getRe(utils.str.perlReToReplacer)
def getNick(irc, msg, args, state):
if ircutils.isNick(args[0]):
@ -494,7 +490,7 @@ def getCommandName(irc, msg, args, state):
state.args.append(callbacks.canonicalName(args.pop(0)))
def getIp(irc, msg, args, state):
if utils.isIP(args[0]):
if utils.net.isIP(args[0]):
state.args.append(args.pop(0))
else:
irc.errorInvalid('ip', args[0])
@ -845,7 +841,7 @@ class Spec(object):
def __init__(self, types, allowExtra=False):
self.types = types
self.allowExtra = allowExtra
utils.mapinto(contextify, self.types)
utils.seq.mapinto(contextify, self.types)
def __call__(self, irc, msg, args, stateAttrs={}):
state = self._state(self.types[:], stateAttrs)

27
src/conf.py
View File

@ -27,8 +27,6 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.fix as fix
import os
import sys
import time
@ -70,6 +68,12 @@ daemonized = False
###
allowDefaultOwner = False
###
# Here we replace values in other modules as appropriate.
###
utils.web.defaultHeaders['User-agent'] = \
'Mozilla/5.0 (Compatible; Supybot %s)' % version
###
# The standard registry.
###
@ -405,7 +409,7 @@ registerChannelValue(supybot.reply, 'showSimpleSyntax',
class ValidPrefixChars(registry.String):
"""Value must contain only ~!@#$%^&*()_-+=[{}]\\|'\";:,<.>/?"""
def setValue(self, v):
if v.translate(string.ascii, '`~!@#$%^&*()_-+=[{}]\\|\'";:,<.>/?'):
if v.translate(utils.str.chars, '`~!@#$%^&*()_-+=[{}]\\|\'";:,<.>/?'):
self.error()
registry.String.setValue(self, v)
@ -579,9 +583,9 @@ registerChannelValue(supybot.commands.nested, 'pipeSyntax',
example: 'bot: foo | bar'."""))
registerGroup(supybot.commands, 'defaultPlugins',
orderAlphabetically=True, help=utils.normalizeWhitespace("""Determines
what commands have default plugins set, and which plugins are set to
be the default for each of those commands."""))
orderAlphabetically=True, help="""Determines what commands have default
plugins set, and which plugins are set to be the default for each of those
commands.""")
registerGlobalValue(supybot.commands.defaultPlugins, 'importantPlugins',
registry.SpaceSeparatedSetOfStrings(
['Admin', 'Channel', 'Config', 'Misc', 'Owner', 'User'],
@ -704,12 +708,8 @@ registerGlobalValue(supybot.directories.data, 'tmp',
DataFilenameDirectory('tmp', """Determines what directory temporary files
are put into."""))
# Remember, we're *meant* to replace this nice little wrapper.
def transactionalFile(*args, **kwargs):
kwargs['tmpDir'] = supybot.directories.data.tmp()
kwargs['backupDir'] = supybot.directories.backup()
return utils.AtomicFile(*args, **kwargs)
utils.transactionalFile = transactionalFile
utils.file.AtomicFile.default.tmpDir = supybot.directories.data.tmp
utils.file.AtomicFile.default.backupDir = supybot.directories.backup
class PluginDirectories(registry.CommaSeparatedListOfStrings):
def __call__(self):
@ -931,6 +931,7 @@ registerGlobalValue(supybot.protocols.http, 'peekSize',
registerGlobalValue(supybot.protocols.http, 'proxy',
registry.String('', """Determines what proxy all HTTP requests should go
through. The value should be of the form 'host:port'."""))
utils.web.proxy = supybot.protocols.http.proxy
###
@ -945,7 +946,7 @@ registerGlobalValue(supybot, 'defaultIgnore',
class IP(registry.String):
"""Value must be a valid IP."""
def setValue(self, v):
if v and not (utils.isIP(v) or utils.isIPV6(v)):
if v and not (utils.net.isIP(v) or utils.net.isIPV6(v)):
self.error()
else:
registry.String.setValue(self, v)

7
src/dbi.py
View File

@ -31,10 +31,6 @@
Module for some slight database-independence for simple databases.
"""
import supybot.fix as fix
import csv
import math
import sets
@ -262,8 +258,7 @@ class FlatfileMapping(MappingInterface):
def vacuum(self):
infd = file(self.filename)
outfd = utils.transactionalFile(self.filename,
makeBackupIfSmaller=False)
outfd = utils.file.AstomicFile(self.filename,makeBackupIfSmaller=False)
outfd.write(infd.readline()) # First line, nextId.
for line in infd:
if not line.startswith('-'):

239
src/fix.py
View File

@ -1,239 +0,0 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
"""
Fixes stuff that Python should have but doesn't.
"""
from __future__ import division
__all__ = []
exported = ['ignore', 'window', 'group', 'partition', 'set', 'frozenset',
'any', 'all', 'rsplit', 'dynamic']
import sys
import new
import atexit
import string
string.ascii = string.maketrans('', '')
import random
_choice = random.choice
def choice(iterable):
if isinstance(iterable, (list, tuple)):
return _choice(iterable)
else:
n = 1
m = new.module('') # Guaranteed unique value.
ret = m
for x in iterable:
if random.random() < 1/n:
ret = x
n += 1
if ret is m:
raise IndexError
return ret
random.choice = choice
def ignore(*args, **kwargs):
"""Simply ignore the arguments sent to it."""
pass
class DynamicScope(object):
def _getLocals(self, name):
f = sys._getframe().f_back.f_back # _getLocals <- __[gs]etattr__ <- ...
while f:
if name in f.f_locals:
return f.f_locals
f = f.f_back
raise NameError, name
def __getattr__(self, name):
try:
return self._getLocals(name)[name]
except (NameError, KeyError):
return None
def __setattr__(self, name, value):
self._getLocals(name)[name] = value
dynamic = DynamicScope()
if sys.version_info < (2, 4, 0):
def reversed(L):
"""Iterates through a sequence in reverse."""
for i in xrange(len(L) - 1, -1, -1):
yield L[i]
exported.append('reversed')
def window(L, size):
"""Returns a sliding 'window' through the list L of size size."""
assert not isinstance(L, int), 'Argument order swapped: window(L, size)'
if size < 1:
raise ValueError, 'size <= 0 disallowed.'
for i in xrange(len(L) - (size-1)):
yield L[i:i+size]
import itertools
def ilen(iterable):
"""Returns the length of an iterator."""
i = 0
for _ in iterable:
i += 1
return i
def trueCycle(iterable):
while 1:
yielded = False
for x in iterable:
yield x
yielded = True
if not yielded:
raise StopIteration
itertools.trueCycle = trueCycle
itertools.ilen = ilen
def groupby(key, iterable):
if key is None:
key = lambda x: x
it = iter(iterable)
value = it.next() # If there are no items, this takes an early exit
oldkey = key(value)
group = [value]
for value in it:
newkey = key(value)
if newkey != oldkey:
yield group
group = []
oldkey = newkey
group.append(value)
yield group
itertools.groupby = groupby
def group(seq, groupSize, noneFill=True):
"""Groups a given sequence into sublists of length groupSize."""
ret = []
L = []
i = groupSize
for elt in seq:
if i > 0:
L.append(elt)
else:
ret.append(L)
i = groupSize
L = []
L.append(elt)
i -= 1
if L:
if noneFill:
while len(L) < groupSize:
L.append(None)
ret.append(L)
return ret
def partition(p, L):
"""Partitions a list L based on a predicate p. Returns a (yes,no) tuple"""
no = []
yes = []
for elt in L:
if p(elt):
yes.append(elt)
else:
no.append(elt)
return (yes, no)
def any(p, seq):
"""Returns true if any element in seq satisfies predicate p."""
for elt in itertools.ifilter(p, seq):
return True
else:
return False
def all(p, seq):
"""Returns true if all elements in seq satisfy predicate p."""
for elt in itertools.ifilterfalse(p, seq):
return False
else:
return True
def rsplit(s, sep=None, maxsplit=-1):
"""Equivalent to str.split, except splitting from the right."""
if sys.version_info < (2, 4, 0):
if sep is not None:
sep = sep[::-1]
L = s[::-1].split(sep, maxsplit)
L.reverse()
return [s[::-1] for s in L]
else:
return s.rsplit(sep, maxsplit)
if sys.version_info < (2, 4, 0):
import operator
def itemgetter(i):
return lambda x: x[i]
def attrgetter(attr):
return lambda x: getattr(x, attr)
operator.itemgetter = itemgetter
operator.attrgetter = attrgetter
import csv
import cStringIO as StringIO
def join(L):
fd = StringIO.StringIO()
writer = csv.writer(fd)
writer.writerow(L)
return fd.getvalue().rstrip('\r\n')
def split(s):
fd = StringIO.StringIO(s)
reader = csv.reader(fd)
return reader.next()
csv.join = join
csv.split = split
import sets
set = sets.Set
frozenset = sets.ImmutableSet
import socket
# Some socket modules don't have sslerror, so we'll just make it an error.
if not hasattr(socket, 'sslerror'):
socket.sslerror = socket.error
g = globals()
for name in exported:
__builtins__[name] = g[name]
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

22
src/ircdb.py
View File

@ -27,18 +27,13 @@
# POSSIBILITY OF SUCH DAMAGE.
###
from __future__ import division
import supybot.fix as fix
import os
import sets
import time
import string
import operator
from itertools import imap, ilen, ifilter
import supybot.log as log
import supybot.conf as conf
@ -47,6 +42,7 @@ import supybot.world as world
import supybot.ircutils as ircutils
import supybot.registry as registry
import supybot.unpreserve as unpreserve
from utils.iter import imap, ilen, ifilter
def isCapability(capability):
return len(capability.split(None, 1)) == 1
@ -114,7 +110,7 @@ def canonicalCapability(capability):
return capability.lower()
def unWildcardHostmask(hostmask):
return hostmask.translate(string.ascii, '!@*?')
return hostmask.translate(utils.str.chars, '!@*?')
_invert = invertCapability
class CapabilitySet(sets.Set):
@ -228,7 +224,7 @@ class IrcUser(object):
return '%s(id=%s, ignore=%s, password="", name=%s, hashed=%r, ' \
'capabilities=%r, hostmasks=[], secure=%r)\n' % \
(self.__class__.__name__, self.id, self.ignore,
utils.quoted(self.name), self.hashed, self.capabilities,
utils.str.quoted(self.name), self.hashed, self.capabilities,
self.secure)
def __hash__(self):
@ -611,7 +607,7 @@ class UsersDictionary(utils.IterableMap):
if self.filename is not None:
L = self.users.items()
L.sort()
fd = utils.transactionalFile(self.filename)
fd = utils.file.AtomicFile(self.filename)
for (id, u) in L:
fd.write('user %s' % id)
fd.write(os.linesep)
@ -657,7 +653,7 @@ class UsersDictionary(utils.IterableMap):
'Removing the offending hostmasks.')
for (id, hostmask) in ids.iteritems():
log.error('Removing %s from user %s.',
utils.quoted(hostmask), id)
utils.str.quoted(hostmask), id)
self.users[id].removeHostmask(hostmask)
raise DuplicateHostmask, 'Ids %r matched.' % ids
else: # Not a hostmask, must be a name.
@ -789,7 +785,7 @@ class ChannelsDictionary(utils.IterableMap):
"""Flushes the channel database to its file."""
if not self.noFlush:
if self.filename is not None:
fd = utils.transactionalFile(self.filename)
fd = utils.file.AtomicFile(self.filename)
for (channel, c) in self.channels.iteritems():
fd.write('channel %s' % channel)
fd.write(os.linesep)
@ -845,7 +841,7 @@ class IgnoresDB(object):
def open(self, filename):
self.filename = filename
fd = file(self.filename)
for line in utils.nonCommentNonEmptyLines(fd):
for line in utils.file.nonCommentNonEmptyLines(fd):
try:
line = line.rstrip('\r\n')
L = line.split()
@ -857,12 +853,12 @@ class IgnoresDB(object):
self.add(hostmask, expiration)
except Exception, e:
log.error('Invalid line in ignores database: %s',
utils.quoted(line))
utils.str.quoted(line))
fd.close()
def flush(self):
if self.filename is not None:
fd = utils.transactionalFile(self.filename)
fd = utils.file.AtomicFile(self.filename)
now = time.time()
for (hostmask, expiration) in self.hostmasks.items():
if now < expiration or not expiration:

10
src/irclib.py
View File

@ -27,16 +27,11 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.fix as fix
import copy
import sets
import time
import random
import operator
from itertools import imap, chain, cycle
import supybot.log as log
import supybot.conf as conf
@ -45,6 +40,7 @@ import supybot.world as world
import supybot.ircdb as ircdb
import supybot.ircmsgs as ircmsgs
import supybot.ircutils as ircutils
from utils.iter import imap, chain, cycle
from supybot.structures import queue, smallqueue, RingBuffer
###
@ -173,7 +169,7 @@ class IrcMsgQueue(object):
not conf.supybot.protocols.irc.queueDuplicateMessages():
s = str(msg).strip()
log.info('Not adding message %s to queue, already added.',
utils.quoted(s))
utils.str.quoted(s))
return False
else:
self.msgs.add(msg)
@ -642,7 +638,7 @@ class Irc(IrcCommandDispatcher):
name = name.lower()
def nameMatches(cb):
return cb.name().lower() == name
(bad, good) = partition(nameMatches, self.callbacks)
(bad, good) = utils.iter.partition(nameMatches, self.callbacks)
self.callbacks[:] = good
return bad

14
src/ircmsgs.py
View File

@ -34,10 +34,6 @@ construct such messages in an easier way than the constructor for the IrcMsg
object (which, as you'll read later, is quite...full-featured :))
"""
import supybot.fix as fix
import re
import time
import string
@ -192,8 +188,8 @@ class IrcMsg(object):
if self._repr is not None:
return self._repr
self._repr = 'IrcMsg(prefix=%s, command=%s, args=%r)' % \
(utils.quoted(self.prefix), utils.quoted(self.command),
self.args)
(utils.str.quoted(self.prefix),
utils.str.quoted(self.command), self.args)
return self._repr
def __reduce__(self):
@ -586,7 +582,8 @@ def join(channel, key=None, prefix='', msg=None):
return IrcMsg(prefix=prefix, command='JOIN', args=(channel,), msg=msg)
else:
if conf.supybot.protocols.irc.strictRfc():
assert key.translate(string.ascii, string.ascii[128:]) == key and \
assert key.translate(utils.str.chars,
utils.str.chars[128:]) == key and \
'\x00' not in key and \
'\r' not in key and \
'\n' not in key and \
@ -613,7 +610,8 @@ def joins(channels, keys=None, prefix='', msg=None):
else:
for key in keys:
if conf.supybot.protocols.irc.strictRfc():
assert key.translate(string.ascii,string.ascii[128:])==key and\
assert key.translate(utils.str.chars,
utils.str.chars[128:])==key and \
'\x00' not in key and \
'\r' not in key and \
'\n' not in key and \

19
src/ircutils.py
View File

@ -34,16 +34,11 @@ nick class to handle nicks (so comparisons and hashing and whatnot work in an
IRC-case-insensitive fashion), and numerous other things.
"""
import supybot.fix as fix
import re
import time
import random
import string
import textwrap
from itertools import imap, ilen
from cStringIO import StringIO as sio
import supybot.utils as utils
@ -109,7 +104,7 @@ def toLower(s, casemapping=None):
elif casemapping == 'ascii': # freenode
return s.lower()
else:
raise ValueError, 'Invalid casemapping: %s' % utils.quoted(casemapping)
raise ValueError, 'Invalid casemapping: %r' % casemapping
def strEqual(nick1, nick2):
"""s1, s2 => bool
@ -197,11 +192,11 @@ def banmask(hostmask):
"""
assert isUserHostmask(hostmask)
host = hostFromHostmask(hostmask)
if utils.isIP(host):
if utils.net.isIP(host):
L = host.split('.')
L[-1] = '*'
return '*!*@' + '.'.join(L)
elif utils.isIPV6(host):
elif utils.net.isIPV6(host):
L = host.split(':')
L[-1] = '*'
return '*!*@' + ':'.join(L)
@ -450,7 +445,7 @@ def safeArgument(s):
if isinstance(s, unicode):
s = s.encode('utf-8')
elif not isinstance(s, basestring):
debug('Got a non-string in safeArgument: %s', utils.quoted(s))
debug('Got a non-string in safeArgument: %r', s)
s = str(s)
if isValidArgument(s):
return s
@ -466,7 +461,7 @@ def replyTo(msg):
def dccIP(ip):
"""Returns in IP in the proper for DCC."""
assert utils.isIP(ip), \
assert utils.net.isIP(ip), \
'argument must be a string ip in xxx.yyy.zzz.www format.'
i = 0
x = 256**3
@ -483,7 +478,7 @@ def unDccIP(i):
L.append(i % 256)
i /= 256
L.reverse()
return '.'.join(imap(str, L))
return '.'.join(utils.iter.imap(str, L))
class IrcString(str):
"""This class does case-insensitive comparison and hashing of nicks."""
@ -662,7 +657,7 @@ def standardSubstitute(irc, msg, text, env=None):
})
if env is not None:
vars.update(env)
return utils.perlVariableSubstitute(vars, text)
return utils.str.perlVariableSubstitute(vars, text)
if __name__ == '__main__':

2
src/log.py
View File

@ -27,8 +27,6 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.fix as fix
import os
import sys
import time

4
src/plugin.py
View File

@ -49,7 +49,7 @@ def loadPluginModule(name, ignoreDeprecation=False):
files.extend(os.listdir(dir))
except EnvironmentError: # OSError, IOError superclass.
log.warning('Invalid plugin directory: %s; removing.',
utils.quoted(dir))
utils.str.quoted(dir))
conf.supybot.directories.plugins().remove(dir)
loweredFiles = map(str.lower, files)
try:
@ -72,7 +72,7 @@ def loadPluginModule(name, ignoreDeprecation=False):
log.warning('Deprecated plugin loaded: %s', name)
else:
raise Deprecated, 'Attempted to load deprecated plugin %s' % \
utils.quoted(name)
utils.str.quoted(name)
if module.__name__ in sys.modules:
sys.modules[module.__name__] = module
linecache.checkcache()

23
src/registry.py
View File

@ -34,7 +34,6 @@ import time
import string
import textwrap
import supybot.fix as fix
import supybot.utils as utils
def error(s):
@ -68,7 +67,7 @@ def open(filename, clear=False):
if clear:
_cache.clear()
_fd = file(filename)
fd = utils.nonCommentNonEmptyLines(_fd)
fd = utils.file.nonCommentNonEmptyLines(_fd)
acc = ''
for line in fd:
line = line.rstrip('\r\n')
@ -94,7 +93,7 @@ def open(filename, clear=False):
def close(registry, filename, private=True):
first = True
fd = utils.transactionalFile(filename)
fd = utils.file.AtomicFile(filename)
for (name, value) in registry.getValues(getChildren=True):
help = value.help()
if help:
@ -162,7 +161,7 @@ class Group(object):
"""A group; it doesn't hold a value unless handled by a subclass."""
def __init__(self, help='', supplyDefault=False,
orderAlphabetically=False, private=False):
self._help = help
self._help = utils.str.normalizeWhitespace(help)
self._name = 'unset'
self._added = []
self._children = utils.InsensitivePreservingDict()
@ -299,7 +298,7 @@ class Value(Group):
self.__parent.__init__(help, **kwargs)
self._default = default
self._showDefault = showDefault
self._help = utils.normalizeWhitespace(help.strip())
self._help = utils.str.normalizeWhitespace(help.strip())
if setDefault:
self.setValue(default)
@ -309,7 +308,7 @@ class Value(Group):
else:
s = """%s has no docstring. If you're getting this message,
report it, because we forgot to put a proper help string here."""
e = InvalidRegistryValue(utils.normalizeWhitespace(s % self._name))
e = InvalidRegistryValue(utils.str.normalizeWhitespace(s % self._name))
e.value = self
raise e
@ -356,7 +355,7 @@ class Boolean(Value):
"""Value must be either True or False (or On or Off)."""
def set(self, s):
try:
v = utils.toBool(s)
v = utils.str.toBool(s)
except ValueError:
if s.strip().lower() == 'toggle':
v = not self.value
@ -440,7 +439,7 @@ class String(Value):
_printable = string.printable[:-4]
def _needsQuoting(self, s):
return s.translate(string.ascii, self._printable) and s.strip() != s
return s.translate(utils.str.chars, self._printable) and s.strip() != s
def __str__(self):
s = self.value
@ -456,12 +455,12 @@ class OnlySomeStrings(String):
self.__parent = super(OnlySomeStrings, self)
self.__parent.__init__(*args, **kwargs)
self.__doc__ = 'Valid values include %s.' % \
utils.commaAndify(map(repr, self.validStrings))
utils.str.commaAndify(map(repr, self.validStrings))
def help(self):
strings = [s for s in self.validStrings if s]
return '%s Valid strings: %s.' % \
(self._help, utils.commaAndify(strings))
(self._help, utils.str.commaAndify(strings))
def normalize(self, s):
lowered = s.lower()
@ -487,7 +486,7 @@ class NormalizedString(String):
self._showDefault = False
def normalize(self, s):
return utils.normalizeWhitespace(s.strip())
return utils.str.normalizeWhitespace(s.strip())
def set(self, s):
s = self.normalize(s)
@ -540,7 +539,7 @@ class Regexp(Value):
def set(self, s):
try:
if s:
self.setValue(utils.perlReToPythonRe(s), sr=s)
self.setValue(utils.str.perlReToPythonRe(s), sr=s)
else:
self.setValue(None)
except ValueError, e:

4
src/schedule.py
View File

@ -32,10 +32,6 @@ Schedule plugin with a subclass of drivers.IrcDriver in order to be run as a
Supybot driver.
"""
import supybot.fix as fix
import time
import heapq

14
src/structures.py
View File

@ -31,13 +31,9 @@
Data structures for Python.
"""
import supybot.fix as fix
import time
import types
from itertools import imap, ilen
from itertools import imap
class RingBuffer(object):
"""Class to represent a fixed-size ring buffer."""
@ -345,8 +341,12 @@ class TimeoutQueue(object):
yield elt
def __len__(self):
return ilen(self)
# No dependency on utils.
# return ilen(self)
i = 0
for _ in self:
i += 1
return i
class MaxLengthQueue(queue):
__slots__ = ('length',)

4
src/test.py
View File

@ -27,8 +27,6 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.fix as fix
import gc
import os
import re
@ -96,7 +94,7 @@ class SupyTestCase(unittest.TestCase):
def setUp(self):
log.critical('Beginning test case %s', self.id())
threads = [t.getName() for t in threading.enumerate()]
log.critical('Threads: %s' % utils.commaAndify(threads))
log.critical('Threads: %s' % utils.str.commaAndify(threads))
unittest.TestCase.setUp(self)
def tearDown(self):

2
src/unpreserve.py
View File

@ -27,8 +27,6 @@
# POSSIBILITY OF SUCH DAMAGE.
###
class Reader(object):
def __init__(self, Creator, *args, **kwargs):
self.Creator = Creator

864
src/utils.py
View File

@ -1,864 +0,0 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
"""
Simple utility functions.
"""
import supybot.fix as fix
import os
import re
import sys
import md5
import new
import sha
import sets
import time
import types
import random
import shutil
import socket
import string
import sgmllib
import compiler
import textwrap
import UserDict
import itertools
import traceback
import htmlentitydefs
from itertools import imap, ifilter
from supybot.structures import TwoWayDictionary
curry = new.instancemethod
def normalizeWhitespace(s):
"""Normalizes the whitespace in a string; \s+ becomes one space."""
return ' '.join(s.split())
class HtmlToText(sgmllib.SGMLParser):
"""Taken from some eff-bot code on c.l.p."""
entitydefs = htmlentitydefs.entitydefs.copy()
entitydefs['nbsp'] = ' '
def __init__(self, tagReplace=' '):
self.data = []
self.tagReplace = tagReplace
sgmllib.SGMLParser.__init__(self)
def unknown_starttag(self, tag, attr):
self.data.append(self.tagReplace)
def unknown_endtag(self, tag):
self.data.append(self.tagReplace)
def handle_data(self, data):
self.data.append(data)
def getText(self):
text = ''.join(self.data).strip()
return normalizeWhitespace(text)
def htmlToText(s, tagReplace=' '):
"""Turns HTML into text. tagReplace is a string to replace HTML tags with.
"""
x = HtmlToText(tagReplace)
x.feed(s)
return x.getText()
def abbrev(strings, d=None):
"""Returns a dictionary mapping unambiguous abbreviations to full forms."""
def eachSubstring(s):
for i in xrange(1, len(s)+1):
yield s[:i]
if len(strings) != len(set(strings)):
raise ValueError, \
'strings given to utils.abbrev have duplicates: %r' % strings
if d is None:
d = {}
for s in strings:
for abbreviation in eachSubstring(s):
if abbreviation not in d:
d[abbreviation] = s
else:
if abbreviation not in strings:
d[abbreviation] = None
removals = []
for key in d:
if d[key] is None:
removals.append(key)
for key in removals:
del d[key]
return d
def timeElapsed(elapsed, short=False, leadingZeroes=False, years=True,
weeks=True, days=True, hours=True, minutes=True, seconds=True):
"""Given <elapsed> seconds, returns a string with an English description of
how much time as passed. leadingZeroes determines whether 0 days, 0 hours,
etc. will be printed; the others determine what larger time periods should
be used.
"""
ret = []
def format(s, i):
if i or leadingZeroes or ret:
if short:
ret.append('%s%s' % (i, s[0]))
else:
ret.append(nItems(s, i))
elapsed = int(elapsed)
assert years or weeks or days or \
hours or minutes or seconds, 'One flag must be True'
if years:
(yrs, elapsed) = (elapsed // 31536000, elapsed % 31536000)
format('year', yrs)
if weeks:
(wks, elapsed) = (elapsed // 604800, elapsed % 604800)
format('week', wks)
if days:
(ds, elapsed) = (elapsed // 86400, elapsed % 86400)
format('day', ds)
if hours:
(hrs, elapsed) = (elapsed // 3600, elapsed % 3600)
format('hour', hrs)
if minutes or seconds:
(mins, secs) = (elapsed // 60, elapsed % 60)
if leadingZeroes or mins:
format('minute', mins)
if seconds:
leadingZeroes = True
format('second', secs)
if not ret:
raise ValueError, 'Time difference not great enough to be noted.'
if short:
return ' '.join(ret)
else:
return commaAndify(ret)
def distance(s, t):
"""Returns the levenshtein edit distance between two strings."""
n = len(s)
m = len(t)
if n == 0:
return m
elif m == 0:
return n
d = []
for i in range(n+1):
d.append([])
for j in range(m+1):
d[i].append(0)
d[0][j] = j
d[i][0] = i
for i in range(1, n+1):
cs = s[i-1]
for j in range(1, m+1):
ct = t[j-1]
cost = int(cs != ct)
d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cost)
return d[n][m]
_soundextrans = string.maketrans(string.ascii_uppercase,
'01230120022455012623010202')
_notUpper = string.ascii.translate(string.ascii, string.ascii_uppercase)
def soundex(s, length=4):
"""Returns the soundex hash of a given string."""
s = s.upper() # Make everything uppercase.
s = s.translate(string.ascii, _notUpper) # Delete non-letters.
if not s:
raise ValueError, 'Invalid string for soundex: %s'
firstChar = s[0] # Save the first character.
s = s.translate(_soundextrans) # Convert to soundex numbers.
s = s.lstrip(s[0]) # Remove all repeated first characters.
L = [firstChar]
for c in s:
if c != L[-1]:
L.append(c)
L = [c for c in L if c != '0'] + (['0']*(length-1))
s = ''.join(L)
return length and s[:length] or s.rstrip('0')
def dqrepr(s):
"""Returns a repr() of s guaranteed to be in double quotes."""
# The wankers-that-be decided not to use double-quotes anymore in 2.3.
# return '"' + repr("'\x00" + s)[6:]
return '"%s"' % s.encode('string_escape').replace('"', '\\"')
def quoted(s):
"""Returns a quoted s."""
return '"%s"' % s
def _getSep(s):
if len(s) < 2:
raise ValueError, 'string given to _getSep is too short: %r' % s
if s.startswith('m') or s.startswith('s'):
separator = s[1]
else:
separator = s[0]
if separator.isalnum() or separator in '{}[]()<>':
raise ValueError, \
'Invalid separator: separator must not be alphanumeric or in ' \
'"{}[]()<>"'
return separator
def _getSplitterRe(s):
separator = _getSep(s)
return re.compile(r'(?<!\\)%s' % re.escape(separator))
def perlReToPythonRe(s):
"""Converts a string representation of a Perl regular expression (i.e.,
m/^foo$/i or /foo|bar/) to a Python regular expression.
"""
sep = _getSep(s)
splitter = _getSplitterRe(s)
try:
(kind, regexp, flags) = splitter.split(s)
except ValueError: # Unpack list of wrong size.
raise ValueError, 'Must be of the form m/.../ or /.../'
regexp = regexp.replace('\\'+sep, sep)
if kind not in ('', 'm'):
raise ValueError, 'Invalid kind: must be in ("", "m")'
flag = 0
try:
for c in flags.upper():
flag |= getattr(re, c)
except AttributeError:
raise ValueError, 'Invalid flag: %s' % c
try:
return re.compile(regexp, flag)
except re.error, e:
raise ValueError, str(e)
def perlReToReplacer(s):
"""Converts a string representation of a Perl regular expression (i.e.,
s/foo/bar/g or s/foo/bar/i) to a Python function doing the equivalent
replacement.
"""
sep = _getSep(s)
splitter = _getSplitterRe(s)
try:
(kind, regexp, replace, flags) = splitter.split(s)
except ValueError: # Unpack list of wrong size.
raise ValueError, 'Must be of the form s/.../.../'
regexp = regexp.replace('\x08', r'\b')
replace = replace.replace('\\'+sep, sep)
for i in xrange(10):
replace = replace.replace(chr(i), r'\%s' % i)
if kind != 's':
raise ValueError, 'Invalid kind: must be "s"'
g = False
if 'g' in flags:
g = True
flags = filter('g'.__ne__, flags)
r = perlReToPythonRe('/'.join(('', regexp, flags)))
if g:
return curry(r.sub, replace)
else:
return lambda s: r.sub(replace, s, 1)
_perlVarSubstituteRe = re.compile(r'\$\{([^}]+)\}|\$([a-zA-Z][a-zA-Z0-9]*)')
def perlVariableSubstitute(vars, text):
def replacer(m):
(braced, unbraced) = m.groups()
var = braced or unbraced
try:
x = vars[var]
if callable(x):
return x()
else:
return str(x)
except KeyError:
if braced:
return '${%s}' % braced
else:
return '$' + unbraced
return _perlVarSubstituteRe.sub(replacer, text)
def findBinaryInPath(s):
"""Return full path of a binary if it's in PATH, otherwise return None."""
cmdLine = None
for dir in os.getenv('PATH').split(':'):
filename = os.path.join(dir, s)
if os.path.exists(filename):
cmdLine = filename
break
return cmdLine
def commaAndify(seq, comma=',', And='and'):
"""Given a a sequence, returns an English clause for that sequence.
I.e., given [1, 2, 3], returns '1, 2, and 3'
"""
L = list(seq)
if len(L) == 0:
return ''
elif len(L) == 1:
return ''.join(L) # We need this because it raises TypeError.
elif len(L) == 2:
L.insert(1, And)
return ' '.join(L)
else:
L[-1] = '%s %s' % (And, L[-1])
sep = '%s ' % comma
return sep.join(L)
_unCommaTheRe = re.compile(r'(.*),\s*(the)$', re.I)
def unCommaThe(s):
"""Takes a string of the form 'foo, the' and turns it into 'the foo'."""
m = _unCommaTheRe.match(s)
if m is not None:
return '%s %s' % (m.group(2), m.group(1))
else:
return s
def ellipsisify(s, n):
"""Returns a shortened version of s. Produces up to the first n chars at
the nearest word boundary.
"""
if len(s) <= n:
return s
else:
return (textwrap.wrap(s, n-3)[0] + '...')
plurals = TwoWayDictionary({})
def matchCase(s1, s2):
"""Matches the case of s1 in s2"""
if s1.isupper():
return s2.upper()
else:
L = list(s2)
for (i, char) in enumerate(s1[:len(s2)]):
if char.isupper():
L[i] = L[i].upper()
return ''.join(L)
consonants = 'bcdfghjklmnpqrstvwxz'
_pluralizeRegex = re.compile('[%s]y$' % consonants)
def pluralize(s, i=2):
"""Returns the plural of s based on its number i. Put any exceptions to
the general English rule of appending 's' in the plurals dictionary.
"""
if i == 1:
return s
else:
lowered = s.lower()
# Exception dictionary
if lowered in plurals:
return matchCase(s, plurals[lowered])
# Words ending with 'ch', 'sh' or 'ss' such as 'punch(es)', 'fish(es)
# and miss(es)
elif any(lowered.endswith, ['x', 'ch', 'sh', 'ss']):
return matchCase(s, s+'es')
# Words ending with a consonant followed by a 'y' such as
# 'try (tries)' or 'spy (spies)'
elif _pluralizeRegex.search(lowered):
return matchCase(s, s[:-1] + 'ies')
# In all other cases, we simply add an 's' to the base word
else:
return matchCase(s, s+'s')
_depluralizeRegex = re.compile('[%s]ies' % consonants)
def depluralize(s):
"""Returns the singular of s."""
lowered = s.lower()
if lowered in plurals:
return matchCase(s, plurals[lowered])
elif any(lowered.endswith, ['ches', 'shes', 'sses']):
return s[:-2]
elif re.search(_depluralizeRegex, lowered):
return s[:-3] + 'y'
else:
if lowered.endswith('s'):
return s[:-1] # Chop off 's'.
else:
return s # Don't know what to do.
def nItems(item, n, between=None):
"""Works like this:
>>> nItems('clock', 1)
'1 clock'
>>> nItems('clock', 10)
'10 clocks'
>>> nItems('clock', 10, between='grandfather')
'10 grandfather clocks'
"""
if between is None:
return '%s %s' % (n, pluralize(item, n))
else:
return '%s %s %s' % (n, between, pluralize(item, n))
def be(i):
"""Returns the form of the verb 'to be' based on the number i."""
if i == 1:
return 'is'
else:
return 'are'
def has(i):
"""Returns the form of the verb 'to have' based on the number i."""
if i == 1:
return 'has'
else:
return 'have'
def sortBy(f, L):
"""Uses the decorate-sort-undecorate pattern to sort L by function f."""
for (i, elt) in enumerate(L):
L[i] = (f(elt), i, elt)
L.sort()
for (i, elt) in enumerate(L):
L[i] = L[i][2]
if sys.version_info < (2, 4, 0):
def sorted(iterable, cmp=None, key=None, reversed=False):
L = list(iterable)
if key is not None:
assert cmp is None, 'Can\'t use both cmp and key.'
sortBy(key, L)
else:
L.sort(cmp)
if reversed:
L.reverse()
return L
__builtins__['sorted'] = sorted
def mktemp(suffix=''):
"""Gives a decent random string, suitable for a filename."""
r = random.Random()
m = md5.md5(suffix)
r.seed(time.time())
s = str(r.getstate())
for x in xrange(0, random.randrange(400), random.randrange(1, 5)):
m.update(str(x))
m.update(s)
m.update(str(time.time()))
s = m.hexdigest()
return sha.sha(s + str(time.time())).hexdigest() + suffix
def itersplit(isSeparator, iterable, maxsplit=-1, yieldEmpty=False):
"""itersplit(isSeparator, iterable, maxsplit=-1, yieldEmpty=False)
Splits an iterator based on a predicate isSeparator."""
if isinstance(isSeparator, basestring):
f = lambda s: s == isSeparator
else:
f = isSeparator
acc = []
for element in iterable:
if maxsplit == 0 or not f(element):
acc.append(element)
else:
maxsplit -= 1
if acc or yieldEmpty:
yield acc
acc = []
if acc or yieldEmpty:
yield acc
def flatten(seq, strings=False):
"""Flattens a list of lists into a single list. See the test for examples.
"""
for elt in seq:
if not strings and type(elt) == str or type(elt) == unicode:
yield elt
else:
try:
for x in flatten(elt):
yield x
except TypeError:
yield elt
def saltHash(password, salt=None, hash='sha'):
if salt is None:
salt = mktemp()[:8]
if hash == 'sha':
hasher = sha.sha
elif hash == 'md5':
hasher = md5.md5
return '|'.join([salt, hasher(salt + password).hexdigest()])
def safeEval(s, namespace={'True': True, 'False': False, 'None': None}):
"""Evaluates s, safely. Useful for turning strings into tuples/lists/etc.
without unsafely using eval()."""
try:
node = compiler.parse(s)
except SyntaxError, e:
raise ValueError, 'Invalid string: %s.' % e
nodes = compiler.parse(s).node.nodes
if not nodes:
if node.__class__ is compiler.ast.Module:
return node.doc
else:
raise ValueError, 'Unsafe string: %s' % quoted(s)
node = nodes[0]
if node.__class__ is not compiler.ast.Discard:
raise ValueError, 'Invalid expression: %s' % quoted(s)
node = node.getChildNodes()[0]
def checkNode(node):
if node.__class__ is compiler.ast.Const:
return True
if node.__class__ in (compiler.ast.List,
compiler.ast.Tuple,
compiler.ast.Dict):
return all(checkNode, node.getChildNodes())
if node.__class__ is compiler.ast.Name:
if node.name in namespace:
return True
else:
return False
else:
return False
if checkNode(node):
return eval(s, namespace, namespace)
else:
raise ValueError, 'Unsafe string: %s' % quoted(s)
def exnToString(e):
"""Turns a simple exception instance into a string (better than str(e))"""
strE = str(e)
if strE:
return '%s: %s' % (e.__class__.__name__, strE)
else:
return e.__class__.__name__
class IterableMap(object):
"""Define .iteritems() in a class and subclass this to get the other iters.
"""
def iteritems(self):
raise NotImplementedError
def iterkeys(self):
for (key, _) in self.iteritems():
yield key
__iter__ = iterkeys
def itervalues(self):
for (_, value) in self.iteritems():
yield value
def items(self):
return list(self.iteritems())
def keys(self):
return list(self.iterkeys())
def values(self):
return list(self.itervalues())
def __len__(self):
ret = 0
for _ in self.iteritems():
ret += 1
return ret
def __nonzero__(self):
for _ in self.iteritems():
return True
return False
def nonCommentLines(fd):
for line in fd:
if not line.startswith('#'):
yield line
def nonEmptyLines(fd):
## for line in fd:
## if line.strip():
## yield line
return ifilter(str.strip, fd)
def nonCommentNonEmptyLines(fd):
return nonEmptyLines(nonCommentLines(fd))
def changeFunctionName(f, name, doc=None):
if doc is None:
doc = f.__doc__
newf = types.FunctionType(f.func_code, f.func_globals, name,
f.func_defaults, f.func_closure)
newf.__doc__ = doc
return newf
def getSocket(host):
"""Returns a socket of the correct AF_INET type (v4 or v6) in order to
communicate with host.
"""
host = socket.gethostbyname(host)
if isIP(host):
return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
elif isIPV6(host):
return socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
raise socket.error, 'Something wonky happened.'
def isIP(s):
"""Returns whether or not a given string is an IPV4 address.
>>> isIP('255.255.255.255')
1
>>> isIP('abc.abc.abc.abc')
0
"""
try:
return bool(socket.inet_aton(s))
except socket.error:
return False
def bruteIsIPV6(s):
if s.count('::') <= 1:
L = s.split(':')
if len(L) <= 8:
for x in L:
if x:
try:
int(x, 16)
except ValueError:
return False
return True
return False
def isIPV6(s):
"""Returns whether or not a given string is an IPV6 address."""
try:
if hasattr(socket, 'inet_pton'):
return bool(socket.inet_pton(socket.AF_INET6, s))
else:
return bruteIsIPV6(s)
except socket.error:
try:
socket.inet_pton(socket.AF_INET6, '::')
except socket.error:
# We gotta fake it.
return bruteIsIPV6(s)
return False
class InsensitivePreservingDict(UserDict.DictMixin, object):
def key(self, s):
"""Override this if you wish."""
if s is not None:
s = s.lower()
return s
def __init__(self, dict=None, key=None):
if key is not None:
self.key = key
self.data = {}
if dict is not None:
self.update(dict)
def __repr__(self):
return '%s(%s)' % (self.__class__.__name__,
super(InsensitivePreservingDict, self).__repr__())
def fromkeys(cls, keys, s=None, dict=None, key=None):
d = cls(dict=dict, key=key)
for key in keys:
d[key] = s
return d
fromkeys = classmethod(fromkeys)
def __getitem__(self, k):
return self.data[self.key(k)][1]
def __setitem__(self, k, v):
self.data[self.key(k)] = (k, v)
def __delitem__(self, k):
del self.data[self.key(k)]
def iteritems(self):
return self.data.itervalues()
def keys(self):
L = []
for (k, _) in self.iteritems():
L.append(k)
return L
def __reduce__(self):
return (self.__class__, (dict(self.data.values()),))
class NormalizingSet(sets.Set):
def __init__(self, iterable=()):
iterable = itertools.imap(self.normalize, iterable)
super(NormalizingSet, self).__init__(iterable)
def normalize(self, x):
return x
def add(self, x):
return super(NormalizingSet, self).add(self.normalize(x))
def remove(self, x):
return super(NormalizingSet, self).remove(self.normalize(x))
def discard(self, x):
return super(NormalizingSet, self).discard(self.normalize(x))
def __contains__(self, x):
return super(NormalizingSet, self).__contains__(self.normalize(x))
has_key = __contains__
def mungeEmailForWeb(s):
s = s.replace('@', ' AT ')
s = s.replace('.', ' DOT ')
return s
class AtomicFile(file):
"""Used for files that need to be atomically written -- i.e., if there's a
failure, the original file remains, unmodified. mode must be 'w' or 'wb'"""
def __init__(self, filename, mode='w', allowEmptyOverwrite=True,
makeBackupIfSmaller=True, tmpDir=None, backupDir=None):
if mode not in ('w', 'wb'):
raise ValueError, 'Invalid mode: %s' % quoted(mode)
self.rolledback = False
self.allowEmptyOverwrite = allowEmptyOverwrite
self.makeBackupIfSmaller = makeBackupIfSmaller
self.filename = filename
self.backupDir = backupDir
if tmpDir is None:
# If not given a tmpDir, we'll just put a random token on the end
# of our filename and put it in the same directory.
self.tempFilename = '%s.%s' % (self.filename, mktemp())
else:
# If given a tmpDir, we'll get the basename (just the filename, no
# directory), put our random token on the end, and put it in tmpDir
tempFilename = '%s.%s' % (os.path.basename(self.filename), mktemp())
self.tempFilename = os.path.join(tmpDir, tempFilename)
# This doesn't work because of the uncollectable garbage effect.
# self.__parent = super(AtomicFile, self)
super(AtomicFile, self).__init__(self.tempFilename, mode)
def rollback(self):
if not self.closed:
super(AtomicFile, self).close()
if os.path.exists(self.tempFilename):
os.remove(self.tempFilename)
self.rolledback = True
def close(self):
if not self.rolledback:
super(AtomicFile, self).close()
# We don't mind writing an empty file if the file we're overwriting
# doesn't exist.
newSize = os.path.getsize(self.tempFilename)
originalExists = os.path.exists(self.filename)
if newSize or self.allowEmptyOverwrite or not originalExists:
if originalExists:
oldSize = os.path.getsize(self.filename)
if self.makeBackupIfSmaller and newSize < oldSize:
now = int(time.time())
backupFilename = '%s.backup.%s' % (self.filename, now)
if self.backupDir is not None:
backupFilename = os.path.basename(backupFilename)
backupFilename = os.path.join(self.backupDir,
backupFilename)
shutil.copy(self.filename, backupFilename)
# We use shutil.move here instead of os.rename because
# the latter doesn't work on Windows when self.filename
# (the target) already exists. shutil.move handles those
# intricacies for us.
# This raises IOError if we can't write to the file. Since
# in *nix, it only takes write perms to the *directory* to
# rename a file (and shutil.move will use os.rename if
# possible), we first check if we have the write permission
# and only then do we write.
fd = file(self.filename, 'a')
fd.close()
shutil.move(self.tempFilename, self.filename)
else:
raise ValueError, 'AtomicFile.close called after rollback.'
def __del__(self):
# We rollback because if we're deleted without being explicitly closed,
# that's bad. We really should log this here, but as of yet we've got
# no logging facility in utils. I've got some ideas for this, though.
self.rollback()
def transactionalFile(*args, **kwargs):
# This exists so it can be replaced by a function that provides the tmpDir.
# We do that replacement in conf.py.
return AtomicFile(*args, **kwargs)
def stackTrace(frame=None, compact=True):
if frame is None:
frame = sys._getframe()
if compact:
L = []
while frame:
lineno = frame.f_lineno
funcname = frame.f_code.co_name
filename = os.path.basename(frame.f_code.co_filename)
L.append('[%s|%s|%s]' % (filename, funcname, lineno))
frame = frame.f_back
return textwrap.fill(' '.join(L))
else:
return traceback.format_stack(frame)
def callTracer(fd=None, basename=True):
if fd is None:
fd = sys.stdout
def tracer(frame, event, _):
if event == 'call':
code = frame.f_code
lineno = frame.f_lineno
funcname = code.co_name
filename = code.co_filename
if basename:
filename = os.path.basename(filename)
print >>fd, '%s: %s(%s)' % (filename, funcname, lineno)
return tracer
def toBool(s):
s = s.strip().lower()
if s in ('true', 'on', 'enable', 'enabled', '1'):
return True
elif s in ('false', 'off', 'disable', 'disabled', '0'):
return False
else:
raise ValueError, 'Invalid string for toBool: %s' % quoted(s)
def mapinto(f, L):
for (i, x) in enumerate(L):
L[i] = f(x)
if __name__ == '__main__':
import doctest
doctest.testmod(sys.modules['__main__'])
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

106
src/utils/__init__.py Normal file
View File

@ -0,0 +1,106 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
import sys
###
# csv.{join,split} -- useful functions that should exist.
###
import csv
import cStringIO as StringIO
def join(L):
fd = StringIO.StringIO()
writer = csv.writer(fd)
writer.writerow(L)
return fd.getvalue().rstrip('\r\n')
def split(s):
fd = StringIO.StringIO(s)
reader = csv.reader(fd)
return reader.next()
csv.join = join
csv.split = split
# We use this often enough that we're going to stick it in builtins.
def force(x):
if callable(x):
return x()
else:
return x
__builtins__['force'] = force
if sys.version_info < (2, 4, 0):
def reversed(L):
"""Iterates through a sequence in reverse."""
for i in xrange(len(L) - 1, -1, -1):
yield L[i]
__builtins__['reversed'] = reversed
def sorted(iterable, cmp=None, key=None, reversed=False):
L = list(iterable)
if key is not None:
assert cmp is None, 'Can\'t use both cmp and key.'
sortBy(key, L)
else:
L.sort(cmp)
if reversed:
L.reverse()
return L
__builtins__['sorted'] = sorted
import operator
def itemgetter(i):
return lambda x: x[i]
def attrgetter(attr):
return lambda x: getattr(x, attr)
operator.itemgetter = itemgetter
operator.attrgetter = attrgetter
import sets
__builtins__['set'] = sets.Set
__builtins__['frozenset'] = sets.ImmutableSet
import socket
# Some socket modules don't have sslerror, so we'll just make it an error.
if not hasattr(socket, 'sslerror'):
socket.sslerror = socket.error
# These imports need to happen below the block above, so things get put into
# __builtins__ appropriately.
from gen import *
import net
import web
import seq
import str
import file
import iter
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

150
src/utils/file.py Normal file
View File

@ -0,0 +1,150 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
import os
import md5
import sha
import time
import random
import shutil
import os.path
from iter import ifilter
def mktemp(suffix=''):
"""Gives a decent random string, suitable for a filename."""
r = random.Random()
m = md5.md5(suffix)
r.seed(time.time())
s = str(r.getstate())
for x in xrange(0, random.randrange(400), random.randrange(1, 5)):
m.update(str(x))
m.update(s)
m.update(str(time.time()))
s = m.hexdigest()
return sha.sha(s + str(time.time())).hexdigest() + suffix
def nonCommentLines(fd):
for line in fd:
if not line.startswith('#'):
yield line
def nonEmptyLines(fd):
return ifilter(str.strip, fd)
def nonCommentNonEmptyLines(fd):
return nonEmptyLines(nonCommentLines(fd))
class AtomicFile(file):
"""Used for files that need to be atomically written -- i.e., if there's a
failure, the original file remains, unmodified. mode must be 'w' or 'wb'"""
class default(object): # Holder for values.
# Callables?
tmpDir = None
backupDir = None
makeBackupIfSmaller = True
allowEmptyOverwrite = True
def __init__(self, filename, mode='w', allowEmptyOverwrite=None,
makeBackupIfSmaller=None, tmpDir=None, backupDir=None):
if tmpDir is None:
tmpDir = force(self.default.tmpDir)
if backupDir is None:
backupDir = force(self.default.backupDir)
if makeBackupIfSmaller is None:
makeBackupIfSmaller = force(self.default.makeBackupIfSmaller)
if allowEmptyOverwrite is None:
allowEmptyOverwrite = force(self.default.allowEmptyOverwrite)
if mode not in ('w', 'wb'):
raise ValueError, 'Invalid mode: %s' % quoted(mode)
self.rolledback = False
self.allowEmptyOverwrite = allowEmptyOverwrite
self.makeBackupIfSmaller = makeBackupIfSmaller
self.filename = filename
self.backupDir = backupDir
if tmpDir is None:
# If not given a tmpDir, we'll just put a random token on the end
# of our filename and put it in the same directory.
self.tempFilename = '%s.%s' % (self.filename, mktemp())
else:
# If given a tmpDir, we'll get the basename (just the filename, no
# directory), put our random token on the end, and put it in tmpDir
tempFilename = '%s.%s' % (os.path.basename(self.filename), mktemp())
self.tempFilename = os.path.join(tmpDir, tempFilename)
# This doesn't work because of the uncollectable garbage effect.
# self.__parent = super(AtomicFile, self)
super(AtomicFile, self).__init__(self.tempFilename, mode)
def rollback(self):
if not self.closed:
super(AtomicFile, self).close()
if os.path.exists(self.tempFilename):
os.remove(self.tempFilename)
self.rolledback = True
def close(self):
if not self.rolledback:
super(AtomicFile, self).close()
# We don't mind writing an empty file if the file we're overwriting
# doesn't exist.
newSize = os.path.getsize(self.tempFilename)
originalExists = os.path.exists(self.filename)
if newSize or self.allowEmptyOverwrite or not originalExists:
if originalExists:
oldSize = os.path.getsize(self.filename)
if self.makeBackupIfSmaller and newSize < oldSize:
now = int(time.time())
backupFilename = '%s.backup.%s' % (self.filename, now)
if self.backupDir is not None:
backupFilename = os.path.basename(backupFilename)
backupFilename = os.path.join(self.backupDir,
backupFilename)
shutil.copy(self.filename, backupFilename)
# We use shutil.move here instead of os.rename because
# the latter doesn't work on Windows when self.filename
# (the target) already exists. shutil.move handles those
# intricacies for us.
# This raises IOError if we can't write to the file. Since
# in *nix, it only takes write perms to the *directory* to
# rename a file (and shutil.move will use os.rename if
# possible), we first check if we have the write permission
# and only then do we write.
fd = file(self.filename, 'a')
fd.close()
shutil.move(self.tempFilename, self.filename)
else:
raise ValueError, 'AtomicFile.close called after rollback.'
def __del__(self):
# We rollback because if we're deleted without being explicitly closed,
# that's bad. We really should log this here, but as of yet we've got
# no logging facility in utils. I've got some ideas for this, though.
self.rollback()
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

326
src/utils/gen.py Normal file
View File

@ -0,0 +1,326 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
import os
import sys
import md5
import new
import sha
import time
import types
import compiler
import textwrap
import UserDict
import traceback
from iter import imap
from file import mktemp
from str import quoted, nItems, commaAndify
def abbrev(strings, d=None):
"""Returns a dictionary mapping unambiguous abbreviations to full forms."""
def eachSubstring(s):
for i in xrange(1, len(s)+1):
yield s[:i]
if len(strings) != len(set(strings)):
raise ValueError, \
'strings given to utils.abbrev have duplicates: %r' % strings
if d is None:
d = {}
for s in strings:
for abbreviation in eachSubstring(s):
if abbreviation not in d:
d[abbreviation] = s
else:
if abbreviation not in strings:
d[abbreviation] = None
removals = []
for key in d:
if d[key] is None:
removals.append(key)
for key in removals:
del d[key]
return d
def timeElapsed(elapsed, short=False, leadingZeroes=False, years=True,
weeks=True, days=True, hours=True, minutes=True, seconds=True):
"""Given <elapsed> seconds, returns a string with an English description of
how much time as passed. leadingZeroes determines whether 0 days, 0 hours,
etc. will be printed; the others determine what larger time periods should
be used.
"""
ret = []
def format(s, i):
if i or leadingZeroes or ret:
if short:
ret.append('%s%s' % (i, s[0]))
else:
ret.append(nItems(s, i))
elapsed = int(elapsed)
assert years or weeks or days or \
hours or minutes or seconds, 'One flag must be True'
if years:
(yrs, elapsed) = (elapsed // 31536000, elapsed % 31536000)
format('year', yrs)
if weeks:
(wks, elapsed) = (elapsed // 604800, elapsed % 604800)
format('week', wks)
if days:
(ds, elapsed) = (elapsed // 86400, elapsed % 86400)
format('day', ds)
if hours:
(hrs, elapsed) = (elapsed // 3600, elapsed % 3600)
format('hour', hrs)
if minutes or seconds:
(mins, secs) = (elapsed // 60, elapsed % 60)
if leadingZeroes or mins:
format('minute', mins)
if seconds:
leadingZeroes = True
format('second', secs)
if not ret:
raise ValueError, 'Time difference not great enough to be noted.'
if short:
return ' '.join(ret)
else:
return commaAndify(ret)
def findBinaryInPath(s):
"""Return full path of a binary if it's in PATH, otherwise return None."""
cmdLine = None
for dir in os.getenv('PATH').split(':'):
filename = os.path.join(dir, s)
if os.path.exists(filename):
cmdLine = filename
break
return cmdLine
def sortBy(f, L):
"""Uses the decorate-sort-undecorate pattern to sort L by function f."""
for (i, elt) in enumerate(L):
L[i] = (f(elt), i, elt)
L.sort()
for (i, elt) in enumerate(L):
L[i] = L[i][2]
def saltHash(password, salt=None, hash='sha'):
if salt is None:
salt = mktemp()[:8]
if hash == 'sha':
hasher = sha.sha
elif hash == 'md5':
hasher = md5.md5
return '|'.join([salt, hasher(salt + password).hexdigest()])
def safeEval(s, namespace={'True': True, 'False': False, 'None': None}):
"""Evaluates s, safely. Useful for turning strings into tuples/lists/etc.
without unsafely using eval()."""
try:
node = compiler.parse(s)
except SyntaxError, e:
raise ValueError, 'Invalid string: %s.' % e
nodes = compiler.parse(s).node.nodes
if not nodes:
if node.__class__ is compiler.ast.Module:
return node.doc
else:
raise ValueError, 'Unsafe string: %s' % quoted(s)
node = nodes[0]
if node.__class__ is not compiler.ast.Discard:
raise ValueError, 'Invalid expression: %s' % quoted(s)
node = node.getChildNodes()[0]
def checkNode(node):
if node.__class__ is compiler.ast.Const:
return True
if node.__class__ in (compiler.ast.List,
compiler.ast.Tuple,
compiler.ast.Dict):
return all(checkNode, node.getChildNodes())
if node.__class__ is compiler.ast.Name:
if node.name in namespace:
return True
else:
return False
else:
return False
if checkNode(node):
return eval(s, namespace, namespace)
else:
raise ValueError, 'Unsafe string: %s' % quoted(s)
def exnToString(e):
"""Turns a simple exception instance into a string (better than str(e))"""
strE = str(e)
if strE:
return '%s: %s' % (e.__class__.__name__, strE)
else:
return e.__class__.__name__
class IterableMap(object):
"""Define .iteritems() in a class and subclass this to get the other iters.
"""
def iteritems(self):
raise NotImplementedError
def iterkeys(self):
for (key, _) in self.iteritems():
yield key
__iter__ = iterkeys
def itervalues(self):
for (_, value) in self.iteritems():
yield value
def items(self):
return list(self.iteritems())
def keys(self):
return list(self.iterkeys())
def values(self):
return list(self.itervalues())
def __len__(self):
ret = 0
for _ in self.iteritems():
ret += 1
return ret
def __nonzero__(self):
for _ in self.iteritems():
return True
return False
def changeFunctionName(f, name, doc=None):
if doc is None:
doc = f.__doc__
newf = types.FunctionType(f.func_code, f.func_globals, name,
f.func_defaults, f.func_closure)
newf.__doc__ = doc
return newf
class InsensitivePreservingDict(UserDict.DictMixin, object):
def key(self, s):
"""Override this if you wish."""
if s is not None:
s = s.lower()
return s
def __init__(self, dict=None, key=None):
if key is not None:
self.key = key
self.data = {}
if dict is not None:
self.update(dict)
def __repr__(self):
return '%s(%s)' % (self.__class__.__name__,
super(InsensitivePreservingDict, self).__repr__())
def fromkeys(cls, keys, s=None, dict=None, key=None):
d = cls(dict=dict, key=key)
for key in keys:
d[key] = s
return d
fromkeys = classmethod(fromkeys)
def __getitem__(self, k):
return self.data[self.key(k)][1]
def __setitem__(self, k, v):
self.data[self.key(k)] = (k, v)
def __delitem__(self, k):
del self.data[self.key(k)]
def iteritems(self):
return self.data.itervalues()
def keys(self):
L = []
for (k, _) in self.iteritems():
L.append(k)
return L
def __reduce__(self):
return (self.__class__, (dict(self.data.values()),))
class NormalizingSet(set):
def __init__(self, iterable=()):
iterable = imap(self.normalize, iterable)
super(NormalizingSet, self).__init__(iterable)
def normalize(self, x):
return x
def add(self, x):
return super(NormalizingSet, self).add(self.normalize(x))
def remove(self, x):
return super(NormalizingSet, self).remove(self.normalize(x))
def discard(self, x):
return super(NormalizingSet, self).discard(self.normalize(x))
def __contains__(self, x):
return super(NormalizingSet, self).__contains__(self.normalize(x))
has_key = __contains__
def stackTrace(frame=None, compact=True):
if frame is None:
frame = sys._getframe()
if compact:
L = []
while frame:
lineno = frame.f_lineno
funcname = frame.f_code.co_name
filename = os.path.basename(frame.f_code.co_filename)
L.append('[%s|%s|%s]' % (filename, funcname, lineno))
frame = frame.f_back
return textwrap.fill(' '.join(L))
else:
return traceback.format_stack(frame)
def callTracer(fd=None, basename=True):
if fd is None:
fd = sys.stdout
def tracer(frame, event, _):
if event == 'call':
code = frame.f_code
lineno = frame.f_lineno
funcname = code.co_name
filename = code.co_filename
if basename:
filename = os.path.basename(filename)
print >>fd, '%s: %s(%s)' % (filename, funcname, lineno)
return tracer
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

147
src/utils/iter.py Normal file
View File

@ -0,0 +1,147 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
import new
import random
from itertools import *
def len(iterable):
"""Returns the length of an iterator."""
i = 0
for _ in iterable:
i += 1
return i
def trueCycle(iterable):
while 1:
yielded = False
for x in iterable:
yield x
yielded = True
if not yielded:
raise StopIteration
def groupby(key, iterable):
if key is None:
key = lambda x: x
it = iter(iterable)
value = it.next() # If there are no items, this takes an early exit
oldkey = key(value)
group = [value]
for value in it:
newkey = key(value)
if newkey != oldkey:
yield group
group = []
oldkey = newkey
group.append(value)
yield group
def partition(p, iterable):
"""Partitions an iterable based on a predicate p.
Returns a (yes,no) tuple"""
no = []
yes = []
for elt in iterable:
if p(elt):
yes.append(elt)
else:
no.append(elt)
return (yes, no)
def any(p, iterable):
"""Returns true if any element in iterable satisfies predicate p."""
for elt in ifilter(p, iterable):
return True
else:
return False
def all(p, iterable):
"""Returns true if all elements in iterable satisfy predicate p."""
for elt in ifilterfalse(p, iterable):
return False
else:
return True
def choice(iterable):
if isinstance(iterable, (list, tuple)):
return random.choice(iterable)
else:
n = 1
m = new.module('') # Guaranteed unique value.
ret = m
for x in iterable:
if random.random() < 1/n:
ret = x
n += 1
if ret is m:
raise IndexError
return ret
def flatten(iterable, strings=False):
"""Flattens a list of lists into a single list. See the test for examples.
"""
for elt in iterable:
if not strings and isinstance(elt, basestring):
yield elt
else:
try:
for x in flatten(elt):
yield x
except TypeError:
yield elt
def split(isSeparator, iterable, maxsplit=-1, yieldEmpty=False):
"""split(isSeparator, iterable, maxsplit=-1, yieldEmpty=False)
Splits an iterator based on a predicate isSeparator."""
if isinstance(isSeparator, basestring):
f = lambda s: s == isSeparator
else:
f = isSeparator
acc = []
for element in iterable:
if maxsplit == 0 or not f(element):
acc.append(element)
else:
maxsplit -= 1
if acc or yieldEmpty:
yield acc
acc = []
if acc or yieldEmpty:
yield acc
def ilen(iterable):
i = 0
for _ in iterable:
i += 1
return i
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

90
src/utils/net.py Normal file
View File

@ -0,0 +1,90 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
"""
Simple utility modules.
"""
import socket
def getSocket(host):
"""Returns a socket of the correct AF_INET type (v4 or v6) in order to
communicate with host.
"""
host = socket.gethostbyname(host)
if isIP(host):
return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
elif isIPV6(host):
return socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
raise socket.error, 'Something wonky happened.'
def isIP(s):
"""Returns whether or not a given string is an IPV4 address.
>>> isIP('255.255.255.255')
1
>>> isIP('abc.abc.abc.abc')
0
"""
try:
return bool(socket.inet_aton(s))
except socket.error:
return False
def bruteIsIPV6(s):
if s.count('::') <= 1:
L = s.split(':')
if len(L) <= 8:
for x in L:
if x:
try:
int(x, 16)
except ValueError:
return False
return True
return False
def isIPV6(s):
"""Returns whether or not a given string is an IPV6 address."""
try:
if hasattr(socket, 'inet_pton'):
return bool(socket.inet_pton(socket.AF_INET6, s))
else:
return bruteIsIPV6(s)
except socket.error:
try:
socket.inet_pton(socket.AF_INET6, '::')
except socket.error:
# We gotta fake it.
return bruteIsIPV6(s)
return False
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

24
test/test_webutils.py → src/utils/seq.py
View File

@ -1,5 +1,5 @@
###
# Copyright (c) 2004-2005, Jeremiah Fincher
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@ -27,21 +27,19 @@
# POSSIBILITY OF SUCH DAMAGE.
###
from supybot.test import *
def window(L, size):
"""Returns a sliding 'window' through the list L of size size."""
assert not isinstance(L, int), 'Argument order swapped: window(L, size)'
if size < 1:
raise ValueError, 'size <= 0 disallowed.'
for i in xrange(len(L) - (size-1)):
yield L[i:i+size]
import supybot.webutils as webutils
def mapinto(f, L):
for (i, x) in enumerate(L):
L[i] = f(x)
class WebutilsTestCase(SupyTestCase):
def testGetDomain(self):
self.assertEqual(webutils.getDomain('http://slashdot.org/foo/bar.exe'),
'slashdot.org')
if network:
def testGetUrlWithSize(self):
url = 'http://slashdot.org/'
self.failUnless(len(webutils.getUrl(url, 1024)) == 1024)
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

354
src/utils/str.py Normal file
View File

@ -0,0 +1,354 @@
###
# Copyright (c) 2002-2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
"""
Simple utility functions related to strings.
"""
import re
import new
import string
import textwrap
import supybot.structures as structures
curry = new.instancemethod
chars = string.maketrans('', '')
def rsplit(s, sep=None, maxsplit=-1):
"""Equivalent to str.split, except splitting from the right."""
if sys.version_info < (2, 4, 0):
if sep is not None:
sep = sep[::-1]
L = s[::-1].split(sep, maxsplit)
L.reverse()
return [s[::-1] for s in L]
else:
return s.rsplit(sep, maxsplit)
def normalizeWhitespace(s):
"""Normalizes the whitespace in a string; \s+ becomes one space."""
return ' '.join(s.split())
def distance(s, t):
"""Returns the levenshtein edit distance between two strings."""
n = len(s)
m = len(t)
if n == 0:
return m
elif m == 0:
return n
d = []
for i in xrange(n+1):
d.append([])
for j in xrange(m+1):
d[i].append(0)
d[0][j] = j
d[i][0] = i
for i in xrange(1, n+1):
cs = s[i-1]
for j in xrange(1, m+1):
ct = t[j-1]
cost = int(cs != ct)
d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cost)
return d[n][m]
_soundextrans = string.maketrans(string.ascii_uppercase,
'01230120022455012623010202')
_notUpper = chars.translate(chars, string.ascii_uppercase)
def soundex(s, length=4):
"""Returns the soundex hash of a given string."""
s = s.upper() # Make everything uppercase.
s = s.translate(chars, _notUpper) # Delete non-letters.
if not s:
raise ValueError, 'Invalid string for soundex: %s'
firstChar = s[0] # Save the first character.
s = s.translate(_soundextrans) # Convert to soundex numbers.
s = s.lstrip(s[0]) # Remove all repeated first characters.
L = [firstChar]
for c in s:
if c != L[-1]:
L.append(c)
L = [c for c in L if c != '0'] + (['0']*(length-1))
s = ''.join(L)
return length and s[:length] or s.rstrip('0')
def dqrepr(s):
"""Returns a repr() of s guaranteed to be in double quotes."""
# The wankers-that-be decided not to use double-quotes anymore in 2.3.
# return '"' + repr("'\x00" + s)[6:]
return '"%s"' % s.encode('string_escape').replace('"', '\\"')
def quoted(s):
"""Returns a quoted s."""
return '"%s"' % s
def _getSep(s):
if len(s) < 2:
raise ValueError, 'string given to _getSep is too short: %r' % s
if s.startswith('m') or s.startswith('s'):
separator = s[1]
else:
separator = s[0]
if separator.isalnum() or separator in '{}[]()<>':
raise ValueError, \
'Invalid separator: separator must not be alphanumeric or in ' \
'"{}[]()<>"'
return separator
def _getSplitterRe(s):
separator = _getSep(s)
return re.compile(r'(?<!\\)%s' % re.escape(separator))
def perlReToPythonRe(s):
"""Converts a string representation of a Perl regular expression (i.e.,
m/^foo$/i or /foo|bar/) to a Python regular expression.
"""
sep = _getSep(s)
splitter = _getSplitterRe(s)
try:
(kind, regexp, flags) = splitter.split(s)
except ValueError: # Unpack list of wrong size.
raise ValueError, 'Must be of the form m/.../ or /.../'
regexp = regexp.replace('\\'+sep, sep)
if kind not in ('', 'm'):
raise ValueError, 'Invalid kind: must be in ("", "m")'
flag = 0
try:
for c in flags.upper():
flag |= getattr(re, c)
except AttributeError:
raise ValueError, 'Invalid flag: %s' % c
try:
return re.compile(regexp, flag)
except re.error, e:
raise ValueError, str(e)
def perlReToReplacer(s):
"""Converts a string representation of a Perl regular expression (i.e.,
s/foo/bar/g or s/foo/bar/i) to a Python function doing the equivalent
replacement.
"""
sep = _getSep(s)
splitter = _getSplitterRe(s)
try:
(kind, regexp, replace, flags) = splitter.split(s)
except ValueError: # Unpack list of wrong size.
raise ValueError, 'Must be of the form s/.../.../'
regexp = regexp.replace('\x08', r'\b')
replace = replace.replace('\\'+sep, sep)
for i in xrange(10):
replace = replace.replace(chr(i), r'\%s' % i)
if kind != 's':
raise ValueError, 'Invalid kind: must be "s"'
g = False
if 'g' in flags:
g = True
flags = filter('g'.__ne__, flags)
r = perlReToPythonRe('/'.join(('', regexp, flags)))
if g:
return curry(r.sub, replace)
else:
return lambda s: r.sub(replace, s, 1)
_perlVarSubstituteRe = re.compile(r'\$\{([^}]+)\}|\$([a-zA-Z][a-zA-Z0-9]*)')
def perlVariableSubstitute(vars, text):
def replacer(m):
(braced, unbraced) = m.groups()
var = braced or unbraced
try:
x = vars[var]
if callable(x):
return x()
else:
return str(x)
except KeyError:
if braced:
return '${%s}' % braced
else:
return '$' + unbraced
return _perlVarSubstituteRe.sub(replacer, text)
def commaAndify(seq, comma=',', And='and'):
"""Given a a sequence, returns an English clause for that sequence.
I.e., given [1, 2, 3], returns '1, 2, and 3'
"""
L = list(seq)
if len(L) == 0:
return ''
elif len(L) == 1:
return ''.join(L) # We need this because it raises TypeError.
elif len(L) == 2:
L.insert(1, And)
return ' '.join(L)
else:
L[-1] = '%s %s' % (And, L[-1])
sep = '%s ' % comma
return sep.join(L)
_unCommaTheRe = re.compile(r'(.*),\s*(the)$', re.I)
def unCommaThe(s):
"""Takes a string of the form 'foo, the' and turns it into 'the foo'."""
m = _unCommaTheRe.match(s)
if m is not None:
return '%s %s' % (m.group(2), m.group(1))
else:
return s
def ellipsisify(s, n):
"""Returns a shortened version of s. Produces up to the first n chars at
the nearest word boundary.
"""
if len(s) <= n:
return s
else:
return (textwrap.wrap(s, n-3)[0] + '...')
plurals = structures.TwoWayDictionary({})
def matchCase(s1, s2):
"""Matches the case of s1 in s2"""
if s1.isupper():
return s2.upper()
else:
L = list(s2)
for (i, char) in enumerate(s1[:len(s2)]):
if char.isupper():
L[i] = L[i].upper()
return ''.join(L)
consonants = 'bcdfghjklmnpqrstvwxz'
_pluralizeRegex = re.compile('[%s]y$' % consonants)
def pluralize(s, i=2):
"""Returns the plural of s based on its number i. Put any exceptions to
the general English rule of appending 's' in the plurals dictionary.
"""
if i == 1:
return s
else:
lowered = s.lower()
# Exception dictionary
if lowered in plurals:
return matchCase(s, plurals[lowered])
# Words ending with 'ch', 'sh' or 'ss' such as 'punch(es)', 'fish(es)
# and miss(es)
elif any(lowered.endswith, ['x', 'ch', 'sh', 'ss']):
return matchCase(s, s+'es')
# Words ending with a consonant followed by a 'y' such as
# 'try (tries)' or 'spy (spies)'
elif _pluralizeRegex.search(lowered):
return matchCase(s, s[:-1] + 'ies')
# In all other cases, we simply add an 's' to the base word
else:
return matchCase(s, s+'s')
_depluralizeRegex = re.compile('[%s]ies' % consonants)
def depluralize(s):
"""Returns the singular of s."""
lowered = s.lower()
if lowered in plurals:
return matchCase(s, plurals[lowered])
elif any(lowered.endswith, ['ches', 'shes', 'sses']):
return s[:-2]
elif re.search(_depluralizeRegex, lowered):
return s[:-3] + 'y'
else:
if lowered.endswith('s'):
return s[:-1] # Chop off 's'.
else:
return s # Don't know what to do.
def nItems(item, n, between=None):
"""Works like this:
>>> nItems('clock', 1)
'1 clock'
>>> nItems('clock', 10)
'10 clocks'
>>> nItems('clock', 10, between='grandfather')
'10 grandfather clocks'
"""
if between is None:
return '%s %s' % (n, pluralize(item, n))
else:
return '%s %s %s' % (n, between, pluralize(item, n))
def be(i):
"""Returns the form of the verb 'to be' based on the number i."""
if i == 1:
return 'is'
else:
return 'are'
def has(i):
"""Returns the form of the verb 'to have' based on the number i."""
if i == 1:
return 'has'
else:
return 'have'
_formatRe = re.compile('%([isfbhL])')
def format(s, *args, **kwargs):
kwargs.setdefault('decimalSeparator', decimalSeparator)
kwargs.setdefault('thousandsSeparator', thousandsSeparator)
args = list(args)
args.reverse() # For more efficiency popping.
def sub(match):
char = match.group(1)
if char == 's': # Plain string.
return str(args.pop())
elif char == 'i': # Integer
# XXX Improve me!
return str(args.pop())
elif char == 'f': # Float
# XXX Improve me!
return str(args.pop())
elif char == 'b': # form of the verb 'to be'
return be(args.pop())
elif char == 'h': # form of the verb 'to have'
return has(args.pop())
elif char == 'L': # commaAndify the list.
return commaAndify(args.pop())
else:
assert False, 'Invalid char in sub (in format).'
return _formatRe.sub(sub, s)
def toBool(s):
s = s.strip().lower()
if s in ('true', 'on', 'enable', 'enabled', '1'):
return True
elif s in ('false', 'off', 'disable', 'disabled', '0'):
return False
else:
raise ValueError, 'Invalid string for toBool: %s' % quoted(s)
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

56
src/webutils.py → src/utils/web.py
View File

@ -27,24 +27,22 @@
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.fix as fix
import re
import socket
import urllib
import urllib2
import httplib
import sgmllib
import urlparse
import htmlentitydefs
import supybot.conf as conf
from str import normalizeWhitespace
Request = urllib2.Request
urlquote = urllib.quote
urlunquote = urllib.unquote
class WebError(Exception):
class Error(Exception):
pass
# XXX We should tighten this up a bit.
@ -75,14 +73,18 @@ def strError(e):
else:
return str(e)
_headers = {
'User-agent': 'Mozilla/4.0 (compatible; Supybot %s)' % conf.version,
defaultHeaders = {
'User-agent': 'Mozilla/5.0 (compatible; utils.web python module)'
}
# Other modules should feel free to replace this with an appropriate
# application-specific function. Feel free to use a callable here.
proxy = None
def getUrlFd(url, headers=None):
"""Gets a file-like object for a url."""
if headers is None:
headers = _headers
headers = defaultHeaders
try:
if not isinstance(url, urllib2.Request):
if '#' in url:
@ -90,7 +92,7 @@ def getUrlFd(url, headers=None):
request = urllib2.Request(url, headers=headers)
else:
request = url
httpProxy = conf.supybot.protocols.http.proxy()
httpProxy = force(proxy)
if httpProxy:
request.set_proxy(httpProxy, 'http')
fd = urllib2.urlopen(request)
@ -125,5 +127,39 @@ def getUrl(url, size=None, headers=None):
def getDomain(url):
return urlparse.urlparse(url)[1]
class HtmlToText(sgmllib.SGMLParser):
"""Taken from some eff-bot code on c.l.p."""
entitydefs = htmlentitydefs.entitydefs.copy()
entitydefs['nbsp'] = ' '
def __init__(self, tagReplace=' '):
self.data = []
self.tagReplace = tagReplace
sgmllib.SGMLParser.__init__(self)
def unknown_starttag(self, tag, attr):
self.data.append(self.tagReplace)
def unknown_endtag(self, tag):
self.data.append(self.tagReplace)
def handle_data(self, data):
self.data.append(data)
def getText(self):
text = ''.join(self.data).strip()
return normalizeWhitespace(text)
def htmlToText(s, tagReplace=' '):
"""Turns HTML into text. tagReplace is a string to replace HTML tags with.
"""
x = HtmlToText(tagReplace)
x.feed(s)
return x.getText()
def mungeEmail(s):
s = s.replace('@', ' AT ')
s = s.replace('.', ' DOT ')
return s
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

6
src/world.py
View File

@ -31,10 +31,6 @@
Module for general worldly stuff, like global variables and whatnot.
"""
import supybot.fix as fix
import gc
import os
import sys
@ -54,8 +50,6 @@ startedAt = time.time() # Just in case it doesn't get set later.
starting = False
mainThread = threading.currentThread()
# ??? Should we do this? What do we gain?
# assert 'MainThread' in repr(mainThread)
def isMainThread():
return mainThread is threading.currentThread()

1
test/__init__.py
View File

@ -27,6 +27,7 @@
# POSSIBILITY OF SUCH DAMAGE.
###
# We're just masquerading as a plugin :)
import test
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

2
test/test_callbacks.py
View File

@ -42,7 +42,7 @@ class TokenizerTestCase(SupyTestCase):
self.assertEqual(tokenize(''), [])
def testNullCharacter(self):
self.assertEqual(tokenize(utils.dqrepr('\0')), ['\0'])
self.assertEqual(tokenize(utils.str.dqrepr('\0')), ['\0'])
def testSingleDQInDQString(self):
self.assertEqual(tokenize('"\\""'), ['"'])

621
test/test_utils.py
View File

@ -28,11 +28,24 @@
###
from supybot.test import *
import sets
import time
import supybot.utils as utils
class UtilsTest(SupyTestCase):
class GenTest(SupyTestCase):
def testInsensitivePreservingDict(self):
ipd = utils.InsensitivePreservingDict
d = ipd(dict(Foo=10))
self.failUnless(d['foo'] == 10)
self.assertEqual(d.keys(), ['Foo'])
self.assertEqual(d.get('foo'), 10)
self.assertEqual(d.get('Foo'), 10)
def testFindBinaryInPath(self):
if os.name == 'posix':
self.assertEqual(None, utils.findBinaryInPath('asdfhjklasdfhjkl'))
self.failUnless(utils.findBinaryInPath('sh').endswith('/bin/sh'))
def testExnToString(self):
try:
raise KeyError, 1
@ -43,270 +56,21 @@ class UtilsTest(SupyTestCase):
except Exception, e:
self.assertEqual(utils.exnToString(e), 'EOFError')
def testMatchCase(self):
f = utils.matchCase
self.assertEqual('bar', f('foo', 'bar'))
self.assertEqual('Bar', f('Foo', 'bar'))
self.assertEqual('BAr', f('FOo', 'bar'))
self.assertEqual('BAR', f('FOO', 'bar'))
self.assertEqual('bAR', f('fOO', 'bar'))
self.assertEqual('baR', f('foO', 'bar'))
self.assertEqual('BaR', f('FoO', 'bar'))
def testSaltHash(self):
s = utils.saltHash('jemfinch')
(salt, hash) = s.split('|')
self.assertEqual(utils.saltHash('jemfinch', salt=salt), s)
def testPluralize(self):
f = utils.pluralize
self.assertEqual('bike', f('bike', 1))
self.assertEqual('bikes', f('bike', 2))
self.assertEqual('BIKE', f('BIKE', 1))
self.assertEqual('BIKES', f('BIKE', 2))
self.assertEqual('match', f('match', 1))
self.assertEqual('matches', f('match', 2))
self.assertEqual('Patch', f('Patch', 1))
self.assertEqual('Patches', f('Patch', 2))
self.assertEqual('fish', f('fish', 1))
self.assertEqual('fishes', f('fish', 2))
self.assertEqual('try', f('try', 1))
self.assertEqual('tries', f('try', 2))
self.assertEqual('day', f('day', 1))
self.assertEqual('days', f('day', 2))
def testSafeEval(self):
for s in ['1', '()', '(1,)', '[]', '{}', '{1:2}', '{1:(2,3)}',
'1.0', '[1,2,3]', 'True', 'False', 'None',
'(True,False,None)', '"foo"', '{"foo": "bar"}']:
self.assertEqual(eval(s), utils.safeEval(s))
for s in ['lambda: 2', 'import foo', 'foo.bar']:
self.assertRaises(ValueError, utils.safeEval, s)
def testDepluralize(self):
f = utils.depluralize
self.assertEqual('bike', f('bikes'))
self.assertEqual('Bike', f('Bikes'))
self.assertEqual('BIKE', f('BIKES'))
self.assertEqual('match', f('matches'))
self.assertEqual('Match', f('Matches'))
self.assertEqual('fish', f('fishes'))
self.assertEqual('try', f('tries'))
def testTimeElapsed(self):
self.assertRaises(ValueError, utils.timeElapsed, 0,
leadingZeroes=False, seconds=False)
then = 0
now = 0
for now, expected in [(0, '0 seconds'),
(1, '1 second'),
(60, '1 minute and 0 seconds'),
(61, '1 minute and 1 second'),
(62, '1 minute and 2 seconds'),
(122, '2 minutes and 2 seconds'),
(3722, '1 hour, 2 minutes, and 2 seconds'),
(7322, '2 hours, 2 minutes, and 2 seconds'),
(90061,'1 day, 1 hour, 1 minute, and 1 second'),
(180122, '2 days, 2 hours, 2 minutes, '
'and 2 seconds')]:
self.assertEqual(utils.timeElapsed(now - then), expected)
def timeElapsedShort(self):
self.assertEqual(utils.timeElapsed(123, short=True), '2m 3s')
def testDistance(self):
self.assertEqual(utils.distance('', ''), 0)
self.assertEqual(utils.distance('a', 'b'), 1)
self.assertEqual(utils.distance('a', 'a'), 0)
self.assertEqual(utils.distance('foobar', 'jemfinch'), 8)
self.assertEqual(utils.distance('a', 'ab'), 1)
self.assertEqual(utils.distance('foo', ''), 3)
self.assertEqual(utils.distance('', 'foo'), 3)
self.assertEqual(utils.distance('appel', 'nappe'), 2)
self.assertEqual(utils.distance('nappe', 'appel'), 2)
def testAbbrev(self):
L = ['abc', 'bcd', 'bbe', 'foo', 'fool']
d = utils.abbrev(L)
def getItem(s):
return d[s]
self.assertRaises(KeyError, getItem, 'f')
self.assertRaises(KeyError, getItem, 'fo')
self.assertRaises(KeyError, getItem, 'b')
self.assertEqual(d['bb'], 'bbe')
self.assertEqual(d['bc'], 'bcd')
self.assertEqual(d['a'], 'abc')
self.assertEqual(d['ab'], 'abc')
self.assertEqual(d['fool'], 'fool')
self.assertEqual(d['foo'], 'foo')
def testAbbrevFailsWithDups(self):
L = ['english', 'english']
self.assertRaises(ValueError, utils.abbrev, L)
def testSoundex(self):
L = [('Euler', 'E460'),
('Ellery', 'E460'),
('Gauss', 'G200'),
('Ghosh', 'G200'),
('Hilbert', 'H416'),
('Heilbronn', 'H416'),
('Knuth', 'K530'),
('Kant', 'K530'),
('Lloyd', 'L300'),
('Ladd', 'L300'),
('Lukasiewicz', 'L222'),
('Lissajous', 'L222')]
for (name, key) in L:
soundex = utils.soundex(name)
self.assertEqual(soundex, key,
'%s was %s, not %s' % (name, soundex, key))
self.assertRaises(ValueError, utils.soundex, '3')
self.assertRaises(ValueError, utils.soundex, "'")
def testDQRepr(self):
L = ['foo', 'foo\'bar', 'foo"bar', '"', '\\', '', '\x00']
for s in L:
r = utils.dqrepr(s)
self.assertEqual(s, eval(r), s)
self.failUnless(r[0] == '"' and r[-1] == '"', s)
## def testQuoted(self):
## s = 'foo'
## t = 'let\'s'
## self.assertEqual("'%s'" % s, utils.quoted(s), s)
## self.assertEqual('"%s"' % t, utils.quoted(t), t)
def testPerlReToPythonRe(self):
r = utils.perlReToPythonRe('m/foo/')
self.failUnless(r.search('foo'))
r = utils.perlReToPythonRe('/foo/')
self.failUnless(r.search('foo'))
r = utils.perlReToPythonRe('m/\\//')
self.failUnless(r.search('/'))
r = utils.perlReToPythonRe('m/cat/i')
self.failUnless(r.search('CAT'))
self.assertRaises(ValueError, utils.perlReToPythonRe, 'm/?/')
def testP2PReDifferentSeparator(self):
r = utils.perlReToPythonRe('m!foo!')
self.failUnless(r.search('foo'))
def testPerlReToReplacer(self):
f = utils.perlReToReplacer('s/foo/bar/')
self.assertEqual(f('foobarbaz'), 'barbarbaz')
f = utils.perlReToReplacer('s/fool/bar/')
self.assertEqual(f('foobarbaz'), 'foobarbaz')
f = utils.perlReToReplacer('s/foo//')
self.assertEqual(f('foobarbaz'), 'barbaz')
f = utils.perlReToReplacer('s/ba//')
self.assertEqual(f('foobarbaz'), 'foorbaz')
f = utils.perlReToReplacer('s/ba//g')
self.assertEqual(f('foobarbaz'), 'foorz')
f = utils.perlReToReplacer('s/ba\\///g')
self.assertEqual(f('fooba/rba/z'), 'foorz')
f = utils.perlReToReplacer('s/cat/dog/i')
self.assertEqual(f('CATFISH'), 'dogFISH')
f = utils.perlReToReplacer('s/foo/foo\/bar/')
self.assertEqual(f('foo'), 'foo/bar')
f = utils.perlReToReplacer('s/^/foo/')
self.assertEqual(f('bar'), 'foobar')
def testPReToReplacerDifferentSeparator(self):
f = utils.perlReToReplacer('s#foo#bar#')
self.assertEqual(f('foobarbaz'), 'barbarbaz')
def testPerlReToReplacerBug850931(self):
f = utils.perlReToReplacer('s/\b(\w+)\b/\1./g')
self.assertEqual(f('foo bar baz'), 'foo. bar. baz.')
def testPerlVariableSubstitute(self):
f = utils.perlVariableSubstitute
vars = {'foo': 'bar', 'b a z': 'baz', 'b': 'c', 'i': 100,
'f': lambda: 'called'}
self.assertEqual(f(vars, '$foo'), 'bar')
self.assertEqual(f(vars, '${foo}'), 'bar')
self.assertEqual(f(vars, '$b'), 'c')
self.assertEqual(f(vars, '${b}'), 'c')
self.assertEqual(f(vars, '$i'), '100')
self.assertEqual(f(vars, '${i}'), '100')
self.assertEqual(f(vars, '$f'), 'called')
self.assertEqual(f(vars, '${f}'), 'called')
self.assertEqual(f(vars, '${b a z}'), 'baz')
self.assertEqual(f(vars, '$b:$i'), 'c:100')
def testFindBinaryInPath(self):
if os.name == 'posix':
self.assertEqual(None, utils.findBinaryInPath('asdfhjklasdfhjkl'))
self.failUnless(utils.findBinaryInPath('sh').endswith('/bin/sh'))
def testCommaAndify(self):
L = ['foo']
original = L[:]
self.assertEqual(utils.commaAndify(L), 'foo')
self.assertEqual(utils.commaAndify(L, And='or'), 'foo')
self.assertEqual(L, original)
L.append('bar')
original = L[:]
self.assertEqual(utils.commaAndify(L), 'foo and bar')
self.assertEqual(utils.commaAndify(L, And='or'), 'foo or bar')
self.assertEqual(L, original)
L.append('baz')
original = L[:]
self.assertEqual(utils.commaAndify(L), 'foo, bar, and baz')
self.assertEqual(utils.commaAndify(L, And='or'), 'foo, bar, or baz')
self.assertEqual(utils.commaAndify(L, comma=';'), 'foo; bar; and baz')
self.assertEqual(utils.commaAndify(L, comma=';', And='or'),
'foo; bar; or baz')
self.assertEqual(L, original)
self.failUnless(utils.commaAndify(sets.Set(L)))
def testCommaAndifyRaisesTypeError(self):
L = [(2,)]
self.assertRaises(TypeError, utils.commaAndify, L)
L.append((3,))
self.assertRaises(TypeError, utils.commaAndify, L)
def testUnCommaThe(self):
self.assertEqual(utils.unCommaThe('foo bar'), 'foo bar')
self.assertEqual(utils.unCommaThe('foo bar, the'), 'the foo bar')
self.assertEqual(utils.unCommaThe('foo bar, The'), 'The foo bar')
self.assertEqual(utils.unCommaThe('foo bar,the'), 'the foo bar')
def testNormalizeWhitespace(self):
self.assertEqual(utils.normalizeWhitespace('foo bar'), 'foo bar')
self.assertEqual(utils.normalizeWhitespace('foo\nbar'), 'foo bar')
self.assertEqual(utils.normalizeWhitespace('foo\tbar'), 'foo bar')
def testSortBy(self):
L = ['abc', 'z', 'AD']
utils.sortBy(len, L)
self.assertEqual(L, ['z', 'AD', 'abc'])
utils.sortBy(str.lower, L)
self.assertEqual(L, ['abc', 'AD', 'z'])
L = ['supybot', 'Supybot']
utils.sortBy(str.lower, L)
self.assertEqual(L, ['supybot', 'Supybot'])
def testSorted(self):
L = ['a', 'c', 'b']
self.assertEqual(utils.sorted(L), ['a', 'b', 'c'])
self.assertEqual(L, ['a', 'c', 'b'])
def mycmp(x, y):
return -cmp(x, y)
self.assertEqual(utils.sorted(L, mycmp), ['c', 'b', 'a'])
def testNItems(self):
self.assertEqual(utils.nItems('tool', 1, 'crazy'), '1 crazy tool')
self.assertEqual(utils.nItems('tool', 1), '1 tool')
self.assertEqual(utils.nItems('tool', 2, 'crazy'), '2 crazy tools')
self.assertEqual(utils.nItems('tool', 2), '2 tools')
def testItersplit(self):
itersplit = utils.itersplit
L = [1, 2, 3] * 3
s = 'foo bar baz'
self.assertEqual(list(itersplit(lambda x: x == 3, L)),
[[1, 2], [1, 2], [1, 2]])
self.assertEqual(list(itersplit(lambda x: x == 3, L, yieldEmpty=True)),
[[1, 2], [1, 2], [1, 2], []])
self.assertEqual(list(itersplit(lambda x: x, [])), [])
self.assertEqual(list(itersplit(lambda c: c.isspace(), s)),
map(list, s.split()))
self.assertEqual(list(itersplit('for'.__eq__, ['foo', 'for', 'bar'])),
[['foo'], ['bar']])
self.assertEqual(list(itersplit('for'.__eq__,
['foo','for','bar','for', 'baz'], 1)),
[['foo'], ['bar', 'for', 'baz']])
def testSafeEvalTurnsSyntaxErrorIntoValueError(self):
self.assertRaises(ValueError, utils.safeEval, '/usr/local/')
def testIterableMap(self):
class alist(utils.IterableMap):
@ -333,9 +97,277 @@ class UtilsTest(SupyTestCase):
self.assertEqual(list(AL.itervalues()), [2, 3, 4])
self.assertEqual(len(AL), 3)
def testSortBy(self):
L = ['abc', 'z', 'AD']
utils.sortBy(len, L)
self.assertEqual(L, ['z', 'AD', 'abc'])
utils.sortBy(str.lower, L)
self.assertEqual(L, ['abc', 'AD', 'z'])
L = ['supybot', 'Supybot']
utils.sortBy(str.lower, L)
self.assertEqual(L, ['supybot', 'Supybot'])
def testSorted(self):
L = ['a', 'c', 'b']
self.assertEqual(utils.sorted(L), ['a', 'b', 'c'])
self.assertEqual(L, ['a', 'c', 'b'])
def mycmp(x, y):
return -cmp(x, y)
self.assertEqual(utils.sorted(L, mycmp), ['c', 'b', 'a'])
def testTimeElapsed(self):
self.assertRaises(ValueError, utils.timeElapsed, 0,
leadingZeroes=False, seconds=False)
then = 0
now = 0
for now, expected in [(0, '0 seconds'),
(1, '1 second'),
(60, '1 minute and 0 seconds'),
(61, '1 minute and 1 second'),
(62, '1 minute and 2 seconds'),
(122, '2 minutes and 2 seconds'),
(3722, '1 hour, 2 minutes, and 2 seconds'),
(7322, '2 hours, 2 minutes, and 2 seconds'),
(90061,'1 day, 1 hour, 1 minute, and 1 second'),
(180122, '2 days, 2 hours, 2 minutes, '
'and 2 seconds')]:
self.assertEqual(utils.timeElapsed(now - then), expected)
def timeElapsedShort(self):
self.assertEqual(utils.timeElapsed(123, short=True), '2m 3s')
def testAbbrev(self):
L = ['abc', 'bcd', 'bbe', 'foo', 'fool']
d = utils.abbrev(L)
def getItem(s):
return d[s]
self.assertRaises(KeyError, getItem, 'f')
self.assertRaises(KeyError, getItem, 'fo')
self.assertRaises(KeyError, getItem, 'b')
self.assertEqual(d['bb'], 'bbe')
self.assertEqual(d['bc'], 'bcd')
self.assertEqual(d['a'], 'abc')
self.assertEqual(d['ab'], 'abc')
self.assertEqual(d['fool'], 'fool')
self.assertEqual(d['foo'], 'foo')
def testAbbrevFailsWithDups(self):
L = ['english', 'english']
self.assertRaises(ValueError, utils.abbrev, L)
class StrTest(SupyTestCase):
def testMatchCase(self):
f = utils.str.matchCase
self.assertEqual('bar', f('foo', 'bar'))
self.assertEqual('Bar', f('Foo', 'bar'))
self.assertEqual('BAr', f('FOo', 'bar'))
self.assertEqual('BAR', f('FOO', 'bar'))
self.assertEqual('bAR', f('fOO', 'bar'))
self.assertEqual('baR', f('foO', 'bar'))
self.assertEqual('BaR', f('FoO', 'bar'))
def testPluralize(self):
f = utils.str.pluralize
self.assertEqual('bike', f('bike', 1))
self.assertEqual('bikes', f('bike', 2))
self.assertEqual('BIKE', f('BIKE', 1))
self.assertEqual('BIKES', f('BIKE', 2))
self.assertEqual('match', f('match', 1))
self.assertEqual('matches', f('match', 2))
self.assertEqual('Patch', f('Patch', 1))
self.assertEqual('Patches', f('Patch', 2))
self.assertEqual('fish', f('fish', 1))
self.assertEqual('fishes', f('fish', 2))
self.assertEqual('try', f('try', 1))
self.assertEqual('tries', f('try', 2))
self.assertEqual('day', f('day', 1))
self.assertEqual('days', f('day', 2))
def testDepluralize(self):
f = utils.str.depluralize
self.assertEqual('bike', f('bikes'))
self.assertEqual('Bike', f('Bikes'))
self.assertEqual('BIKE', f('BIKES'))
self.assertEqual('match', f('matches'))
self.assertEqual('Match', f('Matches'))
self.assertEqual('fish', f('fishes'))
self.assertEqual('try', f('tries'))
def testDistance(self):
self.assertEqual(utils.str.distance('', ''), 0)
self.assertEqual(utils.str.distance('a', 'b'), 1)
self.assertEqual(utils.str.distance('a', 'a'), 0)
self.assertEqual(utils.str.distance('foobar', 'jemfinch'), 8)
self.assertEqual(utils.str.distance('a', 'ab'), 1)
self.assertEqual(utils.str.distance('foo', ''), 3)
self.assertEqual(utils.str.distance('', 'foo'), 3)
self.assertEqual(utils.str.distance('appel', 'nappe'), 2)
self.assertEqual(utils.str.distance('nappe', 'appel'), 2)
def testSoundex(self):
L = [('Euler', 'E460'),
('Ellery', 'E460'),
('Gauss', 'G200'),
('Ghosh', 'G200'),
('Hilbert', 'H416'),
('Heilbronn', 'H416'),
('Knuth', 'K530'),
('Kant', 'K530'),
('Lloyd', 'L300'),
('Ladd', 'L300'),
('Lukasiewicz', 'L222'),
('Lissajous', 'L222')]
for (name, key) in L:
soundex = utils.str.soundex(name)
self.assertEqual(soundex, key,
'%s was %s, not %s' % (name, soundex, key))
self.assertRaises(ValueError, utils.str.soundex, '3')
self.assertRaises(ValueError, utils.str.soundex, "'")
def testDQRepr(self):
L = ['foo', 'foo\'bar', 'foo"bar', '"', '\\', '', '\x00']
for s in L:
r = utils.str.dqrepr(s)
self.assertEqual(s, eval(r), s)
self.failUnless(r[0] == '"' and r[-1] == '"', s)
def testPerlReToPythonRe(self):
f = utils.str.perlReToPythonRe
r = f('m/foo/')
self.failUnless(r.search('foo'))
r = f('/foo/')
self.failUnless(r.search('foo'))
r = f('m/\\//')
self.failUnless(r.search('/'))
r = f('m/cat/i')
self.failUnless(r.search('CAT'))
self.assertRaises(ValueError, f, 'm/?/')
def testP2PReDifferentSeparator(self):
r = utils.str.perlReToPythonRe('m!foo!')
self.failUnless(r.search('foo'))
def testPerlReToReplacer(self):
PRTR = utils.str.perlReToReplacer
f = PRTR('s/foo/bar/')
self.assertEqual(f('foobarbaz'), 'barbarbaz')
f = PRTR('s/fool/bar/')
self.assertEqual(f('foobarbaz'), 'foobarbaz')
f = PRTR('s/foo//')
self.assertEqual(f('foobarbaz'), 'barbaz')
f = PRTR('s/ba//')
self.assertEqual(f('foobarbaz'), 'foorbaz')
f = PRTR('s/ba//g')
self.assertEqual(f('foobarbaz'), 'foorz')
f = PRTR('s/ba\\///g')
self.assertEqual(f('fooba/rba/z'), 'foorz')
f = PRTR('s/cat/dog/i')
self.assertEqual(f('CATFISH'), 'dogFISH')
f = PRTR('s/foo/foo\/bar/')
self.assertEqual(f('foo'), 'foo/bar')
f = PRTR('s/^/foo/')
self.assertEqual(f('bar'), 'foobar')
def testPReToReplacerDifferentSeparator(self):
f = utils.str.perlReToReplacer('s#foo#bar#')
self.assertEqual(f('foobarbaz'), 'barbarbaz')
def testPerlReToReplacerBug850931(self):
f = utils.str.perlReToReplacer('s/\b(\w+)\b/\1./g')
self.assertEqual(f('foo bar baz'), 'foo. bar. baz.')
def testPerlVariableSubstitute(self):
f = utils.str.perlVariableSubstitute
vars = {'foo': 'bar', 'b a z': 'baz', 'b': 'c', 'i': 100,
'f': lambda: 'called'}
self.assertEqual(f(vars, '$foo'), 'bar')
self.assertEqual(f(vars, '${foo}'), 'bar')
self.assertEqual(f(vars, '$b'), 'c')
self.assertEqual(f(vars, '${b}'), 'c')
self.assertEqual(f(vars, '$i'), '100')
self.assertEqual(f(vars, '${i}'), '100')
self.assertEqual(f(vars, '$f'), 'called')
self.assertEqual(f(vars, '${f}'), 'called')
self.assertEqual(f(vars, '${b a z}'), 'baz')
self.assertEqual(f(vars, '$b:$i'), 'c:100')
def testCommaAndify(self):
f = utils.str.commaAndify
L = ['foo']
original = L[:]
self.assertEqual(f(L), 'foo')
self.assertEqual(f(L, And='or'), 'foo')
self.assertEqual(L, original)
L.append('bar')
original = L[:]
self.assertEqual(f(L), 'foo and bar')
self.assertEqual(f(L, And='or'), 'foo or bar')
self.assertEqual(L, original)
L.append('baz')
original = L[:]
self.assertEqual(f(L), 'foo, bar, and baz')
self.assertEqual(f(L, And='or'), 'foo, bar, or baz')
self.assertEqual(f(L, comma=';'), 'foo; bar; and baz')
self.assertEqual(f(L, comma=';', And='or'),
'foo; bar; or baz')
self.assertEqual(L, original)
self.failUnless(f(set(L)))
def testCommaAndifyRaisesTypeError(self):
L = [(2,)]
self.assertRaises(TypeError, utils.str.commaAndify, L)
L.append((3,))
self.assertRaises(TypeError, utils.str.commaAndify, L)
def testUnCommaThe(self):
f = utils.str.unCommaThe
self.assertEqual(f('foo bar'), 'foo bar')
self.assertEqual(f('foo bar, the'), 'the foo bar')
self.assertEqual(f('foo bar, The'), 'The foo bar')
self.assertEqual(f('foo bar,the'), 'the foo bar')
def testNormalizeWhitespace(self):
f = utils.str.normalizeWhitespace
self.assertEqual(f('foo bar'), 'foo bar')
self.assertEqual(f('foo\nbar'), 'foo bar')
self.assertEqual(f('foo\tbar'), 'foo bar')
def testNItems(self):
nItems = utils.str.nItems
self.assertEqual(nItems('tool', 1, 'crazy'), '1 crazy tool')
self.assertEqual(nItems('tool', 1), '1 tool')
self.assertEqual(nItems('tool', 2, 'crazy'), '2 crazy tools')
self.assertEqual(nItems('tool', 2), '2 tools')
def testEllipsisify(self):
f = utils.str.ellipsisify
self.assertEqual(f('x'*30, 30), 'x'*30)
self.failUnless(len(f('x'*35, 30)) <= 30)
self.failUnless(f(' '.join(['xxxx']*10), 30)[:-3].endswith('xxxx'))
class IterTest(SupyTestCase):
def testSplit(self):
itersplit = utils.iter.split
L = [1, 2, 3] * 3
s = 'foo bar baz'
self.assertEqual(list(itersplit(lambda x: x == 3, L)),
[[1, 2], [1, 2], [1, 2]])
self.assertEqual(list(itersplit(lambda x: x == 3, L, yieldEmpty=True)),
[[1, 2], [1, 2], [1, 2], []])
self.assertEqual(list(itersplit(lambda x: x, [])), [])
self.assertEqual(list(itersplit(lambda c: c.isspace(), s)),
map(list, s.split()))
self.assertEqual(list(itersplit('for'.__eq__, ['foo', 'for', 'bar'])),
[['foo'], ['bar']])
self.assertEqual(list(itersplit('for'.__eq__,
['foo','for','bar','for', 'baz'], 1)),
[['foo'], ['bar', 'for', 'baz']])
def testFlatten(self):
def lflatten(seq):
return list(utils.flatten(seq))
return list(utils.iter.flatten(seq))
self.assertEqual(lflatten([]), [])
self.assertEqual(lflatten([1]), [1])
self.assertEqual(lflatten(range(10)), range(10))
@ -347,60 +379,43 @@ class UtilsTest(SupyTestCase):
self.assertEqual(lflatten([1, [2, [3, 4], 5], 6]), [1, 2, 3, 4, 5, 6])
self.assertRaises(TypeError, lflatten, 1)
def testEllipsisify(self):
f = utils.ellipsisify
self.assertEqual(f('x'*30, 30), 'x'*30)
self.failUnless(len(f('x'*35, 30)) <= 30)
self.failUnless(f(' '.join(['xxxx']*10), 30)[:-3].endswith('xxxx'))
def testSaltHash(self):
s = utils.saltHash('jemfinch')
(salt, hash) = s.split('|')
self.assertEqual(utils.saltHash('jemfinch', salt=salt), s)
def testSafeEval(self):
for s in ['1', '()', '(1,)', '[]', '{}', '{1:2}', '{1:(2,3)}',
'1.0', '[1,2,3]', 'True', 'False', 'None',
'(True,False,None)', '"foo"', '{"foo": "bar"}']:
self.assertEqual(eval(s), utils.safeEval(s))
for s in ['lambda: 2', 'import foo', 'foo.bar']:
self.assertRaises(ValueError, utils.safeEval, s)
def testSafeEvalTurnsSyntaxErrorIntoValueError(self):
self.assertRaises(ValueError, utils.safeEval, '/usr/local/')
class FileTest(SupyTestCase):
def testLines(self):
L = ['foo', 'bar', '#baz', ' ', 'biff']
self.assertEqual(list(utils.nonEmptyLines(L)),
self.assertEqual(list(utils.file.nonEmptyLines(L)),
['foo', 'bar', '#baz', 'biff'])
self.assertEqual(list(utils.nonCommentLines(L)),
self.assertEqual(list(utils.file.nonCommentLines(L)),
['foo', 'bar', ' ', 'biff'])
self.assertEqual(list(utils.nonCommentNonEmptyLines(L)),
self.assertEqual(list(utils.file.nonCommentNonEmptyLines(L)),
['foo', 'bar', 'biff'])
class NetTest(SupyTestCase):
def testIsIP(self):
self.failIf(utils.isIP('a.b.c'))
self.failIf(utils.isIP('256.0.0.0'))
self.failUnless(utils.isIP('127.1'))
self.failUnless(utils.isIP('0.0.0.0'))
self.failUnless(utils.isIP('100.100.100.100'))
isIP = utils.net.isIP
self.failIf(isIP('a.b.c'))
self.failIf(isIP('256.0.0.0'))
self.failUnless(isIP('127.1'))
self.failUnless(isIP('0.0.0.0'))
self.failUnless(isIP('100.100.100.100'))
# This test is too flaky to bother with.
# self.failUnless(utils.isIP('255.255.255.255'))
def testIsIPV6(self):
f = utils.isIPV6
f = utils.net.isIPV6
self.failUnless(f('2001::'))
self.failUnless(f('2001:888:0:1::666'))
def testInsensitivePreservingDict(self):
ipd = utils.InsensitivePreservingDict
d = ipd(dict(Foo=10))
self.failUnless(d['foo'] == 10)
self.assertEqual(d.keys(), ['Foo'])
self.assertEqual(d.get('foo'), 10)
self.assertEqual(d.get('Foo'), 10)
class WebTest(SupyTestCase):
def testGetDomain(self):
url = 'http://slashdot.org/foo/bar.exe'
self.assertEqual(utils.web.getDomain(url), 'slashdot.org')
if network:
def testGetUrlWithSize(self):
url = 'http://slashdot.org/'
self.failUnless(len(utils.web.getUrl(url, 1024)) == 1024)
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: