From 5fd6bbb52d1e49445989bbe9dad48a4cdb9108ab Mon Sep 17 00:00:00 2001 From: Jeremy Fincher Date: Thu, 27 Jan 2005 06:59:08 +0000 Subject: [PATCH] Completely restructured our utils modules. Tons of changes. Here's the summary of things that matter most: * There is no more supybot.fix. * There is no more supybot.webutils; now there is supybot.utils.web. * It's no longer webutils.WebError, but just utils.web.Error. * You shouldn't import itertools, ideally, but instead import utils.iter. * No more using imap/ifilter in commands unless absolutely necessary. It's premature optimization and annoying. * utils.str.format isn't quite ready yet, but will be soon. That'll be the next big thing to fix in our code. --- plugins/Admin/plugin.py | 13 +- plugins/Babelfish/plugin.py | 18 +- plugins/Channel/config.py | 4 +- plugins/Channel/plugin.py | 34 +- plugins/Config/plugin.py | 5 +- plugins/Dict/plugin.py | 10 +- plugins/Format/plugin.py | 2 +- plugins/Math/plugin.py | 19 +- plugins/Misc/plugin.py | 60 +- plugins/Network/plugin.py | 18 +- plugins/Owner/plugin.py | 6 +- plugins/Status/plugin.py | 23 +- plugins/User/plugin.py | 14 +- plugins/__init__.py | 25 +- scripts/supybot | 2 +- setup.py | 2 + src/callbacks.py | 32 +- src/cdb.py | 6 +- src/commands.py | 14 +- src/conf.py | 27 +- src/dbi.py | 7 +- src/fix.py | 239 ------ src/ircdb.py | 22 +- src/irclib.py | 10 +- src/ircmsgs.py | 14 +- src/ircutils.py | 19 +- src/log.py | 2 - src/plugin.py | 4 +- src/registry.py | 27 +- src/schedule.py | 4 - src/structures.py | 14 +- src/test.py | 4 +- src/unpreserve.py | 2 - src/utils.py | 864 ---------------------- src/utils/__init__.py | 106 +++ src/utils/file.py | 150 ++++ src/utils/gen.py | 326 ++++++++ src/utils/iter.py | 147 ++++ src/utils/net.py | 90 +++ test/test_webutils.py => src/utils/seq.py | 24 +- src/utils/str.py | 354 +++++++++ src/{webutils.py => utils/web.py} | 56 +- src/world.py | 6 - test/__init__.py | 1 + test/test_callbacks.py | 2 +- test/test_utils.py | 621 ++++++++-------- 46 files changed, 1744 insertions(+), 1705 deletions(-) delete mode 100644 src/fix.py delete mode 100644 src/utils.py create mode 100644 src/utils/__init__.py create mode 100644 src/utils/file.py create mode 100644 src/utils/gen.py create mode 100644 src/utils/iter.py create mode 100644 src/utils/net.py rename test/test_webutils.py => src/utils/seq.py (77%) create mode 100644 src/utils/str.py rename src/{webutils.py => utils/web.py} (74%) diff --git a/plugins/Admin/plugin.py b/plugins/Admin/plugin.py index 28fc4a07f..e66525b7f 100644 --- a/plugins/Admin/plugin.py +++ b/plugins/Admin/plugin.py @@ -27,16 +27,9 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot - -__author__ = supybot.authors.jemfinch - -import supybot.fix as fix - import sys import time import pprint -from itertools import imap import supybot.log as log import supybot.conf as conf @@ -164,7 +157,7 @@ class Admin(callbacks.Privmsg): L = irc.state.channels.keys() if L: utils.sortBy(ircutils.toLower, L) - irc.reply(utils.commaAndify(L)) + irc.reply(utils.str.commaAndify(L)) else: irc.reply('I\'m not currently in any channels.') channels = wrap(channels, ['private']) @@ -194,7 +187,7 @@ class Admin(callbacks.Privmsg): irc = self.pendingNickChanges.get(irc, None) if irc is not None: irc.error('I can\'t change nicks, the server said %s.' % - utils.quoted(msg.args[2]), private=True) + utils.str.quoted(msg.args[2]), private=True) else: self.log.debug('Got 438 without Admin.nick being called.') @@ -332,7 +325,7 @@ class Admin(callbacks.Privmsg): """ # XXX Add the expirations. if ircdb.ignores.hostmasks: - irc.reply(utils.commaAndify(imap(repr, ircdb.ignores.hostmasks))) + irc.reply(utils.str.commaAndify(map(repr,ircdb.ignores.hostmasks))) else: irc.reply('I\'m not currently globally ignoring anyone.') ignores = wrap(ignores) diff --git a/plugins/Babelfish/plugin.py b/plugins/Babelfish/plugin.py index 6095d50dc..376e10919 100644 --- a/plugins/Babelfish/plugin.py +++ b/plugins/Babelfish/plugin.py @@ -27,11 +27,7 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot - -import sets import random -from itertools import imap import babelfish @@ -69,7 +65,7 @@ class Babelfish(callbacks.Privmsg): Returns the languages that Babelfish can translate to/from. """ - irc.reply(utils.commaAndify(babelfish.available_languages)) + irc.reply(utils.str.commaAndify(babelfish.available_languages)) def translate(self, irc, msg, args, fromLang, toLang, text): """ [to] @@ -87,15 +83,15 @@ class Babelfish(callbacks.Privmsg): irc.error('I do not speak any other languages.') return else: - irc.error('I only speak %s.' % utils.commaAndify(langs)) + irc.error('I only speak %s.' % utils.str.commaAndify(langs)) return translation = babelfish.translate(text, fromLang, toLang) - irc.reply(utils.htmlToText(translation)) + irc.reply(utils.web.htmlToText(translation)) except (KeyError, babelfish.LanguageNotAvailableError), e: languages = self.registryValue('languages', chan) if languages: languages = 'Valid languages include %s' % \ - utils.commaAndify(sorted(languages)) + utils.str.commaAndify(sorted(languages)) else: languages = 'I do not speak any other languages.' irc.errorInvalid('language', str(e), languages) @@ -125,16 +121,16 @@ class Babelfish(callbacks.Privmsg): irc.error('I do not speak any other languages.') return else: - irc.error('I only speak %s.' % utils.commaAndify(langs, + irc.error('I only speak %s.' % utils.str.commaAndify(langs, And='or')) return translations = babelfish.babelize(text, fromLang, toLang) - irc.reply(utils.htmlToText(translations[-1])) + irc.reply(utils.web.htmlToText(translations[-1])) except (KeyError, babelfish.LanguageNotAvailableError), e: languages = self.registryValue('languages', chan) if languages: languages = 'Valid languages include %s' % \ - utils.commaAndify(sorted(languages)) + utils.str.commaAndify(sorted(languages)) else: languages = 'I do not speak any other languages.' irc.errorInvalid('language', str(e), languages) diff --git a/plugins/Channel/config.py b/plugins/Channel/config.py index 781cc3f3f..f47688626 100644 --- a/plugins/Channel/config.py +++ b/plugins/Channel/config.py @@ -47,12 +47,12 @@ class BanmaskStyle(registry.SpaceSeparatedSetOfStrings): 'This is a bug.' registry.SpaceSeparatedSetOfStrings.__init__(self, *args, **kwargs) self.__doc__ = 'Valid values include %s.' % \ - utils.commaAndify(map(repr, self.validStrings)) + utils.str.commaAndify(map(repr, self.validStrings)) def help(self): strings = [s for s in self.validStrings if s] return '%s Valid strings: %s.' % \ - (self._help, utils.commaAndify(strings)) + (self._help, utils.str.commaAndify(strings)) def normalize(self, s): lowered = s.lower() diff --git a/plugins/Channel/plugin.py b/plugins/Channel/plugin.py index 1e518467e..d98567e61 100644 --- a/plugins/Channel/plugin.py +++ b/plugins/Channel/plugin.py @@ -30,8 +30,6 @@ import sys import time -from itertools import imap - import supybot.conf as conf import supybot.ircdb as ircdb import supybot.utils as utils @@ -287,12 +285,12 @@ class Channel(callbacks.Privmsg): self.log.debug('In kban') if not irc.isNick(bannedNick): self.log.warning('%s tried to kban a non nick: %s', - utils.quoted(msg.prefix), - utils.quoted(bannedNick)) + utils.str.quoted(msg.prefix), + utils.str.quoted(bannedNick)) raise callbacks.ArgumentError elif bannedNick == irc.nick: self.log.warning('%s tried to make me kban myself.', - utils.quoted(msg.prefix)) + utils.str.quoted(msg.prefix)) irc.error('I cowardly refuse to kickban myself.') return if not reason: @@ -330,7 +328,7 @@ class Channel(callbacks.Privmsg): if ircutils.hostmaskPatternEqual(banmask, irc.prefix): if ircutils.hostmaskPatternEqual(banmask, irc.prefix): self.log.warning('%s tried to make me kban myself.', - utils.quoted(msg.prefix)) + utils.str.quoted(msg.prefix)) irc.error('I cowardly refuse to ban myself.') return else: @@ -354,7 +352,7 @@ class Channel(callbacks.Privmsg): elif ircdb.checkCapability(msg.prefix, capability): if ircdb.checkCapability(bannedHostmask, capability): self.log.warning('%s tried to ban %s, but both have %s', - msg.prefix, utils.quoted(bannedHostmask), + msg.prefix, utils.str.quoted(bannedHostmask), capability) irc.error('%s has %s too, you can\'t ban him/her/it.' % (bannedNick, capability)) @@ -362,7 +360,7 @@ class Channel(callbacks.Privmsg): doBan() else: self.log.warning('%s attempted kban without %s', - utils.quoted(msg.prefix), capability) + utils.str.quoted(msg.prefix), capability) irc.errorNoCapability(capability) exact,nick,user,host kban = wrap(kban, @@ -515,7 +513,7 @@ class Channel(callbacks.Privmsg): # XXX Add the expirations. c = ircdb.channels.getChannel(channel) if c.bans: - irc.reply(utils.commaAndify(map(utils.dqrepr, c.bans))) + irc.reply(utils.str.commaAndify(map(utils.str.dqrepr, c.bans))) else: irc.reply('There are currently no permanent bans on %s' % channel) permbans = wrap(permbans, [('checkChannelCapability', 'op')]) @@ -561,11 +559,11 @@ class Channel(callbacks.Privmsg): c = ircdb.channels.getChannel(channel) if len(c.ignores) == 0: s = 'I\'m not currently ignoring any hostmasks in %s' % \ - utils.quoted(channel) + utils.str.quoted(channel) irc.reply(s) else: L = sorted(c.ignores) - irc.reply(utils.commaAndify(imap(repr, L))) + irc.reply(utils.str.commaAndify(map(repr, L))) ignores = wrap(ignores, [('checkChannelCapability', 'op')]) def addcapability(self, irc, msg, args, channel, user, capabilities): @@ -602,8 +600,9 @@ class Channel(callbacks.Privmsg): ircdb.users.setUser(user) if fail: irc.error('That user didn\'t have the %s %s.' % - (utils.commaAndify(fail), - utils.pluralize('capability', len(fail))), Raise=True) + (utils.str.commaAndify(fail), + utils.str.pluralize('capability', len(fail))), + Raise=True) irc.replySuccess() removecapability = wrap(removecapability, [('checkChannelCapability', 'op'), @@ -662,8 +661,9 @@ class Channel(callbacks.Privmsg): ircdb.channels.setChannel(channel, chan) if fail: irc.error('I do not know about the %s %s.' % - (utils.commaAndify(fail), - utils.pluralize('capability', len(fail))), Raise=True) + (utils.str.commaAndify(fail), + utils.str.pluralize('capability', len(fail))), + Raise=True) irc.replySuccess() unsetcapability = wrap(unsetcapability, [('checkChannelCapability', 'op'), @@ -774,7 +774,7 @@ class Channel(callbacks.Privmsg): L.append(channel) if L: L.sort() - s = 'I\'m currently lobotomized in %s.' % utils.commaAndify(L) + s = 'I\'m currently lobotomized in %s.' % utils.str.commaAndify(L) irc.reply(s) else: irc.reply('I\'m not currently lobotomized in any channels.') @@ -787,7 +787,7 @@ class Channel(callbacks.Privmsg): """ L = list(irc.state.channels[channel].users) utils.sortBy(str.lower, L) - irc.reply(utils.commaAndify(L)) + irc.reply(utils.str.commaAndify(L)) nicks = wrap(nicks, ['inChannel']) # XXX Check that the caller is in chan. def alertOps(self, irc, channel, s, frm=None): diff --git a/plugins/Config/plugin.py b/plugins/Config/plugin.py index fcd881c8f..f48c3846b 100644 --- a/plugins/Config/plugin.py +++ b/plugins/Config/plugin.py @@ -28,7 +28,6 @@ ### import os -import getopt import signal import supybot.log as log @@ -131,7 +130,7 @@ class Config(callbacks.Privmsg): """ L = self._list(group) if L: - irc.reply(utils.commaAndify(L)) + irc.reply(utils.str.commaAndify(L)) else: irc.error('There don\'t seem to be any values in %s.' % group._name) list = wrap(list, ['configVar']) @@ -148,7 +147,7 @@ class Config(callbacks.Privmsg): if not ircutils.isChannel(possibleChannel): L.append(name) if L: - irc.reply(utils.commaAndify(L)) + irc.reply(utils.str.commaAndify(L)) else: irc.reply('There were no matching configuration variables.') search = wrap(search, ['lowered']) # XXX compose with withoutSpaces? diff --git a/plugins/Dict/plugin.py b/plugins/Dict/plugin.py index 8dabba905..08c638902 100644 --- a/plugins/Dict/plugin.py +++ b/plugins/Dict/plugin.py @@ -53,7 +53,7 @@ class Dict(callbacks.Privmsg): conn = dictclient.Connection(server) dbs = conn.getdbdescs().keys() dbs.sort() - irc.reply(utils.commaAndify(dbs)) + irc.reply(utils.str.commaAndify(dbs)) except socket.error, e: irc.error(webutils.strError(e)) dictionaries = wrap(dictionaries) @@ -102,21 +102,21 @@ class Dict(callbacks.Privmsg): if not definitions: if dictionary == '*': irc.reply('No definition for %s could be found.' % - utils.quoted(word)) + utils.str.quoted(word)) else: irc.reply('No definition for %s could be found in %s' % - (utils.quoted(word), ircutils.bold(dictionary))) + (utils.str.quoted(word), ircutils.bold(dictionary))) return L = [] for d in definitions: dbs.add(ircutils.bold(d.getdb().getname())) (db, s) = (d.getdb().getname(), d.getdefstr()) db = ircutils.bold(db) - s = utils.normalizeWhitespace(s).rstrip(';.,') + s = utils.str.normalizeWhitespace(s).rstrip(';.,') L.append('%s: %s' % (db, s)) utils.sortBy(len, L) if dictionary == '*' and len(dbs) > 1: - s = '%s responded: %s' % (utils.commaAndify(dbs), '; '.join(L)) + s = '%s responded: %s' % (utils.str.commaAndify(dbs), '; '.join(L)) else: s = '; '.join(L) irc.reply(s) diff --git a/plugins/Format/plugin.py b/plugins/Format/plugin.py index 741891f1b..bcb3ffeff 100644 --- a/plugins/Format/plugin.py +++ b/plugins/Format/plugin.py @@ -144,7 +144,7 @@ class Format(callbacks.Privmsg): Returns the text surrounded by double quotes. """ - irc.reply(utils.dqrepr(text)) + irc.reply(utils.str.dqrepr(text)) repr = wrap(repr, ['text']) def concat(self, irc, msg, args, first, second): diff --git a/plugins/Math/plugin.py b/plugins/Math/plugin.py index 8512f1e54..0655483bd 100644 --- a/plugins/Math/plugin.py +++ b/plugins/Math/plugin.py @@ -29,14 +29,11 @@ from __future__ import division -import supybot.plugins as plugins - import re import math import cmath import types import string -from itertools import imap import supybot.utils as utils from supybot.commands import * @@ -160,12 +157,12 @@ class Math(callbacks.Privmsg): crash to the bot with something like 10**10**10**10. One consequence is that large values such as 10**24 might not be exact. """ - if text != text.translate(string.ascii, '_[]'): + if text != text.translate(utils.str.chars, '_[]'): irc.error('There\'s really no reason why you should have ' 'underscores or brackets in your mathematical ' 'expression. Please remove them.') return - #text = text.translate(string.ascii, '_[] \t') + #text = text.translate(utils.str.chars, '_[] \t') if 'lambda' in text: irc.error('You can\'t use lambda in this command.') return @@ -188,7 +185,7 @@ class Math(callbacks.Privmsg): text = self._mathRe.sub(handleMatch, text) try: self.log.info('evaluating %s from %s' % - (utils.quoted(text), msg.prefix)) + (utils.str.quoted(text), msg.prefix)) x = complex(eval(text, self._mathEnv, self._mathEnv)) irc.reply(self._complexToString(x)) except OverflowError: @@ -209,21 +206,21 @@ class Math(callbacks.Privmsg): math, and can thus cause the bot to suck up CPU. Hence it requires the 'trusted' capability to use. """ - if text != text.translate(string.ascii, '_[]'): + if text != text.translate(utils.str.chars, '_[]'): irc.error('There\'s really no reason why you should have ' 'underscores or brackets in your mathematical ' 'expression. Please remove them.') return # This removes spaces, too, but we'll leave the removal of _[] for # safety's sake. - text = text.translate(string.ascii, '_[] \t') + text = text.translate(utils.str.chars, '_[] \t') if 'lambda' in text: irc.error('You can\'t use lambda in this command.') return text = text.replace('lambda', '') try: self.log.info('evaluating %s from %s' % - (utils.quoted(text), msg.prefix)) + (utils.str.quoted(text), msg.prefix)) irc.reply(str(eval(text, self._mathEnv, self._mathEnv))) except OverflowError: maxFloat = math.ldexp(0.9999999999999999, 1024) @@ -280,12 +277,12 @@ class Math(callbacks.Privmsg): stack.append(eval(s, self._mathEnv, self._mathEnv)) except SyntaxError: irc.error('%s is not a defined function.' % - utils.quoted(arg)) + utils.str.quoted(arg)) return if len(stack) == 1: irc.reply(str(self._complexToString(complex(stack[0])))) else: - s = ', '.join(imap(self._complexToString, imap(complex, stack))) + s = ', '.join(map(self._complexToString, map(complex, stack))) irc.reply('Stack: [%s]' % s) def convert(self, irc, msg, args, number, unit1, unit2): diff --git a/plugins/Misc/plugin.py b/plugins/Misc/plugin.py index 2f7f365a1..e72ca3348 100644 --- a/plugins/Misc/plugin.py +++ b/plugins/Misc/plugin.py @@ -27,16 +27,10 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot - -import supybot.fix as fix - import os import sys import time -from itertools import imap, ifilter - import supybot.log as log import supybot.conf as conf import supybot.utils as utils @@ -46,9 +40,10 @@ import supybot.ircdb as ircdb import supybot.irclib as irclib import supybot.ircmsgs as ircmsgs import supybot.ircutils as ircutils -import supybot.webutils as webutils import supybot.callbacks as callbacks +from supybot.utils.iter import ifilter + class Misc(callbacks.Privmsg): def __init__(self): super(Misc, self).__init__() @@ -123,7 +118,7 @@ class Misc(callbacks.Privmsg): (not private and isPublic(cb))] names.sort() if names: - irc.reply(utils.commaAndify(names)) + irc.reply(utils.str.commaAndify(names)) else: if private: irc.reply('There are no private plugins.') @@ -148,7 +143,7 @@ class Misc(callbacks.Privmsg): commands.append(s) if commands: commands.sort() - irc.reply(utils.commaAndify(commands)) + irc.reply(utils.str.commaAndify(commands)) else: irc.error('That plugin exists, but it has no ' 'commands with help.') @@ -177,7 +172,7 @@ class Misc(callbacks.Privmsg): L.append('%s %s' % (name, key)) if L: L.sort() - irc.reply(utils.commaAndify(L)) + irc.reply(utils.str.commaAndify(L)) else: irc.reply('No appropriate commands were found.') apropos = wrap(apropos, ['lowered']) @@ -211,7 +206,7 @@ class Misc(callbacks.Privmsg): names = sorted([cb.name() for cb in cbs]) irc.error('That command exists in the %s plugins. ' 'Please specify exactly which plugin command ' - 'you want help with.'% utils.commaAndify(names)) + 'you want help with.'% utils.str.commaAndify(names)) else: getHelp(cbs[0]) else: @@ -235,9 +230,9 @@ class Misc(callbacks.Privmsg): Returns the version of the current bot. """ try: - newest = webutils.getUrl('http://supybot.sf.net/version.txt') + newest = utils.web.getUrl('http://supybot.sf.net/version.txt') newest ='The newest version available online is %s.'%newest.strip() - except webutils.WebError, e: + except utils.web.Error, e: self.log.warning('Couldn\'t get website version: %r', e) newest = 'I couldn\'t fetch the newest version ' \ 'from the Supybot website.' @@ -265,13 +260,13 @@ class Misc(callbacks.Privmsg): if cbs: names = [cb.name() for cb in cbs] names.sort() - plugin = utils.commaAndify(names) + plugin = utils.str.commaAndify(names) if irc.nested: - irc.reply(utils.commaAndify(names)) + irc.reply(utils.str.commaAndify(names)) else: irc.reply('The %s command is available in the %s %s.' % - (utils.quoted(command), plugin, - utils.pluralize('plugin', len(names)))) + (utils.str.quoted(command), plugin, + utils.str.pluralize('plugin', len(names)))) else: irc.error('There is no such command %s.' % command) plugin = wrap(plugin, ['commandName']) @@ -287,7 +282,7 @@ class Misc(callbacks.Privmsg): return module = sys.modules[cb.__class__.__module__] if hasattr(module, '__author__') and module.__author__: - irc.reply(utils.mungeEmailForWeb(str(module.__author__))) + irc.reply(utils.web.mungeEmail(str(module.__author__))) else: irc.reply('That plugin doesn\'t have an author that claims it.') author = wrap(author, [('plugin')]) @@ -317,7 +312,7 @@ class Misc(callbacks.Privmsg): chunk = L.pop() if L: chunk += ' \x02(%s)\x0F' % \ - utils.nItems('message', len(L), 'more') + utils.str.nItems('message', len(L), 'more') irc.reply(chunk, True) except KeyError: irc.error('You haven\'t asked me a command; perhaps you want ' @@ -380,7 +375,7 @@ class Misc(callbacks.Privmsg): nolimit = True iterable = ifilter(self._validLastMsg, reversed(irc.state.history)) iterable.next() # Drop the first message. - predicates = list(utils.flatten(predicates.itervalues())) + predicates = list(utils.iter.flatten(predicates.itervalues())) resp = [] if irc.nested and not \ self.registryValue('last.nested.includeTimestamp'): @@ -409,7 +404,7 @@ class Misc(callbacks.Privmsg): irc.error('I couldn\'t find a message matching that criteria in ' 'my history of %s messages.' % len(irc.state.history)) else: - irc.reply(utils.commaAndify(resp)) + irc.reply(utils.str.commaAndify(resp)) last = wrap(last, [getopts({'nolimit': '', 'on': 'something', 'with': 'something', @@ -496,7 +491,7 @@ class Misc(callbacks.Privmsg): shortname[, shortname and shortname]. """ L = [getShortName(n) for n in longList] - return utils.commaAndify(L) + return utils.str.commaAndify(L) def sortAuthors(): """ Sort the list of 'long names' based on the number of contributions @@ -520,7 +515,7 @@ class Misc(callbacks.Privmsg): hasContribs = False if getattr(module, '__author__', None): author = 'was written by %s' % \ - utils.mungeEmailForWeb(str(module.__author__)) + utils.web.mungeEmail(str(module.__author__)) hasAuthor = True if getattr(module, '__contributors__', None): contribs = sortAuthors() @@ -532,7 +527,7 @@ class Misc(callbacks.Privmsg): if contribs: contrib = '%s %s contributed to it.' % \ (buildContributorsString(contribs), - utils.has(len(contribs))) + utils.str.has(len(contribs))) hasContribs = True elif hasAuthor: contrib = 'has no additional contributors listed' @@ -549,7 +544,7 @@ class Misc(callbacks.Privmsg): if not authorInfo: return 'The nick specified (%s) is not a registered ' \ 'contributor' % nick - fullName = utils.mungeEmailForWeb(str(authorInfo)) + fullName = utils.web.mungeEmail(str(authorInfo)) contributions = [] if hasattr(module, '__contributors__'): if authorInfo not in module.__contributors__: @@ -558,22 +553,21 @@ class Misc(callbacks.Privmsg): contributions = module.__contributors__[authorInfo] if getattr(module, '__author__', False) == authorInfo: isAuthor = True - # XXX Partition needs moved to utils. - (nonCommands, commands) = fix.partition(lambda s: ' ' in s, - contributions) + (nonCommands, commands) = utils.iter.partition(lambda s: ' ' in s, + contributions) results = [] if commands: results.append( - 'the %s %s' %(utils.commaAndify(commands), - utils.pluralize('command',len(commands)))) + 'the %s %s' %(utils.str.commaAndify(commands), + utils.str.pluralize('command',len(commands)))) if nonCommands: - results.append('the %s' % utils.commaAndify(nonCommands)) + results.append('the %s' % utils.str.commaAndify(nonCommands)) if results and isAuthor: return '%s wrote the %s plugin and also contributed %s' % \ - (fullName, cb.name(), utils.commaAndify(results)) + (fullName, cb.name(), utils.str.commaAndify(results)) elif results and not isAuthor: return '%s contributed %s to the %s plugin' % \ - (fullName, utils.commaAndify(results), cb.name()) + (fullName, utils.str.commaAndify(results), cb.name()) elif isAuthor and not results: return '%s wrote the %s plugin' % (fullName, cb.name()) else: diff --git a/plugins/Network/plugin.py b/plugins/Network/plugin.py index 958fbfc85..bc9e74b4d 100644 --- a/plugins/Network/plugin.py +++ b/plugins/Network/plugin.py @@ -27,10 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot - -import supybot.plugins as plugins - import time import supybot.conf as conf @@ -172,19 +168,19 @@ class Network(callbacks.Privmsg): normal.append(channel) L = [] if ops: - L.append('is an op on %s' % utils.commaAndify(ops)) + L.append('is an op on %s' % utils.str.commaAndify(ops)) if halfops: - L.append('is a halfop on %s' % utils.commaAndify(halfops)) + L.append('is a halfop on %s' % utils.str.commaAndify(halfops)) if voices: - L.append('is voiced on %s' % utils.commaAndify(voices)) + L.append('is voiced on %s' % utils.str.commaAndify(voices)) if normal: if L: - L.append('is also on %s' % utils.commaAndify(normal)) + L.append('is also on %s' % utils.str.commaAndify(normal)) else: - L.append('is on %s' % utils.commaAndify(normal)) + L.append('is on %s' % utils.str.commaAndify(normal)) else: L = ['isn\'t on any non-secret channels'] - channels = utils.commaAndify(L) + channels = utils.str.commaAndify(L) if '317' in d: idle = utils.timeElapsed(d['317'].args[2]) signon = time.strftime(conf.supybot.reply.format.time(), @@ -246,7 +242,7 @@ class Network(callbacks.Privmsg): """ L = ['%s: %s' % (ircd.network, ircd.server) for ircd in world.ircs] utils.sortBy(str.lower, L) - irc.reply(utils.commaAndify(L)) + irc.reply(utils.str.commaAndify(L)) networks = wrap(networks) def doPong(self, irc, msg): diff --git a/plugins/Owner/plugin.py b/plugins/Owner/plugin.py index 8c176f5ec..2123f6bd5 100644 --- a/plugins/Owner/plugin.py +++ b/plugins/Owner/plugin.py @@ -27,16 +27,12 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot.fix as fix - import gc import os import imp import sre import sys -import getopt import socket -import logging import linecache import supybot.log as log @@ -424,7 +420,7 @@ class Owner(callbacks.Privmsg): return except ImportError, e: if name in str(e): - irc.error('No plugin named %s exists.' % utils.dqrepr(name)) + irc.error('No plugin named %s exists.' % utils.str.dqrepr(name)) else: irc.error(str(e)) return diff --git a/plugins/Status/plugin.py b/plugins/Status/plugin.py index 1091a4bac..ecda76346 100644 --- a/plugins/Status/plugin.py +++ b/plugins/Status/plugin.py @@ -76,9 +76,9 @@ class Status(callbacks.Privmsg): networks.setdefault(Irc.network, []).append(Irc.nick) networks = networks.items() networks.sort() - networks = ['%s as %s' % (net, utils.commaAndify(nicks)) + networks = ['%s as %s' % (net, utils.str.commaAndify(nicks)) for (net, nicks) in networks] - L = ['I am connected to %s.' % utils.commaAndify(networks)] + L = ['I am connected to %s.' % utils.str.commaAndify(networks)] if world.profiling: L.append('I am currently in code profiling mode.') irc.reply(' '.join(L)) @@ -92,9 +92,10 @@ class Status(callbacks.Privmsg): threads = [t.getName() for t in threading.enumerate()] threads.sort() s = 'I have spawned %s; %s %s still currently active: %s.' % \ - (utils.nItems('thread', world.threadsSpawned), - utils.nItems('thread', len(threads)), utils.be(len(threads)), - utils.commaAndify(threads)) + (utils.str.nItems('thread', world.threadsSpawned), + utils.str.nItems('thread', len(threads)), + utils.str.be(len(threads)), + utils.str.commaAndify(threads)) irc.reply(s) threads = wrap(threads) @@ -137,7 +138,7 @@ class Status(callbacks.Privmsg): 'of system time, for a total of %.2f seconds of CPU ' \ 'time. %s' % (user, system, user + system, children) if self.registryValue('cpu.threads', target): - spawned = utils.nItems('thread', world.threadsSpawned) + spawned = utils.str.nItems('thread', world.threadsSpawned) response += 'I have spawned %s; I currently have %s still ' \ 'running.' % (spawned, activeThreads) if self.registryValue('cpu.memory', target): @@ -159,7 +160,7 @@ class Status(callbacks.Privmsg): response += ' I\'m taking up %s kB of memory.' % mem except Exception: self.log.exception('Uncaught exception in cpu.memory:') - irc.reply(utils.normalizeWhitespace(response)) + irc.reply(utils.str.normalizeWhitespace(response)) cpu = wrap(cpu) def cmd(self, irc, msg, args): @@ -178,9 +179,9 @@ class Status(callbacks.Privmsg): attr == callbacks.canonicalName(attr): commands += 1 s = 'I offer a total of %s in %s. I have processed %s.' % \ - (utils.nItems('command', commands), - utils.nItems('plugin', callbacksPrivmsg, 'command-based'), - utils.nItems('command', world.commandsProcessed)) + (utils.str.nItems('command', commands), + utils.str.nItems('plugin', callbacksPrivmsg, 'command-based'), + utils.str.nItems('command', world.commandsProcessed)) irc.reply(s) cmd = wrap(cmd) @@ -199,7 +200,7 @@ class Status(callbacks.Privmsg): commands.add(attr) commands = list(commands) commands.sort() - irc.reply(utils.commaAndify(commands)) + irc.reply(utils.str.commaAndify(commands)) commands = wrap(commands) def uptime(self, irc, msg, args): diff --git a/plugins/User/plugin.py b/plugins/User/plugin.py index 1a3d740fb..3253a48eb 100644 --- a/plugins/User/plugin.py +++ b/plugins/User/plugin.py @@ -27,12 +27,8 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot.fix as fix - import re -import getopt import fnmatch -from itertools import imap, ifilter import supybot.conf as conf import supybot.utils as utils @@ -75,7 +71,7 @@ class User(callbacks.Privmsg): users.append(u.name) if users: utils.sortBy(str.lower, users) - irc.reply(utils.commaAndify(users)) + irc.reply(utils.str.commaAndify(users)) else: if predicates: irc.reply('There are no matching registered users.') @@ -158,7 +154,7 @@ class User(callbacks.Privmsg): """ try: id = ircdb.users.getUserId(newname) - irc.error('%s is already registered.' % utils.quoted(newname)) + irc.error('%s is already registered.' % utils.str.quoted(newname)) return except KeyError: pass @@ -295,7 +291,7 @@ class User(callbacks.Privmsg): def getHostmasks(user): hostmasks = map(repr, user.hostmasks) hostmasks.sort() - return utils.commaAndify(hostmasks) + return utils.str.commaAndify(hostmasks) try: user = ircdb.users.getUser(msg.prefix) if name: @@ -418,8 +414,8 @@ class User(callbacks.Privmsg): irc.reply('I have %s registered users ' 'with %s registered hostmasks; ' '%s and %s.' % (users, hostmasks, - utils.nItems('owner', owners), - utils.nItems('admin', admins))) + utils.str.nItems('owner', owners), + utils.str.nItems('admin', admins))) stats = wrap(stats) diff --git a/plugins/__init__.py b/plugins/__init__.py index 404e24fd9..2549be6d8 100644 --- a/plugins/__init__.py +++ b/plugins/__init__.py @@ -27,10 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. ### -__revision__ = "$Id: __init__.py,v 1.44 2005/01/08 07:22:46 jamessan Exp $" - -import supybot.fix as fix - import gc import os import re @@ -106,7 +102,7 @@ class NoSuitableDatabase(Exception): return 'No suitable databases were found. Suitable databases ' \ 'include %s. If you have one of these databases installed, ' \ 'make sure it is listed in the supybot.databases ' \ - 'configuration variable.' % utils.commaAndify(self.suitable) + 'configuration variable.' % utils.str.commaAndify(self.suitable) def DB(filename, types): filename = conf.supybot.directories.data.dirize(filename) @@ -325,7 +321,7 @@ class ChannelUserDB(ChannelUserDictionary): log.debug('Exception: %s', utils.exnToString(e)) def flush(self): - fd = utils.transactionalFile(self.filename, makeBackupIfSmaller=False) + fd = utils.file.AtomicFile(self.filename, makeBackupIfSmaller=False) writer = csv.writer(fd) items = self.items() if not items: @@ -389,9 +385,9 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg): def getCommandHelp(self, name): help = self.__parent.getCommandHelp(name) - help = help.replace('$Types', utils.pluralize(self.name())) + help = help.replace('$Types', utils.str.pluralize(self.name())) help = help.replace('$Type', self.name()) - help = help.replace('$types', utils.pluralize(self.name().lower())) + help = help.replace('$types', utils.str.pluralize(self.name().lower())) help = help.replace('$type', self.name().lower()) return help @@ -442,7 +438,7 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg): remove = wrap(remove, ['user', 'channeldb', 'id']) def searchSerializeRecord(self, record): - text = utils.quoted(utils.ellipsisify(record.text, 50)) + text = utils.str.quoted(utils.str.ellipsisify(record.text, 50)) return '#%s: %s' % (record.id, text) def search(self, irc, msg, args, channel, optlist, glob): @@ -471,10 +467,10 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg): L.append(self.searchSerializeRecord(record)) if L: L.sort() - irc.reply('%s found: %s' % (len(L), utils.commaAndify(L))) + irc.reply('%s found: %s' % (len(L), utils.str.commaAndify(L))) else: irc.reply('No matching %s were found.' % - utils.pluralize(self.name().lower())) + utils.str.pluralize(self.name().lower())) search = wrap(search, ['channeldb', getopts({'by': 'otherUser', 'regexp': 'regexpMatcher'}), @@ -485,7 +481,8 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg): at = time.localtime(record.at) timeS = time.strftime(conf.supybot.reply.format.time(), at) return '%s #%s: %s (added by %s at %s)' % \ - (self.name(), record.id, utils.quoted(record.text), name, timeS) + (self.name(), record.id, + utils.str.quoted(record.text), name, timeS) def get(self, irc, msg, args, channel, id): """[] @@ -527,7 +524,7 @@ class ChannelIdDatabasePlugin(callbacks.Privmsg): """ n = self.db.size(channel) irc.reply('There %s %s in my database.' % - (utils.be(n), utils.nItems(self.name().lower(), n))) + (utils.str.be(n), utils.str.nItems(self.name().lower(), n))) stats = wrap(stats, ['channeldb']) @@ -592,7 +589,7 @@ class PeriodicFileDownloader(object): self.log.warning('Error downloading %s: %s', url, e) return confDir = conf.supybot.directories.data() - newFilename = os.path.join(confDir, utils.mktemp()) + newFilename = os.path.join(confDir, utils.file.mktemp()) outfd = file(newFilename, 'wb') start = time.time() s = infd.read(4096) diff --git a/scripts/supybot b/scripts/supybot index 905a2ce9f..7caccecb7 100644 --- a/scripts/supybot +++ b/scripts/supybot @@ -33,7 +33,7 @@ This is the main program to run Supybot. """ - +import supybot import re import os diff --git a/setup.py b/setup.py index af0c96cc5..87c506fe2 100644 --- a/setup.py +++ b/setup.py @@ -103,11 +103,13 @@ if clean: sys.exit(-1) packages = ['supybot', + 'supybot.utils', 'supybot.drivers', 'supybot.plugins',] + \ ['supybot.plugins.'+s for s in plugins] package_dir = {'supybot': 'src', + 'supybot.utils': 'src/utils', 'supybot.plugins': 'plugins', 'supybot.drivers': 'src/drivers',} diff --git a/src/callbacks.py b/src/callbacks.py index 6359f33b3..6c3c69c30 100644 --- a/src/callbacks.py +++ b/src/callbacks.py @@ -38,10 +38,6 @@ how to use them. import supybot - - -import supybot.fix as fix - import re import copy import sets @@ -166,7 +162,7 @@ def canonicalName(command): while command and command[-1] in special: reAppend = command[-1] + reAppend command = command[:-1] - return command.translate(string.ascii, special).lower() + reAppend + return command.translate(utils.str.chars, special).lower() + reAppend def reply(msg, s, prefixName=None, private=None, notice=None, to=None, action=None, error=False): @@ -233,7 +229,7 @@ def getHelp(method, name=None): if doclines: help = ' '.join(doclines) s = '(%s) -- %s' % (ircutils.bold(s), help) - return utils.normalizeWhitespace(s) + return utils.str.normalizeWhitespace(s) def getSyntax(method, name=None): if name is None: @@ -256,10 +252,10 @@ class Tokenizer(object): # # These are the characters valid in a token. Everything printable except # double-quote, left-bracket, and right-bracket. - validChars = string.ascii.translate(string.ascii, '\x00\r\n \t') + validChars = utils.str.chars.translate(utils.str.chars, '\x00\r\n \t') def __init__(self, brackets='', pipe=False, quotes='"'): if brackets: - self.validChars = self.validChars.translate(string.ascii, brackets) + self.validChars=self.validChars.translate(utils.str.chars, brackets) self.left = brackets[0] self.right = brackets[1] else: @@ -267,9 +263,9 @@ class Tokenizer(object): self.right = '' self.pipe = pipe if self.pipe: - self.validChars = self.validChars.translate(string.ascii, '|') + self.validChars = self.validChars.translate(utils.str.chars, '|') self.quotes = quotes - self.validChars = self.validChars.translate(string.ascii, quotes) + self.validChars = self.validChars.translate(utils.str.chars, quotes) def _handleToken(self, token): @@ -459,7 +455,7 @@ class RichReplyMethods(object): if prefixer is None: prefixer = '' if joiner is None: - joiner = utils.commaAndify + joiner = utils.str.commaAndify if isinstance(prefixer, basestring): prefixer = prefixer.__add__ if isinstance(joiner, basestring): @@ -494,7 +490,7 @@ class RichReplyMethods(object): kwargs['Raise'] = True if isinstance(capability, basestring): # checkCommandCapability! log.warning('Denying %s for lacking %s capability.', - self.msg.prefix, utils.quoted(capability)) + self.msg.prefix, utils.str.quoted(capability)) if not self._getConfig(conf.supybot.reply.error.noCapability): v = self._getConfig(conf.supybot.replies.noCapability) s = self.__makeReply(v % capability, s) @@ -750,7 +746,7 @@ class IrcObjectProxy(RichReplyMethods): 'Please specify the plugin whose command you ' 'wish to call by using its name as a command ' 'before %s.' % - (command, utils.commaAndify(names), command)) + (command, utils.str.commaAndify(names), command)) else: cb = cbs[0] del self.args[0] # Remove the command. @@ -870,7 +866,7 @@ class IrcObjectProxy(RichReplyMethods): response = msgs.pop() if msgs: n = ircutils.bold('(%s)') - n %= utils.nItems('message', len(msgs), 'more') + n %= utils.str.nItems('message', len(msgs), 'more') response = '%s %s' % (response, n) prefix = msg.prefix if self.to and ircutils.isNick(self.to): @@ -1123,7 +1119,7 @@ class Privmsg(irclib.IrcCallback): """Gets the given command from this plugin.""" name = canonicalName(name) assert self.isCommand(name), '%s is not a command.' % \ - utils.quoted(name) + utils.str.quoted(name) return getattr(self, name) def callCommand(self, name, irc, msg, *L, **kwargs): @@ -1155,11 +1151,11 @@ class Privmsg(irclib.IrcCallback): command = self.getCommand(name) if hasattr(command, 'isDispatcher') and \ command.isDispatcher and self.__doc__: - return utils.normalizeWhitespace(self.__doc__) + return utils.str.normalizeWhitespace(self.__doc__) elif hasattr(command, '__doc__'): return getHelp(command) else: - return 'The %s command has no help.' % utils.quoted(name) + return 'The %s command has no help.' % utils.str.quoted(name) def registryValue(self, name, channel=None, value=True): plugin = self.name() @@ -1291,7 +1287,7 @@ class PrivmsgRegexp(Privmsg): self.res.append((r, name)) except re.error, e: self.log.warning('Invalid regexp: %s (%s)', - utils.quoted(value.__doc__), e) + utils.str.quoted(value.__doc__), e) utils.sortBy(operator.itemgetter(1), self.res) def callCommand(self, name, irc, msg, *L, **kwargs): diff --git a/src/cdb.py b/src/cdb.py index 8e4f4c531..abf966e46 100644 --- a/src/cdb.py +++ b/src/cdb.py @@ -32,10 +32,6 @@ Database module, similar to dbhash. Uses a format similar to (if not entirely the same as) DJB's CDB . """ - - -import supybot.fix as fix - import os import sys import sets @@ -134,7 +130,7 @@ def make(dbFilename, readFilename=None): class Maker(object): """Class for making CDB databases.""" def __init__(self, filename): - self.fd = utils.transactionalFile(filename) + self.fd = utils.file.AtomicFile(filename) self.filename = filename self.fd.seek(2048) self.hashPointers = [(0, 0)] * 256 diff --git a/src/commands.py b/src/commands.py index 3f59af856..9cb495b9f 100644 --- a/src/commands.py +++ b/src/commands.py @@ -31,10 +31,6 @@ Includes wrappers for commands. """ - - -import supybot.fix as fix - import time import types import getopt @@ -233,7 +229,7 @@ def getExpiry(irc, msg, args, state): def getBoolean(irc, msg, args, state): try: - state.args.append(utils.toBool(args[0])) + state.args.append(utils.str.toBool(args[0])) del args[0] except ValueError: irc.errorInvalid('boolean', args[0]) @@ -324,8 +320,8 @@ def _getRe(f): irc.errorInvalid('regular expression', s) return get -getMatcher = _getRe(utils.perlReToPythonRe) -getReplacer = _getRe(utils.perlReToReplacer) +getMatcher = _getRe(utils.str.perlReToPythonRe) +getReplacer = _getRe(utils.str.perlReToReplacer) def getNick(irc, msg, args, state): if ircutils.isNick(args[0]): @@ -494,7 +490,7 @@ def getCommandName(irc, msg, args, state): state.args.append(callbacks.canonicalName(args.pop(0))) def getIp(irc, msg, args, state): - if utils.isIP(args[0]): + if utils.net.isIP(args[0]): state.args.append(args.pop(0)) else: irc.errorInvalid('ip', args[0]) @@ -845,7 +841,7 @@ class Spec(object): def __init__(self, types, allowExtra=False): self.types = types self.allowExtra = allowExtra - utils.mapinto(contextify, self.types) + utils.seq.mapinto(contextify, self.types) def __call__(self, irc, msg, args, stateAttrs={}): state = self._state(self.types[:], stateAttrs) diff --git a/src/conf.py b/src/conf.py index 66f174951..c9af9b024 100644 --- a/src/conf.py +++ b/src/conf.py @@ -27,8 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot.fix as fix - import os import sys import time @@ -70,6 +68,12 @@ daemonized = False ### allowDefaultOwner = False +### +# Here we replace values in other modules as appropriate. +### +utils.web.defaultHeaders['User-agent'] = \ + 'Mozilla/5.0 (Compatible; Supybot %s)' % version + ### # The standard registry. ### @@ -405,7 +409,7 @@ registerChannelValue(supybot.reply, 'showSimpleSyntax', class ValidPrefixChars(registry.String): """Value must contain only ~!@#$%^&*()_-+=[{}]\\|'\";:,<.>/?""" def setValue(self, v): - if v.translate(string.ascii, '`~!@#$%^&*()_-+=[{}]\\|\'";:,<.>/?'): + if v.translate(utils.str.chars, '`~!@#$%^&*()_-+=[{}]\\|\'";:,<.>/?'): self.error() registry.String.setValue(self, v) @@ -579,9 +583,9 @@ registerChannelValue(supybot.commands.nested, 'pipeSyntax', example: 'bot: foo | bar'.""")) registerGroup(supybot.commands, 'defaultPlugins', - orderAlphabetically=True, help=utils.normalizeWhitespace("""Determines - what commands have default plugins set, and which plugins are set to - be the default for each of those commands.""")) + orderAlphabetically=True, help="""Determines what commands have default + plugins set, and which plugins are set to be the default for each of those + commands.""") registerGlobalValue(supybot.commands.defaultPlugins, 'importantPlugins', registry.SpaceSeparatedSetOfStrings( ['Admin', 'Channel', 'Config', 'Misc', 'Owner', 'User'], @@ -704,12 +708,8 @@ registerGlobalValue(supybot.directories.data, 'tmp', DataFilenameDirectory('tmp', """Determines what directory temporary files are put into.""")) -# Remember, we're *meant* to replace this nice little wrapper. -def transactionalFile(*args, **kwargs): - kwargs['tmpDir'] = supybot.directories.data.tmp() - kwargs['backupDir'] = supybot.directories.backup() - return utils.AtomicFile(*args, **kwargs) -utils.transactionalFile = transactionalFile +utils.file.AtomicFile.default.tmpDir = supybot.directories.data.tmp +utils.file.AtomicFile.default.backupDir = supybot.directories.backup class PluginDirectories(registry.CommaSeparatedListOfStrings): def __call__(self): @@ -931,6 +931,7 @@ registerGlobalValue(supybot.protocols.http, 'peekSize', registerGlobalValue(supybot.protocols.http, 'proxy', registry.String('', """Determines what proxy all HTTP requests should go through. The value should be of the form 'host:port'.""")) +utils.web.proxy = supybot.protocols.http.proxy ### @@ -945,7 +946,7 @@ registerGlobalValue(supybot, 'defaultIgnore', class IP(registry.String): """Value must be a valid IP.""" def setValue(self, v): - if v and not (utils.isIP(v) or utils.isIPV6(v)): + if v and not (utils.net.isIP(v) or utils.net.isIPV6(v)): self.error() else: registry.String.setValue(self, v) diff --git a/src/dbi.py b/src/dbi.py index 44a3df26d..4b0dbb0af 100644 --- a/src/dbi.py +++ b/src/dbi.py @@ -31,10 +31,6 @@ Module for some slight database-independence for simple databases. """ - - -import supybot.fix as fix - import csv import math import sets @@ -262,8 +258,7 @@ class FlatfileMapping(MappingInterface): def vacuum(self): infd = file(self.filename) - outfd = utils.transactionalFile(self.filename, - makeBackupIfSmaller=False) + outfd = utils.file.AstomicFile(self.filename,makeBackupIfSmaller=False) outfd.write(infd.readline()) # First line, nextId. for line in infd: if not line.startswith('-'): diff --git a/src/fix.py b/src/fix.py deleted file mode 100644 index 63aec9840..000000000 --- a/src/fix.py +++ /dev/null @@ -1,239 +0,0 @@ -### -# Copyright (c) 2002-2005, Jeremiah Fincher -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions, and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions, and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the author of this software nor the name of -# contributors to this software may be used to endorse or promote products -# derived from this software without specific prior written consent. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -### - -""" -Fixes stuff that Python should have but doesn't. -""" - -from __future__ import division - -__all__ = [] - -exported = ['ignore', 'window', 'group', 'partition', 'set', 'frozenset', - 'any', 'all', 'rsplit', 'dynamic'] - -import sys -import new -import atexit -import string -string.ascii = string.maketrans('', '') - -import random -_choice = random.choice -def choice(iterable): - if isinstance(iterable, (list, tuple)): - return _choice(iterable) - else: - n = 1 - m = new.module('') # Guaranteed unique value. - ret = m - for x in iterable: - if random.random() < 1/n: - ret = x - n += 1 - if ret is m: - raise IndexError - return ret -random.choice = choice - -def ignore(*args, **kwargs): - """Simply ignore the arguments sent to it.""" - pass - -class DynamicScope(object): - def _getLocals(self, name): - f = sys._getframe().f_back.f_back # _getLocals <- __[gs]etattr__ <- ... - while f: - if name in f.f_locals: - return f.f_locals - f = f.f_back - raise NameError, name - - def __getattr__(self, name): - try: - return self._getLocals(name)[name] - except (NameError, KeyError): - return None - - def __setattr__(self, name, value): - self._getLocals(name)[name] = value -dynamic = DynamicScope() - - -if sys.version_info < (2, 4, 0): - def reversed(L): - """Iterates through a sequence in reverse.""" - for i in xrange(len(L) - 1, -1, -1): - yield L[i] - exported.append('reversed') - -def window(L, size): - """Returns a sliding 'window' through the list L of size size.""" - assert not isinstance(L, int), 'Argument order swapped: window(L, size)' - if size < 1: - raise ValueError, 'size <= 0 disallowed.' - for i in xrange(len(L) - (size-1)): - yield L[i:i+size] - -import itertools -def ilen(iterable): - """Returns the length of an iterator.""" - i = 0 - for _ in iterable: - i += 1 - return i - -def trueCycle(iterable): - while 1: - yielded = False - for x in iterable: - yield x - yielded = True - if not yielded: - raise StopIteration - -itertools.trueCycle = trueCycle -itertools.ilen = ilen - -def groupby(key, iterable): - if key is None: - key = lambda x: x - it = iter(iterable) - value = it.next() # If there are no items, this takes an early exit - oldkey = key(value) - group = [value] - for value in it: - newkey = key(value) - if newkey != oldkey: - yield group - group = [] - oldkey = newkey - group.append(value) - yield group -itertools.groupby = groupby - -def group(seq, groupSize, noneFill=True): - """Groups a given sequence into sublists of length groupSize.""" - ret = [] - L = [] - i = groupSize - for elt in seq: - if i > 0: - L.append(elt) - else: - ret.append(L) - i = groupSize - L = [] - L.append(elt) - i -= 1 - if L: - if noneFill: - while len(L) < groupSize: - L.append(None) - ret.append(L) - return ret - -def partition(p, L): - """Partitions a list L based on a predicate p. Returns a (yes,no) tuple""" - no = [] - yes = [] - for elt in L: - if p(elt): - yes.append(elt) - else: - no.append(elt) - return (yes, no) - -def any(p, seq): - """Returns true if any element in seq satisfies predicate p.""" - for elt in itertools.ifilter(p, seq): - return True - else: - return False - -def all(p, seq): - """Returns true if all elements in seq satisfy predicate p.""" - for elt in itertools.ifilterfalse(p, seq): - return False - else: - return True - -def rsplit(s, sep=None, maxsplit=-1): - """Equivalent to str.split, except splitting from the right.""" - if sys.version_info < (2, 4, 0): - if sep is not None: - sep = sep[::-1] - L = s[::-1].split(sep, maxsplit) - L.reverse() - return [s[::-1] for s in L] - else: - return s.rsplit(sep, maxsplit) - -if sys.version_info < (2, 4, 0): - import operator - def itemgetter(i): - return lambda x: x[i] - - def attrgetter(attr): - return lambda x: getattr(x, attr) - operator.itemgetter = itemgetter - operator.attrgetter = attrgetter - -import csv -import cStringIO as StringIO -def join(L): - fd = StringIO.StringIO() - writer = csv.writer(fd) - writer.writerow(L) - return fd.getvalue().rstrip('\r\n') - -def split(s): - fd = StringIO.StringIO(s) - reader = csv.reader(fd) - return reader.next() -csv.join = join -csv.split = split - -import sets -set = sets.Set -frozenset = sets.ImmutableSet - -import socket -# Some socket modules don't have sslerror, so we'll just make it an error. -if not hasattr(socket, 'sslerror'): - socket.sslerror = socket.error - -g = globals() -for name in exported: - __builtins__[name] = g[name] - - - -# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: - diff --git a/src/ircdb.py b/src/ircdb.py index eb5d8ffca..4b362fcf0 100644 --- a/src/ircdb.py +++ b/src/ircdb.py @@ -27,18 +27,13 @@ # POSSIBILITY OF SUCH DAMAGE. ### - - from __future__ import division -import supybot.fix as fix - import os import sets import time import string import operator -from itertools import imap, ilen, ifilter import supybot.log as log import supybot.conf as conf @@ -47,6 +42,7 @@ import supybot.world as world import supybot.ircutils as ircutils import supybot.registry as registry import supybot.unpreserve as unpreserve +from utils.iter import imap, ilen, ifilter def isCapability(capability): return len(capability.split(None, 1)) == 1 @@ -114,7 +110,7 @@ def canonicalCapability(capability): return capability.lower() def unWildcardHostmask(hostmask): - return hostmask.translate(string.ascii, '!@*?') + return hostmask.translate(utils.str.chars, '!@*?') _invert = invertCapability class CapabilitySet(sets.Set): @@ -228,7 +224,7 @@ class IrcUser(object): return '%s(id=%s, ignore=%s, password="", name=%s, hashed=%r, ' \ 'capabilities=%r, hostmasks=[], secure=%r)\n' % \ (self.__class__.__name__, self.id, self.ignore, - utils.quoted(self.name), self.hashed, self.capabilities, + utils.str.quoted(self.name), self.hashed, self.capabilities, self.secure) def __hash__(self): @@ -611,7 +607,7 @@ class UsersDictionary(utils.IterableMap): if self.filename is not None: L = self.users.items() L.sort() - fd = utils.transactionalFile(self.filename) + fd = utils.file.AtomicFile(self.filename) for (id, u) in L: fd.write('user %s' % id) fd.write(os.linesep) @@ -657,7 +653,7 @@ class UsersDictionary(utils.IterableMap): 'Removing the offending hostmasks.') for (id, hostmask) in ids.iteritems(): log.error('Removing %s from user %s.', - utils.quoted(hostmask), id) + utils.str.quoted(hostmask), id) self.users[id].removeHostmask(hostmask) raise DuplicateHostmask, 'Ids %r matched.' % ids else: # Not a hostmask, must be a name. @@ -789,7 +785,7 @@ class ChannelsDictionary(utils.IterableMap): """Flushes the channel database to its file.""" if not self.noFlush: if self.filename is not None: - fd = utils.transactionalFile(self.filename) + fd = utils.file.AtomicFile(self.filename) for (channel, c) in self.channels.iteritems(): fd.write('channel %s' % channel) fd.write(os.linesep) @@ -845,7 +841,7 @@ class IgnoresDB(object): def open(self, filename): self.filename = filename fd = file(self.filename) - for line in utils.nonCommentNonEmptyLines(fd): + for line in utils.file.nonCommentNonEmptyLines(fd): try: line = line.rstrip('\r\n') L = line.split() @@ -857,12 +853,12 @@ class IgnoresDB(object): self.add(hostmask, expiration) except Exception, e: log.error('Invalid line in ignores database: %s', - utils.quoted(line)) + utils.str.quoted(line)) fd.close() def flush(self): if self.filename is not None: - fd = utils.transactionalFile(self.filename) + fd = utils.file.AtomicFile(self.filename) now = time.time() for (hostmask, expiration) in self.hostmasks.items(): if now < expiration or not expiration: diff --git a/src/irclib.py b/src/irclib.py index 6c512c429..010d6a482 100644 --- a/src/irclib.py +++ b/src/irclib.py @@ -27,16 +27,11 @@ # POSSIBILITY OF SUCH DAMAGE. ### - - -import supybot.fix as fix - import copy import sets import time import random import operator -from itertools import imap, chain, cycle import supybot.log as log import supybot.conf as conf @@ -45,6 +40,7 @@ import supybot.world as world import supybot.ircdb as ircdb import supybot.ircmsgs as ircmsgs import supybot.ircutils as ircutils +from utils.iter import imap, chain, cycle from supybot.structures import queue, smallqueue, RingBuffer ### @@ -173,7 +169,7 @@ class IrcMsgQueue(object): not conf.supybot.protocols.irc.queueDuplicateMessages(): s = str(msg).strip() log.info('Not adding message %s to queue, already added.', - utils.quoted(s)) + utils.str.quoted(s)) return False else: self.msgs.add(msg) @@ -642,7 +638,7 @@ class Irc(IrcCommandDispatcher): name = name.lower() def nameMatches(cb): return cb.name().lower() == name - (bad, good) = partition(nameMatches, self.callbacks) + (bad, good) = utils.iter.partition(nameMatches, self.callbacks) self.callbacks[:] = good return bad diff --git a/src/ircmsgs.py b/src/ircmsgs.py index 6906a0580..1d2843002 100644 --- a/src/ircmsgs.py +++ b/src/ircmsgs.py @@ -34,10 +34,6 @@ construct such messages in an easier way than the constructor for the IrcMsg object (which, as you'll read later, is quite...full-featured :)) """ - - -import supybot.fix as fix - import re import time import string @@ -192,8 +188,8 @@ class IrcMsg(object): if self._repr is not None: return self._repr self._repr = 'IrcMsg(prefix=%s, command=%s, args=%r)' % \ - (utils.quoted(self.prefix), utils.quoted(self.command), - self.args) + (utils.str.quoted(self.prefix), + utils.str.quoted(self.command), self.args) return self._repr def __reduce__(self): @@ -586,7 +582,8 @@ def join(channel, key=None, prefix='', msg=None): return IrcMsg(prefix=prefix, command='JOIN', args=(channel,), msg=msg) else: if conf.supybot.protocols.irc.strictRfc(): - assert key.translate(string.ascii, string.ascii[128:]) == key and \ + assert key.translate(utils.str.chars, + utils.str.chars[128:]) == key and \ '\x00' not in key and \ '\r' not in key and \ '\n' not in key and \ @@ -613,7 +610,8 @@ def joins(channels, keys=None, prefix='', msg=None): else: for key in keys: if conf.supybot.protocols.irc.strictRfc(): - assert key.translate(string.ascii,string.ascii[128:])==key and\ + assert key.translate(utils.str.chars, + utils.str.chars[128:])==key and \ '\x00' not in key and \ '\r' not in key and \ '\n' not in key and \ diff --git a/src/ircutils.py b/src/ircutils.py index f5e5a5f6d..5f43cc40c 100644 --- a/src/ircutils.py +++ b/src/ircutils.py @@ -34,16 +34,11 @@ nick class to handle nicks (so comparisons and hashing and whatnot work in an IRC-case-insensitive fashion), and numerous other things. """ - - -import supybot.fix as fix - import re import time import random import string import textwrap -from itertools import imap, ilen from cStringIO import StringIO as sio import supybot.utils as utils @@ -109,7 +104,7 @@ def toLower(s, casemapping=None): elif casemapping == 'ascii': # freenode return s.lower() else: - raise ValueError, 'Invalid casemapping: %s' % utils.quoted(casemapping) + raise ValueError, 'Invalid casemapping: %r' % casemapping def strEqual(nick1, nick2): """s1, s2 => bool @@ -197,11 +192,11 @@ def banmask(hostmask): """ assert isUserHostmask(hostmask) host = hostFromHostmask(hostmask) - if utils.isIP(host): + if utils.net.isIP(host): L = host.split('.') L[-1] = '*' return '*!*@' + '.'.join(L) - elif utils.isIPV6(host): + elif utils.net.isIPV6(host): L = host.split(':') L[-1] = '*' return '*!*@' + ':'.join(L) @@ -450,7 +445,7 @@ def safeArgument(s): if isinstance(s, unicode): s = s.encode('utf-8') elif not isinstance(s, basestring): - debug('Got a non-string in safeArgument: %s', utils.quoted(s)) + debug('Got a non-string in safeArgument: %r', s) s = str(s) if isValidArgument(s): return s @@ -466,7 +461,7 @@ def replyTo(msg): def dccIP(ip): """Returns in IP in the proper for DCC.""" - assert utils.isIP(ip), \ + assert utils.net.isIP(ip), \ 'argument must be a string ip in xxx.yyy.zzz.www format.' i = 0 x = 256**3 @@ -483,7 +478,7 @@ def unDccIP(i): L.append(i % 256) i /= 256 L.reverse() - return '.'.join(imap(str, L)) + return '.'.join(utils.iter.imap(str, L)) class IrcString(str): """This class does case-insensitive comparison and hashing of nicks.""" @@ -662,7 +657,7 @@ def standardSubstitute(irc, msg, text, env=None): }) if env is not None: vars.update(env) - return utils.perlVariableSubstitute(vars, text) + return utils.str.perlVariableSubstitute(vars, text) if __name__ == '__main__': diff --git a/src/log.py b/src/log.py index af83a7665..ecaa874b9 100644 --- a/src/log.py +++ b/src/log.py @@ -27,8 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot.fix as fix - import os import sys import time diff --git a/src/plugin.py b/src/plugin.py index 19ba2268e..1ed070680 100644 --- a/src/plugin.py +++ b/src/plugin.py @@ -49,7 +49,7 @@ def loadPluginModule(name, ignoreDeprecation=False): files.extend(os.listdir(dir)) except EnvironmentError: # OSError, IOError superclass. log.warning('Invalid plugin directory: %s; removing.', - utils.quoted(dir)) + utils.str.quoted(dir)) conf.supybot.directories.plugins().remove(dir) loweredFiles = map(str.lower, files) try: @@ -72,7 +72,7 @@ def loadPluginModule(name, ignoreDeprecation=False): log.warning('Deprecated plugin loaded: %s', name) else: raise Deprecated, 'Attempted to load deprecated plugin %s' % \ - utils.quoted(name) + utils.str.quoted(name) if module.__name__ in sys.modules: sys.modules[module.__name__] = module linecache.checkcache() diff --git a/src/registry.py b/src/registry.py index 6e022e888..696185c1a 100644 --- a/src/registry.py +++ b/src/registry.py @@ -34,12 +34,11 @@ import time import string import textwrap -import supybot.fix as fix import supybot.utils as utils def error(s): - """Replace me with something better from another module!""" - print '***', s + """Replace me with something better from another module!""" + print '***', s def exception(s): """Ditto!""" @@ -68,7 +67,7 @@ def open(filename, clear=False): if clear: _cache.clear() _fd = file(filename) - fd = utils.nonCommentNonEmptyLines(_fd) + fd = utils.file.nonCommentNonEmptyLines(_fd) acc = '' for line in fd: line = line.rstrip('\r\n') @@ -94,7 +93,7 @@ def open(filename, clear=False): def close(registry, filename, private=True): first = True - fd = utils.transactionalFile(filename) + fd = utils.file.AtomicFile(filename) for (name, value) in registry.getValues(getChildren=True): help = value.help() if help: @@ -162,7 +161,7 @@ class Group(object): """A group; it doesn't hold a value unless handled by a subclass.""" def __init__(self, help='', supplyDefault=False, orderAlphabetically=False, private=False): - self._help = help + self._help = utils.str.normalizeWhitespace(help) self._name = 'unset' self._added = [] self._children = utils.InsensitivePreservingDict() @@ -299,7 +298,7 @@ class Value(Group): self.__parent.__init__(help, **kwargs) self._default = default self._showDefault = showDefault - self._help = utils.normalizeWhitespace(help.strip()) + self._help = utils.str.normalizeWhitespace(help.strip()) if setDefault: self.setValue(default) @@ -309,7 +308,7 @@ class Value(Group): else: s = """%s has no docstring. If you're getting this message, report it, because we forgot to put a proper help string here.""" - e = InvalidRegistryValue(utils.normalizeWhitespace(s % self._name)) + e = InvalidRegistryValue(utils.str.normalizeWhitespace(s % self._name)) e.value = self raise e @@ -356,7 +355,7 @@ class Boolean(Value): """Value must be either True or False (or On or Off).""" def set(self, s): try: - v = utils.toBool(s) + v = utils.str.toBool(s) except ValueError: if s.strip().lower() == 'toggle': v = not self.value @@ -440,7 +439,7 @@ class String(Value): _printable = string.printable[:-4] def _needsQuoting(self, s): - return s.translate(string.ascii, self._printable) and s.strip() != s + return s.translate(utils.str.chars, self._printable) and s.strip() != s def __str__(self): s = self.value @@ -456,12 +455,12 @@ class OnlySomeStrings(String): self.__parent = super(OnlySomeStrings, self) self.__parent.__init__(*args, **kwargs) self.__doc__ = 'Valid values include %s.' % \ - utils.commaAndify(map(repr, self.validStrings)) + utils.str.commaAndify(map(repr, self.validStrings)) def help(self): strings = [s for s in self.validStrings if s] return '%s Valid strings: %s.' % \ - (self._help, utils.commaAndify(strings)) + (self._help, utils.str.commaAndify(strings)) def normalize(self, s): lowered = s.lower() @@ -487,7 +486,7 @@ class NormalizedString(String): self._showDefault = False def normalize(self, s): - return utils.normalizeWhitespace(s.strip()) + return utils.str.normalizeWhitespace(s.strip()) def set(self, s): s = self.normalize(s) @@ -540,7 +539,7 @@ class Regexp(Value): def set(self, s): try: if s: - self.setValue(utils.perlReToPythonRe(s), sr=s) + self.setValue(utils.str.perlReToPythonRe(s), sr=s) else: self.setValue(None) except ValueError, e: diff --git a/src/schedule.py b/src/schedule.py index 6aea33c34..f95ffa251 100644 --- a/src/schedule.py +++ b/src/schedule.py @@ -32,10 +32,6 @@ Schedule plugin with a subclass of drivers.IrcDriver in order to be run as a Supybot driver. """ - - -import supybot.fix as fix - import time import heapq diff --git a/src/structures.py b/src/structures.py index 0cd48585c..13cee87d4 100644 --- a/src/structures.py +++ b/src/structures.py @@ -31,13 +31,9 @@ Data structures for Python. """ - - -import supybot.fix as fix - import time import types -from itertools import imap, ilen +from itertools import imap class RingBuffer(object): """Class to represent a fixed-size ring buffer.""" @@ -345,8 +341,12 @@ class TimeoutQueue(object): yield elt def __len__(self): - return ilen(self) - + # No dependency on utils. + # return ilen(self) + i = 0 + for _ in self: + i += 1 + return i class MaxLengthQueue(queue): __slots__ = ('length',) diff --git a/src/test.py b/src/test.py index 56a4f63e4..e20c9b34a 100644 --- a/src/test.py +++ b/src/test.py @@ -27,8 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. ### -import supybot.fix as fix - import gc import os import re @@ -96,7 +94,7 @@ class SupyTestCase(unittest.TestCase): def setUp(self): log.critical('Beginning test case %s', self.id()) threads = [t.getName() for t in threading.enumerate()] - log.critical('Threads: %s' % utils.commaAndify(threads)) + log.critical('Threads: %s' % utils.str.commaAndify(threads)) unittest.TestCase.setUp(self) def tearDown(self): diff --git a/src/unpreserve.py b/src/unpreserve.py index 5c14f878a..7bbc0e082 100644 --- a/src/unpreserve.py +++ b/src/unpreserve.py @@ -27,8 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. ### - - class Reader(object): def __init__(self, Creator, *args, **kwargs): self.Creator = Creator diff --git a/src/utils.py b/src/utils.py deleted file mode 100644 index b291d1c04..000000000 --- a/src/utils.py +++ /dev/null @@ -1,864 +0,0 @@ -### -# Copyright (c) 2002-2005, Jeremiah Fincher -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions, and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions, and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the author of this software nor the name of -# contributors to this software may be used to endorse or promote products -# derived from this software without specific prior written consent. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -### - -""" -Simple utility functions. -""" - -import supybot.fix as fix - -import os -import re -import sys -import md5 -import new -import sha -import sets -import time -import types -import random -import shutil -import socket -import string -import sgmllib -import compiler -import textwrap -import UserDict -import itertools -import traceback -import htmlentitydefs -from itertools import imap, ifilter - -from supybot.structures import TwoWayDictionary - -curry = new.instancemethod - -def normalizeWhitespace(s): - """Normalizes the whitespace in a string; \s+ becomes one space.""" - return ' '.join(s.split()) - -class HtmlToText(sgmllib.SGMLParser): - """Taken from some eff-bot code on c.l.p.""" - entitydefs = htmlentitydefs.entitydefs.copy() - entitydefs['nbsp'] = ' ' - def __init__(self, tagReplace=' '): - self.data = [] - self.tagReplace = tagReplace - sgmllib.SGMLParser.__init__(self) - - def unknown_starttag(self, tag, attr): - self.data.append(self.tagReplace) - - def unknown_endtag(self, tag): - self.data.append(self.tagReplace) - - def handle_data(self, data): - self.data.append(data) - - def getText(self): - text = ''.join(self.data).strip() - return normalizeWhitespace(text) - -def htmlToText(s, tagReplace=' '): - """Turns HTML into text. tagReplace is a string to replace HTML tags with. - """ - x = HtmlToText(tagReplace) - x.feed(s) - return x.getText() - -def abbrev(strings, d=None): - """Returns a dictionary mapping unambiguous abbreviations to full forms.""" - def eachSubstring(s): - for i in xrange(1, len(s)+1): - yield s[:i] - if len(strings) != len(set(strings)): - raise ValueError, \ - 'strings given to utils.abbrev have duplicates: %r' % strings - if d is None: - d = {} - for s in strings: - for abbreviation in eachSubstring(s): - if abbreviation not in d: - d[abbreviation] = s - else: - if abbreviation not in strings: - d[abbreviation] = None - removals = [] - for key in d: - if d[key] is None: - removals.append(key) - for key in removals: - del d[key] - return d - -def timeElapsed(elapsed, short=False, leadingZeroes=False, years=True, - weeks=True, days=True, hours=True, minutes=True, seconds=True): - """Given seconds, returns a string with an English description of - how much time as passed. leadingZeroes determines whether 0 days, 0 hours, - etc. will be printed; the others determine what larger time periods should - be used. - """ - ret = [] - def format(s, i): - if i or leadingZeroes or ret: - if short: - ret.append('%s%s' % (i, s[0])) - else: - ret.append(nItems(s, i)) - elapsed = int(elapsed) - assert years or weeks or days or \ - hours or minutes or seconds, 'One flag must be True' - if years: - (yrs, elapsed) = (elapsed // 31536000, elapsed % 31536000) - format('year', yrs) - if weeks: - (wks, elapsed) = (elapsed // 604800, elapsed % 604800) - format('week', wks) - if days: - (ds, elapsed) = (elapsed // 86400, elapsed % 86400) - format('day', ds) - if hours: - (hrs, elapsed) = (elapsed // 3600, elapsed % 3600) - format('hour', hrs) - if minutes or seconds: - (mins, secs) = (elapsed // 60, elapsed % 60) - if leadingZeroes or mins: - format('minute', mins) - if seconds: - leadingZeroes = True - format('second', secs) - if not ret: - raise ValueError, 'Time difference not great enough to be noted.' - if short: - return ' '.join(ret) - else: - return commaAndify(ret) - -def distance(s, t): - """Returns the levenshtein edit distance between two strings.""" - n = len(s) - m = len(t) - if n == 0: - return m - elif m == 0: - return n - d = [] - for i in range(n+1): - d.append([]) - for j in range(m+1): - d[i].append(0) - d[0][j] = j - d[i][0] = i - for i in range(1, n+1): - cs = s[i-1] - for j in range(1, m+1): - ct = t[j-1] - cost = int(cs != ct) - d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cost) - return d[n][m] - -_soundextrans = string.maketrans(string.ascii_uppercase, - '01230120022455012623010202') -_notUpper = string.ascii.translate(string.ascii, string.ascii_uppercase) -def soundex(s, length=4): - """Returns the soundex hash of a given string.""" - s = s.upper() # Make everything uppercase. - s = s.translate(string.ascii, _notUpper) # Delete non-letters. - if not s: - raise ValueError, 'Invalid string for soundex: %s' - firstChar = s[0] # Save the first character. - s = s.translate(_soundextrans) # Convert to soundex numbers. - s = s.lstrip(s[0]) # Remove all repeated first characters. - L = [firstChar] - for c in s: - if c != L[-1]: - L.append(c) - L = [c for c in L if c != '0'] + (['0']*(length-1)) - s = ''.join(L) - return length and s[:length] or s.rstrip('0') - -def dqrepr(s): - """Returns a repr() of s guaranteed to be in double quotes.""" - # The wankers-that-be decided not to use double-quotes anymore in 2.3. - # return '"' + repr("'\x00" + s)[6:] - return '"%s"' % s.encode('string_escape').replace('"', '\\"') - -def quoted(s): - """Returns a quoted s.""" - return '"%s"' % s - -def _getSep(s): - if len(s) < 2: - raise ValueError, 'string given to _getSep is too short: %r' % s - if s.startswith('m') or s.startswith('s'): - separator = s[1] - else: - separator = s[0] - if separator.isalnum() or separator in '{}[]()<>': - raise ValueError, \ - 'Invalid separator: separator must not be alphanumeric or in ' \ - '"{}[]()<>"' - return separator - -def _getSplitterRe(s): - separator = _getSep(s) - return re.compile(r'(?>> nItems('clock', 1) - '1 clock' - - >>> nItems('clock', 10) - '10 clocks' - - >>> nItems('clock', 10, between='grandfather') - '10 grandfather clocks' - """ - if between is None: - return '%s %s' % (n, pluralize(item, n)) - else: - return '%s %s %s' % (n, between, pluralize(item, n)) - -def be(i): - """Returns the form of the verb 'to be' based on the number i.""" - if i == 1: - return 'is' - else: - return 'are' - -def has(i): - """Returns the form of the verb 'to have' based on the number i.""" - if i == 1: - return 'has' - else: - return 'have' - -def sortBy(f, L): - """Uses the decorate-sort-undecorate pattern to sort L by function f.""" - for (i, elt) in enumerate(L): - L[i] = (f(elt), i, elt) - L.sort() - for (i, elt) in enumerate(L): - L[i] = L[i][2] - -if sys.version_info < (2, 4, 0): - def sorted(iterable, cmp=None, key=None, reversed=False): - L = list(iterable) - if key is not None: - assert cmp is None, 'Can\'t use both cmp and key.' - sortBy(key, L) - else: - L.sort(cmp) - if reversed: - L.reverse() - return L - - __builtins__['sorted'] = sorted - -def mktemp(suffix=''): - """Gives a decent random string, suitable for a filename.""" - r = random.Random() - m = md5.md5(suffix) - r.seed(time.time()) - s = str(r.getstate()) - for x in xrange(0, random.randrange(400), random.randrange(1, 5)): - m.update(str(x)) - m.update(s) - m.update(str(time.time())) - s = m.hexdigest() - return sha.sha(s + str(time.time())).hexdigest() + suffix - -def itersplit(isSeparator, iterable, maxsplit=-1, yieldEmpty=False): - """itersplit(isSeparator, iterable, maxsplit=-1, yieldEmpty=False) - - Splits an iterator based on a predicate isSeparator.""" - if isinstance(isSeparator, basestring): - f = lambda s: s == isSeparator - else: - f = isSeparator - acc = [] - for element in iterable: - if maxsplit == 0 or not f(element): - acc.append(element) - else: - maxsplit -= 1 - if acc or yieldEmpty: - yield acc - acc = [] - if acc or yieldEmpty: - yield acc - -def flatten(seq, strings=False): - """Flattens a list of lists into a single list. See the test for examples. - """ - for elt in seq: - if not strings and type(elt) == str or type(elt) == unicode: - yield elt - else: - try: - for x in flatten(elt): - yield x - except TypeError: - yield elt - -def saltHash(password, salt=None, hash='sha'): - if salt is None: - salt = mktemp()[:8] - if hash == 'sha': - hasher = sha.sha - elif hash == 'md5': - hasher = md5.md5 - return '|'.join([salt, hasher(salt + password).hexdigest()]) - -def safeEval(s, namespace={'True': True, 'False': False, 'None': None}): - """Evaluates s, safely. Useful for turning strings into tuples/lists/etc. - without unsafely using eval().""" - try: - node = compiler.parse(s) - except SyntaxError, e: - raise ValueError, 'Invalid string: %s.' % e - nodes = compiler.parse(s).node.nodes - if not nodes: - if node.__class__ is compiler.ast.Module: - return node.doc - else: - raise ValueError, 'Unsafe string: %s' % quoted(s) - node = nodes[0] - if node.__class__ is not compiler.ast.Discard: - raise ValueError, 'Invalid expression: %s' % quoted(s) - node = node.getChildNodes()[0] - def checkNode(node): - if node.__class__ is compiler.ast.Const: - return True - if node.__class__ in (compiler.ast.List, - compiler.ast.Tuple, - compiler.ast.Dict): - return all(checkNode, node.getChildNodes()) - if node.__class__ is compiler.ast.Name: - if node.name in namespace: - return True - else: - return False - else: - return False - if checkNode(node): - return eval(s, namespace, namespace) - else: - raise ValueError, 'Unsafe string: %s' % quoted(s) - -def exnToString(e): - """Turns a simple exception instance into a string (better than str(e))""" - strE = str(e) - if strE: - return '%s: %s' % (e.__class__.__name__, strE) - else: - return e.__class__.__name__ - -class IterableMap(object): - """Define .iteritems() in a class and subclass this to get the other iters. - """ - def iteritems(self): - raise NotImplementedError - - def iterkeys(self): - for (key, _) in self.iteritems(): - yield key - __iter__ = iterkeys - - def itervalues(self): - for (_, value) in self.iteritems(): - yield value - - def items(self): - return list(self.iteritems()) - - def keys(self): - return list(self.iterkeys()) - - def values(self): - return list(self.itervalues()) - - def __len__(self): - ret = 0 - for _ in self.iteritems(): - ret += 1 - return ret - - def __nonzero__(self): - for _ in self.iteritems(): - return True - return False - - -def nonCommentLines(fd): - for line in fd: - if not line.startswith('#'): - yield line - -def nonEmptyLines(fd): -## for line in fd: -## if line.strip(): -## yield line - return ifilter(str.strip, fd) - -def nonCommentNonEmptyLines(fd): - return nonEmptyLines(nonCommentLines(fd)) - -def changeFunctionName(f, name, doc=None): - if doc is None: - doc = f.__doc__ - newf = types.FunctionType(f.func_code, f.func_globals, name, - f.func_defaults, f.func_closure) - newf.__doc__ = doc - return newf - -def getSocket(host): - """Returns a socket of the correct AF_INET type (v4 or v6) in order to - communicate with host. - """ - host = socket.gethostbyname(host) - if isIP(host): - return socket.socket(socket.AF_INET, socket.SOCK_STREAM) - elif isIPV6(host): - return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - else: - raise socket.error, 'Something wonky happened.' - -def isIP(s): - """Returns whether or not a given string is an IPV4 address. - - >>> isIP('255.255.255.255') - 1 - - >>> isIP('abc.abc.abc.abc') - 0 - """ - try: - return bool(socket.inet_aton(s)) - except socket.error: - return False - -def bruteIsIPV6(s): - if s.count('::') <= 1: - L = s.split(':') - if len(L) <= 8: - for x in L: - if x: - try: - int(x, 16) - except ValueError: - return False - return True - return False - -def isIPV6(s): - """Returns whether or not a given string is an IPV6 address.""" - try: - if hasattr(socket, 'inet_pton'): - return bool(socket.inet_pton(socket.AF_INET6, s)) - else: - return bruteIsIPV6(s) - except socket.error: - try: - socket.inet_pton(socket.AF_INET6, '::') - except socket.error: - # We gotta fake it. - return bruteIsIPV6(s) - return False - -class InsensitivePreservingDict(UserDict.DictMixin, object): - def key(self, s): - """Override this if you wish.""" - if s is not None: - s = s.lower() - return s - - def __init__(self, dict=None, key=None): - if key is not None: - self.key = key - self.data = {} - if dict is not None: - self.update(dict) - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, - super(InsensitivePreservingDict, self).__repr__()) - - def fromkeys(cls, keys, s=None, dict=None, key=None): - d = cls(dict=dict, key=key) - for key in keys: - d[key] = s - return d - fromkeys = classmethod(fromkeys) - - def __getitem__(self, k): - return self.data[self.key(k)][1] - - def __setitem__(self, k, v): - self.data[self.key(k)] = (k, v) - - def __delitem__(self, k): - del self.data[self.key(k)] - - def iteritems(self): - return self.data.itervalues() - - def keys(self): - L = [] - for (k, _) in self.iteritems(): - L.append(k) - return L - - def __reduce__(self): - return (self.__class__, (dict(self.data.values()),)) - - -class NormalizingSet(sets.Set): - def __init__(self, iterable=()): - iterable = itertools.imap(self.normalize, iterable) - super(NormalizingSet, self).__init__(iterable) - - def normalize(self, x): - return x - - def add(self, x): - return super(NormalizingSet, self).add(self.normalize(x)) - - def remove(self, x): - return super(NormalizingSet, self).remove(self.normalize(x)) - - def discard(self, x): - return super(NormalizingSet, self).discard(self.normalize(x)) - - def __contains__(self, x): - return super(NormalizingSet, self).__contains__(self.normalize(x)) - has_key = __contains__ - -def mungeEmailForWeb(s): - s = s.replace('@', ' AT ') - s = s.replace('.', ' DOT ') - return s - -class AtomicFile(file): - """Used for files that need to be atomically written -- i.e., if there's a - failure, the original file remains, unmodified. mode must be 'w' or 'wb'""" - def __init__(self, filename, mode='w', allowEmptyOverwrite=True, - makeBackupIfSmaller=True, tmpDir=None, backupDir=None): - if mode not in ('w', 'wb'): - raise ValueError, 'Invalid mode: %s' % quoted(mode) - self.rolledback = False - self.allowEmptyOverwrite = allowEmptyOverwrite - self.makeBackupIfSmaller = makeBackupIfSmaller - self.filename = filename - self.backupDir = backupDir - if tmpDir is None: - # If not given a tmpDir, we'll just put a random token on the end - # of our filename and put it in the same directory. - self.tempFilename = '%s.%s' % (self.filename, mktemp()) - else: - # If given a tmpDir, we'll get the basename (just the filename, no - # directory), put our random token on the end, and put it in tmpDir - tempFilename = '%s.%s' % (os.path.basename(self.filename), mktemp()) - self.tempFilename = os.path.join(tmpDir, tempFilename) - # This doesn't work because of the uncollectable garbage effect. - # self.__parent = super(AtomicFile, self) - super(AtomicFile, self).__init__(self.tempFilename, mode) - - def rollback(self): - if not self.closed: - super(AtomicFile, self).close() - if os.path.exists(self.tempFilename): - os.remove(self.tempFilename) - self.rolledback = True - - def close(self): - if not self.rolledback: - super(AtomicFile, self).close() - # We don't mind writing an empty file if the file we're overwriting - # doesn't exist. - newSize = os.path.getsize(self.tempFilename) - originalExists = os.path.exists(self.filename) - if newSize or self.allowEmptyOverwrite or not originalExists: - if originalExists: - oldSize = os.path.getsize(self.filename) - if self.makeBackupIfSmaller and newSize < oldSize: - now = int(time.time()) - backupFilename = '%s.backup.%s' % (self.filename, now) - if self.backupDir is not None: - backupFilename = os.path.basename(backupFilename) - backupFilename = os.path.join(self.backupDir, - backupFilename) - shutil.copy(self.filename, backupFilename) - # We use shutil.move here instead of os.rename because - # the latter doesn't work on Windows when self.filename - # (the target) already exists. shutil.move handles those - # intricacies for us. - - # This raises IOError if we can't write to the file. Since - # in *nix, it only takes write perms to the *directory* to - # rename a file (and shutil.move will use os.rename if - # possible), we first check if we have the write permission - # and only then do we write. - fd = file(self.filename, 'a') - fd.close() - shutil.move(self.tempFilename, self.filename) - - else: - raise ValueError, 'AtomicFile.close called after rollback.' - - def __del__(self): - # We rollback because if we're deleted without being explicitly closed, - # that's bad. We really should log this here, but as of yet we've got - # no logging facility in utils. I've got some ideas for this, though. - self.rollback() - -def transactionalFile(*args, **kwargs): - # This exists so it can be replaced by a function that provides the tmpDir. - # We do that replacement in conf.py. - return AtomicFile(*args, **kwargs) - -def stackTrace(frame=None, compact=True): - if frame is None: - frame = sys._getframe() - if compact: - L = [] - while frame: - lineno = frame.f_lineno - funcname = frame.f_code.co_name - filename = os.path.basename(frame.f_code.co_filename) - L.append('[%s|%s|%s]' % (filename, funcname, lineno)) - frame = frame.f_back - return textwrap.fill(' '.join(L)) - else: - return traceback.format_stack(frame) - -def callTracer(fd=None, basename=True): - if fd is None: - fd = sys.stdout - def tracer(frame, event, _): - if event == 'call': - code = frame.f_code - lineno = frame.f_lineno - funcname = code.co_name - filename = code.co_filename - if basename: - filename = os.path.basename(filename) - print >>fd, '%s: %s(%s)' % (filename, funcname, lineno) - return tracer - - -def toBool(s): - s = s.strip().lower() - if s in ('true', 'on', 'enable', 'enabled', '1'): - return True - elif s in ('false', 'off', 'disable', 'disabled', '0'): - return False - else: - raise ValueError, 'Invalid string for toBool: %s' % quoted(s) - -def mapinto(f, L): - for (i, x) in enumerate(L): - L[i] = f(x) - -if __name__ == '__main__': - import doctest - doctest.testmod(sys.modules['__main__']) - - -# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/utils/__init__.py b/src/utils/__init__.py new file mode 100644 index 000000000..e09b4bcfb --- /dev/null +++ b/src/utils/__init__.py @@ -0,0 +1,106 @@ +### +# Copyright (c) 2002-2005, Jeremiah Fincher +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author of this software nor the name of +# contributors to this software may be used to endorse or promote products +# derived from this software without specific prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +### + +import sys + +### +# csv.{join,split} -- useful functions that should exist. +### +import csv +import cStringIO as StringIO +def join(L): + fd = StringIO.StringIO() + writer = csv.writer(fd) + writer.writerow(L) + return fd.getvalue().rstrip('\r\n') + +def split(s): + fd = StringIO.StringIO(s) + reader = csv.reader(fd) + return reader.next() +csv.join = join +csv.split = split + +# We use this often enough that we're going to stick it in builtins. +def force(x): + if callable(x): + return x() + else: + return x +__builtins__['force'] = force + +if sys.version_info < (2, 4, 0): + def reversed(L): + """Iterates through a sequence in reverse.""" + for i in xrange(len(L) - 1, -1, -1): + yield L[i] + __builtins__['reversed'] = reversed + + def sorted(iterable, cmp=None, key=None, reversed=False): + L = list(iterable) + if key is not None: + assert cmp is None, 'Can\'t use both cmp and key.' + sortBy(key, L) + else: + L.sort(cmp) + if reversed: + L.reverse() + return L + + __builtins__['sorted'] = sorted + + import operator + def itemgetter(i): + return lambda x: x[i] + + def attrgetter(attr): + return lambda x: getattr(x, attr) + operator.itemgetter = itemgetter + operator.attrgetter = attrgetter + + import sets + __builtins__['set'] = sets.Set + __builtins__['frozenset'] = sets.ImmutableSet + + import socket + # Some socket modules don't have sslerror, so we'll just make it an error. + if not hasattr(socket, 'sslerror'): + socket.sslerror = socket.error + +# These imports need to happen below the block above, so things get put into +# __builtins__ appropriately. +from gen import * +import net +import web +import seq +import str +import file +import iter + +# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/utils/file.py b/src/utils/file.py new file mode 100644 index 000000000..3754f59f5 --- /dev/null +++ b/src/utils/file.py @@ -0,0 +1,150 @@ +### +# Copyright (c) 2002-2005, Jeremiah Fincher +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author of this software nor the name of +# contributors to this software may be used to endorse or promote products +# derived from this software without specific prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +### + +import os +import md5 +import sha +import time +import random +import shutil +import os.path +from iter import ifilter + +def mktemp(suffix=''): + """Gives a decent random string, suitable for a filename.""" + r = random.Random() + m = md5.md5(suffix) + r.seed(time.time()) + s = str(r.getstate()) + for x in xrange(0, random.randrange(400), random.randrange(1, 5)): + m.update(str(x)) + m.update(s) + m.update(str(time.time())) + s = m.hexdigest() + return sha.sha(s + str(time.time())).hexdigest() + suffix + +def nonCommentLines(fd): + for line in fd: + if not line.startswith('#'): + yield line + +def nonEmptyLines(fd): + return ifilter(str.strip, fd) + +def nonCommentNonEmptyLines(fd): + return nonEmptyLines(nonCommentLines(fd)) + +class AtomicFile(file): + """Used for files that need to be atomically written -- i.e., if there's a + failure, the original file remains, unmodified. mode must be 'w' or 'wb'""" + class default(object): # Holder for values. + # Callables? + tmpDir = None + backupDir = None + makeBackupIfSmaller = True + allowEmptyOverwrite = True + def __init__(self, filename, mode='w', allowEmptyOverwrite=None, + makeBackupIfSmaller=None, tmpDir=None, backupDir=None): + if tmpDir is None: + tmpDir = force(self.default.tmpDir) + if backupDir is None: + backupDir = force(self.default.backupDir) + if makeBackupIfSmaller is None: + makeBackupIfSmaller = force(self.default.makeBackupIfSmaller) + if allowEmptyOverwrite is None: + allowEmptyOverwrite = force(self.default.allowEmptyOverwrite) + if mode not in ('w', 'wb'): + raise ValueError, 'Invalid mode: %s' % quoted(mode) + self.rolledback = False + self.allowEmptyOverwrite = allowEmptyOverwrite + self.makeBackupIfSmaller = makeBackupIfSmaller + self.filename = filename + self.backupDir = backupDir + if tmpDir is None: + # If not given a tmpDir, we'll just put a random token on the end + # of our filename and put it in the same directory. + self.tempFilename = '%s.%s' % (self.filename, mktemp()) + else: + # If given a tmpDir, we'll get the basename (just the filename, no + # directory), put our random token on the end, and put it in tmpDir + tempFilename = '%s.%s' % (os.path.basename(self.filename), mktemp()) + self.tempFilename = os.path.join(tmpDir, tempFilename) + # This doesn't work because of the uncollectable garbage effect. + # self.__parent = super(AtomicFile, self) + super(AtomicFile, self).__init__(self.tempFilename, mode) + + def rollback(self): + if not self.closed: + super(AtomicFile, self).close() + if os.path.exists(self.tempFilename): + os.remove(self.tempFilename) + self.rolledback = True + + def close(self): + if not self.rolledback: + super(AtomicFile, self).close() + # We don't mind writing an empty file if the file we're overwriting + # doesn't exist. + newSize = os.path.getsize(self.tempFilename) + originalExists = os.path.exists(self.filename) + if newSize or self.allowEmptyOverwrite or not originalExists: + if originalExists: + oldSize = os.path.getsize(self.filename) + if self.makeBackupIfSmaller and newSize < oldSize: + now = int(time.time()) + backupFilename = '%s.backup.%s' % (self.filename, now) + if self.backupDir is not None: + backupFilename = os.path.basename(backupFilename) + backupFilename = os.path.join(self.backupDir, + backupFilename) + shutil.copy(self.filename, backupFilename) + # We use shutil.move here instead of os.rename because + # the latter doesn't work on Windows when self.filename + # (the target) already exists. shutil.move handles those + # intricacies for us. + + # This raises IOError if we can't write to the file. Since + # in *nix, it only takes write perms to the *directory* to + # rename a file (and shutil.move will use os.rename if + # possible), we first check if we have the write permission + # and only then do we write. + fd = file(self.filename, 'a') + fd.close() + shutil.move(self.tempFilename, self.filename) + + else: + raise ValueError, 'AtomicFile.close called after rollback.' + + def __del__(self): + # We rollback because if we're deleted without being explicitly closed, + # that's bad. We really should log this here, but as of yet we've got + # no logging facility in utils. I've got some ideas for this, though. + self.rollback() + +# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/utils/gen.py b/src/utils/gen.py new file mode 100644 index 000000000..d483ab12f --- /dev/null +++ b/src/utils/gen.py @@ -0,0 +1,326 @@ +### +# Copyright (c) 2002-2005, Jeremiah Fincher +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author of this software nor the name of +# contributors to this software may be used to endorse or promote products +# derived from this software without specific prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +### + +import os +import sys +import md5 +import new +import sha +import time +import types +import compiler +import textwrap +import UserDict +import traceback + +from iter import imap +from file import mktemp +from str import quoted, nItems, commaAndify + +def abbrev(strings, d=None): + """Returns a dictionary mapping unambiguous abbreviations to full forms.""" + def eachSubstring(s): + for i in xrange(1, len(s)+1): + yield s[:i] + if len(strings) != len(set(strings)): + raise ValueError, \ + 'strings given to utils.abbrev have duplicates: %r' % strings + if d is None: + d = {} + for s in strings: + for abbreviation in eachSubstring(s): + if abbreviation not in d: + d[abbreviation] = s + else: + if abbreviation not in strings: + d[abbreviation] = None + removals = [] + for key in d: + if d[key] is None: + removals.append(key) + for key in removals: + del d[key] + return d + +def timeElapsed(elapsed, short=False, leadingZeroes=False, years=True, + weeks=True, days=True, hours=True, minutes=True, seconds=True): + """Given seconds, returns a string with an English description of + how much time as passed. leadingZeroes determines whether 0 days, 0 hours, + etc. will be printed; the others determine what larger time periods should + be used. + """ + ret = [] + def format(s, i): + if i or leadingZeroes or ret: + if short: + ret.append('%s%s' % (i, s[0])) + else: + ret.append(nItems(s, i)) + elapsed = int(elapsed) + assert years or weeks or days or \ + hours or minutes or seconds, 'One flag must be True' + if years: + (yrs, elapsed) = (elapsed // 31536000, elapsed % 31536000) + format('year', yrs) + if weeks: + (wks, elapsed) = (elapsed // 604800, elapsed % 604800) + format('week', wks) + if days: + (ds, elapsed) = (elapsed // 86400, elapsed % 86400) + format('day', ds) + if hours: + (hrs, elapsed) = (elapsed // 3600, elapsed % 3600) + format('hour', hrs) + if minutes or seconds: + (mins, secs) = (elapsed // 60, elapsed % 60) + if leadingZeroes or mins: + format('minute', mins) + if seconds: + leadingZeroes = True + format('second', secs) + if not ret: + raise ValueError, 'Time difference not great enough to be noted.' + if short: + return ' '.join(ret) + else: + return commaAndify(ret) + +def findBinaryInPath(s): + """Return full path of a binary if it's in PATH, otherwise return None.""" + cmdLine = None + for dir in os.getenv('PATH').split(':'): + filename = os.path.join(dir, s) + if os.path.exists(filename): + cmdLine = filename + break + return cmdLine + +def sortBy(f, L): + """Uses the decorate-sort-undecorate pattern to sort L by function f.""" + for (i, elt) in enumerate(L): + L[i] = (f(elt), i, elt) + L.sort() + for (i, elt) in enumerate(L): + L[i] = L[i][2] + +def saltHash(password, salt=None, hash='sha'): + if salt is None: + salt = mktemp()[:8] + if hash == 'sha': + hasher = sha.sha + elif hash == 'md5': + hasher = md5.md5 + return '|'.join([salt, hasher(salt + password).hexdigest()]) + +def safeEval(s, namespace={'True': True, 'False': False, 'None': None}): + """Evaluates s, safely. Useful for turning strings into tuples/lists/etc. + without unsafely using eval().""" + try: + node = compiler.parse(s) + except SyntaxError, e: + raise ValueError, 'Invalid string: %s.' % e + nodes = compiler.parse(s).node.nodes + if not nodes: + if node.__class__ is compiler.ast.Module: + return node.doc + else: + raise ValueError, 'Unsafe string: %s' % quoted(s) + node = nodes[0] + if node.__class__ is not compiler.ast.Discard: + raise ValueError, 'Invalid expression: %s' % quoted(s) + node = node.getChildNodes()[0] + def checkNode(node): + if node.__class__ is compiler.ast.Const: + return True + if node.__class__ in (compiler.ast.List, + compiler.ast.Tuple, + compiler.ast.Dict): + return all(checkNode, node.getChildNodes()) + if node.__class__ is compiler.ast.Name: + if node.name in namespace: + return True + else: + return False + else: + return False + if checkNode(node): + return eval(s, namespace, namespace) + else: + raise ValueError, 'Unsafe string: %s' % quoted(s) + +def exnToString(e): + """Turns a simple exception instance into a string (better than str(e))""" + strE = str(e) + if strE: + return '%s: %s' % (e.__class__.__name__, strE) + else: + return e.__class__.__name__ + +class IterableMap(object): + """Define .iteritems() in a class and subclass this to get the other iters. + """ + def iteritems(self): + raise NotImplementedError + + def iterkeys(self): + for (key, _) in self.iteritems(): + yield key + __iter__ = iterkeys + + def itervalues(self): + for (_, value) in self.iteritems(): + yield value + + def items(self): + return list(self.iteritems()) + + def keys(self): + return list(self.iterkeys()) + + def values(self): + return list(self.itervalues()) + + def __len__(self): + ret = 0 + for _ in self.iteritems(): + ret += 1 + return ret + + def __nonzero__(self): + for _ in self.iteritems(): + return True + return False + + +def changeFunctionName(f, name, doc=None): + if doc is None: + doc = f.__doc__ + newf = types.FunctionType(f.func_code, f.func_globals, name, + f.func_defaults, f.func_closure) + newf.__doc__ = doc + return newf + +class InsensitivePreservingDict(UserDict.DictMixin, object): + def key(self, s): + """Override this if you wish.""" + if s is not None: + s = s.lower() + return s + + def __init__(self, dict=None, key=None): + if key is not None: + self.key = key + self.data = {} + if dict is not None: + self.update(dict) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, + super(InsensitivePreservingDict, self).__repr__()) + + def fromkeys(cls, keys, s=None, dict=None, key=None): + d = cls(dict=dict, key=key) + for key in keys: + d[key] = s + return d + fromkeys = classmethod(fromkeys) + + def __getitem__(self, k): + return self.data[self.key(k)][1] + + def __setitem__(self, k, v): + self.data[self.key(k)] = (k, v) + + def __delitem__(self, k): + del self.data[self.key(k)] + + def iteritems(self): + return self.data.itervalues() + + def keys(self): + L = [] + for (k, _) in self.iteritems(): + L.append(k) + return L + + def __reduce__(self): + return (self.__class__, (dict(self.data.values()),)) + + +class NormalizingSet(set): + def __init__(self, iterable=()): + iterable = imap(self.normalize, iterable) + super(NormalizingSet, self).__init__(iterable) + + def normalize(self, x): + return x + + def add(self, x): + return super(NormalizingSet, self).add(self.normalize(x)) + + def remove(self, x): + return super(NormalizingSet, self).remove(self.normalize(x)) + + def discard(self, x): + return super(NormalizingSet, self).discard(self.normalize(x)) + + def __contains__(self, x): + return super(NormalizingSet, self).__contains__(self.normalize(x)) + has_key = __contains__ + +def stackTrace(frame=None, compact=True): + if frame is None: + frame = sys._getframe() + if compact: + L = [] + while frame: + lineno = frame.f_lineno + funcname = frame.f_code.co_name + filename = os.path.basename(frame.f_code.co_filename) + L.append('[%s|%s|%s]' % (filename, funcname, lineno)) + frame = frame.f_back + return textwrap.fill(' '.join(L)) + else: + return traceback.format_stack(frame) + +def callTracer(fd=None, basename=True): + if fd is None: + fd = sys.stdout + def tracer(frame, event, _): + if event == 'call': + code = frame.f_code + lineno = frame.f_lineno + funcname = code.co_name + filename = code.co_filename + if basename: + filename = os.path.basename(filename) + print >>fd, '%s: %s(%s)' % (filename, funcname, lineno) + return tracer + +# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/utils/iter.py b/src/utils/iter.py new file mode 100644 index 000000000..6694e2c38 --- /dev/null +++ b/src/utils/iter.py @@ -0,0 +1,147 @@ +### +# Copyright (c) 2002-2005, Jeremiah Fincher +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author of this software nor the name of +# contributors to this software may be used to endorse or promote products +# derived from this software without specific prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +### + +import new +import random + +from itertools import * + +def len(iterable): + """Returns the length of an iterator.""" + i = 0 + for _ in iterable: + i += 1 + return i + +def trueCycle(iterable): + while 1: + yielded = False + for x in iterable: + yield x + yielded = True + if not yielded: + raise StopIteration + +def groupby(key, iterable): + if key is None: + key = lambda x: x + it = iter(iterable) + value = it.next() # If there are no items, this takes an early exit + oldkey = key(value) + group = [value] + for value in it: + newkey = key(value) + if newkey != oldkey: + yield group + group = [] + oldkey = newkey + group.append(value) + yield group + +def partition(p, iterable): + """Partitions an iterable based on a predicate p. + Returns a (yes,no) tuple""" + no = [] + yes = [] + for elt in iterable: + if p(elt): + yes.append(elt) + else: + no.append(elt) + return (yes, no) + +def any(p, iterable): + """Returns true if any element in iterable satisfies predicate p.""" + for elt in ifilter(p, iterable): + return True + else: + return False + +def all(p, iterable): + """Returns true if all elements in iterable satisfy predicate p.""" + for elt in ifilterfalse(p, iterable): + return False + else: + return True + +def choice(iterable): + if isinstance(iterable, (list, tuple)): + return random.choice(iterable) + else: + n = 1 + m = new.module('') # Guaranteed unique value. + ret = m + for x in iterable: + if random.random() < 1/n: + ret = x + n += 1 + if ret is m: + raise IndexError + return ret + +def flatten(iterable, strings=False): + """Flattens a list of lists into a single list. See the test for examples. + """ + for elt in iterable: + if not strings and isinstance(elt, basestring): + yield elt + else: + try: + for x in flatten(elt): + yield x + except TypeError: + yield elt + +def split(isSeparator, iterable, maxsplit=-1, yieldEmpty=False): + """split(isSeparator, iterable, maxsplit=-1, yieldEmpty=False) + + Splits an iterator based on a predicate isSeparator.""" + if isinstance(isSeparator, basestring): + f = lambda s: s == isSeparator + else: + f = isSeparator + acc = [] + for element in iterable: + if maxsplit == 0 or not f(element): + acc.append(element) + else: + maxsplit -= 1 + if acc or yieldEmpty: + yield acc + acc = [] + if acc or yieldEmpty: + yield acc + +def ilen(iterable): + i = 0 + for _ in iterable: + i += 1 + return i + +# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/utils/net.py b/src/utils/net.py new file mode 100644 index 000000000..8c899af94 --- /dev/null +++ b/src/utils/net.py @@ -0,0 +1,90 @@ +### +# Copyright (c) 2002-2005, Jeremiah Fincher +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author of this software nor the name of +# contributors to this software may be used to endorse or promote products +# derived from this software without specific prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +### + +""" +Simple utility modules. +""" + +import socket + +def getSocket(host): + """Returns a socket of the correct AF_INET type (v4 or v6) in order to + communicate with host. + """ + host = socket.gethostbyname(host) + if isIP(host): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + elif isIPV6(host): + return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + else: + raise socket.error, 'Something wonky happened.' + +def isIP(s): + """Returns whether or not a given string is an IPV4 address. + + >>> isIP('255.255.255.255') + 1 + + >>> isIP('abc.abc.abc.abc') + 0 + """ + try: + return bool(socket.inet_aton(s)) + except socket.error: + return False + +def bruteIsIPV6(s): + if s.count('::') <= 1: + L = s.split(':') + if len(L) <= 8: + for x in L: + if x: + try: + int(x, 16) + except ValueError: + return False + return True + return False + +def isIPV6(s): + """Returns whether or not a given string is an IPV6 address.""" + try: + if hasattr(socket, 'inet_pton'): + return bool(socket.inet_pton(socket.AF_INET6, s)) + else: + return bruteIsIPV6(s) + except socket.error: + try: + socket.inet_pton(socket.AF_INET6, '::') + except socket.error: + # We gotta fake it. + return bruteIsIPV6(s) + return False + +# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/test/test_webutils.py b/src/utils/seq.py similarity index 77% rename from test/test_webutils.py rename to src/utils/seq.py index b283bcfe7..18cecab1b 100644 --- a/test/test_webutils.py +++ b/src/utils/seq.py @@ -1,5 +1,5 @@ ### -# Copyright (c) 2004-2005, Jeremiah Fincher +# Copyright (c) 2002-2005, Jeremiah Fincher # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -27,21 +27,19 @@ # POSSIBILITY OF SUCH DAMAGE. ### -from supybot.test import * +def window(L, size): + """Returns a sliding 'window' through the list L of size size.""" + assert not isinstance(L, int), 'Argument order swapped: window(L, size)' + if size < 1: + raise ValueError, 'size <= 0 disallowed.' + for i in xrange(len(L) - (size-1)): + yield L[i:i+size] -import supybot.webutils as webutils +def mapinto(f, L): + for (i, x) in enumerate(L): + L[i] = f(x) -class WebutilsTestCase(SupyTestCase): - def testGetDomain(self): - self.assertEqual(webutils.getDomain('http://slashdot.org/foo/bar.exe'), - 'slashdot.org') - - if network: - def testGetUrlWithSize(self): - url = 'http://slashdot.org/' - self.failUnless(len(webutils.getUrl(url, 1024)) == 1024) - # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/utils/str.py b/src/utils/str.py new file mode 100644 index 000000000..784d06195 --- /dev/null +++ b/src/utils/str.py @@ -0,0 +1,354 @@ +### +# Copyright (c) 2002-2005, Jeremiah Fincher +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author of this software nor the name of +# contributors to this software may be used to endorse or promote products +# derived from this software without specific prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +### + +""" +Simple utility functions related to strings. +""" + +import re +import new +import string +import textwrap + +import supybot.structures as structures + +curry = new.instancemethod + +chars = string.maketrans('', '') + +def rsplit(s, sep=None, maxsplit=-1): + """Equivalent to str.split, except splitting from the right.""" + if sys.version_info < (2, 4, 0): + if sep is not None: + sep = sep[::-1] + L = s[::-1].split(sep, maxsplit) + L.reverse() + return [s[::-1] for s in L] + else: + return s.rsplit(sep, maxsplit) + +def normalizeWhitespace(s): + """Normalizes the whitespace in a string; \s+ becomes one space.""" + return ' '.join(s.split()) + +def distance(s, t): + """Returns the levenshtein edit distance between two strings.""" + n = len(s) + m = len(t) + if n == 0: + return m + elif m == 0: + return n + d = [] + for i in xrange(n+1): + d.append([]) + for j in xrange(m+1): + d[i].append(0) + d[0][j] = j + d[i][0] = i + for i in xrange(1, n+1): + cs = s[i-1] + for j in xrange(1, m+1): + ct = t[j-1] + cost = int(cs != ct) + d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cost) + return d[n][m] + +_soundextrans = string.maketrans(string.ascii_uppercase, + '01230120022455012623010202') +_notUpper = chars.translate(chars, string.ascii_uppercase) +def soundex(s, length=4): + """Returns the soundex hash of a given string.""" + s = s.upper() # Make everything uppercase. + s = s.translate(chars, _notUpper) # Delete non-letters. + if not s: + raise ValueError, 'Invalid string for soundex: %s' + firstChar = s[0] # Save the first character. + s = s.translate(_soundextrans) # Convert to soundex numbers. + s = s.lstrip(s[0]) # Remove all repeated first characters. + L = [firstChar] + for c in s: + if c != L[-1]: + L.append(c) + L = [c for c in L if c != '0'] + (['0']*(length-1)) + s = ''.join(L) + return length and s[:length] or s.rstrip('0') + +def dqrepr(s): + """Returns a repr() of s guaranteed to be in double quotes.""" + # The wankers-that-be decided not to use double-quotes anymore in 2.3. + # return '"' + repr("'\x00" + s)[6:] + return '"%s"' % s.encode('string_escape').replace('"', '\\"') + +def quoted(s): + """Returns a quoted s.""" + return '"%s"' % s + +def _getSep(s): + if len(s) < 2: + raise ValueError, 'string given to _getSep is too short: %r' % s + if s.startswith('m') or s.startswith('s'): + separator = s[1] + else: + separator = s[0] + if separator.isalnum() or separator in '{}[]()<>': + raise ValueError, \ + 'Invalid separator: separator must not be alphanumeric or in ' \ + '"{}[]()<>"' + return separator + +def _getSplitterRe(s): + separator = _getSep(s) + return re.compile(r'(?>> nItems('clock', 1) + '1 clock' + + >>> nItems('clock', 10) + '10 clocks' + + >>> nItems('clock', 10, between='grandfather') + '10 grandfather clocks' + """ + if between is None: + return '%s %s' % (n, pluralize(item, n)) + else: + return '%s %s %s' % (n, between, pluralize(item, n)) + +def be(i): + """Returns the form of the verb 'to be' based on the number i.""" + if i == 1: + return 'is' + else: + return 'are' + +def has(i): + """Returns the form of the verb 'to have' based on the number i.""" + if i == 1: + return 'has' + else: + return 'have' + +_formatRe = re.compile('%([isfbhL])') +def format(s, *args, **kwargs): + kwargs.setdefault('decimalSeparator', decimalSeparator) + kwargs.setdefault('thousandsSeparator', thousandsSeparator) + args = list(args) + args.reverse() # For more efficiency popping. + def sub(match): + char = match.group(1) + if char == 's': # Plain string. + return str(args.pop()) + elif char == 'i': # Integer + # XXX Improve me! + return str(args.pop()) + elif char == 'f': # Float + # XXX Improve me! + return str(args.pop()) + elif char == 'b': # form of the verb 'to be' + return be(args.pop()) + elif char == 'h': # form of the verb 'to have' + return has(args.pop()) + elif char == 'L': # commaAndify the list. + return commaAndify(args.pop()) + else: + assert False, 'Invalid char in sub (in format).' + return _formatRe.sub(sub, s) + +def toBool(s): + s = s.strip().lower() + if s in ('true', 'on', 'enable', 'enabled', '1'): + return True + elif s in ('false', 'off', 'disable', 'disabled', '0'): + return False + else: + raise ValueError, 'Invalid string for toBool: %s' % quoted(s) + +# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/webutils.py b/src/utils/web.py similarity index 74% rename from src/webutils.py rename to src/utils/web.py index 14574803d..650a06d26 100644 --- a/src/webutils.py +++ b/src/utils/web.py @@ -27,24 +27,22 @@ # POSSIBILITY OF SUCH DAMAGE. ### - - -import supybot.fix as fix - import re import socket import urllib import urllib2 import httplib +import sgmllib import urlparse +import htmlentitydefs -import supybot.conf as conf +from str import normalizeWhitespace Request = urllib2.Request urlquote = urllib.quote urlunquote = urllib.unquote -class WebError(Exception): +class Error(Exception): pass # XXX We should tighten this up a bit. @@ -75,14 +73,18 @@ def strError(e): else: return str(e) -_headers = { - 'User-agent': 'Mozilla/4.0 (compatible; Supybot %s)' % conf.version, +defaultHeaders = { + 'User-agent': 'Mozilla/5.0 (compatible; utils.web python module)' } +# Other modules should feel free to replace this with an appropriate +# application-specific function. Feel free to use a callable here. +proxy = None + def getUrlFd(url, headers=None): """Gets a file-like object for a url.""" if headers is None: - headers = _headers + headers = defaultHeaders try: if not isinstance(url, urllib2.Request): if '#' in url: @@ -90,7 +92,7 @@ def getUrlFd(url, headers=None): request = urllib2.Request(url, headers=headers) else: request = url - httpProxy = conf.supybot.protocols.http.proxy() + httpProxy = force(proxy) if httpProxy: request.set_proxy(httpProxy, 'http') fd = urllib2.urlopen(request) @@ -125,5 +127,39 @@ def getUrl(url, size=None, headers=None): def getDomain(url): return urlparse.urlparse(url)[1] +class HtmlToText(sgmllib.SGMLParser): + """Taken from some eff-bot code on c.l.p.""" + entitydefs = htmlentitydefs.entitydefs.copy() + entitydefs['nbsp'] = ' ' + def __init__(self, tagReplace=' '): + self.data = [] + self.tagReplace = tagReplace + sgmllib.SGMLParser.__init__(self) + + def unknown_starttag(self, tag, attr): + self.data.append(self.tagReplace) + + def unknown_endtag(self, tag): + self.data.append(self.tagReplace) + + def handle_data(self, data): + self.data.append(data) + + def getText(self): + text = ''.join(self.data).strip() + return normalizeWhitespace(text) + +def htmlToText(s, tagReplace=' '): + """Turns HTML into text. tagReplace is a string to replace HTML tags with. + """ + x = HtmlToText(tagReplace) + x.feed(s) + return x.getText() + +def mungeEmail(s): + s = s.replace('@', ' AT ') + s = s.replace('.', ' DOT ') + return s + # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/src/world.py b/src/world.py index 0679ce178..2b9d8eb0d 100644 --- a/src/world.py +++ b/src/world.py @@ -31,10 +31,6 @@ Module for general worldly stuff, like global variables and whatnot. """ - - -import supybot.fix as fix - import gc import os import sys @@ -54,8 +50,6 @@ startedAt = time.time() # Just in case it doesn't get set later. starting = False mainThread = threading.currentThread() -# ??? Should we do this? What do we gain? -# assert 'MainThread' in repr(mainThread) def isMainThread(): return mainThread is threading.currentThread() diff --git a/test/__init__.py b/test/__init__.py index 1f11b3439..74f15fe8e 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -27,6 +27,7 @@ # POSSIBILITY OF SUCH DAMAGE. ### +# We're just masquerading as a plugin :) import test # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: diff --git a/test/test_callbacks.py b/test/test_callbacks.py index 7729b18e0..128fd5072 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -42,7 +42,7 @@ class TokenizerTestCase(SupyTestCase): self.assertEqual(tokenize(''), []) def testNullCharacter(self): - self.assertEqual(tokenize(utils.dqrepr('\0')), ['\0']) + self.assertEqual(tokenize(utils.str.dqrepr('\0')), ['\0']) def testSingleDQInDQString(self): self.assertEqual(tokenize('"\\""'), ['"']) diff --git a/test/test_utils.py b/test/test_utils.py index d637b27a8..65c26feb1 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -28,11 +28,24 @@ ### from supybot.test import * -import sets +import time import supybot.utils as utils -class UtilsTest(SupyTestCase): +class GenTest(SupyTestCase): + def testInsensitivePreservingDict(self): + ipd = utils.InsensitivePreservingDict + d = ipd(dict(Foo=10)) + self.failUnless(d['foo'] == 10) + self.assertEqual(d.keys(), ['Foo']) + self.assertEqual(d.get('foo'), 10) + self.assertEqual(d.get('Foo'), 10) + + def testFindBinaryInPath(self): + if os.name == 'posix': + self.assertEqual(None, utils.findBinaryInPath('asdfhjklasdfhjkl')) + self.failUnless(utils.findBinaryInPath('sh').endswith('/bin/sh')) + def testExnToString(self): try: raise KeyError, 1 @@ -43,270 +56,21 @@ class UtilsTest(SupyTestCase): except Exception, e: self.assertEqual(utils.exnToString(e), 'EOFError') - def testMatchCase(self): - f = utils.matchCase - self.assertEqual('bar', f('foo', 'bar')) - self.assertEqual('Bar', f('Foo', 'bar')) - self.assertEqual('BAr', f('FOo', 'bar')) - self.assertEqual('BAR', f('FOO', 'bar')) - self.assertEqual('bAR', f('fOO', 'bar')) - self.assertEqual('baR', f('foO', 'bar')) - self.assertEqual('BaR', f('FoO', 'bar')) + def testSaltHash(self): + s = utils.saltHash('jemfinch') + (salt, hash) = s.split('|') + self.assertEqual(utils.saltHash('jemfinch', salt=salt), s) - def testPluralize(self): - f = utils.pluralize - self.assertEqual('bike', f('bike', 1)) - self.assertEqual('bikes', f('bike', 2)) - self.assertEqual('BIKE', f('BIKE', 1)) - self.assertEqual('BIKES', f('BIKE', 2)) - self.assertEqual('match', f('match', 1)) - self.assertEqual('matches', f('match', 2)) - self.assertEqual('Patch', f('Patch', 1)) - self.assertEqual('Patches', f('Patch', 2)) - self.assertEqual('fish', f('fish', 1)) - self.assertEqual('fishes', f('fish', 2)) - self.assertEqual('try', f('try', 1)) - self.assertEqual('tries', f('try', 2)) - self.assertEqual('day', f('day', 1)) - self.assertEqual('days', f('day', 2)) + def testSafeEval(self): + for s in ['1', '()', '(1,)', '[]', '{}', '{1:2}', '{1:(2,3)}', + '1.0', '[1,2,3]', 'True', 'False', 'None', + '(True,False,None)', '"foo"', '{"foo": "bar"}']: + self.assertEqual(eval(s), utils.safeEval(s)) + for s in ['lambda: 2', 'import foo', 'foo.bar']: + self.assertRaises(ValueError, utils.safeEval, s) - def testDepluralize(self): - f = utils.depluralize - self.assertEqual('bike', f('bikes')) - self.assertEqual('Bike', f('Bikes')) - self.assertEqual('BIKE', f('BIKES')) - self.assertEqual('match', f('matches')) - self.assertEqual('Match', f('Matches')) - self.assertEqual('fish', f('fishes')) - self.assertEqual('try', f('tries')) - - def testTimeElapsed(self): - self.assertRaises(ValueError, utils.timeElapsed, 0, - leadingZeroes=False, seconds=False) - then = 0 - now = 0 - for now, expected in [(0, '0 seconds'), - (1, '1 second'), - (60, '1 minute and 0 seconds'), - (61, '1 minute and 1 second'), - (62, '1 minute and 2 seconds'), - (122, '2 minutes and 2 seconds'), - (3722, '1 hour, 2 minutes, and 2 seconds'), - (7322, '2 hours, 2 minutes, and 2 seconds'), - (90061,'1 day, 1 hour, 1 minute, and 1 second'), - (180122, '2 days, 2 hours, 2 minutes, ' - 'and 2 seconds')]: - self.assertEqual(utils.timeElapsed(now - then), expected) - - def timeElapsedShort(self): - self.assertEqual(utils.timeElapsed(123, short=True), '2m 3s') - - def testDistance(self): - self.assertEqual(utils.distance('', ''), 0) - self.assertEqual(utils.distance('a', 'b'), 1) - self.assertEqual(utils.distance('a', 'a'), 0) - self.assertEqual(utils.distance('foobar', 'jemfinch'), 8) - self.assertEqual(utils.distance('a', 'ab'), 1) - self.assertEqual(utils.distance('foo', ''), 3) - self.assertEqual(utils.distance('', 'foo'), 3) - self.assertEqual(utils.distance('appel', 'nappe'), 2) - self.assertEqual(utils.distance('nappe', 'appel'), 2) - - def testAbbrev(self): - L = ['abc', 'bcd', 'bbe', 'foo', 'fool'] - d = utils.abbrev(L) - def getItem(s): - return d[s] - self.assertRaises(KeyError, getItem, 'f') - self.assertRaises(KeyError, getItem, 'fo') - self.assertRaises(KeyError, getItem, 'b') - self.assertEqual(d['bb'], 'bbe') - self.assertEqual(d['bc'], 'bcd') - self.assertEqual(d['a'], 'abc') - self.assertEqual(d['ab'], 'abc') - self.assertEqual(d['fool'], 'fool') - self.assertEqual(d['foo'], 'foo') - - def testAbbrevFailsWithDups(self): - L = ['english', 'english'] - self.assertRaises(ValueError, utils.abbrev, L) - - def testSoundex(self): - L = [('Euler', 'E460'), - ('Ellery', 'E460'), - ('Gauss', 'G200'), - ('Ghosh', 'G200'), - ('Hilbert', 'H416'), - ('Heilbronn', 'H416'), - ('Knuth', 'K530'), - ('Kant', 'K530'), - ('Lloyd', 'L300'), - ('Ladd', 'L300'), - ('Lukasiewicz', 'L222'), - ('Lissajous', 'L222')] - for (name, key) in L: - soundex = utils.soundex(name) - self.assertEqual(soundex, key, - '%s was %s, not %s' % (name, soundex, key)) - self.assertRaises(ValueError, utils.soundex, '3') - self.assertRaises(ValueError, utils.soundex, "'") - - - def testDQRepr(self): - L = ['foo', 'foo\'bar', 'foo"bar', '"', '\\', '', '\x00'] - for s in L: - r = utils.dqrepr(s) - self.assertEqual(s, eval(r), s) - self.failUnless(r[0] == '"' and r[-1] == '"', s) - -## def testQuoted(self): -## s = 'foo' -## t = 'let\'s' -## self.assertEqual("'%s'" % s, utils.quoted(s), s) -## self.assertEqual('"%s"' % t, utils.quoted(t), t) - - def testPerlReToPythonRe(self): - r = utils.perlReToPythonRe('m/foo/') - self.failUnless(r.search('foo')) - r = utils.perlReToPythonRe('/foo/') - self.failUnless(r.search('foo')) - r = utils.perlReToPythonRe('m/\\//') - self.failUnless(r.search('/')) - r = utils.perlReToPythonRe('m/cat/i') - self.failUnless(r.search('CAT')) - self.assertRaises(ValueError, utils.perlReToPythonRe, 'm/?/') - - def testP2PReDifferentSeparator(self): - r = utils.perlReToPythonRe('m!foo!') - self.failUnless(r.search('foo')) - - def testPerlReToReplacer(self): - f = utils.perlReToReplacer('s/foo/bar/') - self.assertEqual(f('foobarbaz'), 'barbarbaz') - f = utils.perlReToReplacer('s/fool/bar/') - self.assertEqual(f('foobarbaz'), 'foobarbaz') - f = utils.perlReToReplacer('s/foo//') - self.assertEqual(f('foobarbaz'), 'barbaz') - f = utils.perlReToReplacer('s/ba//') - self.assertEqual(f('foobarbaz'), 'foorbaz') - f = utils.perlReToReplacer('s/ba//g') - self.assertEqual(f('foobarbaz'), 'foorz') - f = utils.perlReToReplacer('s/ba\\///g') - self.assertEqual(f('fooba/rba/z'), 'foorz') - f = utils.perlReToReplacer('s/cat/dog/i') - self.assertEqual(f('CATFISH'), 'dogFISH') - f = utils.perlReToReplacer('s/foo/foo\/bar/') - self.assertEqual(f('foo'), 'foo/bar') - f = utils.perlReToReplacer('s/^/foo/') - self.assertEqual(f('bar'), 'foobar') - - def testPReToReplacerDifferentSeparator(self): - f = utils.perlReToReplacer('s#foo#bar#') - self.assertEqual(f('foobarbaz'), 'barbarbaz') - - def testPerlReToReplacerBug850931(self): - f = utils.perlReToReplacer('s/\b(\w+)\b/\1./g') - self.assertEqual(f('foo bar baz'), 'foo. bar. baz.') - - def testPerlVariableSubstitute(self): - f = utils.perlVariableSubstitute - vars = {'foo': 'bar', 'b a z': 'baz', 'b': 'c', 'i': 100, - 'f': lambda: 'called'} - self.assertEqual(f(vars, '$foo'), 'bar') - self.assertEqual(f(vars, '${foo}'), 'bar') - self.assertEqual(f(vars, '$b'), 'c') - self.assertEqual(f(vars, '${b}'), 'c') - self.assertEqual(f(vars, '$i'), '100') - self.assertEqual(f(vars, '${i}'), '100') - self.assertEqual(f(vars, '$f'), 'called') - self.assertEqual(f(vars, '${f}'), 'called') - self.assertEqual(f(vars, '${b a z}'), 'baz') - self.assertEqual(f(vars, '$b:$i'), 'c:100') - - - def testFindBinaryInPath(self): - if os.name == 'posix': - self.assertEqual(None, utils.findBinaryInPath('asdfhjklasdfhjkl')) - self.failUnless(utils.findBinaryInPath('sh').endswith('/bin/sh')) - - def testCommaAndify(self): - L = ['foo'] - original = L[:] - self.assertEqual(utils.commaAndify(L), 'foo') - self.assertEqual(utils.commaAndify(L, And='or'), 'foo') - self.assertEqual(L, original) - L.append('bar') - original = L[:] - self.assertEqual(utils.commaAndify(L), 'foo and bar') - self.assertEqual(utils.commaAndify(L, And='or'), 'foo or bar') - self.assertEqual(L, original) - L.append('baz') - original = L[:] - self.assertEqual(utils.commaAndify(L), 'foo, bar, and baz') - self.assertEqual(utils.commaAndify(L, And='or'), 'foo, bar, or baz') - self.assertEqual(utils.commaAndify(L, comma=';'), 'foo; bar; and baz') - self.assertEqual(utils.commaAndify(L, comma=';', And='or'), - 'foo; bar; or baz') - self.assertEqual(L, original) - self.failUnless(utils.commaAndify(sets.Set(L))) - - def testCommaAndifyRaisesTypeError(self): - L = [(2,)] - self.assertRaises(TypeError, utils.commaAndify, L) - L.append((3,)) - self.assertRaises(TypeError, utils.commaAndify, L) - - def testUnCommaThe(self): - self.assertEqual(utils.unCommaThe('foo bar'), 'foo bar') - self.assertEqual(utils.unCommaThe('foo bar, the'), 'the foo bar') - self.assertEqual(utils.unCommaThe('foo bar, The'), 'The foo bar') - self.assertEqual(utils.unCommaThe('foo bar,the'), 'the foo bar') - - def testNormalizeWhitespace(self): - self.assertEqual(utils.normalizeWhitespace('foo bar'), 'foo bar') - self.assertEqual(utils.normalizeWhitespace('foo\nbar'), 'foo bar') - self.assertEqual(utils.normalizeWhitespace('foo\tbar'), 'foo bar') - - def testSortBy(self): - L = ['abc', 'z', 'AD'] - utils.sortBy(len, L) - self.assertEqual(L, ['z', 'AD', 'abc']) - utils.sortBy(str.lower, L) - self.assertEqual(L, ['abc', 'AD', 'z']) - L = ['supybot', 'Supybot'] - utils.sortBy(str.lower, L) - self.assertEqual(L, ['supybot', 'Supybot']) - - def testSorted(self): - L = ['a', 'c', 'b'] - self.assertEqual(utils.sorted(L), ['a', 'b', 'c']) - self.assertEqual(L, ['a', 'c', 'b']) - def mycmp(x, y): - return -cmp(x, y) - self.assertEqual(utils.sorted(L, mycmp), ['c', 'b', 'a']) - - def testNItems(self): - self.assertEqual(utils.nItems('tool', 1, 'crazy'), '1 crazy tool') - self.assertEqual(utils.nItems('tool', 1), '1 tool') - self.assertEqual(utils.nItems('tool', 2, 'crazy'), '2 crazy tools') - self.assertEqual(utils.nItems('tool', 2), '2 tools') - - def testItersplit(self): - itersplit = utils.itersplit - L = [1, 2, 3] * 3 - s = 'foo bar baz' - self.assertEqual(list(itersplit(lambda x: x == 3, L)), - [[1, 2], [1, 2], [1, 2]]) - self.assertEqual(list(itersplit(lambda x: x == 3, L, yieldEmpty=True)), - [[1, 2], [1, 2], [1, 2], []]) - self.assertEqual(list(itersplit(lambda x: x, [])), []) - self.assertEqual(list(itersplit(lambda c: c.isspace(), s)), - map(list, s.split())) - self.assertEqual(list(itersplit('for'.__eq__, ['foo', 'for', 'bar'])), - [['foo'], ['bar']]) - self.assertEqual(list(itersplit('for'.__eq__, - ['foo','for','bar','for', 'baz'], 1)), - [['foo'], ['bar', 'for', 'baz']]) + def testSafeEvalTurnsSyntaxErrorIntoValueError(self): + self.assertRaises(ValueError, utils.safeEval, '/usr/local/') def testIterableMap(self): class alist(utils.IterableMap): @@ -333,9 +97,277 @@ class UtilsTest(SupyTestCase): self.assertEqual(list(AL.itervalues()), [2, 3, 4]) self.assertEqual(len(AL), 3) + def testSortBy(self): + L = ['abc', 'z', 'AD'] + utils.sortBy(len, L) + self.assertEqual(L, ['z', 'AD', 'abc']) + utils.sortBy(str.lower, L) + self.assertEqual(L, ['abc', 'AD', 'z']) + L = ['supybot', 'Supybot'] + utils.sortBy(str.lower, L) + self.assertEqual(L, ['supybot', 'Supybot']) + + def testSorted(self): + L = ['a', 'c', 'b'] + self.assertEqual(utils.sorted(L), ['a', 'b', 'c']) + self.assertEqual(L, ['a', 'c', 'b']) + def mycmp(x, y): + return -cmp(x, y) + self.assertEqual(utils.sorted(L, mycmp), ['c', 'b', 'a']) + + def testTimeElapsed(self): + self.assertRaises(ValueError, utils.timeElapsed, 0, + leadingZeroes=False, seconds=False) + then = 0 + now = 0 + for now, expected in [(0, '0 seconds'), + (1, '1 second'), + (60, '1 minute and 0 seconds'), + (61, '1 minute and 1 second'), + (62, '1 minute and 2 seconds'), + (122, '2 minutes and 2 seconds'), + (3722, '1 hour, 2 minutes, and 2 seconds'), + (7322, '2 hours, 2 minutes, and 2 seconds'), + (90061,'1 day, 1 hour, 1 minute, and 1 second'), + (180122, '2 days, 2 hours, 2 minutes, ' + 'and 2 seconds')]: + self.assertEqual(utils.timeElapsed(now - then), expected) + + def timeElapsedShort(self): + self.assertEqual(utils.timeElapsed(123, short=True), '2m 3s') + + def testAbbrev(self): + L = ['abc', 'bcd', 'bbe', 'foo', 'fool'] + d = utils.abbrev(L) + def getItem(s): + return d[s] + self.assertRaises(KeyError, getItem, 'f') + self.assertRaises(KeyError, getItem, 'fo') + self.assertRaises(KeyError, getItem, 'b') + self.assertEqual(d['bb'], 'bbe') + self.assertEqual(d['bc'], 'bcd') + self.assertEqual(d['a'], 'abc') + self.assertEqual(d['ab'], 'abc') + self.assertEqual(d['fool'], 'fool') + self.assertEqual(d['foo'], 'foo') + + def testAbbrevFailsWithDups(self): + L = ['english', 'english'] + self.assertRaises(ValueError, utils.abbrev, L) + + +class StrTest(SupyTestCase): + def testMatchCase(self): + f = utils.str.matchCase + self.assertEqual('bar', f('foo', 'bar')) + self.assertEqual('Bar', f('Foo', 'bar')) + self.assertEqual('BAr', f('FOo', 'bar')) + self.assertEqual('BAR', f('FOO', 'bar')) + self.assertEqual('bAR', f('fOO', 'bar')) + self.assertEqual('baR', f('foO', 'bar')) + self.assertEqual('BaR', f('FoO', 'bar')) + + def testPluralize(self): + f = utils.str.pluralize + self.assertEqual('bike', f('bike', 1)) + self.assertEqual('bikes', f('bike', 2)) + self.assertEqual('BIKE', f('BIKE', 1)) + self.assertEqual('BIKES', f('BIKE', 2)) + self.assertEqual('match', f('match', 1)) + self.assertEqual('matches', f('match', 2)) + self.assertEqual('Patch', f('Patch', 1)) + self.assertEqual('Patches', f('Patch', 2)) + self.assertEqual('fish', f('fish', 1)) + self.assertEqual('fishes', f('fish', 2)) + self.assertEqual('try', f('try', 1)) + self.assertEqual('tries', f('try', 2)) + self.assertEqual('day', f('day', 1)) + self.assertEqual('days', f('day', 2)) + + def testDepluralize(self): + f = utils.str.depluralize + self.assertEqual('bike', f('bikes')) + self.assertEqual('Bike', f('Bikes')) + self.assertEqual('BIKE', f('BIKES')) + self.assertEqual('match', f('matches')) + self.assertEqual('Match', f('Matches')) + self.assertEqual('fish', f('fishes')) + self.assertEqual('try', f('tries')) + + def testDistance(self): + self.assertEqual(utils.str.distance('', ''), 0) + self.assertEqual(utils.str.distance('a', 'b'), 1) + self.assertEqual(utils.str.distance('a', 'a'), 0) + self.assertEqual(utils.str.distance('foobar', 'jemfinch'), 8) + self.assertEqual(utils.str.distance('a', 'ab'), 1) + self.assertEqual(utils.str.distance('foo', ''), 3) + self.assertEqual(utils.str.distance('', 'foo'), 3) + self.assertEqual(utils.str.distance('appel', 'nappe'), 2) + self.assertEqual(utils.str.distance('nappe', 'appel'), 2) + + def testSoundex(self): + L = [('Euler', 'E460'), + ('Ellery', 'E460'), + ('Gauss', 'G200'), + ('Ghosh', 'G200'), + ('Hilbert', 'H416'), + ('Heilbronn', 'H416'), + ('Knuth', 'K530'), + ('Kant', 'K530'), + ('Lloyd', 'L300'), + ('Ladd', 'L300'), + ('Lukasiewicz', 'L222'), + ('Lissajous', 'L222')] + for (name, key) in L: + soundex = utils.str.soundex(name) + self.assertEqual(soundex, key, + '%s was %s, not %s' % (name, soundex, key)) + self.assertRaises(ValueError, utils.str.soundex, '3') + self.assertRaises(ValueError, utils.str.soundex, "'") + + def testDQRepr(self): + L = ['foo', 'foo\'bar', 'foo"bar', '"', '\\', '', '\x00'] + for s in L: + r = utils.str.dqrepr(s) + self.assertEqual(s, eval(r), s) + self.failUnless(r[0] == '"' and r[-1] == '"', s) + + def testPerlReToPythonRe(self): + f = utils.str.perlReToPythonRe + r = f('m/foo/') + self.failUnless(r.search('foo')) + r = f('/foo/') + self.failUnless(r.search('foo')) + r = f('m/\\//') + self.failUnless(r.search('/')) + r = f('m/cat/i') + self.failUnless(r.search('CAT')) + self.assertRaises(ValueError, f, 'm/?/') + + def testP2PReDifferentSeparator(self): + r = utils.str.perlReToPythonRe('m!foo!') + self.failUnless(r.search('foo')) + + def testPerlReToReplacer(self): + PRTR = utils.str.perlReToReplacer + f = PRTR('s/foo/bar/') + self.assertEqual(f('foobarbaz'), 'barbarbaz') + f = PRTR('s/fool/bar/') + self.assertEqual(f('foobarbaz'), 'foobarbaz') + f = PRTR('s/foo//') + self.assertEqual(f('foobarbaz'), 'barbaz') + f = PRTR('s/ba//') + self.assertEqual(f('foobarbaz'), 'foorbaz') + f = PRTR('s/ba//g') + self.assertEqual(f('foobarbaz'), 'foorz') + f = PRTR('s/ba\\///g') + self.assertEqual(f('fooba/rba/z'), 'foorz') + f = PRTR('s/cat/dog/i') + self.assertEqual(f('CATFISH'), 'dogFISH') + f = PRTR('s/foo/foo\/bar/') + self.assertEqual(f('foo'), 'foo/bar') + f = PRTR('s/^/foo/') + self.assertEqual(f('bar'), 'foobar') + + def testPReToReplacerDifferentSeparator(self): + f = utils.str.perlReToReplacer('s#foo#bar#') + self.assertEqual(f('foobarbaz'), 'barbarbaz') + + def testPerlReToReplacerBug850931(self): + f = utils.str.perlReToReplacer('s/\b(\w+)\b/\1./g') + self.assertEqual(f('foo bar baz'), 'foo. bar. baz.') + + def testPerlVariableSubstitute(self): + f = utils.str.perlVariableSubstitute + vars = {'foo': 'bar', 'b a z': 'baz', 'b': 'c', 'i': 100, + 'f': lambda: 'called'} + self.assertEqual(f(vars, '$foo'), 'bar') + self.assertEqual(f(vars, '${foo}'), 'bar') + self.assertEqual(f(vars, '$b'), 'c') + self.assertEqual(f(vars, '${b}'), 'c') + self.assertEqual(f(vars, '$i'), '100') + self.assertEqual(f(vars, '${i}'), '100') + self.assertEqual(f(vars, '$f'), 'called') + self.assertEqual(f(vars, '${f}'), 'called') + self.assertEqual(f(vars, '${b a z}'), 'baz') + self.assertEqual(f(vars, '$b:$i'), 'c:100') + + def testCommaAndify(self): + f = utils.str.commaAndify + L = ['foo'] + original = L[:] + self.assertEqual(f(L), 'foo') + self.assertEqual(f(L, And='or'), 'foo') + self.assertEqual(L, original) + L.append('bar') + original = L[:] + self.assertEqual(f(L), 'foo and bar') + self.assertEqual(f(L, And='or'), 'foo or bar') + self.assertEqual(L, original) + L.append('baz') + original = L[:] + self.assertEqual(f(L), 'foo, bar, and baz') + self.assertEqual(f(L, And='or'), 'foo, bar, or baz') + self.assertEqual(f(L, comma=';'), 'foo; bar; and baz') + self.assertEqual(f(L, comma=';', And='or'), + 'foo; bar; or baz') + self.assertEqual(L, original) + self.failUnless(f(set(L))) + + def testCommaAndifyRaisesTypeError(self): + L = [(2,)] + self.assertRaises(TypeError, utils.str.commaAndify, L) + L.append((3,)) + self.assertRaises(TypeError, utils.str.commaAndify, L) + + def testUnCommaThe(self): + f = utils.str.unCommaThe + self.assertEqual(f('foo bar'), 'foo bar') + self.assertEqual(f('foo bar, the'), 'the foo bar') + self.assertEqual(f('foo bar, The'), 'The foo bar') + self.assertEqual(f('foo bar,the'), 'the foo bar') + + def testNormalizeWhitespace(self): + f = utils.str.normalizeWhitespace + self.assertEqual(f('foo bar'), 'foo bar') + self.assertEqual(f('foo\nbar'), 'foo bar') + self.assertEqual(f('foo\tbar'), 'foo bar') + + def testNItems(self): + nItems = utils.str.nItems + self.assertEqual(nItems('tool', 1, 'crazy'), '1 crazy tool') + self.assertEqual(nItems('tool', 1), '1 tool') + self.assertEqual(nItems('tool', 2, 'crazy'), '2 crazy tools') + self.assertEqual(nItems('tool', 2), '2 tools') + + def testEllipsisify(self): + f = utils.str.ellipsisify + self.assertEqual(f('x'*30, 30), 'x'*30) + self.failUnless(len(f('x'*35, 30)) <= 30) + self.failUnless(f(' '.join(['xxxx']*10), 30)[:-3].endswith('xxxx')) + + +class IterTest(SupyTestCase): + def testSplit(self): + itersplit = utils.iter.split + L = [1, 2, 3] * 3 + s = 'foo bar baz' + self.assertEqual(list(itersplit(lambda x: x == 3, L)), + [[1, 2], [1, 2], [1, 2]]) + self.assertEqual(list(itersplit(lambda x: x == 3, L, yieldEmpty=True)), + [[1, 2], [1, 2], [1, 2], []]) + self.assertEqual(list(itersplit(lambda x: x, [])), []) + self.assertEqual(list(itersplit(lambda c: c.isspace(), s)), + map(list, s.split())) + self.assertEqual(list(itersplit('for'.__eq__, ['foo', 'for', 'bar'])), + [['foo'], ['bar']]) + self.assertEqual(list(itersplit('for'.__eq__, + ['foo','for','bar','for', 'baz'], 1)), + [['foo'], ['bar', 'for', 'baz']]) + def testFlatten(self): def lflatten(seq): - return list(utils.flatten(seq)) + return list(utils.iter.flatten(seq)) self.assertEqual(lflatten([]), []) self.assertEqual(lflatten([1]), [1]) self.assertEqual(lflatten(range(10)), range(10)) @@ -347,60 +379,43 @@ class UtilsTest(SupyTestCase): self.assertEqual(lflatten([1, [2, [3, 4], 5], 6]), [1, 2, 3, 4, 5, 6]) self.assertRaises(TypeError, lflatten, 1) - def testEllipsisify(self): - f = utils.ellipsisify - self.assertEqual(f('x'*30, 30), 'x'*30) - self.failUnless(len(f('x'*35, 30)) <= 30) - self.failUnless(f(' '.join(['xxxx']*10), 30)[:-3].endswith('xxxx')) - - def testSaltHash(self): - s = utils.saltHash('jemfinch') - (salt, hash) = s.split('|') - self.assertEqual(utils.saltHash('jemfinch', salt=salt), s) - - def testSafeEval(self): - for s in ['1', '()', '(1,)', '[]', '{}', '{1:2}', '{1:(2,3)}', - '1.0', '[1,2,3]', 'True', 'False', 'None', - '(True,False,None)', '"foo"', '{"foo": "bar"}']: - self.assertEqual(eval(s), utils.safeEval(s)) - for s in ['lambda: 2', 'import foo', 'foo.bar']: - self.assertRaises(ValueError, utils.safeEval, s) - - - def testSafeEvalTurnsSyntaxErrorIntoValueError(self): - self.assertRaises(ValueError, utils.safeEval, '/usr/local/') +class FileTest(SupyTestCase): def testLines(self): L = ['foo', 'bar', '#baz', ' ', 'biff'] - self.assertEqual(list(utils.nonEmptyLines(L)), + self.assertEqual(list(utils.file.nonEmptyLines(L)), ['foo', 'bar', '#baz', 'biff']) - self.assertEqual(list(utils.nonCommentLines(L)), + self.assertEqual(list(utils.file.nonCommentLines(L)), ['foo', 'bar', ' ', 'biff']) - self.assertEqual(list(utils.nonCommentNonEmptyLines(L)), + self.assertEqual(list(utils.file.nonCommentNonEmptyLines(L)), ['foo', 'bar', 'biff']) + +class NetTest(SupyTestCase): def testIsIP(self): - self.failIf(utils.isIP('a.b.c')) - self.failIf(utils.isIP('256.0.0.0')) - self.failUnless(utils.isIP('127.1')) - self.failUnless(utils.isIP('0.0.0.0')) - self.failUnless(utils.isIP('100.100.100.100')) + isIP = utils.net.isIP + self.failIf(isIP('a.b.c')) + self.failIf(isIP('256.0.0.0')) + self.failUnless(isIP('127.1')) + self.failUnless(isIP('0.0.0.0')) + self.failUnless(isIP('100.100.100.100')) # This test is too flaky to bother with. # self.failUnless(utils.isIP('255.255.255.255')) def testIsIPV6(self): - f = utils.isIPV6 + f = utils.net.isIPV6 self.failUnless(f('2001::')) self.failUnless(f('2001:888:0:1::666')) - def testInsensitivePreservingDict(self): - ipd = utils.InsensitivePreservingDict - d = ipd(dict(Foo=10)) - self.failUnless(d['foo'] == 10) - self.assertEqual(d.keys(), ['Foo']) - self.assertEqual(d.get('foo'), 10) - self.assertEqual(d.get('Foo'), 10) +class WebTest(SupyTestCase): + def testGetDomain(self): + url = 'http://slashdot.org/foo/bar.exe' + self.assertEqual(utils.web.getDomain(url), 'slashdot.org') + if network: + def testGetUrlWithSize(self): + url = 'http://slashdot.org/' + self.failUnless(len(utils.web.getUrl(url, 1024)) == 1024) # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: