###
# Copyright (c) 2002-2004, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions, and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author of this software nor the name of
# contributors to this software may be used to endorse or promote products
# derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###
"""
Keeps track of URLs posted to a channel, along with relevant context. Allows
searching for URLs and returning random URLs. Also provides statistics on the
URLs in the database.
"""
__revision__ = "$Id$"
import supybot.plugins as plugins
import re
import supybot.dbi as dbi
import supybot.conf as conf
import supybot.utils as utils
from supybot.commands import *
import supybot.ircmsgs as ircmsgs
import supybot.ircutils as ircutils
import supybot.webutils as webutils
import supybot.registry as registry
import supybot.callbacks as callbacks
def configure(advanced):
from supybot.questions import output, expect, anything, something, yn
conf.registerPlugin('URL', True)
if yn("""This plugin also offers a snarfer that will try to fetch the
title of URLs that it sees in the channel. Would like you this
snarfer to be enabled?""", default=False):
conf.supybot.plugins.URL.titleSnarfer.setValue(True)
conf.registerPlugin('URL')
conf.registerChannelValue(conf.supybot.plugins.URL, 'titleSnarfer',
registry.Boolean(False, """Determines whether the bot will output the HTML
title of URLs it sees in the channel."""))
conf.registerChannelValue(conf.supybot.plugins.URL, 'nonSnarfingRegexp',
registry.Regexp(None, """Determines what URLs are to be snarfed and stored
in the database in the channel; URLs matching the regexp given will not be
snarfed. Give the empty string if you have no URLs that you'd like to
exclude from being snarfed."""))
class UrlRecord(dbi.Record):
__fields__ = [
('url', eval),
('by', eval),
('near', eval),
('at', eval),
]
class DbiUrlDB(plugins.DbiChannelDB):
class DB(dbi.DB):
Record = UrlRecord
def add(self, url, msg):
record = self.Record(url=url, by=msg.nick,
near=msg.args[1], at=msg.receivedAt)
super(self.__class__, self).add(record)
def urls(self, p):
L = list(self.select(p))
L.reverse()
return L
URLDB = plugins.DB('URL', {'flat': DbiUrlDB})
class URL(callbacks.PrivmsgCommandAndRegexp):
priority = 100 # lower than 99, the normal priority.
regexps = ['titleSnarfer']
_titleRe = re.compile('<title>(.*?)</title>', re.I | re.S)
def __init__(self):
self.__parent = super(URL, self)
self.__parent.__init__()
self.db = URLDB()
def doPrivmsg(self, irc, msg):
channel = msg.args[0]
if irc.isChannel(channel):
if ircmsgs.isAction(msg):
text = ircmsgs.unAction(msg)
else:
text = msg.args[1]
for url in webutils.urlRe.findall(text):
r = self.registryValue('nonSnarfingRegexp', channel)
if r and r.search(url):
self.log.debug('Skipping adding %s to db.',
utils.quoted(url))
continue
self.log.debug('Adding %s to db.', utils.quoted(url))
self.db.add(channel, url, msg)
self.__parent.doPrivmsg(irc, msg)
def titleSnarfer(self, irc, msg, match):
r"https?://[^\])>\s]+"
channel = msg.args[0]
if not irc.isChannel(channel):
return
if callbacks.addressed(irc.nick, msg):
return
if self.registryValue('titleSnarfer', channel):
url = match.group(0)
r = self.registryValue('nonSnarfingRegexp', channel)
if r and r.search(url):
self.log.debug('Not titleSnarfing %s.', utils.quoted(url))
return
try:
size = conf.supybot.protocols.http.peekSize()
text = webutils.getUrl(url, size=size)
except webutils.WebError, e:
self.log.info('Couldn\'t snarf title of %s, %s.', url, e)
return
m = self._titleRe.search(text)
if m is not None:
domain = webutils.getDomain(url)
title = utils.htmlToText(m.group(1).strip())
s = 'Title: %s (at %s)' % (title, domain)
irc.reply(s, prefixName=False)
titleSnarfer = urlSnarfer(titleSnarfer)
def stats(self, irc, msg, args, channel):
"""[<channel>]
Returns the number of URLs in the URL database. <channel> is only
required if the message isn't sent in the channel itself.
"""
self.db.vacuum(channel)
count = self.db.size(channel)
irc.reply('I have %s in my database.' % utils.nItems('URL', count))
stats = wrap(stats, ['channeldb'])
def last(self, irc, msg, args, channel, optlist):
"""[<channel>] [--{from,with,without,near,proto}=<value>] --nolimit
Gives the last URL matching the given criteria. --from is from whom
the URL came; --proto is the protocol the URL used; --with is something
inside the URL; --without is something that should not be in the URL;
--near is something in the same message as the URL; If --nolimit is
given, returns all the URLs that are found. to just the URL.
<channel> is only necessary if the message isn't sent in the channel
itself.
"""
predicates = []
f = None
nolimit = False
for (option, arg) in optlist:
if option == 'nolimit':
nolimit = True
elif option == 'from':
def f(record, arg=arg):
return ircutils.strEqual(record.by, arg)
elif option == 'with':
def f(record, arg=arg):
return arg in record.url
elif option == 'without':
def f(record, arg=arg):
return arg not in record.url
elif option == 'proto':
def f(record, arg=arg):
return record.url.startswith(arg)
elif option == 'near':
def f(record, arg=arg):
return arg in record.near
if f is not None:
predicates.append(f)
def predicate(record):
for predicate in predicates:
if not predicate(record):
return False
return True
urls = [record.url for record in self.db.urls(channel, predicate)]
if not urls:
irc.reply('No URLs matched that criteria.')
else:
if nolimit:
urls = ['<%s>' % url for url in urls]
s = ', '.join(urls)
else:
# We should optimize this with another URLDB method eventually.
s = urls[0]
irc.reply(s)
last = wrap(last, ['channeldb',
getopts({'from': 'something', 'with': 'something',
'near': 'something', 'proto': 'something',
'nolimit': '', 'without': 'something',})])
Class = URL
# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: