This merges an old branch i had for sqlite3 factoids+moobotfactoids.

fix up moobotfactoids+factoids to use the sqlite text_factory=str,
also fixed up a test for factoid search, since it now sorts keys alphabetically.
This commit is contained in:
Daniel Folkinshteyn 2010-03-21 02:25:11 -04:00
commit c2b1c1b143
5 changed files with 186 additions and 151 deletions

View File

@ -40,12 +40,21 @@ import supybot.plugins as plugins
import supybot.ircutils as ircutils
import supybot.callbacks as callbacks
#try:
#import sqlite3 as sqlite
#except ImportError:
#raise callbacks.Error, 'You need to have PySQLite installed to use this ' \
#'plugin. Download it at ' \
#'<http://code.google.com/p/pysqlite/>'
try:
import sqlite
import sqlite3
except ImportError:
raise callbacks.Error, 'You need to have PySQLite installed to use this ' \
'plugin. Download it at ' \
'<http://code.google.com/p/pysqlite/>'
from pysqlite2 import dbapi2 as sqlite3 # for python2.4
# these are needed cuz we are overriding getdb
import threading
import supybot.world as world
def getFactoid(irc, msg, args, state):
assert not state.channel
@ -83,8 +92,11 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
def makeDb(self, filename):
if os.path.exists(filename):
return sqlite.connect(filename)
db = sqlite.connect(filename)
db = sqlite3.connect(filename)
db.text_factory = str
return db
db = sqlite3.connect(filename)
db.text_factory = str
cursor = db.cursor()
cursor.execute("""CREATE TABLE keys (
id INTEGER PRIMARY KEY,
@ -108,6 +120,20 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
db.commit()
return db
# override this because sqlite3 doesn't have autocommit
# use isolation_level instead.
def getDb(self, channel):
"""Use this to get a database for a specific channel."""
currentThread = threading.currentThread()
if channel not in self.dbCache and currentThread == world.mainThread:
self.dbCache[channel] = self.makeDb(self.makeFilename(channel))
if currentThread != world.mainThread:
db = self.makeDb(self.makeFilename(channel))
else:
db = self.dbCache[channel]
db.isolation_level = None
return db
def getCommandHelp(self, command, simpleSyntax=None):
method = self.getCommandMethod(command)
if method.im_func.func_name == 'learn':
@ -129,12 +155,14 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
def learn(self, irc, msg, args, channel, key, factoid):
db = self.getDb(channel)
cursor = db.cursor()
cursor.execute("SELECT id, locked FROM keys WHERE key LIKE %s", key)
if cursor.rowcount == 0:
cursor.execute("""INSERT INTO keys VALUES (NULL, %s, 0)""", key)
cursor.execute("SELECT id, locked FROM keys WHERE key LIKE ?", (key,))
results = cursor.fetchall()
if len(results) == 0:
cursor.execute("""INSERT INTO keys VALUES (NULL, ?, 0)""", (key,))
db.commit()
cursor.execute("SELECT id, locked FROM keys WHERE key LIKE %s",key)
(id, locked) = map(int, cursor.fetchone())
cursor.execute("SELECT id, locked FROM keys WHERE key LIKE ?", (key,))
results = cursor.fetchall()
(id, locked) = map(int, results[0])
capability = ircdb.makeChannelCapability(channel, 'factoids')
if not locked:
if ircdb.users.hasUser(msg.prefix):
@ -142,8 +170,8 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
else:
name = msg.nick
cursor.execute("""INSERT INTO factoids VALUES
(NULL, %s, %s, %s, %s, %s)""",
id, name, int(time.time()), 0, factoid)
(NULL, ?, ?, ?, ?, ?)""",
(id, name, int(time.time()), 0, factoid))
db.commit()
irc.replySuccess()
else:
@ -163,9 +191,9 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
db = self.getDb(channel)
cursor = db.cursor()
cursor.execute("""SELECT factoids.fact, factoids.id FROM factoids, keys
WHERE keys.key LIKE %s AND factoids.key_id=keys.id
WHERE keys.key LIKE ? AND factoids.key_id=keys.id
ORDER BY factoids.id
LIMIT 20""", key)
LIMIT 20""", (key,))
return cursor.fetchall()
#return [t[0] for t in cursor.fetchall()]
@ -176,9 +204,9 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
for (fact,id) in factoids:
cursor.execute("""SELECT factoids.usage_count
FROM factoids
WHERE factoids.id=%s""", id)
WHERE factoids.id=?""", (id,))
old_count = cursor.fetchall()[0][0]
cursor.execute("UPDATE factoids SET usage_count=%s WHERE id=%s", old_count + 1, id)
cursor.execute("UPDATE factoids SET usage_count=? WHERE id=?", (old_count + 1, id,))
db.commit()
def _replyFactoids(self, irc, msg, key, channel, factoids,
@ -253,7 +281,7 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
FROM keys, factoids
WHERE factoids.key_id=keys.id
ORDER BY factoids.usage_count DESC
LIMIT %s""", numfacts)
LIMIT ?""", (numfacts,))
factkeys = cursor.fetchall()
s = [ "#%d %s (%d)" % (i+1, key[0], key[1]) for i, key in enumerate(factkeys) ]
irc.reply(", ".join(s))
@ -268,7 +296,7 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
"""
db = self.getDb(channel)
cursor = db.cursor()
cursor.execute("UPDATE keys SET locked=1 WHERE key LIKE %s", key)
cursor.execute("UPDATE keys SET locked=1 WHERE key LIKE ?", (key,))
db.commit()
irc.replySuccess()
lock = wrap(lock, ['channel', 'text'])
@ -282,7 +310,7 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
"""
db = self.getDb(channel)
cursor = db.cursor()
cursor.execute("UPDATE keys SET locked=0 WHERE key LIKE %s", key)
cursor.execute("UPDATE keys SET locked=0 WHERE key LIKE ?", (key,))
db.commit()
irc.replySuccess()
unlock = wrap(unlock, ['channel', 'text'])
@ -310,32 +338,33 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
cursor = db.cursor()
cursor.execute("""SELECT keys.id, factoids.id
FROM keys, factoids
WHERE key LIKE %s AND
factoids.key_id=keys.id""", key)
if cursor.rowcount == 0:
WHERE key LIKE ? AND
factoids.key_id=keys.id""", (key,))
results = cursor.fetchall()
if len(results) == 0:
irc.error('There is no such factoid.')
elif cursor.rowcount == 1 or number is True:
(id, _) = cursor.fetchone()
cursor.execute("""DELETE FROM factoids WHERE key_id=%s""", id)
cursor.execute("""DELETE FROM keys WHERE key LIKE %s""", key)
elif len(results) == 1 or number is True:
(id, _) = results[0]
cursor.execute("""DELETE FROM factoids WHERE key_id=?""", (id,))
cursor.execute("""DELETE FROM keys WHERE key LIKE ?""", (key,))
db.commit()
irc.replySuccess()
else:
if number is not None:
results = cursor.fetchall()
#results = cursor.fetchall()
try:
(_, id) = results[number-1]
except IndexError:
irc.error('Invalid factoid number.')
return
cursor.execute("DELETE FROM factoids WHERE id=%s", id)
cursor.execute("DELETE FROM factoids WHERE id=?", (id,))
db.commit()
irc.replySuccess()
else:
irc.error('%s factoids have that key. '
'Please specify which one to remove, '
'or use * to designate all of them.' %
cursor.rowcount)
len(results))
forget = wrap(forget, ['channel', many('something')])
def random(self, irc, msg, args, channel):
@ -349,10 +378,11 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
cursor.execute("""SELECT fact, key_id FROM factoids
ORDER BY random()
LIMIT 3""")
if cursor.rowcount != 0:
results = cursor.fetchall()
if len(results) != 0:
L = []
for (factoid, id) in cursor.fetchall():
cursor.execute("""SELECT key FROM keys WHERE id=%s""", id)
for (factoid, id) in results:
cursor.execute("""SELECT key FROM keys WHERE id=?""", (id,))
(key,) = cursor.fetchone()
L.append('"%s": %s' % (ircutils.bold(key), factoid))
irc.reply('; '.join(L))
@ -369,14 +399,15 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
"""
db = self.getDb(channel)
cursor = db.cursor()
cursor.execute("SELECT id, locked FROM keys WHERE key LIKE %s", key)
if cursor.rowcount == 0:
cursor.execute("SELECT id, locked FROM keys WHERE key LIKE ?", (key,))
results = cursor.fetchall()
if len(results) == 0:
irc.error('No factoid matches that key.')
return
(id, locked) = map(int, cursor.fetchone())
(id, locked) = map(int, results[0])
cursor.execute("""SELECT added_by, added_at FROM factoids
WHERE key_id=%s
ORDER BY id""", id)
WHERE key_id=?
ORDER BY id""", (id,))
factoids = cursor.fetchall()
L = []
counter = 0
@ -403,16 +434,17 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
cursor = db.cursor()
cursor.execute("""SELECT factoids.id, factoids.fact
FROM keys, factoids
WHERE keys.key LIKE %s AND
keys.id=factoids.key_id""", key)
if cursor.rowcount == 0:
WHERE keys.key LIKE ? AND
keys.id=factoids.key_id""", (key,))
results = cursor.fetchall()
if len(results) == 0:
irc.error(format('I couldn\'t find any key %q', key))
return
elif cursor.rowcount < number:
elif len(results) < number:
irc.errorInvalid('key id')
(id, fact) = cursor.fetchall()[number-1]
(id, fact) = results[number-1]
newfact = replacer(fact)
cursor.execute("UPDATE factoids SET fact=%s WHERE id=%s", newfact, id)
cursor.execute("UPDATE factoids SET fact=? WHERE id=?", (newfact, id))
db.commit()
irc.replySuccess()
change = wrap(change, ['channel', 'something',
@ -447,7 +479,7 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
db.create_function(predicateName, 1, p)
predicateName += 'p'
for glob in globs:
criteria.append('TARGET LIKE %s')
criteria.append('TARGET LIKE ?')
formats.append(glob.translate(self._sqlTrans))
cursor = db.cursor()
sql = """SELECT keys.key FROM %s WHERE %s""" % \
@ -455,16 +487,17 @@ class Factoids(callbacks.Plugin, plugins.ChannelDBHandler):
sql = sql + " ORDER BY keys.key"
sql = sql.replace('TARGET', target)
cursor.execute(sql, formats)
if cursor.rowcount == 0:
results = cursor.fetchall()
if len(results) == 0:
irc.reply('No keys matched that query.')
elif cursor.rowcount == 1 and \
elif len(results) == 1 and \
self.registryValue('showFactoidIfOnlyOneMatch', channel):
self.whatis(irc, msg, [cursor.fetchone()[0]])
elif cursor.rowcount > 100:
self.whatis(irc, msg, [results[0][0]])
elif len(results) > 100:
irc.reply('More than 100 keys matched that query; '
'please narrow your query.')
else:
keys = [repr(t[0]) for t in cursor.fetchall()]
keys = [repr(t[0]) for t in results]
s = format('%L', keys)
irc.reply(s)
search = wrap(search, ['channel',

View File

@ -99,8 +99,8 @@ if sqlite:
self.assertRegexp('factoids search --regexp m/^j/ *ss*',
'jamessan')
self.assertRegexp('factoids search --regexp /^j/',
'jemfinch.*jamessan')
self.assertRegexp('factoids search j*', 'jemfinch.*jamessan')
'jamessan.*jemfinch')
self.assertRegexp('factoids search j*', 'jamessan.*jemfinch')
self.assertRegexp('factoids search *ke*',
'inkedmn.*strike|strike.*inkedmn')
self.assertRegexp('factoids search ke',

View File

@ -50,7 +50,7 @@ import time
try:
import sqlite3
except:
except ImportError:
from pysqlite2 import dbapi2 as sqlite3 # for python2.4
# these are needed cuz we are overriding getdb

View File

@ -98,19 +98,21 @@ class SqliteMoobotDB(object):
def _getDb(self, channel):
try:
import sqlite
import sqlite3
except ImportError:
raise callbacks.Error, \
'You need to have PySQLite installed to use this ' \
'plugin. Download it at ' \
'<http://code.google.com/p/pysqlite/>'
from pysqlite2 import dbapi2 as sqlite3 # for python2.4
if channel in self.dbs:
return self.dbs[channel]
filename = plugins.makeChannelFilename(self.filename, channel)
if os.path.exists(filename):
self.dbs[channel] = sqlite.connect(filename)
return self.dbs[channel]
db = sqlite.connect(filename)
db = sqlite3.connect(filename)
db.text_factory = str
self.dbs[channel] = db
return db
db = sqlite3.connect(filename)
db.text_factory = str
self.dbs[channel] = db
cursor = db.cursor()
cursor.execute("""CREATE TABLE factoids (
@ -133,11 +135,12 @@ class SqliteMoobotDB(object):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""SELECT fact FROM factoids
WHERE key LIKE %s""", key)
if cursor.rowcount == 0:
WHERE key LIKE ?""", (key,))
results = cursor.fetchall()
if len(results) == 0:
return None
else:
return cursor.fetchall()[0]
return results[0]
def getFactinfo(self, channel, key):
db = self._getDb(channel)
@ -147,63 +150,65 @@ class SqliteMoobotDB(object):
last_requested_by, last_requested_at,
requested_count, locked_by, locked_at
FROM factoids
WHERE key LIKE %s""", key)
if cursor.rowcount == 0:
WHERE key LIKE ?""", (key,))
results = cursor.fetchall()
if len(results) == 0:
return None
else:
return cursor.fetchone()
return results[0]
def randomFactoid(self, channel):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""SELECT fact, key FROM factoids
ORDER BY random() LIMIT 1""")
if cursor.rowcount == 0:
results = cursor.fetchall()
if len(results) == 0:
return None
else:
return cursor.fetchone()
return results[0]
def addFactoid(self, channel, key, value, creator_id):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""INSERT INTO factoids VALUES
(%s, %s, %s, NULL, NULL, NULL, NULL,
NULL, NULL, %s, 0)""",
key, creator_id, int(time.time()), value)
(?, ?, ?, NULL, NULL, NULL, NULL,
NULL, NULL, ?, 0)""",
(key, creator_id, int(time.time()), value))
db.commit()
def updateFactoid(self, channel, key, newvalue, modifier_id):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""UPDATE factoids
SET fact=%s, modified_by=%s,
modified_at=%s WHERE key LIKE %s""",
newvalue, modifier_id, int(time.time()), key)
SET fact=?, modified_by=?,
modified_at=? WHERE key LIKE ?""",
(newvalue, modifier_id, int(time.time()), key))
db.commit()
def updateRequest(self, channel, key, hostmask):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""UPDATE factoids SET
last_requested_by = %s,
last_requested_at = %s,
last_requested_by = ?,
last_requested_at = ?,
requested_count = requested_count + 1
WHERE key = %s""",
hostmask, int(time.time()), key)
WHERE key = ?""",
(hostmask, int(time.time()), key))
db.commit()
def removeFactoid(self, channel, key):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""DELETE FROM factoids WHERE key LIKE %s""",
key)
cursor.execute("""DELETE FROM factoids WHERE key LIKE ?""",
(key,))
db.commit()
def locked(self, channel, key):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute ("""SELECT locked_by FROM factoids
WHERE key LIKE %s""", key)
WHERE key LIKE ?""", (key,))
if cursor.fetchone()[0] is None:
return False
else:
@ -213,17 +218,17 @@ class SqliteMoobotDB(object):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""UPDATE factoids
SET locked_by=%s, locked_at=%s
WHERE key LIKE %s""",
locker_id, int(time.time()), key)
SET locked_by=?, locked_at=?
WHERE key LIKE ?""",
(locker_id, int(time.time()), key))
db.commit()
def unlock(self, channel, key):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""UPDATE factoids
SET locked_by=%s, locked_at=%s
WHERE key LIKE %s""", None, None, key)
SET locked_by=?, locked_at=?
WHERE key LIKE ?""", (None, None, key))
db.commit()
def mostAuthored(self, channel, limit):
@ -231,14 +236,14 @@ class SqliteMoobotDB(object):
cursor = db.cursor()
cursor.execute("""SELECT created_by, count(key) FROM factoids
GROUP BY created_by
ORDER BY count(key) DESC LIMIT %s""", limit)
ORDER BY count(key) DESC LIMIT ?""", (limit,))
return cursor.fetchall()
def mostRecent(self, channel, limit):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""SELECT key FROM factoids
ORDER BY created_at DESC LIMIT %s""", limit)
ORDER BY created_at DESC LIMIT ?""", (limit,))
return cursor.fetchall()
def mostPopular(self, channel, limit):
@ -246,43 +251,35 @@ class SqliteMoobotDB(object):
cursor = db.cursor()
cursor.execute("""SELECT key, requested_count FROM factoids
WHERE requested_count > 0
ORDER BY requested_count DESC LIMIT %s""", limit)
if cursor.rowcount == 0:
return []
else:
return cursor.fetchall()
ORDER BY requested_count DESC LIMIT ?""", (limit,))
results = cursor.fetchall()
return results
def getKeysByAuthor(self, channel, authorId):
db = self._getDb(channel)
cursor = db.cursor()
cursor.execute("""SELECT key FROM factoids WHERE created_by=%s
ORDER BY key""", authorId)
if cursor.rowcount == 0:
return []
else:
return cursor.fetchall()
cursor.execute("""SELECT key FROM factoids WHERE created_by=?
ORDER BY key""", (authorId,))
results = cursor.fetchall()
return results
def getKeysByGlob(self, channel, glob):
db = self._getDb(channel)
cursor = db.cursor()
glob = '%%%s%%' % glob
cursor.execute("""SELECT key FROM factoids WHERE key LIKE %s
ORDER BY key""", glob)
if cursor.rowcount == 0:
return []
else:
return cursor.fetchall()
cursor.execute("""SELECT key FROM factoids WHERE key LIKE ?
ORDER BY key""", (glob,))
results = cursor.fetchall()
return results
def getKeysByValueGlob(self, channel, glob):
db = self._getDb(channel)
cursor = db.cursor()
glob = '%%%s%%' % glob
cursor.execute("""SELECT key FROM factoids WHERE fact LIKE %s
ORDER BY key""", glob)
if cursor.rowcount == 0:
return []
else:
return cursor.fetchall()
cursor.execute("""SELECT key FROM factoids WHERE fact LIKE ?
ORDER BY key""", (glob,))
results = cursor.fetchall()
return results
MoobotDB = plugins.DB('MoobotFactoids', {'sqlite': SqliteMoobotDB})

View File

@ -51,46 +51,51 @@ from supybot.commands import *
import supybot.ircutils as ircutils
import supybot.callbacks as callbacks
## i think we don't need any of this with sqlite3
#try:
## We need to sweep away all that mx.* crap because our code doesn't account
## for PySQLite's arbitrary use of it. Whoever decided to change sqlite's
## behavior based on whether or not that module is installed was a *CRACK*
## **FIEND**, plain and simple.
#mxCrap = {}
#for (name, module) in sys.modules.items():
#if name.startswith('mx'):
#mxCrap[name] = module
#sys.modules.pop(name)
## Now that the mx crap is gone, we can import sqlite.
#import sqlite3 as sqlite
## And now we'll put it back, even though it sucks.
#sys.modules.update(mxCrap)
## Just in case, we'll do this as well. It doesn't seem to work fine by
## itself, though, or else we'd just do this in the first place.
#sqlite.have_datetime = False
#Connection = sqlite.Connection
#class MyConnection(sqlite.Connection):
#def commit(self, *args, **kwargs):
#if self.autocommit:
#return
#else:
#Connection.commit(self, *args, **kwargs)
#def __del__(self):
#try:
#Connection.__del__(self)
#except AttributeError:
#pass
#except Exception, e:
#try:
#log.exception('Uncaught exception in __del__:')
#except:
#pass
#sqlite.Connection = MyConnection
##del Connection.__del__
#except ImportError:
#pass
try:
# We need to sweep away all that mx.* crap because our code doesn't account
# for PySQLite's arbitrary use of it. Whoever decided to change sqlite's
# behavior based on whether or not that module is installed was a *CRACK*
# **FIEND**, plain and simple.
mxCrap = {}
for (name, module) in sys.modules.items():
if name.startswith('mx'):
mxCrap[name] = module
sys.modules.pop(name)
# Now that the mx crap is gone, we can import sqlite.
import sqlite
# And now we'll put it back, even though it sucks.
sys.modules.update(mxCrap)
# Just in case, we'll do this as well. It doesn't seem to work fine by
# itself, though, or else we'd just do this in the first place.
sqlite.have_datetime = False
Connection = sqlite.Connection
class MyConnection(sqlite.Connection):
def commit(self, *args, **kwargs):
if self.autocommit:
return
else:
Connection.commit(self, *args, **kwargs)
def __del__(self):
try:
Connection.__del__(self)
except AttributeError:
pass
except Exception, e:
try:
log.exception('Uncaught exception in __del__:')
except:
pass
sqlite.Connection = MyConnection
#del Connection.__del__
import sqlite3
except ImportError:
pass
from pysqlite2 import dbapi2 as sqlite3 # for python2.4
class NoSuitableDatabase(Exception):
def __init__(self, suitable):
@ -176,7 +181,7 @@ class ChannelDBHandler(object):
db = self.makeDb(self.makeFilename(channel))
else:
db = self.dbCache[channel]
db.autocommit = 1
db.isolation_level = None
return db
def die(self):