From 0e7349cf77a96a1bb2a28640d88b55f2981cd183 Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Thu, 31 Jul 2014 19:37:57 +0000 Subject: [PATCH 1/5] Fix utils.str.timestamp for ints. --- src/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/conf.py b/src/conf.py index 401703fb0..3bfc0685d 100644 --- a/src/conf.py +++ b/src/conf.py @@ -374,7 +374,7 @@ registerChannelValue(supybot.reply.format, 'time', def timestamp(t): if t is None: t = time.time() - elif isinstance(t, float): + elif isinstance(t, float) or isinstance(t, int): t = time.localtime(t) format = get(supybot.reply.format.time, dynamic.channel) return time.strftime(format, t) From e42a3dd6ac2055f6103f5b3a8ab9e04eec15421a Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Thu, 31 Jul 2014 19:53:00 +0000 Subject: [PATCH 2/5] Fix Python 2.6 compatibility. --- src/conf.py | 2 +- test/test_utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/conf.py b/src/conf.py index 3bfc0685d..f0e2d9201 100644 --- a/src/conf.py +++ b/src/conf.py @@ -374,7 +374,7 @@ registerChannelValue(supybot.reply.format, 'time', def timestamp(t): if t is None: t = time.time() - elif isinstance(t, float) or isinstance(t, int): + if isinstance(t, float) or isinstance(t, int): t = time.localtime(t) format = get(supybot.reply.format.time, dynamic.channel) return time.strftime(format, t) diff --git a/test/test_utils.py b/test/test_utils.py index c5b53fe5b..debad420f 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1136,7 +1136,7 @@ class TestCacheDict(SupyTestCase): class TestTruncatableSet(SupyTestCase): def testBasics(self): s = TruncatableSet(['foo', 'bar', 'baz', 'qux']) - self.assertEqual(s, {'foo', 'bar', 'baz', 'qux'}) + self.assertEqual(s, set(['foo', 'bar', 'baz', 'qux'])) self.failUnless('foo' in s) self.failUnless('bar' in s) self.failIf('quux' in s) @@ -1151,13 +1151,13 @@ class TestTruncatableSet(SupyTestCase): s.add('baz') s.add('qux') s.truncate(3) - self.assertEqual(s, {'bar', 'baz', 'qux'}) + self.assertEqual(s, set(['bar', 'baz', 'qux'])) def testTruncateUnion(self): s = TruncatableSet(['bar', 'foo']) - s |= {'baz', 'qux'} + s |= set(['baz', 'qux']) s.truncate(3) - self.assertEqual(s, {'foo', 'baz', 'qux'}) + self.assertEqual(s, set(['foo', 'baz', 'qux'])) # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: From 5f582e3f5246c8a17871a48efc2a52088f0c01b5 Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Thu, 31 Jul 2014 22:52:16 +0200 Subject: [PATCH 3/5] Make AtomicFile support context managers. --- src/utils/file.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/utils/file.py b/src/utils/file.py index b2a698033..6c3f9f326 100644 --- a/src/utils/file.py +++ b/src/utils/file.py @@ -156,13 +156,19 @@ class AtomicFile(object): # self.__parent = super(AtomicFile, self) self._fd = open(self.tempFilename, mode) + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): + if exc_type: + self.rollback() + else: + self.close() + + @property def closed(self): return self._fd.closed - def close(self): - return self._fd.close() - def write(self, data): return self._fd.write(data) From 00e25f86d87ff78078115acaa7feb9269ac1981d Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Thu, 31 Jul 2014 22:52:44 +0200 Subject: [PATCH 4/5] Implement TruncatableSet.__repr__. --- src/utils/structures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/structures.py b/src/utils/structures.py index db2a7706b..65aae4423 100644 --- a/src/utils/structures.py +++ b/src/utils/structures.py @@ -460,6 +460,8 @@ class TruncatableSet(collections.MutableSet): def __init__(self, iterable=[]): self._ordered_items = list(iterable) self._items = set(self._ordered_items) + def __repr__(self): + return 'TruncatableSet({%r})' % self._items def __contains__(self, item): return item in self._items def __iter__(self): From b5911f8489005d1da0920263d25247ba52a7fba1 Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Thu, 31 Jul 2014 22:53:03 +0200 Subject: [PATCH 5/5] RSS: Implement persistence of announced headlines. --- plugins/RSS/plugin.py | 50 +++++++++++++++++++++++++++++++++++++------ plugins/RSS/test.py | 17 +++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/plugins/RSS/plugin.py b/plugins/RSS/plugin.py index 0978599a1..8eb82389f 100644 --- a/plugins/RSS/plugin.py +++ b/plugins/RSS/plugin.py @@ -29,13 +29,15 @@ # POSSIBILITY OF SUCH DAMAGE. ### +import re +import os +import sys +import json import time import types import string import socket import threading -import re -import sys import feedparser import supybot.conf as conf @@ -56,10 +58,13 @@ def get_feedName(irc, msg, args, state): state.args.append(callbacks.canonicalName(args.pop(0))) addConverter('feedName', get_feedName) +announced_headlines_filename = \ + conf.supybot.directories.data.dirize('RSS_announced.flat') + class Feed: __slots__ = ('url', 'name', 'data', 'last_update', 'entries', 'lock', 'announced_entries') - def __init__(self, name, url, plugin_is_loading=False): + def __init__(self, name, url, plugin_is_loading=False, announced=None): assert name, name if not url: assert utils.web.httpUrlRe.match(name), name @@ -72,7 +77,12 @@ class Feed: self.last_update = time.time() if plugin_is_loading else 0 self.entries = [] self.lock = threading.Lock() - self.announced_entries = utils.structures.TruncatableSet() + self.announced_entries = announced or \ + utils.structures.TruncatableSet() + + def __repr__(self): + return 'Feed(%r, %r, , %r)' % \ + (self.name, self.url, self.announced_entries) def get_command(self, plugin): docstring = format(_("""[] @@ -105,6 +115,14 @@ def sort_feed_items(items, order): return items return sitems +def load_announces_db(fd): + return dict((name, utils.structures.TruncatableSet(entries)) + for (name, entries) in json.load(fd).items()) +def save_announces_db(db, fd): + json.dump(dict((name, list(entries)) for (name, entries) in db), fd) + + + class RSS(callbacks.Plugin): """This plugin is useful both for announcing updates to RSS feeds in a channel, and for retrieving the headlines of RSS feeds via command. Use @@ -118,6 +136,11 @@ class RSS(callbacks.Plugin): self.feed_names = callbacks.CanonicalNameDict() # Scheme: {url: feed} self.feeds = {} + if os.path.isfile(announced_headlines_filename): + with open(announced_headlines_filename) as fd: + announced = load_announces_db(fd) + else: + announced = {} for name in self.registryValue('feeds'): self.assert_feed_does_not_exist(name) self.register_feed_config(name) @@ -126,7 +149,20 @@ class RSS(callbacks.Plugin): except registry.NonExistentRegistryEntry: self.log.warning('%s is not a registered feed, removing.',name) continue - self.register_feed(name, url, True) + self.register_feed(name, url, True, announced.get(name, [])) + world.flushers.append(self._flush) + + def die(self): + self._flush() + world.flushers.remove(self._flush) + self.__parent.die() + + def _flush(self): + l = [(f.name, f.announced_entries) for f in self.feeds.values()] + with utils.file.AtomicFile(announced_headlines_filename, 'wb', + backupDir='/dev/null') as fd: + save_announces_db(l, fd) + ################## # Feed registering @@ -141,9 +177,9 @@ class RSS(callbacks.Plugin): group = self.registryValue('feeds', value=False) conf.registerGlobalValue(group, name, registry.String(url, '')) - def register_feed(self, name, url, plugin_is_loading): + def register_feed(self, name, url, plugin_is_loading, announced=[]): self.feed_names[name] = url - self.feeds[url] = Feed(name, url, plugin_is_loading) + self.feeds[url] = Feed(name, url, plugin_is_loading, announced) def remove_feed(self, feed): del self.feed_names[feed.name] diff --git a/plugins/RSS/test.py b/plugins/RSS/test.py index eb20e4f57..ec193615a 100644 --- a/plugins/RSS/test.py +++ b/plugins/RSS/test.py @@ -81,6 +81,23 @@ class RSSTestCase(ChannelPluginTestCase): self._feedMsg('rss remove xkcd') feedparser._open_resource = old_open + def testAnnounceReload(self): + old_open = feedparser._open_resource + feedparser._open_resource = constant(xkcd_old) + try: + with conf.supybot.plugins.RSS.waitPeriod.context(1): + self.assertNotError('rss add xkcd http://xkcd.com/rss.xml') + self.assertNotError('rss announce add xkcd') + self.assertNotError(' ') + self.assertNotError('reload RSS') + self.assertNoResponse(' ') + time.sleep(1.1) + self.assertNoResponse(' ') + finally: + self._feedMsg('rss announce remove xkcd') + self._feedMsg('rss remove xkcd') + feedparser._open_resource = old_open + if network: def testRssinfo(self): self.assertNotError('rss info %s' % url)