Store ignored hostmasks in Expiring HostmaskSet to prevent their pattern cache from expiring too soon

See e0fdcb67c0 for the rationale
(tl;dr: prevents triggering a degenerate case of the LRU cache when
there are over 1000 ignore masks)
This commit is contained in:
Valentin Lorentz 2021-05-30 19:06:19 +02:00
parent 698dfc2fca
commit 43aada5b33
3 changed files with 88 additions and 20 deletions

View File

@ -385,7 +385,7 @@ class IrcChannel(object):
self.defaultAllow = defaultAllow self.defaultAllow = defaultAllow
self.expiredBans = [] self.expiredBans = []
self.bans = bans or {} self.bans = bans or {}
self.ignores = ignores or {} self.ignores = ircutils.ExpiringHostmaskDict(ignores)
self.silences = silences or [] self.silences = silences or []
self.exceptions = exceptions or [] self.exceptions = exceptions or []
self.capabilities = capabilities or CapabilitySet() self.capabilities = capabilities or CapabilitySet()
@ -471,14 +471,8 @@ class IrcChannel(object):
assert ircutils.isUserHostmask(hostmask), 'got %s' % hostmask assert ircutils.isUserHostmask(hostmask), 'got %s' % hostmask
if self.checkBan(hostmask): if self.checkBan(hostmask):
return True return True
now = time.time() if self.ignores.match(hostmask):
for (pattern, expiration) in list(self.ignores.items()):
if now < expiration or not expiration:
if ircutils.hostmaskPatternEqual(pattern, hostmask):
return True return True
else:
del self.ignores[pattern]
# Later we may wish to keep expiredIgnores, but not now.
return False return False
def preserve(self, fd, indent=''): def preserve(self, fd, indent=''):
@ -1059,7 +1053,7 @@ class IgnoresDB(object):
__slots__ = ('filename', 'hostmasks') __slots__ = ('filename', 'hostmasks')
def __init__(self): def __init__(self):
self.filename = None self.filename = None
self.hostmasks = {} self.hostmasks = ircutils.ExpiringHostmaskDict()
def open(self, filename): def open(self, filename):
self.filename = filename self.filename = filename
@ -1098,26 +1092,20 @@ class IgnoresDB(object):
def reload(self): def reload(self):
if self.filename is not None: if self.filename is not None:
oldhostmasks = self.hostmasks.copy() oldhostmasks = list(self.hostmasks.items())
self.hostmasks.clear() self.hostmasks.clear()
try: try:
self.open(self.filename) self.open(self.filename)
except EnvironmentError as e: except EnvironmentError as e:
log.warning('IgnoresDB.reload failed: %s', e) log.warning('IgnoresDB.reload failed: %s', e)
# Let's be somewhat transactional. # Let's be somewhat transactional.
self.hostmasks.update(oldhostmasks) for (hostmask, expiration) in oldhostmasks:
self.hostmasks.add(hostmask, expiration)
else: else:
log.warning('IgnoresDB.reload called without self.filename.') log.warning('IgnoresDB.reload called without self.filename.')
def checkIgnored(self, prefix): def checkIgnored(self, prefix):
now = time.time() return bool(self.hostmasks.match(prefix))
for (hostmask, expiration) in list(self.hostmasks.items()):
if expiration and now > expiration:
del self.hostmasks[hostmask]
else:
if ircutils.hostmaskPatternEqual(hostmask, prefix):
return True
return False
def add(self, hostmask, expiration=0): def add(self, hostmask, expiration=0):
assert ircutils.isUserHostmask(hostmask), 'got %s' % hostmask assert ircutils.isUserHostmask(hostmask), 'got %s' % hostmask

View File

@ -252,6 +252,60 @@ class HostmaskSet(collections.abc.MutableSet):
return pattern return pattern
return None return None
def __repr__(self):
return 'HostmaskSet(%r)' % (list(self.data),)
class ExpiringHostmaskDict(collections.abc.MutableMapping):
"""Like HostmaskSet, but behaves like a dict with expiration timestamps
as values."""
# To keep it thread-safe, add to self.patterns first, then
# self.data; and remove from self.data first.
# And never iterate on self.patterns
def __init__(self, hostmasks=None):
if isinstance(hostmasks, (list, tuple)):
hostmasks = dict(hostmasks)
self.data = hostmasks or {}
self.patterns = HostmaskSet(list(self.data))
def __getitem__(self, hostmask):
return self.data[hostmask]
def __setitem__(self, hostmask, expiration):
"""For backward compatibility, in case any plugin depends on it
being dict-like."""
self.patterns.add(hostmask)
self.data[hostmask] = expiration
def __iter__(self):
return iter(self.data)
def __len__(self):
return len(self.data)
def __delitem__(self, hostmask):
del self.data[hostmask]
self.patterns.discard(hostmask)
def expire(self):
now = time.time()
for (hostmask, expiration) in list(self.data.items()):
if now >= expiration and expiration:
self.pop(hostmask, None)
def match(self, hostname):
self.expire()
return self.patterns.match(hostname)
def clear(self):
self.data.clear()
self.patterns.clear()
def __repr__(self):
return 'ExpiringHostmaskSet(%r)' % (self.expirations,)
def banmask(hostmask): def banmask(hostmask):
"""Returns a properly generic banning hostmask for a hostmask. """Returns a properly generic banning hostmask for a hostmask.

View File

@ -73,6 +73,32 @@ class FunctionsTestCase(SupyTestCase):
hs = ircutils.HostmaskSet(["*!user@host"]) hs = ircutils.HostmaskSet(["*!user@host"])
self.assertEqual(hs.match("nick!user@host"), "*!user@host") self.assertEqual(hs.match("nick!user@host"), "*!user@host")
def testExpiringHostmaskDict(self):
hs = ircutils.ExpiringHostmaskDict()
self.assertEqual(hs.match("nick!user@host"), None)
time1 = time.time() + 15
time2 = time.time() + 10
hs["*!user@host"] = time1
hs["*!user@host2"] = time2
self.assertEqual(hs.match("nick!user@host"), "*!user@host")
self.assertEqual(hs.match("nick!user@host2"), "*!user@host2")
self.assertCountEqual(list(hs.items()),
[("*!user@host", time1), ("*!user@host2", time2)])
del hs["*!user@host2"]
self.assertEqual(hs.match("nick!user@host"), "*!user@host")
self.assertEqual(hs.match("nick!user@host2"), None)
timeFastForward(10)
self.assertEqual(hs.match("nick!user@host"), "*!user@host")
timeFastForward(10)
self.assertEqual(hs.match("nick!user@host"), None)
hs = ircutils.ExpiringHostmaskDict([("*!user@host", time.time() + 10)])
self.assertEqual(hs.match("nick!user@host"), "*!user@host")
self.assertEqual(hs.match("nick!user@host2"), None)
timeFastForward(11)
self.assertEqual(hs.match("nick!user@host"), None)
self.assertEqual(hs.match("nick!user@host2"), None)
def testIsUserHostmask(self): def testIsUserHostmask(self):
self.assertTrue(ircutils.isUserHostmask(self.hostmask)) self.assertTrue(ircutils.isUserHostmask(self.hostmask))
self.assertTrue(ircutils.isUserHostmask('a!b@c')) self.assertTrue(ircutils.isUserHostmask('a!b@c'))