Misc: Make @last handle --regexp in a single process for all messages

Spawning one process for each message was a little silly, considering
there can be thousands of messages.

Plus, some instances do reach the timeout after running for a few weeks,
so we really need to fix this.

Ideally, `regexp_wrapper` should also be removed from other plugins
(Todo, Notes, ...) as they have the same issues, but this will do for
now.
This commit is contained in:
Valentin Lorentz 2022-02-16 21:27:26 +01:00
parent 4b892c2b1d
commit 4e60d8812d

View File

@ -1,7 +1,7 @@
### ###
# Copyright (c) 2002-2005, Jeremiah Fincher # Copyright (c) 2002-2005, Jeremiah Fincher
# Copyright (c) 2009, James McCoy # Copyright (c) 2009, James McCoy
# Copyright (c) 2010-2021, Valentin Lorentz # Copyright (c) 2010-2022, Valentin Lorentz
# All rights reserved. # All rights reserved.
# #
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
@ -34,7 +34,9 @@ import os
import sys import sys
import json import json
import time import time
import queue
import functools import functools
import multiprocessing
import supybot import supybot
@ -43,6 +45,7 @@ import supybot.conf as conf
from supybot import commands from supybot import commands
import supybot.utils as utils import supybot.utils as utils
from supybot.commands import * from supybot.commands import *
from supybot.commands import ProcessTimeoutError
import supybot.ircdb as ircdb import supybot.ircdb as ircdb
import supybot.irclib as irclib import supybot.irclib as irclib
import supybot.utils.minisix as minisix import supybot.utils.minisix as minisix
@ -458,6 +461,9 @@ class Misc(callbacks.Plugin):
msg.channel) msg.channel)
else: else:
skipfirst = False skipfirst = False
final_predicates = []
for (option, arg) in optlist: for (option, arg) in optlist:
if option == 'from': if option == 'from':
def f(m, arg=arg): def f(m, arg=arg):
@ -482,22 +488,34 @@ class Misc(callbacks.Plugin):
return arg.lower() not in m.args[1].lower() return arg.lower() not in m.args[1].lower()
predicates.setdefault('without', []).append(f) predicates.setdefault('without', []).append(f)
elif option == 'regexp': elif option == 'regexp':
def f(m, arg=arg): def f(messages, arg=arg):
def f1(s, arg): reobj = re.compile(arg)
"""Since we can't enqueue match objects into the multiprocessing queue,
we'll just wrap the function to return bools.""" # using a queue to return results, so we can return at
if process(arg.search, s, timeout=0.1) is not None: # least some results in case of timeout
return True q = multiprocessing.Queue()
else:
return False def p(messages):
if ircmsgs.isAction(m): for m in messages:
m1 = ircmsgs.unAction(m) if ircmsgs.isAction(m):
else: s = ircmsgs.unAction(m)
m1 = m.args[1] else:
return regexp_wrapper(m1, reobj=arg, timeout=0.1, s = m.args[1]
plugin_name=self.name(), if reobj.search(s):
fcn_name='last') q.put(m)
predicates.setdefault('regexp', []).append(f) try:
process(p, messages, timeout=3.,
pn=self.name(), cn='last')
except ProcessTimeoutError:
pass
results = []
while True:
try:
results.append(q.get(False))
except queue.Empty:
break
return results
final_predicates.append(f)
elif option == 'nolimit': elif option == 'nolimit':
nolimit = True nolimit = True
iterable = filter(functools.partial(self._validLastMsg, irc), iterable = filter(functools.partial(self._validLastMsg, irc),
@ -531,24 +549,32 @@ class Misc(callbacks.Plugin):
showNick = False showNick = False
else: else:
showNick = True showNick = True
candidates = []
# Run predicates that filter on individual messages
for m in iterable: for m in iterable:
for predicate in predicates: for predicate in predicates:
try: if not predicate(m):
if not predicate(m): break
break
except RegexpTimeout:
irc.error(_('The regular expression timed out.'))
return
else: else:
if nolimit: candidates.append(m)
resp.append(ircmsgs.prettyPrint(m,
timestampFormat=tsf, # Run predicates that filter lists of messages
showNick=showNick)) for predicate in final_predicates:
else: candidates = predicate(candidates)
irc.reply(ircmsgs.prettyPrint(m,
timestampFormat=tsf, for m in candidates:
showNick=showNick)) if nolimit:
return resp.append(ircmsgs.prettyPrint(m,
timestampFormat=tsf,
showNick=showNick))
else:
irc.reply(ircmsgs.prettyPrint(m,
timestampFormat=tsf,
showNick=showNick))
return
if not resp: if not resp:
irc.error(_('I couldn\'t find a message matching that criteria in ' irc.error(_('I couldn\'t find a message matching that criteria in '
'my history of %s messages.') % len(irc.state.history)) 'my history of %s messages.') % len(irc.state.history))