From 3c1c4a69e9927bcc7265b1d77fd1ab49cb55090e Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Fri, 16 Apr 2021 23:38:44 +0200 Subject: [PATCH] callbacks: Make snarfers' output paginatable with @more by moving the 'smart' reply() method from NestedCommandsIrcProxy to ReplyIrcProxy. There is no reason only commands should have a paginated output and not snarfers defined in PluginRegexp. --- plugins/Fediverse/test.py | 32 +++ src/callbacks.py | 549 +++++++++++++++++++------------------- 2 files changed, 309 insertions(+), 272 deletions(-) diff --git a/plugins/Fediverse/test.py b/plugins/Fediverse/test.py index 36b4ca648..6d75a920d 100644 --- a/plugins/Fediverse/test.py +++ b/plugins/Fediverse/test.py @@ -29,6 +29,7 @@ ### import os +import copy import json import functools import contextlib @@ -50,6 +51,7 @@ from .test_data import ( OUTBOX_DATA, STATUS_URL, STATUS_DATA, + STATUS_VALUE, STATUS_WITH_PHOTO_URL, STATUS_WITH_PHOTO_DATA, OUTBOX_FIRSTPAGE_URL, @@ -450,6 +452,36 @@ class NetworklessFediverseTestCase(BaseFediverseTestCase): + "@ FirstAuthor I am replying to you", ) + def testStatusUrlSnarferMore(self): + # Actually this is a test for src/callbacks.py, but it's easier to + # stick it here. + status_value = copy.deepcopy(STATUS_VALUE) + str_long = "l" + ("o" * 400) + "ng" + str_message = "mess" + ("a" * 400) + "ge" + status_obj = status_value["object"] + status_obj["content"] = status_obj["content"].replace( + "to you", "to you with a " + str_long + " " + str_message + ) + with conf.supybot.plugins.Fediverse.snarfers.status.context(True): + expected_requests = [ + (STATUS_URL, json.dumps(status_value).encode()), + (ACTOR_URL, ACTOR_DATA), + ] + + with self.mockWebfingerSupport(True), self.mockRequests( + expected_requests + ): + self.assertSnarfResponse( + "aaa https://example.org/users/someuser/statuses/1234 bbb", + "\x02someuser\x02 (@someuser@example.org): " + + "@ FirstAuthor I am replying to you with a " + + " \x02(2 more messages)\x02", + ) + + self.assertNoResponse(" ") + self.assertResponse("more", str_long + " \x02(1 more message)\x02") + self.assertResponse("more", str_message) + def testStatusUrlSnarferErrors(self): with conf.supybot.plugins.Fediverse.snarfers.status.context(True): expected_requests = [(STATUS_URL, utils.web.Error("blah"))] diff --git a/src/callbacks.py b/src/callbacks.py index 1e380f6f1..9c9b05be0 100644 --- a/src/callbacks.py +++ b/src/callbacks.py @@ -627,6 +627,7 @@ class ReplyIrcProxy(RichReplyMethods): """This class is a thin wrapper around an irclib.Irc object that gives it the reply() and error() methods (as well as everything in RichReplyMethods, based on those two).""" + _mores = ircutils.IrcDict() def __init__(self, irc, msg): self.irc = irc self.msg = msg @@ -660,24 +661,295 @@ class ReplyIrcProxy(RichReplyMethods): self.irc.queueMsg(m) return m + def _defaultPrefixNick(self, msg): + if msg.channel: + return conf.get(conf.supybot.reply.withNickPrefix, + channel=msg.channel, network=self.irc.network) + else: + return conf.supybot.reply.withNickPrefix() + def reply(self, s, msg=None, **kwargs): + """ + Keyword arguments: + + :arg bool noLengthCheck: + True if the length shouldn't be checked (used for 'more' handling) + + :arg bool prefixNick: + False if the nick shouldn't be prefixed to the reply. + + :arg bool action: + True if the reply should be an action. + + :arg bool private: + True if the reply should be in private. + + :arg bool notice: + True if the reply should be noticed when the bot is configured + to do so. + + :arg str to: + The nick or channel the reply should go to. + Defaults to msg.args[0] (or msg.nick if private) + + :arg bool sendImmediately: + True if the reply should use sendMsg() which + bypasses conf.supybot.protocols.irc.throttleTime + and gets sent before any queued messages + """ if msg is None: msg = self.msg assert not isinstance(s, ircmsgs.IrcMsg), \ 'Old code alert: there is no longer a "msg" argument to reply.' kwargs.pop('noLengthCheck', None) - m = _makeReply(self, msg, s, **kwargs) - self.irc.queueMsg(m) - return m + if 'target' not in kwargs: + target = kwargs.get('private', False) and kwargs.get('to', None) \ + or msg.args[0] + if 'prefixNick' not in kwargs: + kwargs['prefixNick'] = self._defaultPrefixNick(msg) + self._sendReply(s, target=target, msg=msg, **kwargs) def __getattr__(self, attr): return getattr(self.irc, attr) + def _replyOverhead(self, target, targetNick, prefixNick): + """Returns the number of bytes added to a PRIVMSG payload, either by + Limnoria itself or by the server. + Ignores tag bytes, as they are accounted for separatly.""" + overhead = ( + len(':') + + len(self.irc.prefix.encode()) + + len(' PRIVMSG ') + + len(target.encode()) + + len(' :') + + len('\r\n') + ) + if prefixNick and targetNick is not None: + overhead += len(targetNick) + len(': ') + return overhead + + def _sendReply(self, s, target, msg, sendImmediately=False, + noLengthCheck=False, **kwargs): + if sendImmediately: + sendMsg = self.irc.sendMsg + else: + sendMsg = self.irc.queueMsg + + if isinstance(self.irc, self.__class__): + s = s[:conf.supybot.reply.maximumLength()] + return self.irc.reply(s, + noLengthCheck=noLengthCheck, + **kwargs) + elif noLengthCheck: + # noLengthCheck only matters to NestedCommandsIrcProxy, so + # it's not used here. Just in case you were wondering. + m = _makeReply(self, msg, s, **kwargs) + sendMsg(m) + return m + else: + s = ircutils.safeArgument(s) + allowedLength = conf.get(conf.supybot.reply.mores.length, + channel=target, network=self.irc.network) + if not allowedLength: # 0 indicates this. + allowedLength = 512 - self._replyOverhead( + target, msg.nick, prefixNick=kwargs['prefixNick']) + maximumMores = conf.get(conf.supybot.reply.mores.maximum, + channel=target, network=self.irc.network) + maximumLength = allowedLength * maximumMores + if len(s) > maximumLength: + log.warning('Truncating to %s bytes from %s bytes.', + maximumLength, len(s)) + s = s[:maximumLength] + s_size = len(s.encode()) if minisix.PY3 else len(s) + if s_size <= allowedLength or \ + not conf.get(conf.supybot.reply.mores, + channel=target, network=self.irc.network): + # There's no need for action=self.action here because + # action implies noLengthCheck, which has already been + # handled. Let's stick an assert in here just in case. + assert not kwargs.get('action') + m = _makeReply(self, msg, s, **kwargs) + sendMsg(m) + return m + # The '(XX more messages)' may have not the same + # length in the current locale + allowedLength -= len(_('(XX more messages)')) + 1 # bold + chunks = ircutils.wrap(s, allowedLength) + + # Last messages to display at the beginning of the list + # (which is used like a stack) + chunks.reverse() + + instant = conf.get(conf.supybot.reply.mores.instant, + channel=target, network=self.irc.network) + + # Big complex loop ahead, with lots of cases and opportunities for + # off-by-one errors. Here is the meaning of each of the variables + # + # * 'i' is the number of chunks after the current one + # + # * 'is_first' is True when the message is the very first message + # (so last iteration of the loop) + # + # * 'is_last' is True when the message is the very last (so first + # iteration of the loop) + # + # * 'is_instant' is True when the message is in one of the messages + # sent immediately when the command is called, ie. without + # calling @misc more. (when supybot.reply.mores.instant is 1, + # which is the default, this is equivalent to 'is_first') + # + # * 'is_last_instant' is True when the message is the last of the + # instant message (so the first iteration of the loop with an + # instant message). + # + # We need all this complexity because pagination is hard, and we + # want: + # + # * the '(XX more messages)' suffix on the last instant message, + # and every other message (mandatory, it's a great feature), + # but not on the other instant messages (mandatory when + # multiline is enabled, but very nice to have in general) + # * the nick prefix on the first message and every other message + # that isn't instant (mandatory), but not on the other instant + # messages (also mandatory only when multiline is enabled) + msgs = [] + for (i, chunk) in enumerate(chunks): + is_first = i == len(chunks) - 1 + is_last = i == 0 + is_instant = len(chunks) - i <= instant + is_last_instant = len(chunks) - i == instant + if is_last: + # last message, no suffix to add + pass + elif is_instant and not is_last_instant: + # one of the first messages, and the next one will + # also be sent immediately, so no suffix + pass + else: + if i == 1: + more = _('more message') + else: + more = _('more messages') + n = ircutils.bold('(%i %s)' % (len(msgs), more)) + chunk = '%s %s' % (chunk, n) + + if is_instant and not is_first: + d = kwargs.copy() + d['prefixNick'] = False + msgs.append(_makeReply(self, msg, chunk, **d)) + else: + msgs.append(_makeReply(self, msg, chunk, **kwargs)) + + instant_messages = [] + + while instant > 0 and msgs: + instant -= 1 + response = msgs.pop() + instant_messages.append(response) + # XXX We should somehow allow these to be returned, but + # until someone complains, we'll be fine :) We + # can't return from here, though, for obvious + # reasons. + # return m + + if conf.supybot.protocols.irc.experimentalExtensions() \ + and 'draft/multiline' in self.state.capabilities_ack \ + and len(instant_messages) > 1: + # More than one message to send now, and we are allowed to use + # multiline batches, so let's do it + self.queueMultilineBatches( + instant_messages, target, msg.nick, concat=True, + allowedLength=allowedLength, sendImmediately=sendImmediately) + else: + for instant_msg in instant_messages: + sendMsg(instant_msg) + + if not msgs: + return + prefix = msg.prefix + if target and ircutils.isNick(target): + try: + state = self.getRealIrc().state + prefix = state.nickToHostmask(target) + except KeyError: + pass # We'll leave it as it is. + mask = prefix.split('!', 1)[1] + self._mores[mask] = msgs + public = bool(self.msg.channel) + private = kwargs.get('private', False) or not public + self._mores[msg.nick] = (private, msgs) + return response + + def queueMultilineBatches(self, msgs, target, targetNick, concat, + allowedLength=0, sendImmediately=False): + """Queues the msgs passed as argument in batches using draft/multiline + batches. + + This errors if experimentalExtensions is disabled or draft/multiline + was not negotiated.""" + assert conf.supybot.protocols.irc.experimentalExtensions() + assert 'draft/multiline' in self.state.capabilities_ack + + if not allowedLength: # 0 indicates this. + # We're only interested in the overhead outside the payload, + # regardless of the entire payload (nick prefix included), + # so prefixNick=False + allowedLength = 512 - self._replyOverhead( + target, targetNick, prefixNick=False) + + multiline_cap_values = ircutils.parseCapabilityKeyValue( + self.state.capabilities_ls['draft/multiline']) + + # All the messages in instant_messages are to be sent + # immediately, in multiline batches. + max_bytes_per_batch = int(multiline_cap_values['max-bytes']) + + # We have to honor max_bytes_per_batch, but I don't want to + # encode messages again here just to have their length, so + # let's assume they all have the maximum length. + # It's not optimal, but close enough and simplifies the code. + messages_per_batch = max_bytes_per_batch // allowedLength + + # "Clients MUST NOT send tags other than draft/multiline-concat and + # batch on messages within the batch. In particular, all client-only + # tags associated with the message must be sent attached to the initial + # BATCH command." + # -- + # So we copy the tags of the first message, discard the tags of all + # other messages, and apply the tags to the opening BATCH + server_tags = msgs[0].server_tags + + for batch_msgs in utils.iter.grouper(msgs, messages_per_batch): + # TODO: should use sendBatch instead of queueBatch if + # sendImmediately is True + batch_name = ircutils.makeLabel() + batch = [] + batch.append(ircmsgs.IrcMsg(command='BATCH', + args=('+' + batch_name, 'draft/multiline', target), + server_tags=server_tags)) + + for (i, batch_msg) in enumerate(batch_msgs): + if batch_msg is None: + continue # 'grouper' generates None at the end + assert 'batch' not in batch_msg.server_tags + + # Discard the existing tags, and add the batch ones. + batch_msg.server_tags = {'batch': batch_name} + if concat and i > 0: + # Tell clients not to add a newline after this + batch_msg.server_tags['draft/multiline-concat'] = None + batch.append(batch_msg) + + batch.append(ircmsgs.IrcMsg( + command='BATCH', args=('-' + batch_name,))) + + self.queueBatch(batch) + SimpleProxy = ReplyIrcProxy # Backwards-compatibility class NestedCommandsIrcProxy(ReplyIrcProxy): "A proxy object to allow proper nesting of commands (even threaded ones)." - _mores = ircutils.IrcDict() def __init__(self, irc, msg, args, nested=0): assert isinstance(args, list), 'Args should be a list, not a string.' super(NestedCommandsIrcProxy, self).__init__(irc, msg) @@ -721,11 +993,7 @@ class NestedCommandsIrcProxy(ReplyIrcProxy): self.notice = None self.private = None self.noLengthCheck = None - if self.msg.channel: - self.prefixNick = conf.get(conf.supybot.reply.withNickPrefix, - channel=self.msg.channel, network=self.irc.network) - else: - self.prefixNick = conf.supybot.reply.withNickPrefix() + self.prefixNick = self._defaultPrefixNick(self.msg) def evalArgs(self, withClass=None): while self.counter < len(self.args): @@ -900,34 +1168,6 @@ class NestedCommandsIrcProxy(ReplyIrcProxy): def reply(self, s, noLengthCheck=False, prefixNick=None, action=None, private=None, notice=None, to=None, msg=None, sendImmediately=False, stripCtcp=True): - """ - Keyword arguments: - - :arg bool noLengthCheck: - True if the length shouldn't be checked (used for 'more' handling) - - :arg bool prefixNick: - False if the nick shouldn't be prefixed to the reply. - - :arg bool action: - True if the reply should be an action. - - :arg bool private: - True if the reply should be in private. - - :arg bool notice: - True if the reply should be noticed when the bot is configured - to do so. - - :arg str to: - The nick or channel the reply should go to. - Defaults to msg.args[0] (or msg.nick if private) - - :arg bool sendImmediately: - True if the reply should use sendMsg() which - bypasses conf.supybot.protocols.irc.throttleTime - and gets sent before any queued messages - """ # These use and or or based on whether or not they default to True or # False. Those that default to True use and; those that default to # False use or. @@ -981,241 +1221,6 @@ class NestedCommandsIrcProxy(ReplyIrcProxy): self.args[self.counter] = s self.evalArgs() - def _replyOverhead(self, target, targetNick, prefixNick): - """Returns the number of bytes added to a PRIVMSG payload, either by - Limnoria itself or by the server. - Ignores tag bytes, as they are accounted for separatly.""" - overhead = ( - len(':') - + len(self.irc.prefix.encode()) - + len(' PRIVMSG ') - + len(target.encode()) - + len(' :') - + len('\r\n') - ) - if prefixNick and targetNick is not None: - overhead += len(targetNick) + len(': ') - return overhead - - def _sendReply(self, s, target, msg, sendImmediately, - noLengthCheck, **kwargs): - if sendImmediately: - sendMsg = self.irc.sendMsg - else: - sendMsg = self.irc.queueMsg - - if isinstance(self.irc, self.__class__): - s = s[:conf.supybot.reply.maximumLength()] - return self.irc.reply(s, - noLengthCheck=self.noLengthCheck, - **kwargs) - elif noLengthCheck: - # noLengthCheck only matters to NestedCommandsIrcProxy, so - # it's not used here. Just in case you were wondering. - m = _makeReply(self, msg, s, **kwargs) - sendMsg(m) - return m - else: - s = ircutils.safeArgument(s) - allowedLength = conf.get(conf.supybot.reply.mores.length, - channel=target, network=self.irc.network) - if not allowedLength: # 0 indicates this. - allowedLength = 512 - self._replyOverhead( - target, msg.nick, prefixNick=kwargs['prefixNick']) - maximumMores = conf.get(conf.supybot.reply.mores.maximum, - channel=target, network=self.irc.network) - maximumLength = allowedLength * maximumMores - if len(s) > maximumLength: - log.warning('Truncating to %s bytes from %s bytes.', - maximumLength, len(s)) - s = s[:maximumLength] - s_size = len(s.encode()) if minisix.PY3 else len(s) - if s_size <= allowedLength or \ - not conf.get(conf.supybot.reply.mores, - channel=target, network=self.irc.network): - # There's no need for action=self.action here because - # action implies noLengthCheck, which has already been - # handled. Let's stick an assert in here just in case. - assert not kwargs.get('action') - m = _makeReply(self, msg, s, **kwargs) - sendMsg(m) - return m - # The '(XX more messages)' may have not the same - # length in the current locale - allowedLength -= len(_('(XX more messages)')) + 1 # bold - chunks = ircutils.wrap(s, allowedLength) - - # Last messages to display at the beginning of the list - # (which is used like a stack) - chunks.reverse() - - instant = conf.get(conf.supybot.reply.mores.instant, - channel=target, network=self.irc.network) - - # Big complex loop ahead, with lots of cases and opportunities for - # off-by-one errors. Here is the meaning of each of the variables - # - # * 'i' is the number of chunks after the current one - # - # * 'is_first' is True when the message is the very first message - # (so last iteration of the loop) - # - # * 'is_last' is True when the message is the very last (so first - # iteration of the loop) - # - # * 'is_instant' is True when the message is in one of the messages - # sent immediately when the command is called, ie. without - # calling @misc more. (when supybot.reply.mores.instant is 1, - # which is the default, this is equivalent to 'is_first') - # - # * 'is_last_instant' is True when the message is the last of the - # instant message (so the first iteration of the loop with an - # instant message). - # - # We need all this complexity because pagination is hard, and we - # want: - # - # * the '(XX more messages)' suffix on the last instant message, - # and every other message (mandatory, it's a great feature), - # but not on the other instant messages (mandatory when - # multiline is enabled, but very nice to have in general) - # * the nick prefix on the first message and every other message - # that isn't instant (mandatory), but not on the other instant - # messages (also mandatory only when multiline is enabled) - msgs = [] - for (i, chunk) in enumerate(chunks): - is_first = i == len(chunks) - 1 - is_last = i == 0 - is_instant = len(chunks) - i <= instant - is_last_instant = len(chunks) - i == instant - if is_last: - # last message, no suffix to add - pass - elif is_instant and not is_last_instant: - # one of the first messages, and the next one will - # also be sent immediately, so no suffix - pass - else: - if i == 1: - more = _('more message') - else: - more = _('more messages') - n = ircutils.bold('(%i %s)' % (len(msgs), more)) - chunk = '%s %s' % (chunk, n) - - if is_instant and not is_first: - d = kwargs.copy() - d['prefixNick'] = False - msgs.append(_makeReply(self, msg, chunk, **d)) - else: - msgs.append(_makeReply(self, msg, chunk, **kwargs)) - - instant_messages = [] - - while instant > 0 and msgs: - instant -= 1 - response = msgs.pop() - instant_messages.append(response) - # XXX We should somehow allow these to be returned, but - # until someone complains, we'll be fine :) We - # can't return from here, though, for obvious - # reasons. - # return m - - if conf.supybot.protocols.irc.experimentalExtensions() \ - and 'draft/multiline' in self.state.capabilities_ack \ - and len(instant_messages) > 1: - # More than one message to send now, and we are allowed to use - # multiline batches, so let's do it - self.queueMultilineBatches( - instant_messages, target, msg.nick, concat=True, - allowedLength=allowedLength, sendImmediately=sendImmediately) - else: - for instant_msg in instant_messages: - sendMsg(instant_msg) - - if not msgs: - return - prefix = msg.prefix - to = kwargs['to'] - if to and ircutils.isNick(to): - try: - state = self.getRealIrc().state - prefix = state.nickToHostmask(to) - except KeyError: - pass # We'll leave it as it is. - mask = prefix.split('!', 1)[1] - self._mores[mask] = msgs - public = bool(self.msg.channel) - private = kwargs['private'] or not public - self._mores[msg.nick] = (private, msgs) - return response - - def queueMultilineBatches(self, msgs, target, targetNick, concat, - allowedLength=0, sendImmediately=False): - """Queues the msgs passed as argument in batches using draft/multiline - batches. - - This errors if experimentalExtensions is disabled or draft/multiline - was not negotiated.""" - assert conf.supybot.protocols.irc.experimentalExtensions() - assert 'draft/multiline' in self.state.capabilities_ack - - if not allowedLength: # 0 indicates this. - # We're only interested in the overhead outside the payload, - # regardless of the entire payload (nick prefix included), - # so prefixNick=False - allowedLength = 512 - self._replyOverhead( - target, targetNick, prefixNick=False) - - multiline_cap_values = ircutils.parseCapabilityKeyValue( - self.state.capabilities_ls['draft/multiline']) - - # All the messages in instant_messages are to be sent - # immediately, in multiline batches. - max_bytes_per_batch = int(multiline_cap_values['max-bytes']) - - # We have to honor max_bytes_per_batch, but I don't want to - # encode messages again here just to have their length, so - # let's assume they all have the maximum length. - # It's not optimal, but close enough and simplifies the code. - messages_per_batch = max_bytes_per_batch // allowedLength - - # "Clients MUST NOT send tags other than draft/multiline-concat and - # batch on messages within the batch. In particular, all client-only - # tags associated with the message must be sent attached to the initial - # BATCH command." - # -- - # So we copy the tags of the first message, discard the tags of all - # other messages, and apply the tags to the opening BATCH - server_tags = msgs[0].server_tags - - for batch_msgs in utils.iter.grouper(msgs, messages_per_batch): - # TODO: should use sendBatch instead of queueBatch if - # sendImmediately is True - batch_name = ircutils.makeLabel() - batch = [] - batch.append(ircmsgs.IrcMsg(command='BATCH', - args=('+' + batch_name, 'draft/multiline', target), - server_tags=server_tags)) - - for (i, batch_msg) in enumerate(batch_msgs): - if batch_msg is None: - continue # 'grouper' generates None at the end - assert 'batch' not in batch_msg.server_tags - - # Discard the existing tags, and add the batch ones. - batch_msg.server_tags = {'batch': batch_name} - if concat and i > 0: - # Tell clients not to add a newline after this - batch_msg.server_tags['draft/multiline-concat'] = None - batch.append(batch_msg) - - batch.append(ircmsgs.IrcMsg( - command='BATCH', args=('-' + batch_name,))) - - self.queueBatch(batch) - def noReply(self, msg=None): if msg is None: msg = self.msg