callbacks: Make snarfers' output paginatable with @more

by moving the 'smart' reply() method from NestedCommandsIrcProxy
to ReplyIrcProxy.

There is no reason only commands should have a paginated output
and not snarfers defined in PluginRegexp.
This commit is contained in:
Valentin Lorentz 2021-04-16 23:38:44 +02:00
parent 24ca278b93
commit 3c1c4a69e9
2 changed files with 309 additions and 272 deletions

View File

@ -29,6 +29,7 @@
### ###
import os import os
import copy
import json import json
import functools import functools
import contextlib import contextlib
@ -50,6 +51,7 @@ from .test_data import (
OUTBOX_DATA, OUTBOX_DATA,
STATUS_URL, STATUS_URL,
STATUS_DATA, STATUS_DATA,
STATUS_VALUE,
STATUS_WITH_PHOTO_URL, STATUS_WITH_PHOTO_URL,
STATUS_WITH_PHOTO_DATA, STATUS_WITH_PHOTO_DATA,
OUTBOX_FIRSTPAGE_URL, OUTBOX_FIRSTPAGE_URL,
@ -450,6 +452,36 @@ class NetworklessFediverseTestCase(BaseFediverseTestCase):
+ "@ FirstAuthor I am replying to you", + "@ FirstAuthor I am replying to you",
) )
def testStatusUrlSnarferMore(self):
# Actually this is a test for src/callbacks.py, but it's easier to
# stick it here.
status_value = copy.deepcopy(STATUS_VALUE)
str_long = "l" + ("o" * 400) + "ng"
str_message = "mess" + ("a" * 400) + "ge"
status_obj = status_value["object"]
status_obj["content"] = status_obj["content"].replace(
"to you", "to you with a " + str_long + " " + str_message
)
with conf.supybot.plugins.Fediverse.snarfers.status.context(True):
expected_requests = [
(STATUS_URL, json.dumps(status_value).encode()),
(ACTOR_URL, ACTOR_DATA),
]
with self.mockWebfingerSupport(True), self.mockRequests(
expected_requests
):
self.assertSnarfResponse(
"aaa https://example.org/users/someuser/statuses/1234 bbb",
"\x02someuser\x02 (@someuser@example.org): "
+ "@ FirstAuthor I am replying to you with a "
+ " \x02(2 more messages)\x02",
)
self.assertNoResponse(" ")
self.assertResponse("more", str_long + " \x02(1 more message)\x02")
self.assertResponse("more", str_message)
def testStatusUrlSnarferErrors(self): def testStatusUrlSnarferErrors(self):
with conf.supybot.plugins.Fediverse.snarfers.status.context(True): with conf.supybot.plugins.Fediverse.snarfers.status.context(True):
expected_requests = [(STATUS_URL, utils.web.Error("blah"))] expected_requests = [(STATUS_URL, utils.web.Error("blah"))]

View File

@ -627,6 +627,7 @@ class ReplyIrcProxy(RichReplyMethods):
"""This class is a thin wrapper around an irclib.Irc object that gives it """This class is a thin wrapper around an irclib.Irc object that gives it
the reply() and error() methods (as well as everything in RichReplyMethods, the reply() and error() methods (as well as everything in RichReplyMethods,
based on those two).""" based on those two)."""
_mores = ircutils.IrcDict()
def __init__(self, irc, msg): def __init__(self, irc, msg):
self.irc = irc self.irc = irc
self.msg = msg self.msg = msg
@ -660,24 +661,295 @@ class ReplyIrcProxy(RichReplyMethods):
self.irc.queueMsg(m) self.irc.queueMsg(m)
return m return m
def _defaultPrefixNick(self, msg):
if msg.channel:
return conf.get(conf.supybot.reply.withNickPrefix,
channel=msg.channel, network=self.irc.network)
else:
return conf.supybot.reply.withNickPrefix()
def reply(self, s, msg=None, **kwargs): def reply(self, s, msg=None, **kwargs):
"""
Keyword arguments:
:arg bool noLengthCheck:
True if the length shouldn't be checked (used for 'more' handling)
:arg bool prefixNick:
False if the nick shouldn't be prefixed to the reply.
:arg bool action:
True if the reply should be an action.
:arg bool private:
True if the reply should be in private.
:arg bool notice:
True if the reply should be noticed when the bot is configured
to do so.
:arg str to:
The nick or channel the reply should go to.
Defaults to msg.args[0] (or msg.nick if private)
:arg bool sendImmediately:
True if the reply should use sendMsg() which
bypasses conf.supybot.protocols.irc.throttleTime
and gets sent before any queued messages
"""
if msg is None: if msg is None:
msg = self.msg msg = self.msg
assert not isinstance(s, ircmsgs.IrcMsg), \ assert not isinstance(s, ircmsgs.IrcMsg), \
'Old code alert: there is no longer a "msg" argument to reply.' 'Old code alert: there is no longer a "msg" argument to reply.'
kwargs.pop('noLengthCheck', None) kwargs.pop('noLengthCheck', None)
m = _makeReply(self, msg, s, **kwargs) if 'target' not in kwargs:
self.irc.queueMsg(m) target = kwargs.get('private', False) and kwargs.get('to', None) \
return m or msg.args[0]
if 'prefixNick' not in kwargs:
kwargs['prefixNick'] = self._defaultPrefixNick(msg)
self._sendReply(s, target=target, msg=msg, **kwargs)
def __getattr__(self, attr): def __getattr__(self, attr):
return getattr(self.irc, attr) return getattr(self.irc, attr)
def _replyOverhead(self, target, targetNick, prefixNick):
"""Returns the number of bytes added to a PRIVMSG payload, either by
Limnoria itself or by the server.
Ignores tag bytes, as they are accounted for separatly."""
overhead = (
len(':')
+ len(self.irc.prefix.encode())
+ len(' PRIVMSG ')
+ len(target.encode())
+ len(' :')
+ len('\r\n')
)
if prefixNick and targetNick is not None:
overhead += len(targetNick) + len(': ')
return overhead
def _sendReply(self, s, target, msg, sendImmediately=False,
noLengthCheck=False, **kwargs):
if sendImmediately:
sendMsg = self.irc.sendMsg
else:
sendMsg = self.irc.queueMsg
if isinstance(self.irc, self.__class__):
s = s[:conf.supybot.reply.maximumLength()]
return self.irc.reply(s,
noLengthCheck=noLengthCheck,
**kwargs)
elif noLengthCheck:
# noLengthCheck only matters to NestedCommandsIrcProxy, so
# it's not used here. Just in case you were wondering.
m = _makeReply(self, msg, s, **kwargs)
sendMsg(m)
return m
else:
s = ircutils.safeArgument(s)
allowedLength = conf.get(conf.supybot.reply.mores.length,
channel=target, network=self.irc.network)
if not allowedLength: # 0 indicates this.
allowedLength = 512 - self._replyOverhead(
target, msg.nick, prefixNick=kwargs['prefixNick'])
maximumMores = conf.get(conf.supybot.reply.mores.maximum,
channel=target, network=self.irc.network)
maximumLength = allowedLength * maximumMores
if len(s) > maximumLength:
log.warning('Truncating to %s bytes from %s bytes.',
maximumLength, len(s))
s = s[:maximumLength]
s_size = len(s.encode()) if minisix.PY3 else len(s)
if s_size <= allowedLength or \
not conf.get(conf.supybot.reply.mores,
channel=target, network=self.irc.network):
# There's no need for action=self.action here because
# action implies noLengthCheck, which has already been
# handled. Let's stick an assert in here just in case.
assert not kwargs.get('action')
m = _makeReply(self, msg, s, **kwargs)
sendMsg(m)
return m
# The '(XX more messages)' may have not the same
# length in the current locale
allowedLength -= len(_('(XX more messages)')) + 1 # bold
chunks = ircutils.wrap(s, allowedLength)
# Last messages to display at the beginning of the list
# (which is used like a stack)
chunks.reverse()
instant = conf.get(conf.supybot.reply.mores.instant,
channel=target, network=self.irc.network)
# Big complex loop ahead, with lots of cases and opportunities for
# off-by-one errors. Here is the meaning of each of the variables
#
# * 'i' is the number of chunks after the current one
#
# * 'is_first' is True when the message is the very first message
# (so last iteration of the loop)
#
# * 'is_last' is True when the message is the very last (so first
# iteration of the loop)
#
# * 'is_instant' is True when the message is in one of the messages
# sent immediately when the command is called, ie. without
# calling @misc more. (when supybot.reply.mores.instant is 1,
# which is the default, this is equivalent to 'is_first')
#
# * 'is_last_instant' is True when the message is the last of the
# instant message (so the first iteration of the loop with an
# instant message).
#
# We need all this complexity because pagination is hard, and we
# want:
#
# * the '(XX more messages)' suffix on the last instant message,
# and every other message (mandatory, it's a great feature),
# but not on the other instant messages (mandatory when
# multiline is enabled, but very nice to have in general)
# * the nick prefix on the first message and every other message
# that isn't instant (mandatory), but not on the other instant
# messages (also mandatory only when multiline is enabled)
msgs = []
for (i, chunk) in enumerate(chunks):
is_first = i == len(chunks) - 1
is_last = i == 0
is_instant = len(chunks) - i <= instant
is_last_instant = len(chunks) - i == instant
if is_last:
# last message, no suffix to add
pass
elif is_instant and not is_last_instant:
# one of the first messages, and the next one will
# also be sent immediately, so no suffix
pass
else:
if i == 1:
more = _('more message')
else:
more = _('more messages')
n = ircutils.bold('(%i %s)' % (len(msgs), more))
chunk = '%s %s' % (chunk, n)
if is_instant and not is_first:
d = kwargs.copy()
d['prefixNick'] = False
msgs.append(_makeReply(self, msg, chunk, **d))
else:
msgs.append(_makeReply(self, msg, chunk, **kwargs))
instant_messages = []
while instant > 0 and msgs:
instant -= 1
response = msgs.pop()
instant_messages.append(response)
# XXX We should somehow allow these to be returned, but
# until someone complains, we'll be fine :) We
# can't return from here, though, for obvious
# reasons.
# return m
if conf.supybot.protocols.irc.experimentalExtensions() \
and 'draft/multiline' in self.state.capabilities_ack \
and len(instant_messages) > 1:
# More than one message to send now, and we are allowed to use
# multiline batches, so let's do it
self.queueMultilineBatches(
instant_messages, target, msg.nick, concat=True,
allowedLength=allowedLength, sendImmediately=sendImmediately)
else:
for instant_msg in instant_messages:
sendMsg(instant_msg)
if not msgs:
return
prefix = msg.prefix
if target and ircutils.isNick(target):
try:
state = self.getRealIrc().state
prefix = state.nickToHostmask(target)
except KeyError:
pass # We'll leave it as it is.
mask = prefix.split('!', 1)[1]
self._mores[mask] = msgs
public = bool(self.msg.channel)
private = kwargs.get('private', False) or not public
self._mores[msg.nick] = (private, msgs)
return response
def queueMultilineBatches(self, msgs, target, targetNick, concat,
allowedLength=0, sendImmediately=False):
"""Queues the msgs passed as argument in batches using draft/multiline
batches.
This errors if experimentalExtensions is disabled or draft/multiline
was not negotiated."""
assert conf.supybot.protocols.irc.experimentalExtensions()
assert 'draft/multiline' in self.state.capabilities_ack
if not allowedLength: # 0 indicates this.
# We're only interested in the overhead outside the payload,
# regardless of the entire payload (nick prefix included),
# so prefixNick=False
allowedLength = 512 - self._replyOverhead(
target, targetNick, prefixNick=False)
multiline_cap_values = ircutils.parseCapabilityKeyValue(
self.state.capabilities_ls['draft/multiline'])
# All the messages in instant_messages are to be sent
# immediately, in multiline batches.
max_bytes_per_batch = int(multiline_cap_values['max-bytes'])
# We have to honor max_bytes_per_batch, but I don't want to
# encode messages again here just to have their length, so
# let's assume they all have the maximum length.
# It's not optimal, but close enough and simplifies the code.
messages_per_batch = max_bytes_per_batch // allowedLength
# "Clients MUST NOT send tags other than draft/multiline-concat and
# batch on messages within the batch. In particular, all client-only
# tags associated with the message must be sent attached to the initial
# BATCH command."
# -- <https://ircv3.net/specs/extensions/multiline>
# So we copy the tags of the first message, discard the tags of all
# other messages, and apply the tags to the opening BATCH
server_tags = msgs[0].server_tags
for batch_msgs in utils.iter.grouper(msgs, messages_per_batch):
# TODO: should use sendBatch instead of queueBatch if
# sendImmediately is True
batch_name = ircutils.makeLabel()
batch = []
batch.append(ircmsgs.IrcMsg(command='BATCH',
args=('+' + batch_name, 'draft/multiline', target),
server_tags=server_tags))
for (i, batch_msg) in enumerate(batch_msgs):
if batch_msg is None:
continue # 'grouper' generates None at the end
assert 'batch' not in batch_msg.server_tags
# Discard the existing tags, and add the batch ones.
batch_msg.server_tags = {'batch': batch_name}
if concat and i > 0:
# Tell clients not to add a newline after this
batch_msg.server_tags['draft/multiline-concat'] = None
batch.append(batch_msg)
batch.append(ircmsgs.IrcMsg(
command='BATCH', args=('-' + batch_name,)))
self.queueBatch(batch)
SimpleProxy = ReplyIrcProxy # Backwards-compatibility SimpleProxy = ReplyIrcProxy # Backwards-compatibility
class NestedCommandsIrcProxy(ReplyIrcProxy): class NestedCommandsIrcProxy(ReplyIrcProxy):
"A proxy object to allow proper nesting of commands (even threaded ones)." "A proxy object to allow proper nesting of commands (even threaded ones)."
_mores = ircutils.IrcDict()
def __init__(self, irc, msg, args, nested=0): def __init__(self, irc, msg, args, nested=0):
assert isinstance(args, list), 'Args should be a list, not a string.' assert isinstance(args, list), 'Args should be a list, not a string.'
super(NestedCommandsIrcProxy, self).__init__(irc, msg) super(NestedCommandsIrcProxy, self).__init__(irc, msg)
@ -721,11 +993,7 @@ class NestedCommandsIrcProxy(ReplyIrcProxy):
self.notice = None self.notice = None
self.private = None self.private = None
self.noLengthCheck = None self.noLengthCheck = None
if self.msg.channel: self.prefixNick = self._defaultPrefixNick(self.msg)
self.prefixNick = conf.get(conf.supybot.reply.withNickPrefix,
channel=self.msg.channel, network=self.irc.network)
else:
self.prefixNick = conf.supybot.reply.withNickPrefix()
def evalArgs(self, withClass=None): def evalArgs(self, withClass=None):
while self.counter < len(self.args): while self.counter < len(self.args):
@ -900,34 +1168,6 @@ class NestedCommandsIrcProxy(ReplyIrcProxy):
def reply(self, s, noLengthCheck=False, prefixNick=None, action=None, def reply(self, s, noLengthCheck=False, prefixNick=None, action=None,
private=None, notice=None, to=None, msg=None, private=None, notice=None, to=None, msg=None,
sendImmediately=False, stripCtcp=True): sendImmediately=False, stripCtcp=True):
"""
Keyword arguments:
:arg bool noLengthCheck:
True if the length shouldn't be checked (used for 'more' handling)
:arg bool prefixNick:
False if the nick shouldn't be prefixed to the reply.
:arg bool action:
True if the reply should be an action.
:arg bool private:
True if the reply should be in private.
:arg bool notice:
True if the reply should be noticed when the bot is configured
to do so.
:arg str to:
The nick or channel the reply should go to.
Defaults to msg.args[0] (or msg.nick if private)
:arg bool sendImmediately:
True if the reply should use sendMsg() which
bypasses conf.supybot.protocols.irc.throttleTime
and gets sent before any queued messages
"""
# These use and or or based on whether or not they default to True or # These use and or or based on whether or not they default to True or
# False. Those that default to True use and; those that default to # False. Those that default to True use and; those that default to
# False use or. # False use or.
@ -981,241 +1221,6 @@ class NestedCommandsIrcProxy(ReplyIrcProxy):
self.args[self.counter] = s self.args[self.counter] = s
self.evalArgs() self.evalArgs()
def _replyOverhead(self, target, targetNick, prefixNick):
"""Returns the number of bytes added to a PRIVMSG payload, either by
Limnoria itself or by the server.
Ignores tag bytes, as they are accounted for separatly."""
overhead = (
len(':')
+ len(self.irc.prefix.encode())
+ len(' PRIVMSG ')
+ len(target.encode())
+ len(' :')
+ len('\r\n')
)
if prefixNick and targetNick is not None:
overhead += len(targetNick) + len(': ')
return overhead
def _sendReply(self, s, target, msg, sendImmediately,
noLengthCheck, **kwargs):
if sendImmediately:
sendMsg = self.irc.sendMsg
else:
sendMsg = self.irc.queueMsg
if isinstance(self.irc, self.__class__):
s = s[:conf.supybot.reply.maximumLength()]
return self.irc.reply(s,
noLengthCheck=self.noLengthCheck,
**kwargs)
elif noLengthCheck:
# noLengthCheck only matters to NestedCommandsIrcProxy, so
# it's not used here. Just in case you were wondering.
m = _makeReply(self, msg, s, **kwargs)
sendMsg(m)
return m
else:
s = ircutils.safeArgument(s)
allowedLength = conf.get(conf.supybot.reply.mores.length,
channel=target, network=self.irc.network)
if not allowedLength: # 0 indicates this.
allowedLength = 512 - self._replyOverhead(
target, msg.nick, prefixNick=kwargs['prefixNick'])
maximumMores = conf.get(conf.supybot.reply.mores.maximum,
channel=target, network=self.irc.network)
maximumLength = allowedLength * maximumMores
if len(s) > maximumLength:
log.warning('Truncating to %s bytes from %s bytes.',
maximumLength, len(s))
s = s[:maximumLength]
s_size = len(s.encode()) if minisix.PY3 else len(s)
if s_size <= allowedLength or \
not conf.get(conf.supybot.reply.mores,
channel=target, network=self.irc.network):
# There's no need for action=self.action here because
# action implies noLengthCheck, which has already been
# handled. Let's stick an assert in here just in case.
assert not kwargs.get('action')
m = _makeReply(self, msg, s, **kwargs)
sendMsg(m)
return m
# The '(XX more messages)' may have not the same
# length in the current locale
allowedLength -= len(_('(XX more messages)')) + 1 # bold
chunks = ircutils.wrap(s, allowedLength)
# Last messages to display at the beginning of the list
# (which is used like a stack)
chunks.reverse()
instant = conf.get(conf.supybot.reply.mores.instant,
channel=target, network=self.irc.network)
# Big complex loop ahead, with lots of cases and opportunities for
# off-by-one errors. Here is the meaning of each of the variables
#
# * 'i' is the number of chunks after the current one
#
# * 'is_first' is True when the message is the very first message
# (so last iteration of the loop)
#
# * 'is_last' is True when the message is the very last (so first
# iteration of the loop)
#
# * 'is_instant' is True when the message is in one of the messages
# sent immediately when the command is called, ie. without
# calling @misc more. (when supybot.reply.mores.instant is 1,
# which is the default, this is equivalent to 'is_first')
#
# * 'is_last_instant' is True when the message is the last of the
# instant message (so the first iteration of the loop with an
# instant message).
#
# We need all this complexity because pagination is hard, and we
# want:
#
# * the '(XX more messages)' suffix on the last instant message,
# and every other message (mandatory, it's a great feature),
# but not on the other instant messages (mandatory when
# multiline is enabled, but very nice to have in general)
# * the nick prefix on the first message and every other message
# that isn't instant (mandatory), but not on the other instant
# messages (also mandatory only when multiline is enabled)
msgs = []
for (i, chunk) in enumerate(chunks):
is_first = i == len(chunks) - 1
is_last = i == 0
is_instant = len(chunks) - i <= instant
is_last_instant = len(chunks) - i == instant
if is_last:
# last message, no suffix to add
pass
elif is_instant and not is_last_instant:
# one of the first messages, and the next one will
# also be sent immediately, so no suffix
pass
else:
if i == 1:
more = _('more message')
else:
more = _('more messages')
n = ircutils.bold('(%i %s)' % (len(msgs), more))
chunk = '%s %s' % (chunk, n)
if is_instant and not is_first:
d = kwargs.copy()
d['prefixNick'] = False
msgs.append(_makeReply(self, msg, chunk, **d))
else:
msgs.append(_makeReply(self, msg, chunk, **kwargs))
instant_messages = []
while instant > 0 and msgs:
instant -= 1
response = msgs.pop()
instant_messages.append(response)
# XXX We should somehow allow these to be returned, but
# until someone complains, we'll be fine :) We
# can't return from here, though, for obvious
# reasons.
# return m
if conf.supybot.protocols.irc.experimentalExtensions() \
and 'draft/multiline' in self.state.capabilities_ack \
and len(instant_messages) > 1:
# More than one message to send now, and we are allowed to use
# multiline batches, so let's do it
self.queueMultilineBatches(
instant_messages, target, msg.nick, concat=True,
allowedLength=allowedLength, sendImmediately=sendImmediately)
else:
for instant_msg in instant_messages:
sendMsg(instant_msg)
if not msgs:
return
prefix = msg.prefix
to = kwargs['to']
if to and ircutils.isNick(to):
try:
state = self.getRealIrc().state
prefix = state.nickToHostmask(to)
except KeyError:
pass # We'll leave it as it is.
mask = prefix.split('!', 1)[1]
self._mores[mask] = msgs
public = bool(self.msg.channel)
private = kwargs['private'] or not public
self._mores[msg.nick] = (private, msgs)
return response
def queueMultilineBatches(self, msgs, target, targetNick, concat,
allowedLength=0, sendImmediately=False):
"""Queues the msgs passed as argument in batches using draft/multiline
batches.
This errors if experimentalExtensions is disabled or draft/multiline
was not negotiated."""
assert conf.supybot.protocols.irc.experimentalExtensions()
assert 'draft/multiline' in self.state.capabilities_ack
if not allowedLength: # 0 indicates this.
# We're only interested in the overhead outside the payload,
# regardless of the entire payload (nick prefix included),
# so prefixNick=False
allowedLength = 512 - self._replyOverhead(
target, targetNick, prefixNick=False)
multiline_cap_values = ircutils.parseCapabilityKeyValue(
self.state.capabilities_ls['draft/multiline'])
# All the messages in instant_messages are to be sent
# immediately, in multiline batches.
max_bytes_per_batch = int(multiline_cap_values['max-bytes'])
# We have to honor max_bytes_per_batch, but I don't want to
# encode messages again here just to have their length, so
# let's assume they all have the maximum length.
# It's not optimal, but close enough and simplifies the code.
messages_per_batch = max_bytes_per_batch // allowedLength
# "Clients MUST NOT send tags other than draft/multiline-concat and
# batch on messages within the batch. In particular, all client-only
# tags associated with the message must be sent attached to the initial
# BATCH command."
# -- <https://ircv3.net/specs/extensions/multiline>
# So we copy the tags of the first message, discard the tags of all
# other messages, and apply the tags to the opening BATCH
server_tags = msgs[0].server_tags
for batch_msgs in utils.iter.grouper(msgs, messages_per_batch):
# TODO: should use sendBatch instead of queueBatch if
# sendImmediately is True
batch_name = ircutils.makeLabel()
batch = []
batch.append(ircmsgs.IrcMsg(command='BATCH',
args=('+' + batch_name, 'draft/multiline', target),
server_tags=server_tags))
for (i, batch_msg) in enumerate(batch_msgs):
if batch_msg is None:
continue # 'grouper' generates None at the end
assert 'batch' not in batch_msg.server_tags
# Discard the existing tags, and add the batch ones.
batch_msg.server_tags = {'batch': batch_name}
if concat and i > 0:
# Tell clients not to add a newline after this
batch_msg.server_tags['draft/multiline-concat'] = None
batch.append(batch_msg)
batch.append(ircmsgs.IrcMsg(
command='BATCH', args=('-' + batch_name,)))
self.queueBatch(batch)
def noReply(self, msg=None): def noReply(self, msg=None):
if msg is None: if msg is None:
msg = self.msg msg = self.msg