From d1854cfc9bc8e82bc7c3cc2d78b682a9c7c85c66 Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Sun, 10 May 2020 14:52:55 +0200 Subject: [PATCH] Fediverse: Add URL snarfer. --- plugins/Fediverse/activitypub.py | 4 +- plugins/Fediverse/config.py | 27 ++++- plugins/Fediverse/plugin.py | 62 ++++++++-- plugins/Fediverse/test.py | 192 ++++++++++++++++++++++--------- 4 files changed, 217 insertions(+), 68 deletions(-) diff --git a/plugins/Fediverse/activitypub.py b/plugins/Fediverse/activitypub.py index fc4d0750b..99d556cbd 100644 --- a/plugins/Fediverse/activitypub.py +++ b/plugins/Fediverse/activitypub.py @@ -232,10 +232,10 @@ def actor_url(localuser, hostname): def get_actor(localuser, hostname): url = actor_url(localuser, hostname) - return get_actor_from_url(url) + return get_resource_from_url(url) -def get_actor_from_url(url): +def get_resource_from_url(url): content = signed_request(url, headers={"Accept": ACTIVITY_MIMETYPE}) assert content is not None diff --git a/plugins/Fediverse/config.py b/plugins/Fediverse/config.py index ec7e816ee..844e33daa 100644 --- a/plugins/Fediverse/config.py +++ b/plugins/Fediverse/config.py @@ -51,9 +51,10 @@ def configure(advanced): Fediverse = conf.registerPlugin("Fediverse") +conf.registerGroup(Fediverse, "snarfers") conf.registerChannelValue( - Fediverse, - "usernameSnarfer", + Fediverse.snarfers, + "username", registry.Boolean( False, _( @@ -62,6 +63,28 @@ conf.registerChannelValue( ), ), ) +conf.registerChannelValue( + Fediverse.snarfers, + "profile", + registry.Boolean( + False, + _( + """Determines whether the bot will output the profile of + URLs to Fediverse accounts it sees in channel messages.""" + ), + ), +) +conf.registerChannelValue( + Fediverse.snarfers, + "status", + registry.Boolean( + False, + _( + """Determines whether the bot will output the content of + statuses whose URLs it sees in channel messages.""" + ), + ), +) conf.registerGroup(Fediverse, "format") conf.registerGroup(Fediverse.format, "statuses") diff --git a/plugins/Fediverse/plugin.py b/plugins/Fediverse/plugin.py index 2174be1d5..ec82c55b6 100644 --- a/plugins/Fediverse/plugin.py +++ b/plugins/Fediverse/plugin.py @@ -34,7 +34,7 @@ import importlib import urllib.parse from supybot import utils, callbacks, httpserver -from supybot.commands import wrap +from supybot.commands import urlSnarfer, wrap from supybot.i18n import PluginInternationalization from . import activitypub as ap @@ -102,7 +102,7 @@ class FediverseHttp(httpserver.SupyHTTPServerCallback): ], "id": actor_url, "preferredUsername": hostname, - "type": "Person", + "type": "Service", "publicKey": { "id": actor_url + "#main-key", "owner": actor_url, @@ -117,7 +117,7 @@ class Fediverse(callbacks.PluginRegexp): """Fetches information from ActivityPub servers.""" threaded = True - regexps = ["usernameSnarfer"] + regexps = ["usernameSnarfer", "urlSnarfer_"] def __init__(self, irc): super().__init__(irc) @@ -152,7 +152,7 @@ class Fediverse(callbacks.PluginRegexp): match = utils.web.urlRe.match(username) if match: # TODO: error handling - actor = ap.get_actor_from_url(match.group(0)) + actor = ap.get_resource_from_url(match.group(0)) username = self._format_actor_username(actor) else: irc.errorInvalid("fediverse username", username) @@ -232,26 +232,64 @@ class Fediverse(callbacks.PluginRegexp): ) ) + def _format_profile(self, irc, msg, actor): + return _("\x02%s\x02 (%s): %s") % ( + actor["name"], + self._format_actor_username(actor), + utils.web.htmlToText(actor["summary"]), + ) + def usernameSnarfer(self, irc, msg, match): if callbacks.addressed(irc, msg): return + if not self.registryValue( + "snarfers.username", msg.channel, irc.network + ): + return try: actor = self._get_actor(irc, match.group(0)) except ap.ActivityPubError: # Be silent on errors return - irc.reply( - _("\x02%s\x02 (%s): %s") - % ( - actor["name"], - self._format_actor_username(actor), - utils.web.htmlToText(actor["summary"]), - ) - ) + irc.reply(self._format_profile(irc, msg, actor)) usernameSnarfer.__doc__ = _username_regexp.pattern + @urlSnarfer + def urlSnarfer_(self, irc, msg, match): + channel = msg.channel + network = irc.network + url = match.group(0) + if not channel: + return + if callbacks.addressed(irc, msg): + return + snarf_profile = self.registryValue( + "snarfers.profile", channel, network + ) + snarf_status = self.registryValue("snarfers.status", channel, network) + if not snarf_profile and not snarf_status: + return + try: + resource = ap.get_resource_from_url(url) + except ap.ActivityPubError: + return + + try: + if snarf_profile and resource["type"] in ("Person", "Service"): + irc.reply(self._format_profile(irc, msg, resource)) + elif snarf_status and resource["type"] in ( + "Create", + "Note", + "Announce", + ): + irc.reply(self._format_status(irc, msg, resource)) + except ap.ActivityPubError: + return + + urlSnarfer_.__doc__ = utils.web._httpUrlRe + @wrap(["somethingWithoutSpaces"]) def featured(self, irc, msg, args, username): """<@user@instance> diff --git a/plugins/Fediverse/test.py b/plugins/Fediverse/test.py index bf5be295f..8b13050d9 100644 --- a/plugins/Fediverse/test.py +++ b/plugins/Fediverse/test.py @@ -166,6 +166,60 @@ OUTBOX_VALUE = { } OUTBOX_DATA = json.dumps(OUTBOX_VALUE).encode() +STATUS_URL = "https://example.org/users/someuser/statuses/1234" +STATUS_VALUE = { + "id": "https://example.org/users/someuser/statuses/1234/activity", + "type": "Create", + "actor": "https://example.org/users/someuser", + "published": "2020-05-08T01:23:45Z", + "to": ["https://example.org/users/someuser/followers"], + "cc": [ + "https://www.w3.org/ns/activitystreams#Public", + "https://example.com/users/FirstAuthor", + ], + "object": { + "id": "https://example.org/users/someuser/statuses/1234", + "type": "Note", + "summary": None, + "inReplyTo": "https://example.com/users/FirstAuthor/statuses/42", + "published": "2020-05-08T01:23:45Z", + "url": "https://example.org/@FirstAuthor/42", + "attributedTo": "https://example.org/users/someuser", + "to": ["https://example.org/users/someuser/followers"], + "cc": [ + "https://www.w3.org/ns/activitystreams#Public", + "https://example.com/users/FirstAuthor", + ], + "sensitive": False, + "atomUri": "https://example.org/users/someuser/statuses/1234", + "inReplyToAtomUri": "https://example.com/users/FirstAuthor/statuses/42", + "conversation": "tag:example.com,2020-05-08:objectId=aaaa:objectType=Conversation", + "content": '

@FirstAuthor I am replying to you

', + "contentMap": { + "en": '

@FirstAuthor I am replying to you

' + }, + "attachment": [], + "tag": [ + { + "type": "Mention", + "href": "https://example.com/users/FirstAuthor", + "name": "@FirstAuthor@example.com", + } + ], + "replies": { + "id": "https://example.org/users/someuser/statuses/1234/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "next": "https://example.org/users/someuser/statuses/1234/replies?only_other_accounts=true&page=true", + "partOf": "https://example.org/users/someuser/statuses/1234/replies", + "items": [], + }, + }, + }, +} +STATUS_DATA = json.dumps(STATUS_VALUE).encode() + OUTBOX_FIRSTPAGE_URL = "https://example.org/users/someuser/outbox?page=true" OUTBOX_FIRSTPAGE_VALUE = { "@context": [ @@ -188,57 +242,7 @@ OUTBOX_FIRSTPAGE_VALUE = { "prev": "https://example.org/users/someuser/outbox?min_id=104135036335976677&page=true", "partOf": "https://example.org/users/someuser/outbox", "orderedItems": [ - { - "id": "https://example.org/users/someuser/statuses/1234/activity", - "type": "Create", - "actor": "https://example.org/users/someuser", - "published": "2020-05-08T01:23:45Z", - "to": ["https://example.org/users/someuser/followers"], - "cc": [ - "https://www.w3.org/ns/activitystreams#Public", - "https://example.com/users/FirstAuthor", - ], - "object": { - "id": "https://example.org/users/someuser/statuses/1234", - "type": "Note", - "summary": None, - "inReplyTo": "https://example.com/users/FirstAuthor/statuses/42", - "published": "2020-05-08T01:23:45Z", - "url": "https://example.org/@FirstAuthor/42", - "attributedTo": "https://example.org/users/someuser", - "to": ["https://example.org/users/someuser/followers"], - "cc": [ - "https://www.w3.org/ns/activitystreams#Public", - "https://example.com/users/FirstAuthor", - ], - "sensitive": False, - "atomUri": "https://example.org/users/someuser/statuses/1234", - "inReplyToAtomUri": "https://example.com/users/FirstAuthor/statuses/42", - "conversation": "tag:example.com,2020-05-08:objectId=aaaa:objectType=Conversation", - "content": '

@FirstAuthor I am replying to you

', - "contentMap": { - "en": '

@FirstAuthor I am replying to you

' - }, - "attachment": [], - "tag": [ - { - "type": "Mention", - "href": "https://example.com/users/FirstAuthor", - "name": "@FirstAuthor@example.com", - } - ], - "replies": { - "id": "https://example.org/users/someuser/statuses/1234/replies", - "type": "Collection", - "first": { - "type": "CollectionPage", - "next": "https://example.org/users/someuser/statuses/1234/replies?only_other_accounts=true&page=true", - "partOf": "https://example.org/users/someuser/statuses/1234/replies", - "items": [], - }, - }, - }, - }, + STATUS_VALUE, { "id": "https://example.org/users/someuser/statuses/1235/activity", "type": "Create", @@ -517,7 +521,12 @@ class FediverseTestCase(ChannelPluginTestCase): ) def testProfileSnarfer(self): - with conf.supybot.plugins.Fediverse.usernameSnarfer.context(True): + with self.mockRequests([]): + self.assertSnarfNoResponse( + "aaa @nonexistinguser@example.org bbb", timeout=1 + ) + + with conf.supybot.plugins.Fediverse.snarfers.username.context(True): expected_requests = [ (HOSTMETA_URL, HOSTMETA_DATA), (WEBFINGER_URL, WEBFINGER_DATA), @@ -540,6 +549,28 @@ class FediverseTestCase(ChannelPluginTestCase): "aaa @nonexistinguser@example.org bbb", timeout=1 ) + def testProfileUrlSnarfer(self): + with self.mockRequests([]): + self.assertSnarfNoResponse( + "aaa https://example.org/users/someuser bbb", timeout=1 + ) + + with conf.supybot.plugins.Fediverse.snarfers.profile.context(True): + expected_requests = [(ACTOR_URL, utils.web.Error("blah"))] + + with self.mockRequests(expected_requests): + self.assertSnarfNoResponse( + "aaa https://example.org/users/someuser bbb", timeout=1 + ) + + expected_requests = [(ACTOR_URL, ACTOR_DATA)] + + with self.mockRequests(expected_requests): + self.assertSnarfResponse( + "aaa https://example.org/users/someuser bbb", + "\x02someuser\x02 (@someuser@example.org): My Biography", + ) + def testProfileUnknown(self): expected_requests = [ (HOSTMETA_URL, HOSTMETA_DATA), @@ -596,5 +627,62 @@ class FediverseTestCase(ChannelPluginTestCase): + "Status Content", ) + def testStatusUrlSnarfer(self): + with self.mockRequests([]): + self.assertSnarfNoResponse( + "aaa https://example.org/users/someuser/statuses/1234 bbb", + timeout=1, + ) + + with conf.supybot.plugins.Fediverse.snarfers.status.context(True): + expected_requests = [ + (STATUS_URL, STATUS_DATA), + (ACTOR_URL, utils.web.Error("blah")), + ] + + with self.mockRequests(expected_requests): + self.assertSnarfNoResponse( + "aaa https://example.org/users/someuser/statuses/1234 bbb", + timeout=1, + ) + + expected_requests = [ + (STATUS_URL, STATUS_DATA), + (ACTOR_URL, ACTOR_DATA), + ] + + with self.mockRequests(expected_requests): + self.assertSnarfResponse( + "aaa https://example.org/users/someuser/statuses/1234 bbb", + "\x02someuser (@someuser@example.org)\x02: " + + "@ FirstAuthor I am replying to you", + ) + + + def testSnarferType(self): + # Sends a request, notices it's a status, gives up + with conf.supybot.plugins.Fediverse.snarfers.profile.context(True): + expected_requests = [ + (STATUS_URL, STATUS_DATA), + ] + + with self.mockRequests(expected_requests): + self.assertSnarfNoResponse( + "aaa https://example.org/users/someuser/statuses/1234 bbb", + timeout=1, + ) + + # Sends a request, notices it's a profile, gives up + with conf.supybot.plugins.Fediverse.snarfers.profile.context(True): + expected_requests = [ + (ACTOR_URL, ACTOR_DATA), + ] + + with self.mockRequests(expected_requests): + self.assertSnarfNoResponse( + "aaa https://example.org/users/someuser/ bbb", + timeout=1, + ) + # vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: