From 87639ddeb22ad0fbad788fdc210a097d80323bc1 Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 30 Aug 2017 19:16:54 -0700 Subject: [PATCH 1/9] classes: add a has_eob attribute to each server object --- classes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/classes.py b/classes.py index 9536746..f6cd3f7 100644 --- a/classes.py +++ b/classes.py @@ -1619,6 +1619,9 @@ class Server(): self.desc = desc self._irc = irc + # Has the server finished bursting yet? + self.has_eob = False + def __repr__(self): return 'Server(%s)' % self.name From 9a84dbde71c526061a77224226255cf527e8b7b3 Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 30 Aug 2017 19:18:39 -0700 Subject: [PATCH 2/9] protocols: consistently track ENDBURST on sub-servers too --- protocols/hybrid.py | 11 +++++------ protocols/inspircd.py | 1 + protocols/ngircd.py | 18 +++++++++++------- protocols/p10.py | 9 ++++++--- protocols/ts6.py | 10 +++------- protocols/unreal.py | 1 + 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/protocols/hybrid.py b/protocols/hybrid.py index f5672ae..f09e1a6 100644 --- a/protocols/hybrid.py +++ b/protocols/hybrid.py @@ -17,13 +17,11 @@ class HybridProtocol(TS6Protocol): self.casemapping = 'ascii' self.caps = {} self.hook_map = {'EOB': 'ENDBURST', 'TBURST': 'TOPIC', 'SJOIN': 'JOIN'} - self.has_eob = False self.protocol_caps -= {'slash-in-hosts'} def post_connect(self): """Initializes a connection to a server.""" ts = self.start_ts - self.has_eob = False f = self.send # https://github.com/grawity/irc-docs/blob/master/server/ts6.txt#L80 @@ -227,12 +225,13 @@ class HybridProtocol(TS6Protocol): return {'channel': channel, 'setter': setter, 'ts': ts, 'text': topic} def handle_eob(self, numeric, command, args): - log.debug('(%s) end of burst received', self.name) - if not self.has_eob: # Only call ENDBURST hooks if we haven't already. + """EOB (end-of-burst) handler.""" + log.debug('(%s) end of burst received from %s', self.name, numeric) + if not self.servers[numeric].has_eob: + # Don't fight with TS6's generic PING-as-EOB + self.servers[numeric].has_eob = True return {} - self.has_eob = True - def handle_svsmode(self, numeric, command, args): """ Handles SVSMODE, which is used for sending services metadata diff --git a/protocols/inspircd.py b/protocols/inspircd.py index 38fc127..8fd9087 100644 --- a/protocols/inspircd.py +++ b/protocols/inspircd.py @@ -716,6 +716,7 @@ class InspIRCdProtocol(TS6BaseProtocol): def handle_endburst(self, numeric, command, args): """ENDBURST handler; sends a hook with empty contents.""" + self.servers[numeric].has_eob = True return {} def handle_away(self, numeric, command, args): diff --git a/protocols/ngircd.py b/protocols/ngircd.py index 8714683..7dbc0ea 100644 --- a/protocols/ngircd.py +++ b/protocols/ngircd.py @@ -504,16 +504,20 @@ class NgIRCdProtocol(IRCS2SProtocol): assert 'IRC+' in args[1], "Linking to non-ngIRCd server using this protocol module is not supported" def handle_ping(self, source, command, args): - if source == self.uplink: - self._send_with_prefix(self.sid, 'PONG %s :%s' % (self._expandPUID(self.sid), args[-1]), queue=False) + """ + Handles incoming PINGs (and implicit end of burst). + """ + self._send_with_prefix(self.sid, 'PONG %s :%s' % (self._expandPUID(self.sid), args[-1]), queue=False) - if not self.has_eob: - # Treat the first PING we receive as end of burst. - self.has_eob = True + if not self.servers[source].has_eob: + # Treat the first PING we receive as end of burst. + self.servers[source].has_eob = True + + if source == self.uplink: self.connected.set() - # Return the endburst hook. - return {'parse_as': 'ENDBURST'} + # Return the endburst hook. + return {'parse_as': 'ENDBURST'} def handle_server(self, source, command, args): """ diff --git a/protocols/p10.py b/protocols/p10.py index 264a52c..abd3cbb 100644 --- a/protocols/p10.py +++ b/protocols/p10.py @@ -1110,15 +1110,18 @@ class P10Protocol(IRCS2SProtocol): return {'channel': channel, 'users': [source], 'modes': self._channels[channel].modes, 'ts': ts or int(time.time())} - handle_create = handle_join + def handle_end_of_burst(self, source, command, args): - """Handles end of burst from our uplink.""" + """Handles end of burst from servers.""" + # Send EOB acknowledgement; this is required by the P10 specification, # and needed if we want to be able to receive channel messages, etc. if source == self.uplink: self._send_with_prefix(self.sid, 'EA') - return {} + + self.servers[source].has_eob = True + return {} def handle_kick(self, source, command, args): """Handles incoming KICKs.""" diff --git a/protocols/ts6.py b/protocols/ts6.py index 2c42bd4..93c83c0 100644 --- a/protocols/ts6.py +++ b/protocols/ts6.py @@ -20,9 +20,6 @@ class TS6Protocol(TS6BaseProtocol): self.hook_map = {'SJOIN': 'JOIN', 'TB': 'TOPIC', 'TMODE': 'MODE', 'BMASK': 'MODE', 'EUID': 'UID', 'RSFNC': 'SVSNICK', 'ETB': 'TOPIC', 'USERMODE': 'MODE'} - # Track whether we've received end-of-burst from the uplink. - self.has_eob = False - self.required_caps = {'EUID', 'SAVE', 'TB', 'ENCAP', 'QS', 'CHW'} # From ChatIRCd: https://github.com/ChatLounge/ChatIRCd/blob/master/doc/technical/ChatIRCd-extra.txt @@ -264,7 +261,6 @@ class TS6Protocol(TS6BaseProtocol): def post_connect(self): """Initializes a connection to a server.""" ts = self.start_ts - self.has_eob = False f = self.send @@ -445,10 +441,10 @@ class TS6Protocol(TS6BaseProtocol): if self.is_internal_server(destination): self._send_with_prefix(destination, 'PONG %s %s' % (destination, source), queue=False) - if destination == self.sid and not self.has_eob: - # Charybdis' endburst is just sending a PING to the other server. + if not self.servers[source].has_eob: + # TS6 endburst is just sending a PING to the other server. # https://github.com/charybdis-ircd/charybdis/blob/dc336d1/modules/core/m_server.c#L484-L485 - self.has_eob = True + self.servers[source].has_eob = True # Return the endburst hook. return {'parse_as': 'ENDBURST'} diff --git a/protocols/unreal.py b/protocols/unreal.py index 18444b9..985faa8 100644 --- a/protocols/unreal.py +++ b/protocols/unreal.py @@ -379,6 +379,7 @@ class UnrealProtocol(TS6BaseProtocol): def handle_eos(self, numeric, command, args): """EOS is used to denote end of burst.""" + self.servers[numeric].has_eob = True return {} def handle_uid(self, numeric, command, args): From 21b8b51cba98f0f8e8208748758f4d00347374d4 Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 30 Aug 2017 19:21:43 -0700 Subject: [PATCH 3/9] conf: fix getDatabaseName calling the wrong variable name --- conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.py b/conf.py index 2d81270..310c935 100644 --- a/conf.py +++ b/conf.py @@ -144,6 +144,6 @@ def getDatabaseName(dbname): if this is called from an instance running as './pylink testing.yml', it would return '-testing.db'.""" if confname != 'pylink': - dbname += '-%s' % conf.confname + dbname += '-%s' % confname dbname += '.db' return dbname From cad55097f14d44a71911ffcb57e5aa140c81b77d Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 30 Aug 2017 19:29:26 -0700 Subject: [PATCH 4/9] core: reuse existing service client UIDs for all service bots This prevents nick collision wars caused by spawn_service when an ENDBURST hook for the uplink is received multiple times. --- classes.py | 20 ++++++++++---------- coremods/service_support.py | 9 +++++---- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/classes.py b/classes.py index f6cd3f7..f98697f 100644 --- a/classes.py +++ b/classes.py @@ -468,16 +468,13 @@ class PyLinkNetworkCore(structures.DeprecatedAttributesObject, structures.CamelC if not userobj: return False - # Look for the "service" attribute in the User object, if one exists. - try: - sname = userobj.service - # Warn if the service name we fetched isn't a registered service. - if sname not in world.services.keys(): - log.warning("(%s) User %s / %s had a service bot record to a service that doesn't " - "exist (%s)!", self.name, uid, userobj.nick, sname) - return world.services.get(sname) - except AttributeError: - return False + # Look for the "service" attribute in the User object,sname = userobj.service + # Warn if the service name we fetched isn't a registered service. + sname = userobj.service + if sname not in world.services.keys(): + log.warning("(%s) User %s / %s had a service bot record to a service that doesn't " + "exist (%s)!", self.name, uid, userobj.nick, sname) + return world.services.get(sname) structures._BLACKLISTED_COPY_TYPES.append(PyLinkNetworkCore) @@ -1596,6 +1593,9 @@ class User(): # Cloaked host for IRCds that use it self.cloaked_host = None + # Stores service bot name if applicable + self.service = None + def __repr__(self): return 'User(%s/%s)' % (self.uid, self.nick) IrcUser = User diff --git a/coremods/service_support.py b/coremods/service_support.py index 9b8b2bf..9031520 100644 --- a/coremods/service_support.py +++ b/coremods/service_support.py @@ -22,10 +22,11 @@ def spawn_service(irc, source, command, args): # Get the ServiceBot object. sbot = world.services[name] - if name == 'pylink' and irc.pseudoclient: - # irc.pseudoclient already exists, for protocols like clientbot - log.debug('(%s) spawn_service: Using existing nick %r for service %r', irc.name, irc.pseudoclient.nick, name) - userobj = irc.pseudoclient + old_userobj = irc.users.get(sbot.uids.get(irc.name)) + if old_userobj and old_userobj.service: + # A client already exists, so reuse it. + log.debug('(%s) spawn_service: Using existing nick %r for service %r', irc.name, old_userobj.nick, name) + userobj = old_userobj userobj.opertype = "PyLink Service" userobj.manipulatable = sbot.manipulatable else: From 8170e777e8193478ba5be9279fb7501cbecbbbfa Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 30 Aug 2017 19:39:57 -0700 Subject: [PATCH 5/9] protocols: move setting irc.connected to endburst --- protocols/clientbot.py | 7 +++---- protocols/hybrid.py | 3 +++ protocols/inspircd.py | 5 ++--- protocols/p10.py | 2 +- protocols/ts6.py | 7 ++++--- protocols/unreal.py | 5 ++--- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/protocols/clientbot.py b/protocols/clientbot.py index 00b1daf..b763e14 100644 --- a/protocols/clientbot.py +++ b/protocols/clientbot.py @@ -633,10 +633,9 @@ class ClientbotWrapperProtocol(IRCCommonProtocol): self.send(line) # Virtual endburst hook. - self.connected.set() # Note, this should always be set before sending ENDBURST - if not self.has_eob: - self.has_eob = True - return {'parse_as': 'ENDBURST'} + self.connected.set() # Note, this should always be set before the actual ENDBURST hook + self.servers[self.uplink].has_eob = True + return {'parse_as': 'ENDBURST'} handle_422 = handle_376 diff --git a/protocols/hybrid.py b/protocols/hybrid.py index f09e1a6..60df8de 100644 --- a/protocols/hybrid.py +++ b/protocols/hybrid.py @@ -230,6 +230,9 @@ class HybridProtocol(TS6Protocol): if not self.servers[numeric].has_eob: # Don't fight with TS6's generic PING-as-EOB self.servers[numeric].has_eob = True + + if numeric == self.uplink: + self.connected.set() return {} def handle_svsmode(self, numeric, command, args): diff --git a/protocols/inspircd.py b/protocols/inspircd.py index 8fd9087..1e3bfa0 100644 --- a/protocols/inspircd.py +++ b/protocols/inspircd.py @@ -520,9 +520,6 @@ class InspIRCdProtocol(TS6BaseProtocol): log.debug('(%s) self.prefixmodes set to %r', self.name, self.prefixmodes) - # Finally, set the irc.connected (protocol negotiation complete) - # state to True. - self.connected.set() elif args[0] == 'MODSUPPORT': # <- CAPAB MODSUPPORT :m_alltime.so m_check.so m_chghost.so m_chgident.so m_chgname.so m_fullversion.so m_gecosban.so m_knock.so m_muteban.so m_nicklock.so m_nopartmsg.so m_opmoderated.so m_sajoin.so m_sanick.so m_sapart.so m_serverban.so m_services_account.so m_showwhois.so m_silence.so m_swhois.so m_uninvite.so m_watch.so self.modsupport = args[-1].split() @@ -717,6 +714,8 @@ class InspIRCdProtocol(TS6BaseProtocol): def handle_endburst(self, numeric, command, args): """ENDBURST handler; sends a hook with empty contents.""" self.servers[numeric].has_eob = True + if numeric == self.uplink: + self.connected.set() return {} def handle_away(self, numeric, command, args): diff --git a/protocols/p10.py b/protocols/p10.py index abd3cbb..4af3421 100644 --- a/protocols/p10.py +++ b/protocols/p10.py @@ -824,7 +824,6 @@ class P10Protocol(IRCS2SProtocol): self.send('SERVER %s 1 %s %s J10 %s]]] +s6 :%s' % (name, ts, ts, sid, desc)) self._send_with_prefix(sid, "EB") - self.connected.set() def handle_server(self, source, command, args): """Handles incoming server introductions.""" @@ -1119,6 +1118,7 @@ class P10Protocol(IRCS2SProtocol): # and needed if we want to be able to receive channel messages, etc. if source == self.uplink: self._send_with_prefix(self.sid, 'EA') + self.connected.set() self.servers[source].has_eob = True return {} diff --git a/protocols/ts6.py b/protocols/ts6.py index 93c83c0..aab7dd4 100644 --- a/protocols/ts6.py +++ b/protocols/ts6.py @@ -419,9 +419,6 @@ class TS6Protocol(TS6BaseProtocol): if 'SERVICES' in caps: self.cmodes['regonly'] = 'r' - log.debug('(%s) self.connected set!', self.name) - self.connected.set() - def handle_ping(self, source, command, args): """Handles incoming PING commands.""" # PING: @@ -446,6 +443,10 @@ class TS6Protocol(TS6BaseProtocol): # https://github.com/charybdis-ircd/charybdis/blob/dc336d1/modules/core/m_server.c#L484-L485 self.servers[source].has_eob = True + if source == self.uplink: + log.debug('(%s) self.connected set!', self.name) + self.connected.set() + # Return the endburst hook. return {'parse_as': 'ENDBURST'} diff --git a/protocols/unreal.py b/protocols/unreal.py index 985faa8..ad9b2d2 100644 --- a/protocols/unreal.py +++ b/protocols/unreal.py @@ -380,6 +380,8 @@ class UnrealProtocol(TS6BaseProtocol): def handle_eos(self, numeric, command, args): """EOS is used to denote end of burst.""" self.servers[numeric].has_eob = True + if numeric == self.uplink: + self.connected.set() return {} def handle_uid(self, numeric, command, args): @@ -483,9 +485,6 @@ class UnrealProtocol(TS6BaseProtocol): "(Unreal 4.x), got %s)" % (self.min_proto_ver, protover)) self.servers[numeric] = Server(self, None, sname, desc=sdesc) - # Set irc.connected to True, meaning that protocol negotiation passed. - log.debug('(%s) self.connected set!', self.name) - self.connected.set() else: # Legacy (non-SID) servers can still be introduced using the SERVER command. # <- :services.int SERVER a.bc 2 :(H) [GL] a From bc48709595d8c1808ff42e3a08d74f36ebd7aafb Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 30 Aug 2017 19:48:46 -0700 Subject: [PATCH 6/9] PyLinkNetworkCore: fix extraneous warnings in get_service_bot --- classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes.py b/classes.py index f98697f..df03bdf 100644 --- a/classes.py +++ b/classes.py @@ -471,7 +471,7 @@ class PyLinkNetworkCore(structures.DeprecatedAttributesObject, structures.CamelC # Look for the "service" attribute in the User object,sname = userobj.service # Warn if the service name we fetched isn't a registered service. sname = userobj.service - if sname not in world.services.keys(): + if sname is not None and sname not in world.services.keys(): log.warning("(%s) User %s / %s had a service bot record to a service that doesn't " "exist (%s)!", self.name, uid, userobj.nick, sname) return world.services.get(sname) From 94e05a623314e9b0607de4eb01fab28be2e0c7e1 Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 30 Aug 2017 19:50:25 -0700 Subject: [PATCH 7/9] services_support: fix clientbot service spawning when irc.pseudoclient exists but isn't in the user index --- coremods/service_support.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/coremods/service_support.py b/coremods/service_support.py index 9031520..c29e073 100644 --- a/coremods/service_support.py +++ b/coremods/service_support.py @@ -24,9 +24,17 @@ def spawn_service(irc, source, command, args): old_userobj = irc.users.get(sbot.uids.get(irc.name)) if old_userobj and old_userobj.service: - # A client already exists, so reuse it. - log.debug('(%s) spawn_service: Using existing nick %r for service %r', irc.name, old_userobj.nick, name) - userobj = old_userobj + # A client already exists, so don't respawn it. + log.debug('(%s) spawn_service: Not respawning service %r as service client %r already exists.', irc.name, name, + irc.pseudoclient.nick) + return + + if name == 'pylink' and irc.pseudoclient: + # irc.pseudoclient already exists, reuse values from it but + # spawn a new client. This is used for protocols like Clientbot, + # so that they can override the main service nick, among other things. + log.debug('(%s) spawn_service: Using existing nick %r for service %r', irc.name, irc.pseudoclient.nick, name) + userobj = irc.pseudoclient userobj.opertype = "PyLink Service" userobj.manipulatable = sbot.manipulatable else: From 450718cce62f24efdca98b292c980c2a3815b661 Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 31 Aug 2017 12:27:52 -0700 Subject: [PATCH 8/9] relay: don't block on client spawning Just fail instantly if the remote isn't ready. --- plugins/relay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/relay.py b/plugins/relay.py index 6d4f7d3..66a5e2c 100644 --- a/plugins/relay.py +++ b/plugins/relay.py @@ -217,7 +217,7 @@ def get_prefix_modes(irc, remoteirc, channel, user, mlist=None): return modes def spawn_relay_server(irc, remoteirc): - if irc.connected.wait(TCONDITION_TIMEOUT): + if irc.connected.is_set(): try: # ENDBURST is delayed by 3 secs on supported IRCds to prevent # triggering join-flood protection and the like. @@ -358,7 +358,7 @@ def get_remote_user(irc, remoteirc, user, spawn_if_missing=True, times_tagged=0) spawning one if it doesn't exist and spawn_if_missing is True.""" # Wait until the network is working before trying to spawn anything. - if irc.connected.wait(TCONDITION_TIMEOUT): + if irc.connected.is_set(): # Don't spawn clones for registered service bots. sbot = irc.get_service_bot(user) if sbot: From ac89f456833b95826782378be86e49ca7905241f Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 31 Aug 2017 13:16:43 -0700 Subject: [PATCH 9/9] ngircd: rework NJOIN code - Fix "Internal NJOIN error"s caused by joining users already in the channel again - Fix NJOIN being sent from the wrong internal server - Condense two iterations over the user list into one --- protocols/ngircd.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/protocols/ngircd.py b/protocols/ngircd.py index 7dbc0ea..15a92e0 100644 --- a/protocols/ngircd.py +++ b/protocols/ngircd.py @@ -229,26 +229,32 @@ class NgIRCdProtocol(IRCS2SProtocol): raise LookupError('No such PyLink client exists.') log.debug('(%s) sjoin: got %r for users', self.name, users) - njoin_prefix = ':%s NJOIN %s :' % (self._expandPUID(self.sid), channel) + njoin_prefix = ':%s NJOIN %s :' % (self._expandPUID(server), channel) # Format the user list into strings such as @user1, +user2, user3, etc. - nicks_to_send = ['%s%s' % (''.join(self.prefixmodes[modechar] for modechar in userpair[0] if modechar in self.prefixmodes), - self._expandPUID(userpair[1])) for userpair in users] - - # Use 13 args max per line: this is equal to the max of 15 minus the command name and target channel. - for message in utils.wrapArguments(njoin_prefix, nicks_to_send, self.S2S_BUFSIZE, separator=',', max_args_per_line=13): - self.send(message) - - # Add the affected users to our state. + nicks_to_send = [] for userpair in users: - uid = userpair[1] + prefixes, uid = userpair + + if uid not in self.users: + log.warning('(%s) Trying to NJOIN missing user %s?', self.name, uid) + continue + elif uid in self._channels[channel].users: + # Don't rejoin users already in the channel, this causes errors with ngIRCd. + continue + self._channels[channel].users.add(uid) - try: - self.users[uid].channels.add(channel) - except KeyError: # Not initialized yet? - log.warning("(%s) sjoin: KeyError trying to add %r to %r's channel list?", self.name, channel, uid) + self.users[uid].channels.add(channel) self.apply_modes(channel, (('+%s' % prefix, uid) for prefix in userpair[0])) + nicks_to_send.append(''.join(self.prefixmodes[modechar] for modechar in userpair[0]) + \ + self._expandPUID(userpair[1])) + + if nicks_to_send: + # Use 13 args max per line: this is equal to the max of 15 minus the command name and target channel. + for message in utils.wrapArguments(njoin_prefix, nicks_to_send, self.S2S_BUFSIZE, separator=',', max_args_per_line=13): + self.send(message) + if modes: # Burst modes separately if there are any. log.debug("(%s) sjoin: bursting modes %r for channel %r now", self.name, modes, channel)