diff --git a/plugins/relay.py b/plugins/relay.py
index bc13eb7..a1b72f9 100644
--- a/plugins/relay.py
+++ b/plugins/relay.py
@@ -475,10 +475,11 @@ def relayModes(irc, remoteirc, sender, channel, modes=None):
     if supported_modes:
         # Check if the sender is a user; remember servers are allowed to set modes too.
         if sender in irc.users:
-            u = getRemoteUser(irc, remoteirc, sender)
-            remoteirc.proto.modeClient(remoteirc, u, remotechan, supported_modes)
-        else:
-            remoteirc.proto.modeServer(remoteirc, remoteirc.sid, remotechan, supported_modes)
+            u = getRemoteUser(irc, remoteirc, sender, spawnIfMissing=False)
+            if u:
+                remoteirc.proto.modeClient(remoteirc, u, remotechan, supported_modes)
+                return
+        remoteirc.proto.modeServer(remoteirc, remoteirc.sid, remotechan, supported_modes)
 
 def getSupportedUmodes(irc, remoteirc, modes):
     supported_modes = []