diff --git a/irc/channel.go b/irc/channel.go
index 337102fd..75cab1a2 100644
--- a/irc/channel.go
+++ b/irc/channel.go
@@ -239,9 +239,12 @@ func (channel *Channel) Names(client *Client, rb *ResponseBuffer) {
 // ClientIsAtLeast returns whether the client has at least the given channel privilege.
 func (channel *Channel) ClientIsAtLeast(client *Client, permission modes.Mode) bool {
 	channel.stateMutex.RLock()
-	defer channel.stateMutex.RUnlock()
-
 	clientModes := channel.members[client]
+	channel.stateMutex.RUnlock()
+
+	if clientModes == nil {
+		return false
+	}
 
 	for _, mode := range modes.ChannelUserModes {
 		if clientModes.HasMode(mode) {
diff --git a/irc/modes.go b/irc/modes.go
index 0fee6d78..461f8d78 100644
--- a/irc/modes.go
+++ b/irc/modes.go
@@ -108,7 +108,6 @@ func (channel *Channel) ApplyChannelModeChanges(client *Client, isSamode bool, c
 	// so we only output one warning for each list type when full
 	listFullWarned := make(map[modes.Mode]bool)
 
-	clientIsOp := channel.ClientIsAtLeast(client, modes.ChannelOperator)
 	var alreadySentPrivError bool
 
 	applied := make(modes.ModeChanges, 0)
@@ -141,9 +140,9 @@ func (channel *Channel) ApplyChannelModeChanges(client *Client, isSamode bool, c
 			return channel.ClientIsAtLeast(client, change.Mode)
 		case modes.BanMask:
 			// #163: allow unprivileged users to list ban masks
-			return clientIsOp || isListOp(change)
+			return isListOp(change) || channel.ClientIsAtLeast(client, modes.ChannelOperator)
 		default:
-			return clientIsOp
+			return channel.ClientIsAtLeast(client, modes.ChannelOperator)
 		}
 	}