diff --git a/irc/channel.go b/irc/channel.go
index 12ad927f..b082926c 100644
--- a/irc/channel.go
+++ b/irc/channel.go
@@ -748,10 +748,15 @@ func (channel *Channel) applyModeMask(client *Client, mode modes.Mode, op modes.
 func (channel *Channel) Quit(client *Client) {
 	channel.stateMutex.Lock()
 	channel.members.Remove(client)
+	empty := len(channel.members) == 0
 	channel.stateMutex.Unlock()
 	channel.regenerateMembersCache(false)
 
 	client.removeChannel(channel)
+
+	if empty {
+		client.server.channels.Cleanup(channel)
+	}
 }
 
 func (channel *Channel) Kick(client *Client, target *Client, comment string, rb *ResponseBuffer) {
diff --git a/irc/channelmanager.go b/irc/channelmanager.go
index 72fde389..8d2fa737 100644
--- a/irc/channelmanager.go
+++ b/irc/channelmanager.go
@@ -76,18 +76,20 @@ func (cm *ChannelManager) Join(client *Client, name string, key string, rb *Resp
 
 	entry.channel.Join(client, key, rb)
 
-	cm.maybeCleanup(entry, true)
+	cm.maybeCleanup(entry.channel, true)
 
 	return nil
 }
 
-func (cm *ChannelManager) maybeCleanup(entry *channelManagerEntry, afterJoin bool) {
+func (cm *ChannelManager) maybeCleanup(channel *Channel, afterJoin bool) {
 	cm.Lock()
 	defer cm.Unlock()
 
-	if entry.channel == nil {
+	entry := cm.chans[channel.NameCasefolded()]
+	if entry == nil || entry.channel != channel {
 		return
 	}
+
 	if afterJoin {
 		entry.pendingJoins -= 1
 	}
@@ -121,10 +123,13 @@ func (cm *ChannelManager) Part(client *Client, name string, message string, rb *
 		return errNoSuchChannel
 	}
 	entry.channel.Part(client, message, rb)
-	cm.maybeCleanup(entry, false)
 	return nil
 }
 
+func (cm *ChannelManager) Cleanup(channel *Channel) {
+	cm.maybeCleanup(channel, false)
+}
+
 // Rename renames a channel (but does not notify the members)
 func (cm *ChannelManager) Rename(name string, newname string) error {
 	cfname, err := CasefoldChannel(name)