diff --git a/src/irclib.py b/src/irclib.py
index 4502e850a..b16422d6c 100644
--- a/src/irclib.py
+++ b/src/irclib.py
@@ -280,7 +280,7 @@ class Irc(object):
                 toRemove.append(i)
         if toRemove:
             for i in reviter(range(len(self.callbacks))):
-                if toRemove[-1] == i:
+                if toRemove and toRemove[-1] == i:
                     toRemove.pop()
                     ret.append(self.callbacks.pop(i))
         return ret