diff --git a/irc/server.go b/irc/server.go
index ef2302ff..ba2ed34c 100644
--- a/irc/server.go
+++ b/irc/server.go
@@ -44,11 +44,14 @@ func NewServer(config *Config) *Server {
 		name:      config.Server.Name,
 		newConns:  make(chan net.Conn, 16),
 		operators: config.Operators(),
-		password:  config.Server.PasswordBytes(),
 		signals:   make(chan os.Signal, 1),
 		timeout:   make(chan *Client, 16),
 	}
 
+	if config.Server.Password != "" {
+		server.password = config.Server.PasswordBytes()
+	}
+
 	server.loadChannels()
 
 	for _, addr := range config.Server.Listen {