diff --git a/src/device.c b/src/device.c
index 3826987b..445aec9f 100644
--- a/src/device.c
+++ b/src/device.c
@@ -320,10 +320,6 @@ void device_connect_network(struct device *device, struct network *network,
 		else
 			eapol_sm_set_8021x_config(sm,
 					network_get_settings(network));
-
-		eapol_sm_set_user_data(sm, device);
-		eapol_sm_set_tx_user_data(sm,
-				L_INT_TO_PTR(l_io_get_fd(device->eapol_io)));
 	}
 
 	device->connect_pending = l_dbus_message_ref(message);
diff --git a/src/device.h b/src/device.h
index dab6a0ba..cde32379 100644
--- a/src/device.h
+++ b/src/device.h
@@ -51,7 +51,6 @@ struct device {
 	struct l_queue *autoconnect_list;
 	struct l_dbus_message *connect_pending;
 	struct l_dbus_message *disconnect_pending;
-	struct l_io *eapol_io;
 
 	struct wiphy *wiphy;
 	struct netdev *netdev;
diff --git a/src/netdev.c b/src/netdev.c
index 7c60346c..86ffbb07 100644
--- a/src/netdev.c
+++ b/src/netdev.c
@@ -28,7 +28,9 @@
 #include <linux/rtnetlink.h>
 #include <net/if_arp.h>
 #include <linux/if.h>
+#include <linux/if_packet.h>
 #include <linux/if_ether.h>
+#include <sys/socket.h>
 #include <errno.h>
 
 #include <ell/ell.h>
@@ -51,6 +53,8 @@ struct netdev {
 	uint32_t type;
 	uint8_t addr[ETH_ALEN];
 
+	struct l_io *eapol_io;
+
 	netdev_event_func_t event_filter;
 	netdev_connect_cb_t connect_cb;
 	void *user_data;
@@ -74,6 +78,31 @@ static void do_debug(const char *str, void *user_data)
 	l_info("%s%s", prefix, str);
 }
 
+static bool eapol_read(struct l_io *io, void *user_data)
+{
+	struct netdev *netdev = user_data;
+	int fd = l_io_get_fd(io);
+	struct sockaddr_ll sll;
+	socklen_t sll_len;
+	ssize_t bytes;
+	uint8_t frame[2304]; /* IEEE Std 802.11 ch. 8.2.3 */
+
+	memset(&sll, 0, sizeof(sll));
+	sll_len = sizeof(sll);
+
+	bytes = recvfrom(fd, frame, sizeof(frame), 0,
+				(struct sockaddr *) &sll, &sll_len);
+	if (bytes <= 0) {
+		l_error("EAPoL read socket: %s", strerror(errno));
+		return false;
+	}
+
+	__eapol_rx_packet(netdev->index, netdev->addr, sll.sll_addr,
+								frame, bytes);
+
+	return true;
+}
+
 struct cb_data {
 	netdev_command_func_t callback;
 	void *user_data;
@@ -175,6 +204,9 @@ static void netdev_free(void *data)
 		netdev->associate_msg = NULL;
 	}
 
+	l_io_destroy(netdev->eapol_io);
+	netdev->eapol_io = NULL;
+
 	netdev_set_linkmode_and_operstate(netdev->index, 0, IF_OPER_DOWN,
 						NULL, NULL);
 
@@ -665,7 +697,10 @@ static void netdev_associate_event(struct l_genl_msg *msg,
 	}
 
 	if (netdev->sm) {
+		eapol_sm_set_tx_user_data(netdev->sm,
+				L_INT_TO_PTR(l_io_get_fd(netdev->eapol_io)));
 		eapol_start(netdev->index, netdev->sm);
+
 		netdev->sm = NULL;
 
 		if (netdev->event_filter)
@@ -1015,6 +1050,13 @@ static void netdev_get_interface_callback(struct l_genl_msg *msg,
 	memcpy(netdev->addr, ifaddr, sizeof(netdev->addr));
 	memcpy(netdev->name, ifname, ifname_len);
 
+	netdev->eapol_io = eapol_open_pae(netdev->index);
+	if (netdev->eapol_io)
+		l_io_set_read_handler(netdev->eapol_io, eapol_read,
+								netdev, NULL);
+	else
+		l_error("Failed to open PAE socket");
+
 	l_queue_push_tail(netdev_list, netdev);
 
 	netdev_set_linkmode_and_operstate(netdev->index, 1,
diff --git a/src/wiphy.c b/src/wiphy.c
index 517b5b02..8e65b207 100644
--- a/src/wiphy.c
+++ b/src/wiphy.c
@@ -27,9 +27,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
-#include <sys/socket.h>
-#include <linux/if.h>
-#include <linux/if_packet.h>
 #include <linux/if_ether.h>
 #include <time.h>
 
@@ -76,33 +73,6 @@ static struct l_queue *device_list;
 static bool new_scan_results(uint32_t wiphy_id, uint32_t ifindex,
 				struct l_queue *bss_list, void *userdata);
 
-static bool eapol_read(struct l_io *io, void *user_data)
-{
-	struct device *device = user_data;
-	struct netdev *netdev = device->netdev;
-	int fd = l_io_get_fd(io);
-	struct sockaddr_ll sll;
-	socklen_t sll_len;
-	ssize_t bytes;
-	uint8_t frame[2304]; /* IEEE Std 802.11 ch. 8.2.3 */
-
-	memset(&sll, 0, sizeof(sll));
-	sll_len = sizeof(sll);
-
-	bytes = recvfrom(fd, frame, sizeof(frame), 0,
-				(struct sockaddr *) &sll, &sll_len);
-	if (bytes <= 0) {
-		l_error("EAPoL read socket: %s", strerror(errno));
-		return false;
-	}
-
-	__eapol_rx_packet(netdev_get_ifindex(netdev),
-				netdev_get_address(netdev),
-				sll.sll_addr, frame, bytes);
-
-	return true;
-}
-
 static const char *iwd_network_get_path(struct device *device,
 					const uint8_t *ssid, size_t ssid_len,
 					enum security security)
@@ -658,13 +628,6 @@ struct device *device_create(struct wiphy *wiphy, struct netdev *netdev)
 	scan_ifindex_add(device->index);
 	device_enter_state(device, DEVICE_STATE_AUTOCONNECT);
 
-	device->eapol_io = eapol_open_pae(device->index);
-	if (device->eapol_io)
-		l_io_set_read_handler(device->eapol_io, eapol_read,
-								device, NULL);
-	else
-		l_error("Failed to open PAE socket");
-
 	return device;
 }
 
@@ -694,7 +657,6 @@ static void device_free(void *user)
 	l_queue_destroy(device->bss_list, bss_free);
 	l_queue_destroy(device->old_bss_list, bss_free);
 	l_queue_destroy(device->autoconnect_list, l_free);
-	l_io_destroy(device->eapol_io);
 
 	scan_ifindex_remove(device->index);
 	l_free(device);