diff --git a/Makefile b/Makefile
index 3a8ea6750e..0e9609f69c 100644
--- a/Makefile
+++ b/Makefile
@@ -271,11 +271,7 @@ LIBS-y += drivers/power/libpower.o \
 	drivers/power/pmic/libpmic.o \
 	drivers/power/battery/libbattery.o
 LIBS-y += drivers/spi/libspi.o
-ifeq ($(CPU),mpc83xx)
-LIBS-y += drivers/qe/libqe.o
-endif
 ifeq ($(CPU),mpc85xx)
-LIBS-y += drivers/qe/libqe.o
 LIBS-y += drivers/net/fm/libfm.o
 endif
 LIBS-y += drivers/serial/libserial.o
diff --git a/drivers/Makefile b/drivers/Makefile
index 95a4b80724..5d03f37a18 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -13,3 +13,4 @@ obj-y += tpm/
 obj-y += twserial/
 obj-y += video/
 obj-y += watchdog/
+obj-$(CONFIG_QE) += qe/
diff --git a/drivers/qe/Makefile b/drivers/qe/Makefile
index b8c15f8e1f..7f1bd06922 100644
--- a/drivers/qe/Makefile
+++ b/drivers/qe/Makefile
@@ -4,5 +4,5 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
-obj-$(and $(CONFIG_QE),$(CONFIG_OF_LIBFDT)) += fdt.o
-obj-$(CONFIG_QE) += qe.o uccf.o uec.o uec_phy.o
+obj-y := qe.o uccf.o uec.o uec_phy.o
+obj-$(CONFIG_OF_LIBFDT) += fdt.o